Go to the documentation of this file.
30 #undef ZYPP_BASE_LOGGER_LOGGROUP
31 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
67 if ( lhs->location.medianr() == rhs->location.medianr() )
68 return lhs->location.filename() < rhs->location.filename();
70 return lhs->location.medianr() < rhs->location.medianr();
118 return str << obj->location;
136 Fetcher::Options
options()
const;
154 static shared_ptr<Impl> _nullimpl(
new Impl );
196 void validate(
const Pathname & localfile_r,
const std::list<FileChecker> & checkers_r );
202 const Pathname &dest_dir, FetcherJob::Flags flags );
217 friend Impl * rwcowClone<Impl>(
const Impl * rhs );
220 {
return new Impl( *
this ); }
223 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
237 job.reset(
new FetcherJob(resource, deltafile));
260 job->checkers.push_back(checker);
265 _resources.push_back(job);
275 job->checkers.push_back(checker);
281 _resources.push_back(job);
290 job->checkers.push_back(checker);
291 _resources.push_back(job);
296 MIL <<
"adding index " << resource << endl;
316 DBG <<
"Adding fetcher cache: '" << cache_dir <<
"'." << endl;
317 _caches.insert(cache_dir);
322 ERR <<
"Not adding cache: '" << cache_dir <<
"'. Not a directory." << endl;
327 ERR <<
"Not adding cache '" << cache_dir <<
"'. Path does not exists." << endl;
343 swap( ret, cacheLocation );
347 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
348 for(
const Pathname & cacheDir : _caches )
350 cacheLocation = cacheDir / resource_r.
filename();
353 MIL <<
"file " << resource_r.
filename() <<
" found in cache " << cacheDir << endl;
354 swap( ret, cacheLocation );
366 MIL <<
"Checking job [" << localfile_r <<
"] (" << checkers_r.size() <<
" checkers )" << endl;
371 chkfnc( localfile_r );
373 ERR <<
"Invalid checker for '" << localfile_r <<
"'" << endl;
396 auto fnc_addIfInContent( [&](
const std::string & index_r ) ->
bool
405 downloadAndReadIndexList( media, dest_dir );
411 fnc_addIfInContent(
"CHECKSUMS" ) || fnc_addIfInContent(
"SHA1SUMS" );
415 fnc_addIfInContent(
"content" );
424 != _dircontent.end() )
428 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
437 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
444 const Pathname &dest_dir, FetcherJob::Flags flags )
448 MIL <<
"Adding directory " << resource.
filename() << endl;
451 getDirectoryContent(media, resource, content);
456 WAR <<
"Skiping subtree hidden at " << resource.
filename() << endl;
462 autoaddIndexes(content, media, resource, dest_dir);
464 for ( filesystem::DirContent::const_iterator it = content.begin();
480 if ( _checksums.find(filename.
asString()) != _checksums.end() )
483 chksm = _checksums[filename.
asString()];
487 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
497 addDirJobs(media, filename, dest_dir, flags);
512 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard;
515 Pathname tmpFile = locateInCache( resource, destDir_r );
516 if ( tmpFile.
empty() )
518 MIL <<
"Not found in cache, retrieving..." << endl;
527 if ( tmpFile == destFullPath )
531 validate( tmpFile, jobp_r->checkers );
534 if ( tmpFile == destFullPath )
538 if (
assert_dir( destFullPath->dirname() ) != 0 )
550 WAR <<
"optional resource " << resource <<
" could not be transferred." << endl;
570 { _repoindex = data_r; }
579 readChecksumsIndex(index, basedir);
580 else if ( index.
basename() ==
"content" )
581 readContentFileIndex(index, basedir);
583 WAR << index <<
": index file format not known" << endl;
591 MIL << index <<
" contains " << reader.
_repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
595 _checksums[(basedir + it->first).
asString()] = it->second;
602 std::ifstream in( index.
c_str() );
606 while (
getline( in, buffer ) )
609 if ( buffer[0] ==
'#' )
616 if ( buffer.empty() )
618 WAR <<
"Missing filename in CHECKSUMS file: " << index.
asString() <<
" (" <<
checksum <<
")" << endl;
631 MIL <<
"downloading index " << resource << endl;
657 fetcher.
start( dest_dir, media );
666 fetcher.
start( dest_dir, media );
673 WAR <<
"No public key specified by user for index '" << keyloc.
filename() <<
"'"<< endl;
677 fetcher.
start( dest_dir, media );
688 if ( _indexes.empty() )
690 MIL <<
"No indexes to read." << endl;
694 for_( it_idx, _indexes.begin(), _indexes.end() )
696 if ( (*it_idx)->read )
698 DBG <<
"Already read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
703 downloadIndex( media, (*it_idx)->location, dest_dir );
705 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().
dirname() );
707 MIL <<
"Remember read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
708 (*it_idx)->read =
true;
711 MIL <<
"done reading indexes" << endl;
721 progress.
sendTo(progress_receiver);
723 downloadAndReadIndexList(media, dest_dir);
730 addDirJobs(media, location, dest_dir, jobp->flags);
744 getDirectoryContent(media, jobp->location.filename().dirname(), content);
747 MIL <<
"Autodiscovering signed indexes on '"
748 << jobp->location.filename().dirname() <<
"' for '"
749 << jobp->location.filename() <<
"'" << endl;
751 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
755 getDirectoryContent(media,
Pathname(
"/"), content);
758 MIL <<
"Autodiscovering signed indexes on '"
760 << jobp->location.filename() <<
"'" << endl;
762 autoaddIndexes(content, media,
Pathname(
"/"), dest_dir);
767 if ( jobp->location.checksum().empty() )
769 if ( _checksums.find(jobp->location.filename().asString())
770 != _checksums.end() )
772 CheckSum chksm = _checksums[jobp->location.filename().asString()];
774 jobp->checkers.push_back(digest_check);
784 jobp->checkers.push_back(digest_check);
792 jobp->checkers.push_back(digest_check);
797 provideToDest( media, dest_dir, jobp );
799 if ( ! progress.
incr() )
807 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
Class to encapsulate the OnMediaLocation object and the FileChecker together.
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Impl * clone() const
clone for RWCOW_pointer
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
OnMediaLocation location
Index localtion.
bool isExist() const
Return whether valid stat info exists.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Base class for Exception.
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
DefaultIntegral< bool, false > read
Whether we read this index.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
shared_ptr< FetcherJob > FetcherJob_Ptr
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
void addCachePath(const Pathname &cache_dir)
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Checks for the validity of a signature.
FetcherIndex(const OnMediaLocation &loc)
std::set ordering (less semantic)
Parse repoindex part from a content file.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
int unlink(const Pathname &path)
Like 'unlink'.
void setOptions(Options options)
Set the Fetcher options.
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Options options() const
Get current options.
std::string getline(std::istream &str)
Read one line from stream.
Listentry returned by readdir.
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
static shared_ptr< Impl > nullimpl()
Offer default Impl.
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
void remember(const Exception &old_r)
Store an other Exception as history.
Wrapper class for ::stat/::lstat.
ZYPP_DECLARE_FLAGS(Flags, Flag)
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
std::list< FileChecker > checkers
const char * c_str() const
String representation.
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Fetcher::Options _options
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Easy-to use interface to the ZYPP dependency resolver.
std::list< DirEntry > DirContent
Returned by readdir.
std::list< FetcherJob_Ptr > _resources
std::string stripFirstWord(std::string &line, const bool ltrim_first)
std::map< std::string, filesystem::DirContent > _dircontent
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Fetcher::Options options() const
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
void setOptions(Fetcher::Options options)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
parser::susetags::RepoIndex_Ptr _repoindex
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
std::ostream & operator<<(std::ostream &str, const Exception &obj)
Pathname dirname() const
Return all but the last component od this path.
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
void reset()
Reset the transfer (jobs) list.
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
bool empty() const
Test for an empty path.
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
std::set< Pathname > _caches
std::string basename() const
Return the last component of this path.
std::map< std::string, CheckSum > _checksums
String related utilities and Regular expression matching.
void resetDispose()
Set no dispose function.
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
const std::string & asString() const
String representation.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Stream output.
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Maintain [min,max] and counter (value) for progress counting.
void addIndex(const OnMediaLocation &resource)
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
bool incr(value_type val_r=1)
Increment counter value (default by 1).