30 #undef ZYPP_BASE_LOGGER_LOGGROUP
31 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
67 if ( lhs->location.medianr() == rhs->location.medianr() )
68 return lhs->location.filename() < rhs->location.filename();
70 return lhs->location.medianr() < rhs->location.medianr();
118 return str << obj->location;
136 Fetcher::Options
options()
const;
147 void start(
const Pathname &dest_dir,
154 static shared_ptr<Impl> _nullimpl(
new Impl );
175 void readIndex(
const Pathname &index,
const Pathname &basedir );
196 void validate(
const Pathname & localfile_r,
const std::list<FileChecker> & checkers_r );
202 const Pathname &dest_dir, FetcherJob::Flags flags );
210 const Pathname &dest_dir );
217 friend Impl * rwcowClone<Impl>(
const Impl * rhs );
220 {
return new Impl( *
this ); }
223 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
237 job.reset(
new FetcherJob(resource, deltafile));
260 job->checkers.push_back(checker);
265 _resources.push_back(job);
275 job->checkers.push_back(checker);
281 _resources.push_back(job);
290 job->checkers.push_back(checker);
291 _resources.push_back(job);
296 MIL <<
"adding index " << resource << endl;
311 PathInfo info(cache_dir);
312 if ( info.isExist() )
316 DBG <<
"Adding fetcher cache: '" << cache_dir <<
"'." << endl;
317 _caches.insert(cache_dir);
322 ERR <<
"Not adding cache: '" << cache_dir <<
"'. Not a directory." << endl;
327 ERR <<
"Not adding cache '" << cache_dir <<
"'. Path does not exists." << endl;
340 Pathname cacheLocation = destDir_r / resource_r.
filename();
341 if ( PathInfo(cacheLocation).isExist() &&
is_checksum( cacheLocation, resource_r.
checksum() ) )
343 swap( ret, cacheLocation );
347 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
348 for(
const Pathname & cacheDir : _caches )
350 cacheLocation = cacheDir / resource_r.
filename();
351 if ( PathInfo(cacheLocation).isExist() &&
is_checksum( cacheLocation, resource_r.
checksum() ) )
353 MIL <<
"file " << resource_r.
filename() <<
" found in cache " << cacheDir << endl;
354 swap( ret, cacheLocation );
366 MIL <<
"Checking job [" << localfile_r <<
"] (" << checkers_r.size() <<
" checkers )" << endl;
371 chkfnc( localfile_r );
373 ERR <<
"Invalid checker for '" << localfile_r <<
"'" << endl;
394 const Pathname &dest_dir )
396 auto fnc_addIfInContent( [&](
const std::string & index_r ) ->
bool
405 downloadAndReadIndexList( media, dest_dir );
411 fnc_addIfInContent(
"CHECKSUMS" ) || fnc_addIfInContent(
"SHA1SUMS" );
415 fnc_addIfInContent(
"content" );
423 if ( _dircontent.find(resource.
filename().asString())
424 != _dircontent.end() )
428 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
437 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
438 _dircontent[resource.
filename().asString()] = tofill;
444 const Pathname &dest_dir, FetcherJob::Flags flags )
448 MIL <<
"Adding directory " << resource.
filename() << endl;
451 getDirectoryContent(media, resource, content);
456 WAR <<
"Skiping subtree hidden at " << resource.
filename() << endl;
462 autoaddIndexes(content, media, resource, dest_dir);
464 for ( filesystem::DirContent::const_iterator it = content.begin();
472 Pathname filename = resource.
filename() + it->name;
480 if ( _checksums.find(filename.asString()) != _checksums.end() )
483 chksm = _checksums[filename.asString()];
487 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
497 addDirJobs(media, filename, dest_dir, flags);
512 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard;
515 Pathname tmpFile = locateInCache( resource, destDir_r );
516 if ( tmpFile.empty() )
518 MIL <<
"Not found in cache, retrieving..." << endl;
527 if ( tmpFile == destFullPath )
531 validate( tmpFile, jobp_r->checkers );
534 if ( tmpFile == destFullPath )
535 destFullPath.resetDispose();
538 if (
assert_dir( destFullPath->dirname() ) != 0 )
542 ZYPP_THROW(
Exception(
"Can't hardlink/copy " + tmpFile.asString() +
" to " + destDir_r.asString() ) );
550 WAR <<
"optional resource " << resource <<
" could not be transferred." << endl;
570 { _repoindex = data_r; }
578 if ( index.basename() ==
"CHECKSUMS" || index.basename() ==
"SHA1SUMS" )
579 readChecksumsIndex(index, basedir);
580 else if ( index.basename() ==
"content" )
581 readContentFileIndex(index, basedir);
583 WAR << index <<
": index file format not known" << endl;
591 MIL << index <<
" contains " << reader.
_repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
595 _checksums[(basedir + it->first).
asString()] = it->second;
602 std::ifstream in( index.c_str() );
606 while (
getline( in, buffer ) )
609 if ( buffer[0] ==
'#' )
613 if ( checksum.
empty() )
616 if ( buffer.empty() )
618 WAR <<
"Missing filename in CHECKSUMS file: " << index.asString() <<
" (" << checksum <<
")" << endl;
631 MIL <<
"downloading index " << resource << endl;
657 fetcher.
start( dest_dir, media );
659 if ( PathInfo(dest_dir + sigloc.
filename()).isExist() )
666 fetcher.
start( dest_dir, media );
670 if ( PathInfo(dest_dir + keyloc.
filename()).isExist() )
673 WAR <<
"No public key specified by user for index '" << keyloc.
filename() <<
"'"<< endl;
677 fetcher.
start( dest_dir, media );
688 if ( _indexes.empty() )
690 MIL <<
"No indexes to read." << endl;
694 for_( it_idx, _indexes.begin(), _indexes.end() )
696 if ( (*it_idx)->read )
698 DBG <<
"Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
703 downloadIndex( media, (*it_idx)->location, dest_dir );
705 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
707 MIL <<
"Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
708 (*it_idx)->read =
true;
711 MIL <<
"done reading indexes" << endl;
721 progress.
sendTo(progress_receiver);
723 downloadAndReadIndexList(media, dest_dir);
730 addDirJobs(media, location, dest_dir, jobp->flags);
744 getDirectoryContent(media, jobp->location.filename().dirname(), content);
747 MIL <<
"Autodiscovering signed indexes on '"
748 << jobp->location.filename().dirname() <<
"' for '"
749 << jobp->location.filename() <<
"'" << endl;
751 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
755 getDirectoryContent(media, Pathname(
"/"), content);
758 MIL <<
"Autodiscovering signed indexes on '"
760 << jobp->location.filename() <<
"'" << endl;
762 autoaddIndexes(content, media, Pathname(
"/"), dest_dir);
767 if ( jobp->location.checksum().empty() )
769 if ( _checksums.find(jobp->location.filename().asString())
770 != _checksums.end() )
772 CheckSum chksm = _checksums[jobp->location.filename().asString()];
774 jobp->checkers.push_back(digest_check);
784 jobp->checkers.push_back(digest_check);
792 jobp->checkers.push_back(digest_check);
797 provideToDest( media, dest_dir, jobp );
799 if ( ! progress.incr() )
807 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
881 return str << *obj.
_pimpl;
std::string asString(const Patch::Category &obj)
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Fetcher::Options options() const
shared_ptr< FetcherIndex > FetcherIndex_Ptr
void setOptions(Options options)
Set the Fetcher options.
Listentry returned by readdir.
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Checks for the validity of a signature.
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Options options() const
Get current options.
parser::susetags::RepoIndex_Ptr _repoindex
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory)...
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
If a content file is found, it is downloaded and read.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
void addCachePath(const Pathname &cache_dir)
OnMediaLocation location
Index localtion.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
std::list< FileChecker > checkers
static shared_ptr< Impl > nullimpl()
Offer default Impl.
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
void remember(const Exception &old_r)
Store an other Exception as history.
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
std::string getline(std::istream &str)
Read one line from stream.
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Fetcher::Options _options
std::ostream & operator<<(std::ostream &str, const Exception &obj)
int unlink(const Pathname &path)
Like 'unlink'.
void reset()
Reset the transfer (jobs) list.
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Parse repoindex part from a content file.
std::set ordering (less semantic)
Maintain [min,max] and counter (value) for progress counting.
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
std::list< DirEntry > DirContent
Returned by readdir.
DefaultIntegral< bool, false > read
Whether we read this index.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
std::map< std::string, filesystem::DirContent > _dircontent
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
std::list< FetcherJob_Ptr > _resources
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
ZYPP_DECLARE_FLAGS(Flags, Flag)
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
If a CHECKSUMS file is found, it is downloaded and read.
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Class to encapsulate the OnMediaLocation object and the FileChecker together.
void setOptions(Fetcher::Options options)
FetcherIndex(const OnMediaLocation &loc)
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Base class for Exception.
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
std::map< std::string, CheckSum > _checksums
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
std::set< Pathname > _caches
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
shared_ptr< FetcherJob > FetcherJob_Ptr
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs)
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Impl * clone() const
clone for RWCOW_pointer
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
void addIndex(const OnMediaLocation &resource)
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)