32 #undef ZYPP_BASE_LOGGER_LOGGROUP
33 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
69 if ( lhs->location.medianr() == rhs->location.medianr() )
70 return lhs->location.filename() < rhs->location.filename();
72 return lhs->location.medianr() < rhs->location.medianr();
87 RecursiveDirectory = Directory | Recursive,
90 AlwaysVerifyChecksum = 0x0004,
120 return str << obj->location;
137 void setOptions( Fetcher::Options options );
138 Fetcher::Options options()
const;
147 void addCachePath(
const Pathname &cache_dir );
149 void start(
const Pathname &dest_dir,
154 static shared_ptr<Impl> nullimpl()
156 static shared_ptr<Impl> _nullimpl(
new Impl );
163 void downloadAndReadIndexList(
MediaSetAccess &media,
const Pathname &dest_dir);
177 void readIndex(
const Pathname &index,
const Pathname &basedir );
180 void readChecksumsIndex(
const Pathname &index,
const Pathname &basedir );
183 void readContentFileIndex(
const Pathname &index,
const Pathname &basedir );
193 bool provideFromCache(
const OnMediaLocation &resource,
const Pathname &dest_dir );
199 void validate(
const OnMediaLocation &resource,
const Pathname &dest_dir,
const list<FileChecker> &checkers );
205 const Pathname &dest_dir, FetcherJob::Flags flags );
213 const Pathname &dest_dir );
220 friend Impl * rwcowClone<Impl>(
const Impl * rhs );
223 {
return new Impl( *
this ); }
226 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
240 job.reset(
new FetcherJob(resource, deltafile));
241 job->flags |= FetcherJob:: AlwaysVerifyChecksum;
242 _resources.push_back(job);
245 Fetcher::Impl::Impl()
263 job->checkers.push_back(checker);
268 _resources.push_back(job);
278 job->checkers.push_back(checker);
284 _resources.push_back(job);
293 job->checkers.push_back(checker);
294 _resources.push_back(job);
299 MIL <<
"adding index " << resource << endl;
314 PathInfo info(cache_dir);
315 if ( info.isExist() )
319 DBG <<
"Adding fetcher cache: '" << cache_dir <<
"'." << endl;
320 _caches.insert(cache_dir);
325 ERR <<
"Not adding cache: '" << cache_dir <<
"'. Not a directory." << endl;
330 ERR <<
"Not adding cache '" << cache_dir <<
"'. Path does not exists." << endl;
340 Pathname dest_full_path = dest_dir + resource.
filename();
343 if ( PathInfo(dest_full_path).isExist() )
350 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
351 for_ ( it_cache, _caches.begin(), _caches.end() )
354 Pathname cached_file = *it_cache + resource.
filename();
355 if ( PathInfo( cached_file ).isExist() )
357 DBG <<
"File '" << cached_file <<
"' exist, testing checksum " << resource.
checksum() << endl;
362 MIL <<
"file " << resource.
filename() <<
" found in previous cache. Using cached copy." << endl;
366 if( dest_full_path != cached_file )
368 if (
assert_dir( dest_full_path.dirname() ) != 0 )
373 ERR <<
"Can't hardlink/copy " << cached_file +
" to " + dest_dir << endl;
388 Pathname localfile = dest_dir + resource.
filename();
392 MIL <<
"Checking job [" << localfile <<
"] (" << checkers.size() <<
" checkers )" << endl;
394 for ( list<FileChecker>::const_iterator it = checkers.begin();
395 it != checkers.end();
404 ERR <<
"Invalid checker for '" << localfile <<
"'" << endl;
426 const Pathname &dest_dir )
428 auto fnc_addIfInContent( [&](
const std::string & index_r ) ->
bool
437 downloadAndReadIndexList( media, dest_dir );
443 fnc_addIfInContent(
"CHECKSUMS" ) || fnc_addIfInContent(
"SHA1SUMS" );
447 fnc_addIfInContent(
"content" );
455 if ( _dircontent.find(resource.
filename().asString())
456 != _dircontent.end() )
460 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
469 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
470 _dircontent[resource.
filename().asString()] = tofill;
476 const Pathname &dest_dir, FetcherJob::Flags flags )
480 MIL <<
"Adding directory " << resource.
filename() << endl;
483 getDirectoryContent(media, resource, content);
488 WAR <<
"Skiping subtree hidden at " << resource.
filename() << endl;
494 autoaddIndexes(content, media, resource, dest_dir);
496 for ( filesystem::DirContent::const_iterator it = content.begin();
504 Pathname filename = resource.
filename() + it->name;
512 if ( _checksums.find(filename.asString()) != _checksums.end() )
515 chksm = _checksums[filename.asString()];
519 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
529 addDirJobs(media, filename, dest_dir, flags);
540 bool got_from_cache =
false;
543 got_from_cache = provideFromCache(resource, dest_dir);
545 if ( ! got_from_cache )
547 MIL <<
"Not found in cache, downloading" << endl;
554 Pathname dest_full_path = dest_dir + resource.
filename();
556 if (
assert_dir( dest_full_path.dirname() ) != 0 )
560 if ( ! PathInfo(tmp_file).isExist() )
561 ERR << tmp_file <<
" does not exist" << endl;
562 if ( ! PathInfo(dest_full_path.dirname()).isExist() )
563 ERR << dest_full_path.dirname() <<
" does not exist" << endl;
566 ZYPP_THROW(
Exception(
"Can't hardlink/copy " + tmp_file.asString() +
" to " + dest_dir.asString()));
576 WAR <<
"optional resource " << resource <<
" could not be transfered" << endl;
602 void consumeIndex(
const parser::susetags::RepoIndex_Ptr & data_r )
603 { _repoindex = data_r; }
611 if ( index.basename() ==
"CHECKSUMS" || index.basename() ==
"SHA1SUMS" )
612 readChecksumsIndex(index, basedir);
613 else if ( index.basename() ==
"content" )
614 readContentFileIndex(index, basedir);
616 WAR << index <<
": index file format not known" << endl;
624 MIL << index <<
" contains " << reader.
_repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
628 _checksums[(basedir + it->first).
asString()] = it->second;
635 std::ifstream in( index.c_str() );
639 while (
getline( in, buffer ) )
642 if ( buffer[0] ==
'#' )
646 if ( checksum.
empty() )
649 if ( buffer.empty() )
651 WAR <<
"Missing filename in CHECKSUMS file: " << index.asString() <<
" (" << checksum <<
")" << endl;
664 MIL <<
"downloading index " << resource << endl;
690 fetcher.
start( dest_dir, media );
692 if ( PathInfo(dest_dir + sigloc.
filename()).isExist() )
699 fetcher.
start( dest_dir, media );
703 if ( PathInfo(dest_dir + keyloc.
filename()).isExist() )
706 WAR <<
"No public key specified by user for index '" << keyloc.
filename() <<
"'"<< endl;
710 fetcher.
start( dest_dir, media );
721 if ( _indexes.empty() )
723 MIL <<
"No indexes to read." << endl;
727 for_( it_idx, _indexes.begin(), _indexes.end() )
729 if ( (*it_idx)->read )
731 DBG <<
"Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
736 downloadIndex( media, (*it_idx)->location, dest_dir );
738 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
740 MIL <<
"Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
741 (*it_idx)->read =
true;
744 MIL <<
"done reading indexes" << endl;
754 progress.
sendTo(progress_receiver);
756 downloadAndReadIndexList(media, dest_dir);
758 for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
764 addDirJobs(media, location, dest_dir, (*it_res)->flags);
778 getDirectoryContent(media, (*it_res)->location.filename().dirname(), content);
781 MIL <<
"Autodiscovering signed indexes on '"
782 << (*it_res)->location.filename().dirname() <<
"' for '"
783 << (*it_res)->location.filename() <<
"'" << endl;
785 autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir);
789 getDirectoryContent(media, Pathname(
"/"), content);
792 MIL <<
"Autodiscovering signed indexes on '"
794 << (*it_res)->location.filename() <<
"'" << endl;
796 autoaddIndexes(content, media, Pathname(
"/"), dest_dir);
799 provideToDest(media, (*it_res)->location, dest_dir, (*it_res)->deltafile);
803 if ( ! PathInfo(dest_dir + (*it_res)->location.filename()).isExist() )
808 if ( (*it_res)->location.checksum().empty() )
810 if ( _checksums.find((*it_res)->location.filename().asString())
811 != _checksums.end() )
813 CheckSum chksm = _checksums[(*it_res)->location.filename().asString()];
815 (*it_res)->checkers.push_back(digest_check);
825 (*it_res)->checkers.push_back(digest_check);
833 (*it_res)->checkers.push_back(digest_check);
837 validate((*it_res)->location, dest_dir, (*it_res)->checkers);
839 if ( ! progress.incr() )
847 for ( list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
921 return str << *obj.
_pimpl;