libzypp
10.5.0
|
00001 /*---------------------------------------------------------------------\ 00002 | ____ _ __ __ ___ | 00003 | |__ / \ / / . \ . \ | 00004 | / / \ V /| _/ _/ | 00005 | / /__ | | | | | | | 00006 | /_____||_| |_| |_| | 00007 | | 00008 \---------------------------------------------------------------------*/ 00012 #include <iostream> 00013 #include <fstream> 00014 #include <list> 00015 #include <map> 00016 00017 #include "zypp/base/Easy.h" 00018 #include "zypp/base/LogControl.h" 00019 #include "zypp/base/LogTools.h" 00020 #include "zypp/base/PtrTypes.h" 00021 #include "zypp/base/DefaultIntegral.h" 00022 #include "zypp/base/String.h" 00023 #include "zypp/Fetcher.h" 00024 #include "zypp/ZYppFactory.h" 00025 #include "zypp/CheckSum.h" 00026 #include "zypp/base/UserRequestException.h" 00027 #include "zypp/parser/susetags/ContentFileReader.h" 00028 #include "zypp/parser/susetags/RepoIndex.h" 00029 00030 using namespace std; 00031 00032 #undef ZYPP_BASE_LOGGER_LOGGROUP 00033 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher" 00034 00036 namespace zypp 00037 { 00038 00044 struct FetcherIndex 00045 { 00046 FetcherIndex( const OnMediaLocation &loc ) 00047 : location(loc) 00048 {} 00050 OnMediaLocation location; 00052 DefaultIntegral<bool,false> read; 00053 }; 00054 00055 typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr; 00056 00058 struct SameFetcherIndex 00059 { 00060 bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) 00061 { 00062 if ( lhs == rhs ) 00063 return false; // incl. NULL == NULL 00064 if ( ! lhs ) 00065 return true; // NULL < nonNULL 00066 if ( ! rhs ) 00067 return false; // nonNULL > NULL 00068 // both nonNULL ==> compare medianr and path 00069 if ( lhs->location.medianr() == rhs->location.medianr() ) 00070 return lhs->location.filename() < rhs->location.filename(); 00071 //else 00072 return lhs->location.medianr() < rhs->location.medianr(); 00073 } 00074 }; 00075 00080 struct FetcherJob 00081 { 00082 enum Flag 00083 { 00084 None = 0x0000, 00085 Directory = 0x0001, 00086 Recursive = 0x0002, 00087 RecursiveDirectory = Directory | Recursive, 00088 // check checksums even if there is no such 00089 // checksum (warns of no checksum) 00090 AlwaysVerifyChecksum = 0x0004, 00091 }; 00092 ZYPP_DECLARE_FLAGS(Flags, Flag); 00093 00094 00095 FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname()) 00096 : location(loc) 00097 , deltafile(dfile) 00098 , flags(None) 00099 { 00100 //MIL << location << endl; 00101 } 00102 00103 ~FetcherJob() 00104 { 00105 //MIL << location << " | * " << checkers.size() << endl; 00106 } 00107 00108 OnMediaLocation location; 00109 Pathname deltafile; 00110 //CompositeFileChecker checkers; 00111 list<FileChecker> checkers; 00112 Flags flags; 00113 }; 00114 00115 ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags); 00116 typedef shared_ptr<FetcherJob> FetcherJob_Ptr; 00117 00118 std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj ) 00119 { 00120 return str << obj->location; 00121 } 00122 00124 // 00125 // CLASS NAME : Fetcher::Impl 00126 // 00128 class Fetcher::Impl 00129 { 00130 friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj ); 00131 00132 public: 00133 Impl(); 00134 00135 ~Impl() {} 00136 00137 void setOptions( Fetcher::Options options ); 00138 Fetcher::Options options() const; 00139 00140 void addIndex( const OnMediaLocation &resource ); 00141 00142 void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() ); 00143 void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() ); 00144 00145 void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() ); 00146 void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() ); 00147 void addCachePath( const Pathname &cache_dir ); 00148 void reset(); 00149 void start( const Pathname &dest_dir, 00150 MediaSetAccess &media, 00151 const ProgressData::ReceiverFnc & progress_receiver ); 00152 00154 static shared_ptr<Impl> nullimpl() 00155 { 00156 static shared_ptr<Impl> _nullimpl( new Impl ); 00157 return _nullimpl; 00158 } 00159 private: 00163 void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir); 00164 00168 void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir); 00169 00177 void readIndex( const Pathname &index, const Pathname &basedir ); 00178 00180 void readSha1sumsIndex( const Pathname &index, const Pathname &basedir ); 00181 00183 void readContentFileIndex( const Pathname &index, const Pathname &basedir ); 00184 00186 void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content ); 00187 00193 bool provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir ); 00199 void validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers ); 00200 00204 void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource, 00205 const Pathname &dest_dir, FetcherJob::Flags flags ); 00206 00210 void autoaddIndexes( const filesystem::DirContent &content, 00211 MediaSetAccess &media, 00212 const OnMediaLocation &resource, 00213 const Pathname &dest_dir ); 00217 void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir , const Pathname &deltafile); 00218 00219 private: 00220 friend Impl * rwcowClone<Impl>( const Impl * rhs ); 00222 Impl * clone() const 00223 { return new Impl( *this ); } 00224 00225 list<FetcherJob_Ptr> _resources; 00226 std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes; 00227 std::set<Pathname> _caches; 00228 // checksums read from the indexes 00229 map<string, CheckSum> _checksums; 00230 // cache of dir contents 00231 map<string, filesystem::DirContent> _dircontent; 00232 00233 Fetcher::Options _options; 00234 }; 00236 00237 void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile ) 00238 { 00239 FetcherJob_Ptr job; 00240 job.reset(new FetcherJob(resource, deltafile)); 00241 job->flags |= FetcherJob:: AlwaysVerifyChecksum; 00242 _resources.push_back(job); 00243 } 00244 00245 Fetcher::Impl::Impl() 00246 : _options(0) 00247 { 00248 } 00249 00250 void Fetcher::Impl::setOptions( Fetcher::Options options ) 00251 { _options = options; } 00252 00253 Fetcher::Options Fetcher::Impl::options() const 00254 { return _options; } 00255 00256 void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource, 00257 bool recursive, 00258 const FileChecker &checker ) 00259 { 00260 FetcherJob_Ptr job; 00261 job.reset(new FetcherJob(resource)); 00262 if ( checker ) 00263 job->checkers.push_back(checker); 00264 if ( recursive ) 00265 job->flags |= FetcherJob::Recursive; 00266 job->flags |= FetcherJob::Directory; 00267 00268 _resources.push_back(job); 00269 } 00270 00271 void Fetcher::Impl::enqueueDigestedDir( const OnMediaLocation &resource, 00272 bool recursive, 00273 const FileChecker &checker ) 00274 { 00275 FetcherJob_Ptr job; 00276 job.reset(new FetcherJob(resource)); 00277 if ( checker ) 00278 job->checkers.push_back(checker); 00279 if ( recursive ) 00280 job->flags |= FetcherJob::Recursive; 00281 job->flags |= FetcherJob::Directory; 00282 job->flags |= FetcherJob::AlwaysVerifyChecksum; 00283 00284 _resources.push_back(job); 00285 00286 } 00287 00288 void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker ) 00289 { 00290 FetcherJob_Ptr job; 00291 job.reset(new FetcherJob(resource)); 00292 if ( checker ) 00293 job->checkers.push_back(checker); 00294 _resources.push_back(job); 00295 } 00296 00297 void Fetcher::Impl::addIndex( const OnMediaLocation &resource ) 00298 { 00299 MIL << "adding index " << resource << endl; 00300 _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource))); 00301 } 00302 00303 00304 void Fetcher::Impl::reset() 00305 { 00306 _resources.clear(); 00307 _indexes.clear(); 00308 _checksums.clear(); 00309 _dircontent.clear(); 00310 } 00311 00312 void Fetcher::Impl::addCachePath( const Pathname &cache_dir ) 00313 { 00314 PathInfo info(cache_dir); 00315 if ( info.isExist() ) 00316 { 00317 if ( info.isDir() ) 00318 { 00319 DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl; 00320 _caches.insert(cache_dir); 00321 } 00322 else 00323 { 00324 // don't add bad cache directory, just log the error 00325 ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl; 00326 } 00327 } 00328 else 00329 { 00330 ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl; 00331 } 00332 00333 } 00334 00335 // tries to provide resource to dest_dir from any of the configured additional 00336 // cache paths where the file may already be present. returns true if the 00337 // file was provided from the cache. 00338 bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir ) 00339 { 00340 Pathname dest_full_path = dest_dir + resource.filename(); 00341 00342 // first check in the destination directory 00343 if ( PathInfo(dest_full_path).isExist() ) 00344 { 00345 if ( is_checksum( dest_full_path, resource.checksum() ) 00346 && (! resource.checksum().empty() ) ) 00347 return true; 00348 } 00349 00350 MIL << "start fetcher with " << _caches.size() << " cache directories." << endl; 00351 for_ ( it_cache, _caches.begin(), _caches.end() ) 00352 { 00353 // does the current file exists in the current cache? 00354 Pathname cached_file = *it_cache + resource.filename(); 00355 if ( PathInfo( cached_file ).isExist() ) 00356 { 00357 DBG << "File '" << cached_file << "' exist, testing checksum " << resource.checksum() << endl; 00358 // check the checksum 00359 if ( is_checksum( cached_file, resource.checksum() ) && (! resource.checksum().empty() ) ) 00360 { 00361 // cached 00362 MIL << "file " << resource.filename() << " found in previous cache. Using cached copy." << endl; 00363 // checksum is already checked. 00364 // we could later implement double failover and try to download if file copy fails. 00365 // replicate the complete path in the target directory 00366 if( dest_full_path != cached_file ) 00367 { 00368 if ( assert_dir( dest_full_path.dirname() ) != 0 ) 00369 ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString())); 00370 00371 if ( filesystem::hardlinkCopy(cached_file, dest_full_path ) != 0 ) 00372 { 00373 ERR << "Can't hardlink/copy " << cached_file + " to " + dest_dir << endl; 00374 continue; 00375 } 00376 } 00377 // found in cache 00378 return true; 00379 } 00380 } 00381 } // iterate over caches 00382 return false; 00383 } 00384 00385 void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers ) 00386 { 00387 // no matter where did we got the file, try to validate it: 00388 Pathname localfile = dest_dir + resource.filename(); 00389 // call the checker function 00390 try 00391 { 00392 MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl; 00393 00394 for ( list<FileChecker>::const_iterator it = checkers.begin(); 00395 it != checkers.end(); 00396 ++it ) 00397 { 00398 if (*it) 00399 { 00400 (*it)(localfile); 00401 } 00402 else 00403 { 00404 ERR << "Invalid checker for '" << localfile << "'" << endl; 00405 } 00406 } 00407 00408 } 00409 catch ( const FileCheckException &e ) 00410 { 00411 ZYPP_RETHROW(e); 00412 } 00413 catch ( const Exception &e ) 00414 { 00415 ZYPP_RETHROW(e); 00416 } 00417 catch (...) 00418 { 00419 ZYPP_THROW(Exception("Unknown error while validating " + resource.filename().asString())); 00420 } 00421 } 00422 00423 void Fetcher::Impl::autoaddIndexes( const filesystem::DirContent &content, 00424 MediaSetAccess &media, 00425 const OnMediaLocation &resource, 00426 const Pathname &dest_dir ) 00427 { 00428 if ( _options & AutoAddSha1sumsIndexes ) 00429 { 00430 // only try to add an index if it exists 00431 filesystem::DirEntry shafile; 00432 shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE; 00433 if ( find( content.begin(), content.end(), shafile ) != content.end() ) 00434 { 00435 // add the index of this directory 00436 OnMediaLocation indexloc(resource); 00437 indexloc.changeFilename(resource.filename() + "SHA1SUMS"); 00438 addIndex(indexloc); 00439 // we need to read it now 00440 downloadAndReadIndexList(media, dest_dir); 00441 } 00442 } 00443 if ( _options & AutoAddContentFileIndexes ) 00444 { 00445 // only try to add an index if it exists 00446 filesystem::DirEntry contentfile; 00447 contentfile.name = "content"; contentfile.type = filesystem::FT_FILE; 00448 if ( find( content.begin(), content.end(), contentfile ) != content.end() ) 00449 { 00450 // add the index of this directory 00451 OnMediaLocation indexloc(resource); 00452 indexloc.changeFilename(resource.filename() + "content"); 00453 addIndex(indexloc); 00454 // we need to read it now 00455 downloadAndReadIndexList(media, dest_dir); 00456 } 00457 } 00458 } 00459 00460 void Fetcher::Impl::getDirectoryContent( MediaSetAccess &media, 00461 const OnMediaLocation &resource, 00462 filesystem::DirContent &content ) 00463 { 00464 if ( _dircontent.find(resource.filename().asString()) 00465 != _dircontent.end() ) 00466 { 00467 filesystem::DirContent filled(_dircontent[resource.filename().asString()]); 00468 00469 std::copy(filled.begin(), filled.end(), std::back_inserter(content)); 00470 } 00471 else 00472 { 00473 filesystem::DirContent tofill; 00474 media.dirInfo( tofill, 00475 resource.filename(), 00476 false /* dots */, 00477 resource.medianr()); 00478 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content)); 00479 _dircontent[resource.filename().asString()] = tofill; 00480 } 00481 } 00482 00483 void Fetcher::Impl::addDirJobs( MediaSetAccess &media, 00484 const OnMediaLocation &resource, 00485 const Pathname &dest_dir, FetcherJob::Flags flags ) 00486 { 00487 // first get the content of the directory so we can add 00488 // individual transfer jobs 00489 MIL << "Adding directory " << resource.filename() << endl; 00490 filesystem::DirContent content; 00491 getDirectoryContent(media, resource, content); 00492 00493 // this method test for the option flags so indexes are added 00494 // only if the options are enabled 00495 autoaddIndexes(content, media, resource, dest_dir); 00496 00497 for ( filesystem::DirContent::const_iterator it = content.begin(); 00498 it != content.end(); 00499 ++it ) 00500 { 00501 // skip SHA1SUMS* as they were already retrieved 00502 if ( str::hasPrefix(it->name, "SHA1SUMS") ) 00503 continue; 00504 00505 Pathname filename = resource.filename() + it->name; 00506 00507 switch ( it->type ) 00508 { 00509 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all 00510 case filesystem::FT_FILE: 00511 { 00512 CheckSum chksm(resource.checksum()); 00513 if ( _checksums.find(filename.asString()) != _checksums.end() ) 00514 { 00515 // the checksum can be replaced with the one in the index. 00516 chksm = _checksums[filename.asString()]; 00517 //MIL << "resource " << filename << " has checksum in the index file." << endl; 00518 } 00519 else 00520 WAR << "Resource " << filename << " has no checksum in the index either." << endl; 00521 00522 if ( flags & FetcherJob::AlwaysVerifyChecksum ) 00523 enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm)); 00524 else 00525 enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm)); 00526 break; 00527 } 00528 case filesystem::FT_DIR: // newer directory.yast contain at least directory info 00529 if ( flags & FetcherJob::Recursive ) 00530 addDirJobs(media, filename, dest_dir, flags); 00531 break; 00532 default: 00533 // don't provide devices, sockets, etc. 00534 break; 00535 } 00536 } 00537 } 00538 00539 void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, const Pathname &deltafile ) 00540 { 00541 bool got_from_cache = false; 00542 00543 // start look in cache 00544 got_from_cache = provideFromCache(resource, dest_dir); 00545 00546 if ( ! got_from_cache ) 00547 { 00548 MIL << "Not found in cache, downloading" << endl; 00549 00550 // try to get the file from the net 00551 try 00552 { 00553 Pathname tmp_file = media.provideFile(resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, deltafile ); 00554 00555 Pathname dest_full_path = dest_dir + resource.filename(); 00556 00557 if ( assert_dir( dest_full_path.dirname() ) != 0 ) 00558 ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString())); 00559 if ( filesystem::hardlinkCopy( tmp_file, dest_full_path ) != 0 ) 00560 { 00561 if ( ! PathInfo(tmp_file).isExist() ) 00562 ERR << tmp_file << " does not exist" << endl; 00563 if ( ! PathInfo(dest_full_path.dirname()).isExist() ) 00564 ERR << dest_full_path.dirname() << " does not exist" << endl; 00565 00566 media.releaseFile(resource); //not needed anymore, only eat space 00567 ZYPP_THROW( Exception("Can't hardlink/copy " + tmp_file.asString() + " to " + dest_dir.asString())); 00568 } 00569 00570 media.releaseFile(resource); //not needed anymore, only eat space 00571 } 00572 catch (Exception & excpt_r) 00573 { 00574 if ( resource.optional() ) 00575 { 00576 ZYPP_CAUGHT(excpt_r); 00577 WAR << "optional resource " << resource << " could not be transfered" << endl; 00578 return; 00579 } 00580 else 00581 { 00582 excpt_r.remember("Can't provide " + resource.filename().asString() ); 00583 ZYPP_RETHROW(excpt_r); 00584 } 00585 } 00586 } 00587 else 00588 { 00589 // We got the file from cache 00590 // continue with next file 00591 return; 00592 } 00593 } 00594 00595 // helper class to consume a content file 00596 struct ContentReaderHelper : public parser::susetags::ContentFileReader 00597 { 00598 ContentReaderHelper() 00599 { 00600 setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) ); 00601 } 00602 00603 void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r ) 00604 { _repoindex = data_r; } 00605 00606 parser::susetags::RepoIndex_Ptr _repoindex; 00607 }; 00608 00609 // generic function for reading indexes 00610 void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir ) 00611 { 00612 if ( index.basename() == "SHA1SUMS" ) 00613 readSha1sumsIndex(index, basedir); 00614 else if ( index.basename() == "content" ) 00615 readContentFileIndex(index, basedir); 00616 else 00617 WAR << index << ": index file format not known" << endl; 00618 } 00619 00620 // reads a content file index 00621 void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir ) 00622 { 00623 ContentReaderHelper reader; 00624 reader.parse(index); 00625 MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl; 00626 for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() ) 00627 { 00628 // content file entries don't start with / 00629 _checksums[(basedir + it->first).asString()] = it->second; 00630 } 00631 } 00632 00633 // reads a SHA1SUMS file index 00634 void Fetcher::Impl::readSha1sumsIndex( const Pathname &index, const Pathname &basedir ) 00635 { 00636 std::ifstream in( index.c_str() ); 00637 string buffer; 00638 if ( ! in.fail() ) 00639 { 00640 while ( getline(in, buffer) ) 00641 { 00642 vector<string> words; 00643 str::split( buffer, back_inserter(words) ); 00644 if ( words.size() != 2 ) 00645 ZYPP_THROW(Exception("Wrong format for SHA1SUMS file")); 00646 //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl; 00647 if ( ! words[1].empty() ) 00648 _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]); 00649 } 00650 } 00651 else 00652 ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString())); 00653 } 00654 00655 void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir) 00656 { 00657 MIL << "downloading index " << resource << endl; 00658 // create a new fetcher with a different state to transfer the 00659 // file containing checksums and its signature 00660 Fetcher fetcher; 00661 // signature checker for index. We havent got the signature from 00662 // the nextwork yet. 00663 SignatureFileChecker sigchecker; 00664 00665 // build the name of the index and the signature 00666 OnMediaLocation idxloc(resource); 00667 OnMediaLocation sigloc(resource); 00668 OnMediaLocation keyloc(resource); 00669 00670 // we should not fail the download if those don't exists 00671 // the checking will warn later 00672 sigloc.setOptional(true); 00673 keyloc.setOptional(true); 00674 00675 // calculate signature and key name 00676 sigloc.changeFilename( sigloc.filename().extend(".asc") ); 00677 keyloc.changeFilename( keyloc.filename().extend(".key") ); 00678 00679 //assert_dir(dest_dir + idxloc.filename().dirname()); 00680 00681 // transfer the signature 00682 fetcher.enqueue(sigloc); 00683 fetcher.start( dest_dir, media ); 00684 // if we get the signature, update the checker 00685 if ( PathInfo(dest_dir + sigloc.filename()).isExist() ) 00686 sigchecker = SignatureFileChecker(dest_dir + sigloc.filename()); 00687 00688 fetcher.reset(); 00689 00690 // now the key 00691 fetcher.enqueue(keyloc); 00692 fetcher.start( dest_dir, media ); 00693 fetcher.reset(); 00694 00695 // try to import the key 00696 if ( PathInfo(dest_dir + keyloc.filename()).isExist() ) 00697 getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false); 00698 else 00699 WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl; 00700 00701 // now the index itself 00702 fetcher.enqueue( idxloc, FileChecker(sigchecker) ); 00703 fetcher.start( dest_dir, media ); 00704 fetcher.reset(); 00705 } 00706 00707 // this method takes all the user pointed indexes, gets them and also tries to 00708 // download their signature, and verify them. After that, its parses each one 00709 // to fill the checksum cache. 00710 void Fetcher::Impl::downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir) 00711 { 00712 // if there is no indexes, then just return to avoid 00713 // the directory listing 00714 if ( _indexes.empty() ) 00715 { 00716 MIL << "No indexes to read." << endl; 00717 return; 00718 } 00719 00720 for_( it_idx, _indexes.begin(), _indexes.end() ) 00721 { 00722 if ( (*it_idx)->read ) 00723 { 00724 DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl; 00725 } 00726 else 00727 { 00728 // base::LogControl::TmpLineWriter shutUp; 00729 downloadIndex( media, (*it_idx)->location, dest_dir ); 00730 // now we have the indexes in dest_dir 00731 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() ); 00732 // Take care we don't process it again 00733 MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl; 00734 (*it_idx)->read = true; 00735 } 00736 } 00737 MIL << "done reading indexes" << endl; 00738 } 00739 00740 // start processing all fetcher jobs. 00741 // it processes any user pointed index first 00742 void Fetcher::Impl::start( const Pathname &dest_dir, 00743 MediaSetAccess &media, 00744 const ProgressData::ReceiverFnc & progress_receiver ) 00745 { 00746 ProgressData progress(_resources.size()); 00747 progress.sendTo(progress_receiver); 00748 00749 downloadAndReadIndexList(media, dest_dir); 00750 00751 for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res ) 00752 { 00753 00754 if ( (*it_res)->flags & FetcherJob::Directory ) 00755 { 00756 const OnMediaLocation location((*it_res)->location); 00757 addDirJobs(media, location, dest_dir, (*it_res)->flags); 00758 continue; 00759 } 00760 00761 // may be this code can be factored out 00762 // together with the autodiscovery of indexes 00763 // of addDirJobs 00764 if ( ( _options & AutoAddSha1sumsIndexes ) || 00765 ( _options & AutoAddContentFileIndexes ) ) 00766 { 00767 // if auto indexing is enabled, then we need to read the 00768 // index for each file. We look only in the directory 00769 // where the file is. this is expensive of course. 00770 filesystem::DirContent content; 00771 getDirectoryContent(media, (*it_res)->location.filename().dirname(), content); 00772 // this method test for the option flags so indexes are added 00773 // only if the options are enabled 00774 MIL << "Autodiscovering signed indexes on '" 00775 << (*it_res)->location.filename().dirname() << "' for '" 00776 << (*it_res)->location.filename() << "'" << endl; 00777 00778 autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir); 00779 00780 // also look in the root of the media 00781 content.clear(); 00782 getDirectoryContent(media, Pathname("/"), content); 00783 // this method test for the option flags so indexes are added 00784 // only if the options are enabled 00785 MIL << "Autodiscovering signed indexes on '" 00786 << "/" << "' for '" 00787 << (*it_res)->location.filename() << "'" << endl; 00788 00789 autoaddIndexes(content, media, Pathname("/"), dest_dir); 00790 } 00791 00792 provideToDest(media, (*it_res)->location, dest_dir, (*it_res)->deltafile); 00793 00794 // if the file was not transfered, and no exception, just 00795 // return, as it was an optional file 00796 if ( ! PathInfo(dest_dir + (*it_res)->location.filename()).isExist() ) 00797 return; 00798 00799 // if the checksum is empty, but the checksum is in one of the 00800 // indexes checksum, then add a checker 00801 if ( (*it_res)->location.checksum().empty() ) 00802 { 00803 if ( _checksums.find((*it_res)->location.filename().asString()) 00804 != _checksums.end() ) 00805 { 00806 CheckSum chksm = _checksums[(*it_res)->location.filename().asString()]; 00807 ChecksumFileChecker digest_check(chksm); 00808 (*it_res)->checkers.push_back(digest_check); 00809 } 00810 else 00811 { 00812 // if the index checksum is empty too, we only add the checker 00813 // if the AlwaysVerifyChecksum option is set on 00814 if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum ) 00815 { 00816 // add the checker with the empty checksum 00817 ChecksumFileChecker digest_check((*it_res)->location.checksum()); 00818 (*it_res)->checkers.push_back(digest_check); 00819 } 00820 } 00821 } 00822 else 00823 { 00824 // checksum is not empty, so add a checksum checker 00825 ChecksumFileChecker digest_check((*it_res)->location.checksum()); 00826 (*it_res)->checkers.push_back(digest_check); 00827 } 00828 00829 // validate job, this throws if not valid 00830 validate((*it_res)->location, dest_dir, (*it_res)->checkers); 00831 00832 if ( ! progress.incr() ) 00833 ZYPP_THROW(AbortRequestException()); 00834 } // for each job 00835 } 00836 00838 inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj ) 00839 { 00840 for ( list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res ) 00841 { 00842 str << *it_res; 00843 } 00844 return str; 00845 } 00846 00847 Fetcher::Fetcher() 00848 : _pimpl( new Impl() ) 00849 {} 00850 00851 Fetcher::~Fetcher() 00852 {} 00853 00854 void Fetcher::setOptions( Fetcher::Options options ) 00855 { 00856 _pimpl->setOptions(options); 00857 } 00858 00859 Fetcher::Options Fetcher::options() const 00860 { 00861 return _pimpl->options(); 00862 } 00863 00864 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile ) 00865 { 00866 _pimpl->enqueueDigested(resource, checker, deltafile); 00867 } 00868 00869 void Fetcher::enqueueDir( const OnMediaLocation &resource, 00870 bool recursive, 00871 const FileChecker &checker ) 00872 { 00873 _pimpl->enqueueDir(resource, recursive, checker); 00874 } 00875 00876 void Fetcher::enqueueDigestedDir( const OnMediaLocation &resource, 00877 bool recursive, 00878 const FileChecker &checker ) 00879 { 00880 _pimpl->enqueueDigestedDir(resource, recursive, checker); 00881 } 00882 00883 00884 void Fetcher::addIndex( const OnMediaLocation &resource ) 00885 { 00886 _pimpl->addIndex(resource); 00887 } 00888 00889 00890 void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker ) 00891 { 00892 _pimpl->enqueue(resource, checker); 00893 } 00894 00895 void Fetcher::addCachePath( const Pathname &cache_dir ) 00896 { 00897 _pimpl->addCachePath(cache_dir); 00898 } 00899 00900 void Fetcher::reset() 00901 { 00902 _pimpl->reset(); 00903 } 00904 00905 void Fetcher::start( const Pathname &dest_dir, 00906 MediaSetAccess &media, 00907 const ProgressData::ReceiverFnc & progress_receiver ) 00908 { 00909 _pimpl->start(dest_dir, media, progress_receiver); 00910 } 00911 00912 std::ostream & operator<<( std::ostream & str, const Fetcher & obj ) 00913 { 00914 return str << *obj._pimpl; 00915 } 00916 00918 } // namespace zypp 00920