libzypp  10.5.0
Fetcher.cc
Go to the documentation of this file.
00001 /*---------------------------------------------------------------------\
00002 |                          ____ _   __ __ ___                          |
00003 |                         |__  / \ / / . \ . \                         |
00004 |                           / / \ V /|  _/  _/                         |
00005 |                          / /__ | | | | | |                           |
00006 |                         /_____||_| |_| |_|                           |
00007 |                                                                      |
00008 \---------------------------------------------------------------------*/
00012 #include <iostream>
00013 #include <fstream>
00014 #include <list>
00015 #include <map>
00016 
00017 #include "zypp/base/Easy.h"
00018 #include "zypp/base/LogControl.h"
00019 #include "zypp/base/LogTools.h"
00020 #include "zypp/base/PtrTypes.h"
00021 #include "zypp/base/DefaultIntegral.h"
00022 #include "zypp/base/String.h"
00023 #include "zypp/Fetcher.h"
00024 #include "zypp/ZYppFactory.h"
00025 #include "zypp/CheckSum.h"
00026 #include "zypp/base/UserRequestException.h"
00027 #include "zypp/parser/susetags/ContentFileReader.h"
00028 #include "zypp/parser/susetags/RepoIndex.h"
00029 
00030 using namespace std;
00031 
00032 #undef ZYPP_BASE_LOGGER_LOGGROUP
00033 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
00034 
00036 namespace zypp
00037 { 
00038 
00044   struct FetcherIndex
00045   {
00046     FetcherIndex( const OnMediaLocation &loc )
00047       : location(loc)
00048     {}
00050     OnMediaLocation             location;
00052     DefaultIntegral<bool,false> read;
00053   };
00054 
00055   typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
00056 
00058   struct SameFetcherIndex
00059   {
00060     bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs )
00061     {
00062       if ( lhs == rhs )
00063         return false; // incl. NULL == NULL
00064       if ( ! lhs )
00065         return true;  // NULL < nonNULL
00066       if ( ! rhs )
00067         return false; // nonNULL > NULL
00068       // both nonNULL ==> compare medianr and path
00069       if ( lhs->location.medianr() == rhs->location.medianr() )
00070         return lhs->location.filename() < rhs->location.filename();
00071       //else
00072         return lhs->location.medianr() < rhs->location.medianr();
00073     }
00074   };
00075 
00080   struct FetcherJob
00081   {
00082     enum Flag
00083     {
00084         None      = 0x0000,
00085         Directory = 0x0001,
00086         Recursive = 0x0002,
00087         RecursiveDirectory = Directory | Recursive,
00088         // check checksums even if there is no such
00089         // checksum (warns of no checksum)
00090         AlwaysVerifyChecksum = 0x0004,
00091     };
00092     ZYPP_DECLARE_FLAGS(Flags, Flag);
00093 
00094 
00095     FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname())
00096       : location(loc)
00097       , deltafile(dfile)
00098       , flags(None)
00099     {
00100       //MIL << location << endl;
00101     }
00102 
00103     ~FetcherJob()
00104     {
00105       //MIL << location << " | * " << checkers.size() << endl;
00106     }
00107 
00108     OnMediaLocation location;
00109     Pathname deltafile;
00110     //CompositeFileChecker checkers;
00111     list<FileChecker> checkers;
00112     Flags flags;
00113   };
00114 
00115   ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
00116   typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
00117 
00118   std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
00119   {
00120     return str << obj->location;
00121   }
00122 
00124   //
00125   //    CLASS NAME : Fetcher::Impl
00126   //
00128   class Fetcher::Impl
00129   {
00130     friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
00131 
00132   public:
00133     Impl();
00134 
00135     ~Impl() {}
00136 
00137     void setOptions( Fetcher::Options options );
00138     Fetcher::Options options() const;
00139 
00140     void addIndex( const OnMediaLocation &resource );
00141 
00142     void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
00143     void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
00144 
00145     void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker()  );
00146     void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() );
00147     void addCachePath( const Pathname &cache_dir );
00148     void reset();
00149     void start( const Pathname &dest_dir,
00150                 MediaSetAccess &media,
00151                 const ProgressData::ReceiverFnc & progress_receiver );
00152 
00154     static shared_ptr<Impl> nullimpl()
00155     {
00156       static shared_ptr<Impl> _nullimpl( new Impl );
00157       return _nullimpl;
00158     }
00159   private:
00163       void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
00164 
00168       void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
00169 
00177       void readIndex( const Pathname &index, const Pathname &basedir );
00178 
00180       void readSha1sumsIndex( const Pathname &index, const Pathname &basedir );
00181 
00183       void readContentFileIndex( const Pathname &index, const Pathname &basedir );
00184 
00186       void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
00187 
00193       bool provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir );
00199       void validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers );
00200 
00204        void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
00205                         const Pathname &dest_dir, FetcherJob::Flags flags );
00206 
00210       void autoaddIndexes( const filesystem::DirContent &content,
00211                            MediaSetAccess &media,
00212                            const OnMediaLocation &resource,
00213                            const Pathname &dest_dir );
00217       void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir , const Pathname &deltafile);
00218 
00219   private:
00220     friend Impl * rwcowClone<Impl>( const Impl * rhs );
00222     Impl * clone() const
00223     { return new Impl( *this ); }
00224 
00225     list<FetcherJob_Ptr>   _resources;
00226     std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
00227     std::set<Pathname> _caches;
00228     // checksums read from the indexes
00229     map<string, CheckSum> _checksums;
00230     // cache of dir contents
00231     map<string, filesystem::DirContent> _dircontent;
00232 
00233     Fetcher::Options _options;
00234   };
00236 
00237   void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
00238   {
00239     FetcherJob_Ptr job;
00240     job.reset(new FetcherJob(resource, deltafile));
00241     job->flags |= FetcherJob:: AlwaysVerifyChecksum;
00242     _resources.push_back(job);
00243   }
00244 
00245   Fetcher::Impl::Impl()
00246       : _options(0)
00247   {
00248   }
00249 
00250   void Fetcher::Impl::setOptions( Fetcher::Options options )
00251   { _options = options; }
00252 
00253   Fetcher::Options Fetcher::Impl::options() const
00254   { return _options; }
00255 
00256   void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource,
00257                                   bool recursive,
00258                                   const FileChecker &checker )
00259   {
00260     FetcherJob_Ptr job;
00261     job.reset(new FetcherJob(resource));
00262     if ( checker )
00263         job->checkers.push_back(checker);
00264     if ( recursive )
00265         job->flags |= FetcherJob::Recursive;
00266     job->flags |= FetcherJob::Directory;
00267 
00268     _resources.push_back(job);
00269   }
00270 
00271   void Fetcher::Impl::enqueueDigestedDir( const OnMediaLocation &resource,
00272                                           bool recursive,
00273                                           const FileChecker &checker )
00274   {
00275     FetcherJob_Ptr job;
00276     job.reset(new FetcherJob(resource));
00277     if ( checker )
00278         job->checkers.push_back(checker);
00279     if ( recursive )
00280         job->flags |= FetcherJob::Recursive;
00281     job->flags |= FetcherJob::Directory;
00282     job->flags |= FetcherJob::AlwaysVerifyChecksum;
00283 
00284     _resources.push_back(job);
00285 
00286   }
00287 
00288   void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
00289   {
00290     FetcherJob_Ptr job;
00291     job.reset(new FetcherJob(resource));
00292     if ( checker )
00293       job->checkers.push_back(checker);
00294     _resources.push_back(job);
00295   }
00296 
00297   void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
00298   {
00299     MIL << "adding index " << resource << endl;
00300     _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
00301   }
00302 
00303 
00304   void Fetcher::Impl::reset()
00305   {
00306     _resources.clear();
00307     _indexes.clear();
00308     _checksums.clear();
00309     _dircontent.clear();
00310   }
00311 
00312   void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
00313   {
00314     PathInfo info(cache_dir);
00315     if ( info.isExist() )
00316     {
00317       if ( info.isDir() )
00318       {
00319         DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
00320         _caches.insert(cache_dir);
00321       }
00322       else
00323       {
00324         // don't add bad cache directory, just log the error
00325         ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
00326       }
00327     }
00328     else
00329     {
00330         ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
00331     }
00332 
00333   }
00334 
00335   // tries to provide resource to dest_dir from any of the configured additional
00336   // cache paths where the file may already be present. returns true if the
00337   // file was provided from the cache.
00338   bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir )
00339   {
00340     Pathname dest_full_path = dest_dir + resource.filename();
00341 
00342     // first check in the destination directory
00343     if ( PathInfo(dest_full_path).isExist() )
00344     {
00345       if ( is_checksum( dest_full_path, resource.checksum() )
00346            && (! resource.checksum().empty() ) )
00347           return true;
00348     }
00349 
00350     MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
00351     for_ ( it_cache, _caches.begin(), _caches.end() )
00352     {
00353       // does the current file exists in the current cache?
00354       Pathname cached_file = *it_cache + resource.filename();
00355       if ( PathInfo( cached_file ).isExist() )
00356       {
00357         DBG << "File '" << cached_file << "' exist, testing checksum " << resource.checksum() << endl;
00358          // check the checksum
00359         if ( is_checksum( cached_file, resource.checksum() ) && (! resource.checksum().empty() ) )
00360         {
00361           // cached
00362           MIL << "file " << resource.filename() << " found in previous cache. Using cached copy." << endl;
00363           // checksum is already checked.
00364           // we could later implement double failover and try to download if file copy fails.
00365            // replicate the complete path in the target directory
00366           if( dest_full_path != cached_file )
00367           {
00368             if ( assert_dir( dest_full_path.dirname() ) != 0 )
00369               ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
00370 
00371             if ( filesystem::hardlinkCopy(cached_file, dest_full_path ) != 0 )
00372             {
00373               ERR << "Can't hardlink/copy " << cached_file + " to " + dest_dir << endl;
00374               continue;
00375             }
00376           }
00377           // found in cache
00378           return true;
00379         }
00380       }
00381     } // iterate over caches
00382     return false;
00383   }
00384 
00385     void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers )
00386   {
00387     // no matter where did we got the file, try to validate it:
00388     Pathname localfile = dest_dir + resource.filename();
00389     // call the checker function
00390     try
00391     {
00392       MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl;
00393 
00394       for ( list<FileChecker>::const_iterator it = checkers.begin();
00395             it != checkers.end();
00396             ++it )
00397       {
00398         if (*it)
00399         {
00400           (*it)(localfile);
00401         }
00402         else
00403         {
00404           ERR << "Invalid checker for '" << localfile << "'" << endl;
00405         }
00406       }
00407 
00408     }
00409     catch ( const FileCheckException &e )
00410     {
00411       ZYPP_RETHROW(e);
00412     }
00413     catch ( const Exception &e )
00414     {
00415       ZYPP_RETHROW(e);
00416     }
00417     catch (...)
00418     {
00419       ZYPP_THROW(Exception("Unknown error while validating " + resource.filename().asString()));
00420     }
00421   }
00422 
00423   void Fetcher::Impl::autoaddIndexes( const filesystem::DirContent &content,
00424                                       MediaSetAccess &media,
00425                                       const OnMediaLocation &resource,
00426                                       const Pathname &dest_dir )
00427   {
00428       if ( _options & AutoAddSha1sumsIndexes )
00429       {
00430           // only try to add an index if it exists
00431           filesystem::DirEntry shafile;
00432           shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE;
00433           if ( find( content.begin(), content.end(), shafile ) != content.end() )
00434           {
00435               // add the index of this directory
00436               OnMediaLocation indexloc(resource);
00437               indexloc.changeFilename(resource.filename() + "SHA1SUMS");
00438               addIndex(indexloc);
00439               // we need to read it now
00440               downloadAndReadIndexList(media, dest_dir);
00441           }
00442       }
00443       if ( _options & AutoAddContentFileIndexes )
00444       {
00445           // only try to add an index if it exists
00446           filesystem::DirEntry contentfile;
00447           contentfile.name = "content"; contentfile.type = filesystem::FT_FILE;
00448           if ( find( content.begin(), content.end(), contentfile ) != content.end() )
00449           {
00450               // add the index of this directory
00451               OnMediaLocation indexloc(resource);
00452               indexloc.changeFilename(resource.filename() + "content");
00453               addIndex(indexloc);
00454               // we need to read it now
00455               downloadAndReadIndexList(media, dest_dir);
00456           }
00457       }
00458   }
00459 
00460   void Fetcher::Impl::getDirectoryContent( MediaSetAccess &media,
00461                                            const OnMediaLocation &resource,
00462                                            filesystem::DirContent &content )
00463   {
00464       if ( _dircontent.find(resource.filename().asString())
00465            != _dircontent.end() )
00466       {
00467           filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
00468 
00469           std::copy(filled.begin(), filled.end(), std::back_inserter(content));
00470       }
00471       else
00472       {
00473           filesystem::DirContent tofill;
00474           media.dirInfo( tofill,
00475                          resource.filename(),
00476                          false /* dots */,
00477                          resource.medianr());
00478           std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
00479           _dircontent[resource.filename().asString()] = tofill;
00480       }
00481   }
00482 
00483   void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
00484                                   const OnMediaLocation &resource,
00485                                   const Pathname &dest_dir, FetcherJob::Flags flags  )
00486   {
00487       // first get the content of the directory so we can add
00488       // individual transfer jobs
00489       MIL << "Adding directory " << resource.filename() << endl;
00490       filesystem::DirContent content;
00491       getDirectoryContent(media, resource, content);
00492 
00493       // this method test for the option flags so indexes are added
00494       // only if the options are enabled
00495       autoaddIndexes(content, media, resource, dest_dir);
00496 
00497       for ( filesystem::DirContent::const_iterator it = content.begin();
00498             it != content.end();
00499             ++it )
00500       {
00501           // skip SHA1SUMS* as they were already retrieved
00502           if ( str::hasPrefix(it->name, "SHA1SUMS") )
00503               continue;
00504 
00505           Pathname filename = resource.filename() + it->name;
00506 
00507           switch ( it->type )
00508           {
00509           case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
00510           case filesystem::FT_FILE:
00511           {
00512               CheckSum chksm(resource.checksum());
00513               if ( _checksums.find(filename.asString()) != _checksums.end() )
00514               {
00515                   // the checksum can be replaced with the one in the index.
00516                   chksm = _checksums[filename.asString()];
00517                   //MIL << "resource " << filename << " has checksum in the index file." << endl;
00518               }
00519               else
00520                   WAR << "Resource " << filename << " has no checksum in the index either." << endl;
00521 
00522               if ( flags & FetcherJob::AlwaysVerifyChecksum )
00523                   enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
00524               else
00525                   enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
00526               break;
00527           }
00528           case filesystem::FT_DIR: // newer directory.yast contain at least directory info
00529               if ( flags & FetcherJob::Recursive )
00530                   addDirJobs(media, filename, dest_dir, flags);
00531               break;
00532           default:
00533               // don't provide devices, sockets, etc.
00534               break;
00535           }
00536       }
00537   }
00538 
00539   void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, const Pathname &deltafile )
00540   {
00541     bool got_from_cache = false;
00542 
00543     // start look in cache
00544     got_from_cache = provideFromCache(resource, dest_dir);
00545 
00546     if ( ! got_from_cache )
00547     {
00548       MIL << "Not found in cache, downloading" << endl;
00549 
00550       // try to get the file from the net
00551       try
00552       {
00553         Pathname tmp_file = media.provideFile(resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, deltafile );
00554 
00555         Pathname dest_full_path = dest_dir + resource.filename();
00556 
00557         if ( assert_dir( dest_full_path.dirname() ) != 0 )
00558               ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
00559         if ( filesystem::hardlinkCopy( tmp_file, dest_full_path ) != 0 )
00560         {
00561           if ( ! PathInfo(tmp_file).isExist() )
00562               ERR << tmp_file << " does not exist" << endl;
00563           if ( ! PathInfo(dest_full_path.dirname()).isExist() )
00564               ERR << dest_full_path.dirname() << " does not exist" << endl;
00565 
00566           media.releaseFile(resource); //not needed anymore, only eat space
00567           ZYPP_THROW( Exception("Can't hardlink/copy " + tmp_file.asString() + " to " + dest_dir.asString()));
00568         }
00569 
00570         media.releaseFile(resource); //not needed anymore, only eat space
00571       }
00572       catch (Exception & excpt_r)
00573       {
00574         if ( resource.optional() )
00575         {
00576             ZYPP_CAUGHT(excpt_r);
00577             WAR << "optional resource " << resource << " could not be transfered" << endl;
00578             return;
00579         }
00580         else
00581         {
00582             excpt_r.remember("Can't provide " + resource.filename().asString() );
00583             ZYPP_RETHROW(excpt_r);
00584         }
00585       }
00586     }
00587     else
00588     {
00589       // We got the file from cache
00590       // continue with next file
00591         return;
00592     }
00593   }
00594 
00595   // helper class to consume a content file
00596   struct ContentReaderHelper : public parser::susetags::ContentFileReader
00597   {
00598     ContentReaderHelper()
00599     {
00600       setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
00601     }
00602 
00603     void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
00604     { _repoindex = data_r; }
00605 
00606     parser::susetags::RepoIndex_Ptr _repoindex;
00607   };
00608 
00609   // generic function for reading indexes
00610   void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
00611   {
00612     if ( index.basename() == "SHA1SUMS" )
00613       readSha1sumsIndex(index, basedir);
00614     else if ( index.basename() == "content" )
00615       readContentFileIndex(index, basedir);
00616     else
00617       WAR << index << ": index file format not known" << endl;
00618   }
00619 
00620   // reads a content file index
00621   void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
00622   {
00623       ContentReaderHelper reader;
00624       reader.parse(index);
00625       MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
00626       for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
00627       {
00628           // content file entries don't start with /
00629           _checksums[(basedir + it->first).asString()] = it->second;
00630       }
00631   }
00632 
00633   // reads a SHA1SUMS file index
00634   void Fetcher::Impl::readSha1sumsIndex( const Pathname &index, const Pathname &basedir )
00635   {
00636       std::ifstream in( index.c_str() );
00637       string buffer;
00638       if ( ! in.fail() )
00639       {
00640           while ( getline(in, buffer) )
00641           {
00642               vector<string> words;
00643               str::split( buffer, back_inserter(words) );
00644               if ( words.size() != 2 )
00645                   ZYPP_THROW(Exception("Wrong format for SHA1SUMS file"));
00646               //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl;
00647               if ( ! words[1].empty() )
00648                   _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]);
00649           }
00650       }
00651       else
00652           ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString()));
00653   }
00654 
00655   void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
00656   {
00657     MIL << "downloading index " << resource << endl;
00658     // create a new fetcher with a different state to transfer the
00659     // file containing checksums and its signature
00660     Fetcher fetcher;
00661     // signature checker for index. We havent got the signature from
00662     // the nextwork yet.
00663     SignatureFileChecker sigchecker;
00664 
00665     // build the name of the index and the signature
00666     OnMediaLocation idxloc(resource);
00667     OnMediaLocation sigloc(resource);
00668     OnMediaLocation keyloc(resource);
00669 
00670     // we should not fail the download if those don't exists
00671     // the checking will warn later
00672     sigloc.setOptional(true);
00673     keyloc.setOptional(true);
00674 
00675     // calculate signature and key name
00676     sigloc.changeFilename( sigloc.filename().extend(".asc") );
00677     keyloc.changeFilename( keyloc.filename().extend(".key") );
00678 
00679     //assert_dir(dest_dir + idxloc.filename().dirname());
00680 
00681     // transfer the signature
00682     fetcher.enqueue(sigloc);
00683     fetcher.start( dest_dir, media );
00684     // if we get the signature, update the checker
00685     if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
00686         sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
00687 
00688     fetcher.reset();
00689 
00690     // now the key
00691     fetcher.enqueue(keyloc);
00692     fetcher.start( dest_dir, media );
00693     fetcher.reset();
00694 
00695     // try to import the key
00696     if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
00697         getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
00698     else
00699         WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
00700 
00701     // now the index itself
00702     fetcher.enqueue( idxloc, FileChecker(sigchecker) );
00703     fetcher.start( dest_dir, media );
00704     fetcher.reset();
00705  }
00706 
00707   // this method takes all the user pointed indexes, gets them and also tries to
00708   // download their signature, and verify them. After that, its parses each one
00709   // to fill the checksum cache.
00710   void Fetcher::Impl::downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir)
00711   {
00712       // if there is no indexes, then just return to avoid
00713       // the directory listing
00714       if ( _indexes.empty() )
00715       {
00716           MIL << "No indexes to read." << endl;
00717           return;
00718       }
00719 
00720       for_( it_idx, _indexes.begin(), _indexes.end() )
00721       {
00722         if ( (*it_idx)->read )
00723         {
00724           DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
00725         }
00726         else
00727         {
00728           // base::LogControl::TmpLineWriter shutUp;
00729           downloadIndex( media, (*it_idx)->location, dest_dir );
00730           // now we have the indexes in dest_dir
00731           readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
00732           // Take care we don't process it again
00733           MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
00734           (*it_idx)->read = true;
00735         }
00736       }
00737       MIL << "done reading indexes" << endl;
00738   }
00739 
00740   // start processing all fetcher jobs.
00741   // it processes any user pointed index first
00742   void Fetcher::Impl::start( const Pathname &dest_dir,
00743                              MediaSetAccess &media,
00744                              const ProgressData::ReceiverFnc & progress_receiver )
00745   {
00746     ProgressData progress(_resources.size());
00747     progress.sendTo(progress_receiver);
00748 
00749     downloadAndReadIndexList(media, dest_dir);
00750 
00751     for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
00752     {
00753 
00754       if ( (*it_res)->flags & FetcherJob::Directory )
00755       {
00756           const OnMediaLocation location((*it_res)->location);
00757           addDirJobs(media, location, dest_dir, (*it_res)->flags);
00758           continue;
00759       }
00760 
00761       // may be this code can be factored out
00762       // together with the autodiscovery of indexes
00763       // of addDirJobs
00764       if ( ( _options & AutoAddSha1sumsIndexes ) ||
00765            ( _options & AutoAddContentFileIndexes ) )
00766       {
00767           // if auto indexing is enabled, then we need to read the
00768           // index for each file. We look only in the directory
00769           // where the file is. this is expensive of course.
00770           filesystem::DirContent content;
00771           getDirectoryContent(media, (*it_res)->location.filename().dirname(), content);
00772           // this method test for the option flags so indexes are added
00773           // only if the options are enabled
00774           MIL << "Autodiscovering signed indexes on '"
00775               << (*it_res)->location.filename().dirname() << "' for '"
00776               << (*it_res)->location.filename() << "'" << endl;
00777 
00778           autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir);
00779 
00780           // also look in the root of the media
00781           content.clear();
00782           getDirectoryContent(media, Pathname("/"), content);
00783           // this method test for the option flags so indexes are added
00784           // only if the options are enabled
00785           MIL << "Autodiscovering signed indexes on '"
00786               << "/" << "' for '"
00787               << (*it_res)->location.filename() << "'" << endl;
00788 
00789           autoaddIndexes(content, media, Pathname("/"), dest_dir);
00790       }
00791 
00792       provideToDest(media, (*it_res)->location, dest_dir, (*it_res)->deltafile);
00793 
00794       // if the file was not transfered, and no exception, just
00795       // return, as it was an optional file
00796       if ( ! PathInfo(dest_dir + (*it_res)->location.filename()).isExist() )
00797           return;
00798 
00799       // if the checksum is empty, but the checksum is in one of the
00800       // indexes checksum, then add a checker
00801       if ( (*it_res)->location.checksum().empty() )
00802       {
00803           if ( _checksums.find((*it_res)->location.filename().asString())
00804                != _checksums.end() )
00805           {
00806               CheckSum chksm = _checksums[(*it_res)->location.filename().asString()];
00807               ChecksumFileChecker digest_check(chksm);
00808               (*it_res)->checkers.push_back(digest_check);
00809           }
00810           else
00811           {
00812               // if the index checksum is empty too, we only add the checker
00813               // if the  AlwaysVerifyChecksum option is set on
00814               if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum )
00815               {
00816                   // add the checker with the empty checksum
00817                   ChecksumFileChecker digest_check((*it_res)->location.checksum());
00818                   (*it_res)->checkers.push_back(digest_check);
00819               }
00820           }
00821       }
00822       else
00823       {
00824           // checksum is not empty, so add a checksum checker
00825           ChecksumFileChecker digest_check((*it_res)->location.checksum());
00826           (*it_res)->checkers.push_back(digest_check);
00827       }
00828 
00829       // validate job, this throws if not valid
00830       validate((*it_res)->location, dest_dir, (*it_res)->checkers);
00831 
00832       if ( ! progress.incr() )
00833         ZYPP_THROW(AbortRequestException());
00834     } // for each job
00835   }
00836 
00838   inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
00839   {
00840       for ( list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
00841       {
00842           str << *it_res;
00843       }
00844       return str;
00845   }
00846 
00847   Fetcher::Fetcher()
00848   : _pimpl( new Impl() )
00849   {}
00850 
00851   Fetcher::~Fetcher()
00852   {}
00853 
00854   void Fetcher::setOptions( Fetcher::Options options )
00855   {
00856     _pimpl->setOptions(options);
00857   }
00858 
00859   Fetcher::Options Fetcher::options() const
00860   {
00861     return _pimpl->options();
00862   }
00863 
00864   void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
00865   {
00866     _pimpl->enqueueDigested(resource, checker, deltafile);
00867   }
00868 
00869   void Fetcher::enqueueDir( const OnMediaLocation &resource,
00870                             bool recursive,
00871                             const FileChecker &checker )
00872   {
00873       _pimpl->enqueueDir(resource, recursive, checker);
00874   }
00875 
00876   void Fetcher::enqueueDigestedDir( const OnMediaLocation &resource,
00877                                     bool recursive,
00878                                     const FileChecker &checker )
00879   {
00880       _pimpl->enqueueDigestedDir(resource, recursive, checker);
00881   }
00882 
00883 
00884   void Fetcher::addIndex( const OnMediaLocation &resource )
00885   {
00886     _pimpl->addIndex(resource);
00887   }
00888 
00889 
00890   void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker  )
00891   {
00892     _pimpl->enqueue(resource, checker);
00893   }
00894 
00895   void Fetcher::addCachePath( const Pathname &cache_dir )
00896   {
00897     _pimpl->addCachePath(cache_dir);
00898   }
00899 
00900   void Fetcher::reset()
00901   {
00902     _pimpl->reset();
00903   }
00904 
00905   void Fetcher::start( const Pathname &dest_dir,
00906                        MediaSetAccess &media,
00907                        const ProgressData::ReceiverFnc & progress_receiver )
00908   {
00909     _pimpl->start(dest_dir, media, progress_receiver);
00910   }
00911 
00912   std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
00913   {
00914     return str << *obj._pimpl;
00915   }
00916 
00918 } // namespace zypp
00920