libzypp  17.1.1
Fetcher.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include <iostream>
13 #include <fstream>
14 #include <list>
15 #include <map>
16 
17 #include "zypp/base/Easy.h"
18 #include "zypp/base/LogControl.h"
19 #include "zypp/base/LogTools.h"
20 #include "zypp/base/PtrTypes.h"
22 #include "zypp/base/String.h"
23 #include "zypp/Fetcher.h"
24 #include "zypp/ZYppFactory.h"
25 #include "zypp/CheckSum.h"
29 
30 using namespace std;
31 
32 #undef ZYPP_BASE_LOGGER_LOGGROUP
33 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
34 
36 namespace zypp
37 {
38 
44  struct FetcherIndex
45  {
47  : location(loc)
48  {}
53  };
54 
55  typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
56 
59  {
60  bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs )
61  {
62  if ( lhs == rhs )
63  return false; // incl. NULL == NULL
64  if ( ! lhs )
65  return true; // NULL < nonNULL
66  if ( ! rhs )
67  return false; // nonNULL > NULL
68  // both nonNULL ==> compare medianr and path
69  if ( lhs->location.medianr() == rhs->location.medianr() )
70  return lhs->location.filename() < rhs->location.filename();
71  //else
72  return lhs->location.medianr() < rhs->location.medianr();
73  }
74  };
75 
80  struct FetcherJob
81  {
82  enum Flag
83  {
84  None = 0x0000,
85  Directory = 0x0001,
86  Recursive = 0x0002,
87  RecursiveDirectory = Directory | Recursive,
88  // check checksums even if there is no such
89  // checksum (warns of no checksum)
90  AlwaysVerifyChecksum = 0x0004,
91  };
92  ZYPP_DECLARE_FLAGS(Flags, Flag);
93 
94 
95  FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname())
96  : location(loc)
97  , deltafile(dfile)
98  , flags(None)
99  {
100  //MIL << location << endl;
101  }
102 
104  {
105  //MIL << location << " | * " << checkers.size() << endl;
106  }
107 
110  //CompositeFileChecker checkers;
111  list<FileChecker> checkers;
112  Flags flags;
113  };
114 
115  ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
116  typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
117 
118  std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
119  {
120  return str << obj->location;
121  }
122 
124  //
125  // CLASS NAME : Fetcher::Impl
126  //
129  {
130  friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
131 
132  public:
133  Impl();
134 
135  ~Impl() {}
136 
137  void setOptions( Fetcher::Options options );
138  Fetcher::Options options() const;
139 
140  void addIndex( const OnMediaLocation &resource );
141 
142  void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
143  void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
144 
145  void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
146  void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() );
147  void addCachePath( const Pathname &cache_dir );
148  void reset();
149  void start( const Pathname &dest_dir,
150  MediaSetAccess &media,
151  const ProgressData::ReceiverFnc & progress_receiver );
152 
154  static shared_ptr<Impl> nullimpl()
155  {
156  static shared_ptr<Impl> _nullimpl( new Impl );
157  return _nullimpl;
158  }
159  private:
163  void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
164 
168  void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
169 
177  void readIndex( const Pathname &index, const Pathname &basedir );
178 
180  void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
181 
183  void readContentFileIndex( const Pathname &index, const Pathname &basedir );
184 
186  void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
187 
193  bool provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir );
199  void validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers );
200 
204  void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
205  const Pathname &dest_dir, FetcherJob::Flags flags );
206 
210  void autoaddIndexes( const filesystem::DirContent &content,
211  MediaSetAccess &media,
212  const OnMediaLocation &resource,
213  const Pathname &dest_dir );
217  void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir , const Pathname &deltafile);
218 
219  private:
220  friend Impl * rwcowClone<Impl>( const Impl * rhs );
222  Impl * clone() const
223  { return new Impl( *this ); }
224 
225  list<FetcherJob_Ptr> _resources;
226  std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
227  std::set<Pathname> _caches;
228  // checksums read from the indexes
229  map<string, CheckSum> _checksums;
230  // cache of dir contents
231  map<string, filesystem::DirContent> _dircontent;
232 
233  Fetcher::Options _options;
234  };
236 
237  void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
238  {
239  FetcherJob_Ptr job;
240  job.reset(new FetcherJob(resource, deltafile));
241  job->flags |= FetcherJob:: AlwaysVerifyChecksum;
242  _resources.push_back(job);
243  }
244 
245  Fetcher::Impl::Impl()
246  : _options(0)
247  {
248  }
249 
250  void Fetcher::Impl::setOptions( Fetcher::Options options )
251  { _options = options; }
252 
253  Fetcher::Options Fetcher::Impl::options() const
254  { return _options; }
255 
257  bool recursive,
258  const FileChecker &checker )
259  {
260  FetcherJob_Ptr job;
261  job.reset(new FetcherJob(resource));
262  if ( checker )
263  job->checkers.push_back(checker);
264  if ( recursive )
265  job->flags |= FetcherJob::Recursive;
266  job->flags |= FetcherJob::Directory;
267 
268  _resources.push_back(job);
269  }
270 
272  bool recursive,
273  const FileChecker &checker )
274  {
275  FetcherJob_Ptr job;
276  job.reset(new FetcherJob(resource));
277  if ( checker )
278  job->checkers.push_back(checker);
279  if ( recursive )
280  job->flags |= FetcherJob::Recursive;
281  job->flags |= FetcherJob::Directory;
282  job->flags |= FetcherJob::AlwaysVerifyChecksum;
283 
284  _resources.push_back(job);
285 
286  }
287 
288  void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
289  {
290  FetcherJob_Ptr job;
291  job.reset(new FetcherJob(resource));
292  if ( checker )
293  job->checkers.push_back(checker);
294  _resources.push_back(job);
295  }
296 
297  void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
298  {
299  MIL << "adding index " << resource << endl;
300  _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
301  }
302 
303 
305  {
306  _resources.clear();
307  _indexes.clear();
308  _checksums.clear();
309  _dircontent.clear();
310  }
311 
312  void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
313  {
314  PathInfo info(cache_dir);
315  if ( info.isExist() )
316  {
317  if ( info.isDir() )
318  {
319  DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
320  _caches.insert(cache_dir);
321  }
322  else
323  {
324  // don't add bad cache directory, just log the error
325  ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
326  }
327  }
328  else
329  {
330  ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
331  }
332 
333  }
334 
335  // tries to provide resource to dest_dir from any of the configured additional
336  // cache paths where the file may already be present. returns true if the
337  // file was provided from the cache.
338  bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir )
339  {
340  Pathname dest_full_path = dest_dir + resource.filename();
341 
342  // first check in the destination directory
343  if ( PathInfo(dest_full_path).isExist() )
344  {
345  if ( is_checksum( dest_full_path, resource.checksum() )
346  && (! resource.checksum().empty() ) )
347  return true;
348  }
349 
350  MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
351  for_ ( it_cache, _caches.begin(), _caches.end() )
352  {
353  // does the current file exists in the current cache?
354  Pathname cached_file = *it_cache + resource.filename();
355  if ( PathInfo( cached_file ).isExist() )
356  {
357  DBG << "File '" << cached_file << "' exist, testing checksum " << resource.checksum() << endl;
358  // check the checksum
359  if ( is_checksum( cached_file, resource.checksum() ) && (! resource.checksum().empty() ) )
360  {
361  // cached
362  MIL << "file " << resource.filename() << " found in previous cache. Using cached copy." << endl;
363  // checksum is already checked.
364  // we could later implement double failover and try to download if file copy fails.
365  // replicate the complete path in the target directory
366  if( dest_full_path != cached_file )
367  {
368  if ( assert_dir( dest_full_path.dirname() ) != 0 )
369  ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
370 
371  if ( filesystem::hardlinkCopy(cached_file, dest_full_path ) != 0 )
372  {
373  ERR << "Can't hardlink/copy " << cached_file + " to " + dest_dir << endl;
374  continue;
375  }
376  }
377  // found in cache
378  return true;
379  }
380  }
381  } // iterate over caches
382  return false;
383  }
384 
385  void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers )
386  {
387  // no matter where did we got the file, try to validate it:
388  Pathname localfile = dest_dir + resource.filename();
389  // call the checker function
390  try
391  {
392  MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl;
393 
394  for ( list<FileChecker>::const_iterator it = checkers.begin();
395  it != checkers.end();
396  ++it )
397  {
398  if (*it)
399  {
400  (*it)(localfile);
401  }
402  else
403  {
404  ERR << "Invalid checker for '" << localfile << "'" << endl;
405  }
406  }
407 
408  }
409  catch ( const FileCheckException &e )
410  {
411  ZYPP_RETHROW(e);
412  }
413  catch ( const Exception &e )
414  {
415  ZYPP_RETHROW(e);
416  }
417  catch (...)
418  {
419  ZYPP_THROW(Exception("Unknown error while validating " + resource.filename().asString()));
420  }
421  }
422 
424  MediaSetAccess &media,
425  const OnMediaLocation &resource,
426  const Pathname &dest_dir )
427  {
428  auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
429  {
430  if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
431  return false;
432  // add the index of this directory
433  OnMediaLocation indexloc( resource );
434  indexloc.changeFilename( resource.filename() + index_r );
435  addIndex( indexloc );
436  // we need to read it now
437  downloadAndReadIndexList( media, dest_dir );
438  return true;
439  } );
440 
442  {
443  fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
444  }
446  {
447  fnc_addIfInContent( "content" );
448  }
449  }
450 
452  const OnMediaLocation &resource,
453  filesystem::DirContent &content )
454  {
455  if ( _dircontent.find(resource.filename().asString())
456  != _dircontent.end() )
457  {
458  filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
459 
460  std::copy(filled.begin(), filled.end(), std::back_inserter(content));
461  }
462  else
463  {
464  filesystem::DirContent tofill;
465  media.dirInfo( tofill,
466  resource.filename(),
467  false /* dots */,
468  resource.medianr());
469  std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
470  _dircontent[resource.filename().asString()] = tofill;
471  }
472  }
473 
475  const OnMediaLocation &resource,
476  const Pathname &dest_dir, FetcherJob::Flags flags )
477  {
478  // first get the content of the directory so we can add
479  // individual transfer jobs
480  MIL << "Adding directory " << resource.filename() << endl;
481  filesystem::DirContent content;
482  try {
483  getDirectoryContent(media, resource, content);
484  }
485  catch ( media::MediaFileNotFoundException & exception )
486  {
487  ZYPP_CAUGHT( exception );
488  WAR << "Skiping subtree hidden at " << resource.filename() << endl;
489  return;
490  }
491 
492  // this method test for the option flags so indexes are added
493  // only if the options are enabled
494  autoaddIndexes(content, media, resource, dest_dir);
495 
496  for ( filesystem::DirContent::const_iterator it = content.begin();
497  it != content.end();
498  ++it )
499  {
500  // skip CHECKSUMS* as they were already retrieved
501  if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
502  continue;
503 
504  Pathname filename = resource.filename() + it->name;
505 
506  switch ( it->type )
507  {
508  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
509  case filesystem::FT_FILE:
510  {
511  CheckSum chksm(resource.checksum());
512  if ( _checksums.find(filename.asString()) != _checksums.end() )
513  {
514  // the checksum can be replaced with the one in the index.
515  chksm = _checksums[filename.asString()];
516  //MIL << "resource " << filename << " has checksum in the index file." << endl;
517  }
518  else
519  WAR << "Resource " << filename << " has no checksum in the index either." << endl;
520 
521  if ( flags & FetcherJob::AlwaysVerifyChecksum )
522  enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
523  else
524  enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
525  break;
526  }
527  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
528  if ( flags & FetcherJob::Recursive )
529  addDirJobs(media, filename, dest_dir, flags);
530  break;
531  default:
532  // don't provide devices, sockets, etc.
533  break;
534  }
535  }
536  }
537 
538  void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, const Pathname &deltafile )
539  {
540  bool got_from_cache = false;
541 
542  // start look in cache
543  got_from_cache = provideFromCache(resource, dest_dir);
544 
545  if ( ! got_from_cache )
546  {
547  MIL << "Not found in cache, downloading" << endl;
548 
549  // try to get the file from the net
550  try
551  {
552  Pathname tmp_file = media.provideFile(resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, deltafile );
553 
554  Pathname dest_full_path = dest_dir + resource.filename();
555 
556  if ( assert_dir( dest_full_path.dirname() ) != 0 )
557  ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
558  if ( filesystem::hardlinkCopy( tmp_file, dest_full_path ) != 0 )
559  {
560  if ( ! PathInfo(tmp_file).isExist() )
561  ERR << tmp_file << " does not exist" << endl;
562  if ( ! PathInfo(dest_full_path.dirname()).isExist() )
563  ERR << dest_full_path.dirname() << " does not exist" << endl;
564 
565  media.releaseFile(resource); //not needed anymore, only eat space
566  ZYPP_THROW( Exception("Can't hardlink/copy " + tmp_file.asString() + " to " + dest_dir.asString()));
567  }
568 
569  media.releaseFile(resource); //not needed anymore, only eat space
570  }
571  catch (Exception & excpt_r)
572  {
573  if ( resource.optional() )
574  {
575  ZYPP_CAUGHT(excpt_r);
576  WAR << "optional resource " << resource << " could not be transferred" << endl;
577  return;
578  }
579  else
580  {
581  excpt_r.remember("Can't provide " + resource.filename().asString() );
582  ZYPP_RETHROW(excpt_r);
583  }
584  }
585  }
586  else
587  {
588  // We got the file from cache
589  // continue with next file
590  return;
591  }
592  }
593 
594  // helper class to consume a content file
596  {
598  {
599  setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
600  }
601 
602  void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
603  { _repoindex = data_r; }
604 
605  parser::susetags::RepoIndex_Ptr _repoindex;
606  };
607 
608  // generic function for reading indexes
609  void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
610  {
611  if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
612  readChecksumsIndex(index, basedir);
613  else if ( index.basename() == "content" )
614  readContentFileIndex(index, basedir);
615  else
616  WAR << index << ": index file format not known" << endl;
617  }
618 
619  // reads a content file index
620  void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
621  {
622  ContentReaderHelper reader;
623  reader.parse(index);
624  MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
625  for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
626  {
627  // content file entries don't start with /
628  _checksums[(basedir + it->first).asString()] = it->second;
629  }
630  }
631 
632  // reads a CHECKSUMS (old SHA1SUMS) file index
633  void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
634  {
635  std::ifstream in( index.c_str() );
636  if ( ! in.fail() )
637  {
638  std::string buffer;
639  while ( getline( in, buffer ) )
640  {
641 
642  if ( buffer[0] == '#' )
643  continue; // simple comment
644 
645  CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
646  if ( checksum.empty() )
647  continue; // empty line | unknown cheksum format
648 
649  if ( buffer.empty() )
650  {
651  WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
652  continue;
653  }
654 
655  _checksums[(basedir/buffer).asString()] = checksum;
656  }
657  }
658  else
659  ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
660  }
661 
662  void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
663  {
664  MIL << "downloading index " << resource << endl;
665  // create a new fetcher with a different state to transfer the
666  // file containing checksums and its signature
667  Fetcher fetcher;
668  // signature checker for index. We havent got the signature from
669  // the nextwork yet.
670  SignatureFileChecker sigchecker;
671 
672  // build the name of the index and the signature
673  OnMediaLocation idxloc(resource);
674  OnMediaLocation sigloc(resource);
675  OnMediaLocation keyloc(resource);
676 
677  // we should not fail the download if those don't exists
678  // the checking will warn later
679  sigloc.setOptional(true);
680  keyloc.setOptional(true);
681 
682  // calculate signature and key name
683  sigloc.changeFilename( sigloc.filename().extend(".asc") );
684  keyloc.changeFilename( keyloc.filename().extend(".key") );
685 
686  //assert_dir(dest_dir + idxloc.filename().dirname());
687 
688  // transfer the signature
689  fetcher.enqueue(sigloc);
690  fetcher.start( dest_dir, media );
691  // if we get the signature, update the checker
692  if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
693  sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
694 
695  fetcher.reset();
696 
697  // now the key
698  fetcher.enqueue(keyloc);
699  fetcher.start( dest_dir, media );
700  fetcher.reset();
701 
702  // try to import the key
703  if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
704  getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
705  else
706  WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
707 
708  // now the index itself
709  fetcher.enqueue( idxloc, FileChecker(sigchecker) );
710  fetcher.start( dest_dir, media );
711  fetcher.reset();
712  }
713 
714  // this method takes all the user pointed indexes, gets them and also tries to
715  // download their signature, and verify them. After that, its parses each one
716  // to fill the checksum cache.
718  {
719  // if there is no indexes, then just return to avoid
720  // the directory listing
721  if ( _indexes.empty() )
722  {
723  MIL << "No indexes to read." << endl;
724  return;
725  }
726 
727  for_( it_idx, _indexes.begin(), _indexes.end() )
728  {
729  if ( (*it_idx)->read )
730  {
731  DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
732  }
733  else
734  {
735  // base::LogControl::TmpLineWriter shutUp;
736  downloadIndex( media, (*it_idx)->location, dest_dir );
737  // now we have the indexes in dest_dir
738  readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
739  // Take care we don't process it again
740  MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
741  (*it_idx)->read = true;
742  }
743  }
744  MIL << "done reading indexes" << endl;
745  }
746 
747  // start processing all fetcher jobs.
748  // it processes any user pointed index first
749  void Fetcher::Impl::start( const Pathname &dest_dir,
750  MediaSetAccess &media,
751  const ProgressData::ReceiverFnc & progress_receiver )
752  {
753  ProgressData progress(_resources.size());
754  progress.sendTo(progress_receiver);
755 
756  downloadAndReadIndexList(media, dest_dir);
757 
758  for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
759  {
760 
761  if ( (*it_res)->flags & FetcherJob::Directory )
762  {
763  const OnMediaLocation location((*it_res)->location);
764  addDirJobs(media, location, dest_dir, (*it_res)->flags);
765  continue;
766  }
767 
768  // may be this code can be factored out
769  // together with the autodiscovery of indexes
770  // of addDirJobs
771  if ( ( _options & AutoAddChecksumsIndexes ) ||
773  {
774  // if auto indexing is enabled, then we need to read the
775  // index for each file. We look only in the directory
776  // where the file is. this is expensive of course.
777  filesystem::DirContent content;
778  getDirectoryContent(media, (*it_res)->location.filename().dirname(), content);
779  // this method test for the option flags so indexes are added
780  // only if the options are enabled
781  MIL << "Autodiscovering signed indexes on '"
782  << (*it_res)->location.filename().dirname() << "' for '"
783  << (*it_res)->location.filename() << "'" << endl;
784 
785  autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir);
786 
787  // also look in the root of the media
788  content.clear();
789  getDirectoryContent(media, Pathname("/"), content);
790  // this method test for the option flags so indexes are added
791  // only if the options are enabled
792  MIL << "Autodiscovering signed indexes on '"
793  << "/" << "' for '"
794  << (*it_res)->location.filename() << "'" << endl;
795 
796  autoaddIndexes(content, media, Pathname("/"), dest_dir);
797  }
798 
799  provideToDest(media, (*it_res)->location, dest_dir, (*it_res)->deltafile);
800 
801  // if the file was not transferred, and no exception, just
802  // return, as it was an optional file
803  if ( ! PathInfo(dest_dir + (*it_res)->location.filename()).isExist() )
804  continue;
805 
806  // if the checksum is empty, but the checksum is in one of the
807  // indexes checksum, then add a checker
808  if ( (*it_res)->location.checksum().empty() )
809  {
810  if ( _checksums.find((*it_res)->location.filename().asString())
811  != _checksums.end() )
812  {
813  CheckSum chksm = _checksums[(*it_res)->location.filename().asString()];
814  ChecksumFileChecker digest_check(chksm);
815  (*it_res)->checkers.push_back(digest_check);
816  }
817  else
818  {
819  // if the index checksum is empty too, we only add the checker
820  // if the AlwaysVerifyChecksum option is set on
821  if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum )
822  {
823  // add the checker with the empty checksum
824  ChecksumFileChecker digest_check((*it_res)->location.checksum());
825  (*it_res)->checkers.push_back(digest_check);
826  }
827  }
828  }
829  else
830  {
831  // checksum is not empty, so add a checksum checker
832  ChecksumFileChecker digest_check((*it_res)->location.checksum());
833  (*it_res)->checkers.push_back(digest_check);
834  }
835 
836  // validate job, this throws if not valid
837  validate((*it_res)->location, dest_dir, (*it_res)->checkers);
838 
839  if ( ! progress.incr() )
840  ZYPP_THROW(AbortRequestException());
841  } // for each job
842  }
843 
845  inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
846  {
847  for ( list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
848  {
849  str << *it_res;
850  }
851  return str;
852  }
853 
855  : _pimpl( new Impl() )
856  {}
857 
859  {}
860 
861  void Fetcher::setOptions( Fetcher::Options options )
862  {
863  _pimpl->setOptions(options);
864  }
865 
866  Fetcher::Options Fetcher::options() const
867  {
868  return _pimpl->options();
869  }
870 
871  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
872  {
873  _pimpl->enqueueDigested(resource, checker, deltafile);
874  }
875 
876  void Fetcher::enqueueDir( const OnMediaLocation &resource,
877  bool recursive,
878  const FileChecker &checker )
879  {
880  _pimpl->enqueueDir(resource, recursive, checker);
881  }
882 
884  bool recursive,
885  const FileChecker &checker )
886  {
887  _pimpl->enqueueDigestedDir(resource, recursive, checker);
888  }
889 
890 
891  void Fetcher::addIndex( const OnMediaLocation &resource )
892  {
893  _pimpl->addIndex(resource);
894  }
895 
896 
897  void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
898  {
899  _pimpl->enqueue(resource, checker);
900  }
901 
902  void Fetcher::addCachePath( const Pathname &cache_dir )
903  {
904  _pimpl->addCachePath(cache_dir);
905  }
906 
908  {
909  _pimpl->reset();
910  }
911 
912  void Fetcher::start( const Pathname &dest_dir,
913  MediaSetAccess &media,
914  const ProgressData::ReceiverFnc & progress_receiver )
915  {
916  _pimpl->start(dest_dir, media, progress_receiver);
917  }
918 
919  std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
920  {
921  return str << *obj._pimpl;
922  }
923 
925 } // namespace zypp
927 
bool provideFromCache(const OnMediaLocation &resource, const Pathname &dest_dir)
tries to provide the file represented by job into dest_dir by looking at the cache.
Definition: Fetcher.cc:338
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:320
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:602
#define MIL
Definition: Logger.h:64
Pathname deltafile
Definition: Fetcher.cc:109
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:620
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:55
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:861
Listentry returned by readdir.
Definition: PathInfo.h:532
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: ProgressData.h:226
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
Describes a path on a certain media amongs as the information required to download it...
Checks for the validity of a signature.
Definition: FileChecker.h:93
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:891
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:605
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:271
OnMediaLocation & setOptional(bool val)
Set the whether the resource is optional or not.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:166
list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:225
const char * c_str() const
String representation.
Definition: Pathname.h:109
String related utilities and Regular expression matching.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:288
Definition: Arch.h:344
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:902
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:27
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:127
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:883
void addCachePath(const Pathname &cache_dir)
Definition: Fetcher.cc:312
const bool optional() const
whether this is an optional resource.
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:50
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
void provideToDest(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, const Pathname &deltafile)
Provide the resource to dest_dir.
Definition: Fetcher.cc:538
#define ERR
Definition: Logger.h:66
void releaseFile(const OnMediaLocation &resource)
Release file from media.
unsigned medianr() const
media number where the resource is located.
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:154
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:325
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:871
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:105
void validate(const OnMediaLocation &resource, const Pathname &dest_dir, const list< FileChecker > &checkers)
Validates the job against is checkers, by using the file instance on dest_dir.
Definition: Fetcher.cc:385
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:226
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:400
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:261
Fetcher::Options _options
Definition: Fetcher.cc:233
const std::string & asString() const
String representation.
Definition: Pathname.h:90
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:281
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:907
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:123
#define WAR
Definition: Logger.h:65
list< FileChecker > checkers
Definition: Fetcher.cc:111
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:609
Parse repoindex part from a content file.
std::set ordering (less semantic)
Definition: Fetcher.cc:58
Fetcher::Options options() const
Definition: Fetcher.cc:253
Maintain [min,max] and counter (value) for progress counting.
Definition: ProgressData.h:130
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:256
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
friend std::ostream & operator<<(std::ostream &str, const Fetcher &obj)
Definition: Fetcher.cc:919
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:52
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:836
OnMediaLocation location
Definition: Fetcher.cc:108
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:662
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
Definition: Fetcher.cc:912
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Definition: Fetcher.cc:237
Fetcher()
Default ctor.
Definition: Fetcher.cc:854
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:876
std::ostream & operator<<(std::ostream &str, const zypp::shared_ptr< void > &obj)
Definition: PtrTypes.h:134
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:50
map< string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:231
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:451
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:396
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:80
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:250
const Pathname & filename() const
The path to the resource relatve to the url and path.
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:46
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition: PublicKey.h:282
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:633
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:717
Fetcher implementation.
Definition: Fetcher.cc:128
#define ZYPP_DECLARE_OPERATORS_FOR_FLAGS(Name)
Definition: Flags.h:177
Base class for Exception.
Definition: Exception.h:145
#define ZYPP_DECLARE_FLAGS(Name, Enum)
Definition: Flags.h:174
Built in file checkers.
Definition: FileChecker.h:70
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
Definition: Fetcher.cc:95
bool empty() const
Definition: CheckSum.cc:173
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:1004
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
OnMediaLocation & changeFilename(const Pathname &val_r)
Individual manipulation of filename (prefer setLocation).
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:44
std::set< Pathname > _caches
Definition: Fetcher.cc:227
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
Definition: Fetcher.cc:749
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:28
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Definition: Fetcher.cc:845
Options options() const
Get current options.
Definition: Fetcher.cc:866
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:116
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:897
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:858
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs)
Definition: Fetcher.cc:60
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT, const Pathname &deltafile=Pathname())
Provides a file from a media location.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:423
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1028
map< string, CheckSum > _checksums
Definition: Fetcher.cc:229
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:105
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:297
Media access layer responsible for handling files distributed on a set of media with media change and...
#define DBG
Definition: Logger.h:63
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1016
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:474
const CheckSum & checksum() const
the checksum of the resource
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:222
The user is not asked anything, and the error exception is just propagated.