libzypp  17.28.4
Fetcher.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include <iostream>
13 #include <fstream>
14 #include <list>
15 #include <map>
16 
17 #include <zypp/base/Easy.h>
18 #include <zypp/base/LogControl.h>
19 #include <zypp/base/LogTools.h>
20 #include <zypp/base/PtrTypes.h>
22 #include <zypp/base/String.h>
24 #include <zypp/Fetcher.h>
25 #include <zypp/ZYppFactory.h>
26 #include <zypp/CheckSum.h>
30 
31 #undef ZYPP_BASE_LOGGER_LOGGROUP
32 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher"
33 
35 namespace zypp
36 {
37 
43  struct FetcherIndex
44  {
46  : location(loc)
47  {}
52  };
53 
54  typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
55 
58  {
59  bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const
60  {
61  if ( lhs == rhs )
62  return false; // incl. NULL == NULL
63  if ( ! lhs )
64  return true; // NULL < nonNULL
65  if ( ! rhs )
66  return false; // nonNULL > NULL
67  // both nonNULL ==> compare medianr and path
68  if ( lhs->location.medianr() == rhs->location.medianr() )
69  return lhs->location.filename() < rhs->location.filename();
70  //else
71  return lhs->location.medianr() < rhs->location.medianr();
72  }
73  };
74 
79  struct FetcherJob
80  {
81  enum Flag
82  {
83  None = 0x0000,
84  Directory = 0x0001,
85  Recursive = 0x0002,
87  // check checksums even if there is no such
88  // checksum (warns of no checksum)
90  };
92 
93 
95  : location(loc)
96  , flags(None)
97  {
98  //MIL << location << endl;
99  }
100 
102  {
103  //MIL << location << " | * " << checkers.size() << endl;
104  }
105 
107  //CompositeFileChecker checkers;
108  std::list<FileChecker> checkers;
109  Flags flags;
110  };
111 
113  typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
114 
115  std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
116  {
117  return str << obj->location;
118  }
119 
121  //
122  // CLASS NAME : Fetcher::Impl
123  //
126  {
127  friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
128 
129  public:
130  Impl();
131 
132  ~Impl() {}
133 
134  void setOptions( Fetcher::Options options );
135  Fetcher::Options options() const;
136 
137  void addIndex( const OnMediaLocation &resource );
138 
139  void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
140  void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
141 
142  void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
143  void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
144  void addCachePath( const Pathname &cache_dir );
145  void reset();
146  void setMediaSetAccess ( MediaSetAccess &media );
147  void start( const Pathname &dest_dir,
148  const ProgressData::ReceiverFnc &progress );
149  void start( const Pathname &dest_dir,
150  MediaSetAccess &media,
151  const ProgressData::ReceiverFnc & progress_receiver );
152 
154  static shared_ptr<Impl> nullimpl()
155  {
156  static shared_ptr<Impl> _nullimpl( new Impl );
157  return _nullimpl;
158  }
159  private:
163  void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
164 
168  void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
169 
177  void readIndex( const Pathname &index, const Pathname &basedir );
178 
180  void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
181 
183  void readContentFileIndex( const Pathname &index, const Pathname &basedir );
184 
186  void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
187 
193  Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
198  void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
199 
203  void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
204  const Pathname &dest_dir, FetcherJob::Flags flags );
205 
209  void autoaddIndexes( const filesystem::DirContent &content,
210  MediaSetAccess &media,
211  const OnMediaLocation &resource,
212  const Pathname &dest_dir );
216  void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
217 
218  private:
219  friend Impl * rwcowClone<Impl>( const Impl * rhs );
221  Impl * clone() const
222  { return new Impl( *this ); }
223 
224  std::list<FetcherJob_Ptr> _resources;
225  std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
226  std::set<Pathname> _caches;
227  // checksums read from the indexes
228  std::map<std::string, CheckSum> _checksums;
229  // cache of dir contents
230  std::map<std::string, filesystem::DirContent> _dircontent;
231 
233 
234  Fetcher::Options _options;
235  };
237 
239  {
240  if ( _mediaSetAccess )
241  _mediaSetAccess->precacheFiles( {resource} );
242 
243  FetcherJob_Ptr job;
244  job.reset(new FetcherJob(resource));
245  job->flags |= FetcherJob:: AlwaysVerifyChecksum;
246  _resources.push_back(job);
247  }
248 
250  : _options(0)
251  {
252  }
253 
254  void Fetcher::Impl::setOptions( Fetcher::Options options )
255  { _options = options; }
256 
257  Fetcher::Options Fetcher::Impl::options() const
258  { return _options; }
259 
261  bool recursive,
262  const FileChecker &checker )
263  {
264  FetcherJob_Ptr job;
265  job.reset(new FetcherJob(resource));
266  if ( checker )
267  job->checkers.push_back(checker);
268  if ( recursive )
269  job->flags |= FetcherJob::Recursive;
270  job->flags |= FetcherJob::Directory;
271 
272  _resources.push_back(job);
273  }
274 
276  bool recursive,
277  const FileChecker &checker )
278  {
279  FetcherJob_Ptr job;
280  job.reset(new FetcherJob(resource));
281  if ( checker )
282  job->checkers.push_back(checker);
283  if ( recursive )
284  job->flags |= FetcherJob::Recursive;
285  job->flags |= FetcherJob::Directory;
286  job->flags |= FetcherJob::AlwaysVerifyChecksum;
287 
288  _resources.push_back(job);
289 
290  }
291 
292  void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
293  {
294  if ( _mediaSetAccess )
295  _mediaSetAccess->precacheFiles( {resource} );
296 
297  FetcherJob_Ptr job;
298  job.reset(new FetcherJob(resource));
299  if ( checker )
300  job->checkers.push_back(checker);
301  _resources.push_back(job);
302  }
303 
304  void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
305  {
306  MIL << "adding index " << resource << endl;
307  _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
308  }
309 
310 
312  {
313  _resources.clear();
314  _indexes.clear();
315  _checksums.clear();
316  _dircontent.clear();
317  }
318 
320  {
321  _mediaSetAccess = &media;
322  }
323 
324  void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
325  {
326  PathInfo info(cache_dir);
327  if ( info.isExist() )
328  {
329  if ( info.isDir() )
330  {
331  DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
332  _caches.insert(cache_dir);
333  }
334  else
335  {
336  // don't add bad cache directory, just log the error
337  ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
338  }
339  }
340  else
341  {
342  ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
343  }
344 
345  }
346 
347  Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
348  {
349  Pathname ret;
350  // No checksum - no match
351  if ( resource_r.checksum().empty() )
352  return ret;
353 
354  // first check in the destination directory
355  Pathname cacheLocation = destDir_r / resource_r.filename();
356  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
357  {
358  swap( ret, cacheLocation );
359  return ret;
360  }
361 
362  MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
363  for( const Pathname & cacheDir : _caches )
364  {
365  cacheLocation = cacheDir / resource_r.filename();
366  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
367  {
368  MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl;
369  swap( ret, cacheLocation );
370  return ret;
371  }
372  }
373 
374  return ret;
375  }
376 
377  void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
378  {
379  try
380  {
381  MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
382 
383  for ( const FileChecker & chkfnc : checkers_r )
384  {
385  if ( chkfnc )
386  chkfnc( localfile_r );
387  else
388  ERR << "Invalid checker for '" << localfile_r << "'" << endl;
389  }
390 
391  }
392  catch ( const FileCheckException &e )
393  {
394  ZYPP_RETHROW(e);
395  }
396  catch ( const Exception &e )
397  {
398  ZYPP_RETHROW(e);
399  }
400  catch (...)
401  {
402  ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
403  }
404  }
405 
407  MediaSetAccess &media,
408  const OnMediaLocation &resource,
409  const Pathname &dest_dir )
410  {
411  auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
412  {
413  if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
414  return false;
415  // add the index of this directory
416  OnMediaLocation indexloc( resource );
417  indexloc.changeFilename( resource.filename() + index_r );
418  addIndex( indexloc );
419  // we need to read it now
420  downloadAndReadIndexList( media, dest_dir );
421  return true;
422  } );
423 
424  if ( _options & AutoAddChecksumsIndexes )
425  {
426  fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
427  }
428  if ( _options & AutoAddContentFileIndexes )
429  {
430  fnc_addIfInContent( "content" );
431  }
432  }
433 
435  const OnMediaLocation &resource,
436  filesystem::DirContent &content )
437  {
438  if ( _dircontent.find(resource.filename().asString())
439  != _dircontent.end() )
440  {
441  filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
442 
443  std::copy(filled.begin(), filled.end(), std::back_inserter(content));
444  }
445  else
446  {
447  filesystem::DirContent tofill;
448  media.dirInfo( tofill,
449  resource.filename(),
450  false /* dots */,
451  resource.medianr());
452  std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
453  _dircontent[resource.filename().asString()] = tofill;
454  }
455  }
456 
458  const OnMediaLocation &resource,
459  const Pathname &dest_dir, FetcherJob::Flags flags )
460  {
461  // first get the content of the directory so we can add
462  // individual transfer jobs
463  MIL << "Adding directory " << resource.filename() << endl;
464  filesystem::DirContent content;
465  try {
466  getDirectoryContent(media, resource, content);
467  }
468  catch ( media::MediaFileNotFoundException & exception )
469  {
470  ZYPP_CAUGHT( exception );
471  WAR << "Skiping subtree hidden at " << resource.filename() << endl;
472  return;
473  }
474 
475  // this method test for the option flags so indexes are added
476  // only if the options are enabled
477  autoaddIndexes(content, media, resource, dest_dir);
478 
479  for ( filesystem::DirContent::const_iterator it = content.begin();
480  it != content.end();
481  ++it )
482  {
483  // skip CHECKSUMS* as they were already retrieved
484  if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
485  continue;
486 
487  Pathname filename = resource.filename() + it->name;
488 
489  switch ( it->type )
490  {
491  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
492  case filesystem::FT_FILE:
493  {
494  CheckSum chksm(resource.checksum());
495  if ( _checksums.find(filename.asString()) != _checksums.end() )
496  {
497  // the checksum can be replaced with the one in the index.
498  chksm = _checksums[filename.asString()];
499  //MIL << "resource " << filename << " has checksum in the index file." << endl;
500  }
501  else
502  WAR << "Resource " << filename << " has no checksum in the index either." << endl;
503 
504  if ( flags & FetcherJob::AlwaysVerifyChecksum )
505  enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
506  else
507  enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
508  break;
509  }
510  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
511  if ( flags & FetcherJob::Recursive )
512  addDirJobs(media, filename, dest_dir, flags);
513  break;
514  default:
515  // don't provide devices, sockets, etc.
516  break;
517  }
518  }
519  }
520 
521  void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
522  {
523  const OnMediaLocation & resource( jobp_r->location );
524 
525  try
526  {
527  scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
528 
529  // get cached file (by checksum) or provide from media
530  Pathname tmpFile = locateInCache( resource, destDir_r );
531  if ( tmpFile.empty() )
532  {
533  MIL << "Not found in cache, retrieving..." << endl;
535  releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
536  }
537 
538  // The final destination: locateInCache also checks destFullPath!
539  // If we find a cache match (by checksum) at destFullPath, take
540  // care it gets deleted, in case the validation fails.
541  ManagedFile destFullPath( destDir_r / resource.filename() );
542  if ( tmpFile == destFullPath )
543  destFullPath.setDispose( filesystem::unlink );
544 
545  // validate the file (throws if not valid)
546  validate( tmpFile, jobp_r->checkers );
547 
548  // move it to the final destination
549  if ( tmpFile == destFullPath )
550  destFullPath.resetDispose(); // keep it!
551  else
552  {
553  if ( assert_dir( destFullPath->dirname() ) != 0 )
554  ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
555 
556  if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
557  ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
558  }
559  }
560  catch ( Exception & excpt )
561  {
562  if ( resource.optional() )
563  {
564  ZYPP_CAUGHT( excpt );
565  WAR << "optional resource " << resource << " could not be transferred." << endl;
566  return;
567  }
568  else
569  {
570  excpt.remember( "Can't provide " + resource.filename().asString() );
571  ZYPP_RETHROW( excpt );
572  }
573  }
574  }
575 
576  // helper class to consume a content file
578  {
580  {
581  setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
582  }
583 
584  void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
585  { _repoindex = data_r; }
586 
587  parser::susetags::RepoIndex_Ptr _repoindex;
588  };
589 
590  // generic function for reading indexes
591  void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
592  {
593  if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
594  readChecksumsIndex(index, basedir);
595  else if ( index.basename() == "content" )
596  readContentFileIndex(index, basedir);
597  else
598  WAR << index << ": index file format not known" << endl;
599  }
600 
601  // reads a content file index
602  void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
603  {
604  ContentReaderHelper reader;
605  reader.parse(index);
606  MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
607  for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
608  {
609  // content file entries don't start with /
610  _checksums[(basedir + it->first).asString()] = it->second;
611  }
612  }
613 
614  // reads a CHECKSUMS (old SHA1SUMS) file index
615  void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
616  {
617  std::ifstream in( index.c_str() );
618  if ( ! in.fail() )
619  {
620  std::string buffer;
621  while ( getline( in, buffer ) )
622  {
623 
624  if ( buffer[0] == '#' )
625  continue; // simple comment
626 
627  CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
628  if ( checksum.empty() )
629  continue; // empty line | unknown cheksum format
630 
631  if ( buffer.empty() )
632  {
633  WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
634  continue;
635  }
636 
637  _checksums[(basedir/buffer).asString()] = checksum;
638  }
639  }
640  else
641  ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
642  }
643 
644  void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
645  {
646  MIL << "downloading index " << resource << endl;
647  // create a new fetcher with a different state to transfer the
648  // file containing checksums and its signature
649  Fetcher fetcher;
650  // signature checker for index. We havent got the signature from
651  // the nextwork yet.
652  SignatureFileChecker sigchecker;
653 
654  // build the name of the index and the signature
655  OnMediaLocation idxloc(resource);
656  OnMediaLocation sigloc(resource);
657  OnMediaLocation keyloc(resource);
658 
659  // we should not fail the download if those don't exists
660  // the checking will warn later
661  sigloc.setOptional(true);
662  keyloc.setOptional(true);
663 
664  // calculate signature and key name
665  sigloc.changeFilename( sigloc.filename().extend(".asc") );
666  keyloc.changeFilename( keyloc.filename().extend(".key") );
667 
668  //assert_dir(dest_dir + idxloc.filename().dirname());
669 
670  // transfer the signature
671  fetcher.enqueue(sigloc);
672  fetcher.start( dest_dir, media );
673  // if we get the signature, update the checker
674  if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
675  sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
676 
677  fetcher.reset();
678 
679  // now the key
680  fetcher.enqueue(keyloc);
681  fetcher.start( dest_dir, media );
682  fetcher.reset();
683 
684  // try to import the key
685  if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
686  getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
687  else
688  WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
689 
690  // now the index itself
691  fetcher.enqueue( idxloc, FileChecker(sigchecker) );
692  fetcher.start( dest_dir, media );
693  fetcher.reset();
694  }
695 
696  // this method takes all the user pointed indexes, gets them and also tries to
697  // download their signature, and verify them. After that, its parses each one
698  // to fill the checksum cache.
700  {
701  // if there is no indexes, then just return to avoid
702  // the directory listing
703  if ( _indexes.empty() )
704  {
705  MIL << "No indexes to read." << endl;
706  return;
707  }
708 
709  for_( it_idx, _indexes.begin(), _indexes.end() )
710  {
711  if ( (*it_idx)->read )
712  {
713  DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
714  }
715  else
716  {
717  // base::LogControl::TmpLineWriter shutUp;
718  downloadIndex( media, (*it_idx)->location, dest_dir );
719  // now we have the indexes in dest_dir
720  readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
721  // Take care we don't process it again
722  MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
723  (*it_idx)->read = true;
724  }
725  }
726  MIL << "done reading indexes" << endl;
727  }
728 
729  void Fetcher::Impl::start( const Pathname &dest_dir,
730  const ProgressData::ReceiverFnc & progress )
731  {
732  if ( !_mediaSetAccess )
733  ZYPP_THROW( zypp::Exception("Called Fetcher::start without setting MediaSetAccess before.") );
734  start( dest_dir, *_mediaSetAccess, progress );
735  }
736 
737  // start processing all fetcher jobs.
738  // it processes any user pointed index first
739  void Fetcher::Impl::start( const Pathname &dest_dir,
740  MediaSetAccess &media,
741  const ProgressData::ReceiverFnc & progress_receiver )
742  {
743  _mediaSetAccess = nullptr; //reset the internally stored MediaSetAccess
744 
745  ProgressData progress(_resources.size());
746  progress.sendTo(progress_receiver);
747 
748  downloadAndReadIndexList(media, dest_dir);
749 
750  for ( const FetcherJob_Ptr & jobp : _resources )
751  {
752  if ( jobp->flags & FetcherJob::Directory )
753  {
754  const OnMediaLocation location(jobp->location);
755  addDirJobs(media, location, dest_dir, jobp->flags);
756  continue;
757  }
758 
759  // may be this code can be factored out
760  // together with the autodiscovery of indexes
761  // of addDirJobs
762  if ( ( _options & AutoAddChecksumsIndexes ) ||
763  ( _options & AutoAddContentFileIndexes ) )
764  {
765  // if auto indexing is enabled, then we need to read the
766  // index for each file. We look only in the directory
767  // where the file is. this is expensive of course.
768  filesystem::DirContent content;
769  getDirectoryContent(media, jobp->location.filename().dirname(), content);
770  // this method test for the option flags so indexes are added
771  // only if the options are enabled
772  MIL << "Autodiscovering signed indexes on '"
773  << jobp->location.filename().dirname() << "' for '"
774  << jobp->location.filename() << "'" << endl;
775 
776  autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
777 
778  // also look in the root of the media
779  content.clear();
780  getDirectoryContent(media, Pathname("/"), content);
781  // this method test for the option flags so indexes are added
782  // only if the options are enabled
783  MIL << "Autodiscovering signed indexes on '"
784  << "/" << "' for '"
785  << jobp->location.filename() << "'" << endl;
786 
787  autoaddIndexes(content, media, Pathname("/"), dest_dir);
788  }
789 
790  // if the checksum is empty, but the checksum is in one of the
791  // indexes checksum, then add a checker
792  if ( jobp->location.checksum().empty() )
793  {
794  if ( _checksums.find(jobp->location.filename().asString())
795  != _checksums.end() )
796  {
797  CheckSum chksm = _checksums[jobp->location.filename().asString()];
798  ChecksumFileChecker digest_check(chksm);
799  jobp->checkers.push_back(digest_check);
800  }
801  else
802  {
803  // if the index checksum is empty too, we only add the checker
804  // if the AlwaysVerifyChecksum option is set on
805  if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
806  {
807  // add the checker with the empty checksum
808  ChecksumFileChecker digest_check(jobp->location.checksum());
809  jobp->checkers.push_back(digest_check);
810  }
811  }
812  }
813  else
814  {
815  // checksum is not empty, so add a checksum checker
816  ChecksumFileChecker digest_check(jobp->location.checksum());
817  jobp->checkers.push_back(digest_check);
818  }
819 
820  // Provide and validate the file. If the file was not transferred
821  // and no exception was thrown, it was an optional file.
822  provideToDest( media, dest_dir, jobp );
823 
824  if ( ! progress.incr() )
825  ZYPP_THROW(AbortRequestException());
826  } // for each job
827  }
828 
830  inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
831  {
832  for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
833  {
834  str << *it_res;
835  }
836  return str;
837  }
838 
840  : _pimpl( new Impl() )
841  {}
842 
844  {}
845 
846  void Fetcher::setOptions( Fetcher::Options options )
847  {
849  }
850 
851  Fetcher::Options Fetcher::options() const
852  {
853  return _pimpl->options();
854  }
855 
856  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
857  {
858  enqueueDigested( OnMediaLocation(resource).setDeltafile(deltafile), checker );
859  }
860 
861  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
862  {
863  _pimpl->enqueueDigested( resource, checker );
864  }
865 
866  void Fetcher::enqueueDir( const OnMediaLocation &resource,
867  bool recursive,
868  const FileChecker &checker )
869  {
870  _pimpl->enqueueDir(resource, recursive, checker);
871  }
872 
874  bool recursive,
875  const FileChecker &checker )
876  {
877  _pimpl->enqueueDigestedDir(resource, recursive, checker);
878  }
879 
880 
881  void Fetcher::addIndex( const OnMediaLocation &resource )
882  {
883  _pimpl->addIndex(resource);
884  }
885 
886 
887  void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
888  {
889  _pimpl->enqueue(resource, checker);
890  }
891 
892 
893  void Fetcher::addCachePath( const Pathname &cache_dir )
894  {
895  _pimpl->addCachePath(cache_dir);
896  }
897 
899  {
900  _pimpl->reset();
901  }
902 
904  {
905  _pimpl->setMediaSetAccess( media );
906  }
907 
909  {
910  _pimpl->start( dest_dir, progress );
911  }
912 
913  void Fetcher::start( const Pathname &dest_dir,
914  MediaSetAccess &media,
915  const ProgressData::ReceiverFnc & progress_receiver )
916  {
917  _pimpl->start(dest_dir, media, progress_receiver);
918  }
919 
920  std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
921  {
922  return str << *obj._pimpl;
923  }
924 
926 } // namespace zypp
928 
Pathname deltafile
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:93
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:162
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Definition: AutoDispose.h:158
bool empty() const
Definition: CheckSum.cc:173
Built in file checkers.
Definition: FileChecker.h:71
Base class for Exception.
Definition: Exception.h:146
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:105
Fetcher implementation.
Definition: Fetcher.cc:126
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:238
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:406
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:644
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:602
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:591
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:225
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Stream output.
Definition: Fetcher.cc:830
std::map< std::string, CheckSum > _checksums
Definition: Fetcher.cc:228
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:221
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:699
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:457
void addCachePath(const Pathname &cache_dir)
Definition: Fetcher.cc:324
std::list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:224
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:615
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:260
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:304
std::map< std::string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:230
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition: Fetcher.cc:521
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:292
MediaSetAccess * _mediaSetAccess
Definition: Fetcher.cc:232
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
Definition: Fetcher.cc:347
std::set< Pathname > _caches
Definition: Fetcher.cc:226
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:275
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition: Fetcher.cc:377
Fetcher::Options options() const
Definition: Fetcher.cc:257
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:434
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
Definition: Fetcher.cc:729
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:254
Fetcher::Options _options
Definition: Fetcher.cc:234
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:154
void setMediaSetAccess(MediaSetAccess &media)
Definition: Fetcher.cc:319
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:106
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:346
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:881
Options options() const
Get current options.
Definition: Fetcher.cc:851
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:898
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:887
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:866
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:846
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
Fetcher()
Default ctor.
Definition: Fetcher.cc:839
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:873
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:861
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:893
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:903
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:843
Media access layer responsible for handling files distributed on a set of media with media change and...
void precacheFiles(const std::vector< OnMediaLocation > &files)
Tries to fetch the given files and precaches them.
@ PROVIDE_DEFAULT
The user is not asked anything, and the error exception is just propagated.
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT)
Provides a file from a media location.
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
Describes a resource file located on a medium.
bool optional() const
Whether this is an optional resource.
const Pathname & filename() const
The path to the resource on the medium.
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
unsigned medianr() const
The media number the resource is located on.
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
const CheckSum & checksum() const
The checksum of the resource on the server.
OnMediaLocation & setOptional(bool val)
Set whether the resource is optional.
Maintain [min,max] and counter (value) for progress counting.
Definition: ProgressData.h:131
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: ProgressData.h:226
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
bool incr(value_type val_r=1)
Increment counter value (default by 1).
Definition: ProgressData.h:261
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition: PublicKey.h:284
Checks for the validity of a signature.
Definition: FileChecker.h:94
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:219
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:279
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:170
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const std::string & asString() const
String representation.
Definition: Pathname.h:91
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
const char * c_str() const
String representation.
Definition: Pathname.h:110
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
String related utilities and Regular expression matching.
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:991
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:640
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1003
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:319
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:823
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:514
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:50
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:263
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:54
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:113
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:28
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
std::ostream & operator<<(std::ostream &str, const InputStream &obj)
Definition: InputStream.cc:166
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:587
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:584
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:44
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:49
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:45
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:51
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:80
std::list< FileChecker > checkers
Definition: Fetcher.cc:108
@ AlwaysVerifyChecksum
Definition: Fetcher.cc:89
OnMediaLocation location
Definition: Fetcher.cc:106
ZYPP_DECLARE_FLAGS(Flags, Flag)
FetcherJob(const OnMediaLocation &loc)
Definition: Fetcher.cc:94
std::set ordering (less semantic)
Definition: Fetcher.cc:58
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Definition: Fetcher.cc:59
Listentry returned by readdir.
Definition: PathInfo.h:499
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:28
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:400
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:396
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
#define DBG
Definition: Logger.h:95
#define MIL
Definition: Logger.h:96
#define ERR
Definition: Logger.h:98
#define WAR
Definition: Logger.h:97