libzypp  17.5.1
Fetcher.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include <iostream>
13 #include <fstream>
14 #include <list>
15 #include <map>
16 
17 #include "zypp/base/Easy.h"
18 #include "zypp/base/LogControl.h"
19 #include "zypp/base/LogTools.h"
20 #include "zypp/base/PtrTypes.h"
22 #include "zypp/base/String.h"
23 #include "zypp/Fetcher.h"
24 #include "zypp/ZYppFactory.h"
25 #include "zypp/CheckSum.h"
29 
30 #undef ZYPP_BASE_LOGGER_LOGGROUP
31 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
32 
34 namespace zypp
35 {
36 
42  struct FetcherIndex
43  {
45  : location(loc)
46  {}
51  };
52 
53  typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
54 
57  {
58  bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs )
59  {
60  if ( lhs == rhs )
61  return false; // incl. NULL == NULL
62  if ( ! lhs )
63  return true; // NULL < nonNULL
64  if ( ! rhs )
65  return false; // nonNULL > NULL
66  // both nonNULL ==> compare medianr and path
67  if ( lhs->location.medianr() == rhs->location.medianr() )
68  return lhs->location.filename() < rhs->location.filename();
69  //else
70  return lhs->location.medianr() < rhs->location.medianr();
71  }
72  };
73 
78  struct FetcherJob
79  {
80  enum Flag
81  {
82  None = 0x0000,
83  Directory = 0x0001,
84  Recursive = 0x0002,
86  // check checksums even if there is no such
87  // checksum (warns of no checksum)
89  };
90  ZYPP_DECLARE_FLAGS(Flags, Flag);
91 
92 
93  FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname())
94  : location(loc)
95  , deltafile(dfile)
96  , flags(None)
97  {
98  //MIL << location << endl;
99  }
100 
102  {
103  //MIL << location << " | * " << checkers.size() << endl;
104  }
105 
108  //CompositeFileChecker checkers;
109  std::list<FileChecker> checkers;
110  Flags flags;
111  };
112 
113  ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
114  typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
115 
116  std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
117  {
118  return str << obj->location;
119  }
120 
122  //
123  // CLASS NAME : Fetcher::Impl
124  //
127  {
128  friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
129 
130  public:
131  Impl();
132 
133  ~Impl() {}
134 
135  void setOptions( Fetcher::Options options );
136  Fetcher::Options options() const;
137 
138  void addIndex( const OnMediaLocation &resource );
139 
140  void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
141  void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
142 
143  void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
144  void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() );
145  void addCachePath( const Pathname &cache_dir );
146  void reset();
147  void start( const Pathname &dest_dir,
148  MediaSetAccess &media,
149  const ProgressData::ReceiverFnc & progress_receiver );
150 
152  static shared_ptr<Impl> nullimpl()
153  {
154  static shared_ptr<Impl> _nullimpl( new Impl );
155  return _nullimpl;
156  }
157  private:
161  void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
162 
166  void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
167 
175  void readIndex( const Pathname &index, const Pathname &basedir );
176 
178  void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
179 
181  void readContentFileIndex( const Pathname &index, const Pathname &basedir );
182 
184  void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
185 
191  Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
196  void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
197 
201  void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
202  const Pathname &dest_dir, FetcherJob::Flags flags );
203 
207  void autoaddIndexes( const filesystem::DirContent &content,
208  MediaSetAccess &media,
209  const OnMediaLocation &resource,
210  const Pathname &dest_dir );
214  void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
215 
216  private:
217  friend Impl * rwcowClone<Impl>( const Impl * rhs );
219  Impl * clone() const
220  { return new Impl( *this ); }
221 
222  std::list<FetcherJob_Ptr> _resources;
223  std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
224  std::set<Pathname> _caches;
225  // checksums read from the indexes
226  std::map<std::string, CheckSum> _checksums;
227  // cache of dir contents
228  std::map<std::string, filesystem::DirContent> _dircontent;
229 
230  Fetcher::Options _options;
231  };
233 
234  void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
235  {
236  FetcherJob_Ptr job;
237  job.reset(new FetcherJob(resource, deltafile));
238  job->flags |= FetcherJob:: AlwaysVerifyChecksum;
239  _resources.push_back(job);
240  }
241 
243  : _options(0)
244  {
245  }
246 
247  void Fetcher::Impl::setOptions( Fetcher::Options options )
248  { _options = options; }
249 
250  Fetcher::Options Fetcher::Impl::options() const
251  { return _options; }
252 
254  bool recursive,
255  const FileChecker &checker )
256  {
257  FetcherJob_Ptr job;
258  job.reset(new FetcherJob(resource));
259  if ( checker )
260  job->checkers.push_back(checker);
261  if ( recursive )
262  job->flags |= FetcherJob::Recursive;
263  job->flags |= FetcherJob::Directory;
264 
265  _resources.push_back(job);
266  }
267 
269  bool recursive,
270  const FileChecker &checker )
271  {
272  FetcherJob_Ptr job;
273  job.reset(new FetcherJob(resource));
274  if ( checker )
275  job->checkers.push_back(checker);
276  if ( recursive )
277  job->flags |= FetcherJob::Recursive;
278  job->flags |= FetcherJob::Directory;
279  job->flags |= FetcherJob::AlwaysVerifyChecksum;
280 
281  _resources.push_back(job);
282 
283  }
284 
285  void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
286  {
287  FetcherJob_Ptr job;
288  job.reset(new FetcherJob(resource));
289  if ( checker )
290  job->checkers.push_back(checker);
291  _resources.push_back(job);
292  }
293 
294  void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
295  {
296  MIL << "adding index " << resource << endl;
297  _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
298  }
299 
300 
302  {
303  _resources.clear();
304  _indexes.clear();
305  _checksums.clear();
306  _dircontent.clear();
307  }
308 
309  void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
310  {
311  PathInfo info(cache_dir);
312  if ( info.isExist() )
313  {
314  if ( info.isDir() )
315  {
316  DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
317  _caches.insert(cache_dir);
318  }
319  else
320  {
321  // don't add bad cache directory, just log the error
322  ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
323  }
324  }
325  else
326  {
327  ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
328  }
329 
330  }
331 
332  Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
333  {
334  Pathname ret;
335  // No checksum - no match
336  if ( resource_r.checksum().empty() )
337  return ret;
338 
339  // first check in the destination directory
340  Pathname cacheLocation = destDir_r / resource_r.filename();
341  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
342  {
343  swap( ret, cacheLocation );
344  return ret;
345  }
346 
347  MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
348  for( const Pathname & cacheDir : _caches )
349  {
350  cacheLocation = cacheDir / resource_r.filename();
351  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
352  {
353  MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl;
354  swap( ret, cacheLocation );
355  return ret;
356  }
357  }
358 
359  return ret;
360  }
361 
362  void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
363  {
364  try
365  {
366  MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
367 
368  for ( const FileChecker & chkfnc : checkers_r )
369  {
370  if ( chkfnc )
371  chkfnc( localfile_r );
372  else
373  ERR << "Invalid checker for '" << localfile_r << "'" << endl;
374  }
375 
376  }
377  catch ( const FileCheckException &e )
378  {
379  ZYPP_RETHROW(e);
380  }
381  catch ( const Exception &e )
382  {
383  ZYPP_RETHROW(e);
384  }
385  catch (...)
386  {
387  ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
388  }
389  }
390 
392  MediaSetAccess &media,
393  const OnMediaLocation &resource,
394  const Pathname &dest_dir )
395  {
396  auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
397  {
398  if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
399  return false;
400  // add the index of this directory
401  OnMediaLocation indexloc( resource );
402  indexloc.changeFilename( resource.filename() + index_r );
403  addIndex( indexloc );
404  // we need to read it now
405  downloadAndReadIndexList( media, dest_dir );
406  return true;
407  } );
408 
409  if ( _options & AutoAddChecksumsIndexes )
410  {
411  fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
412  }
413  if ( _options & AutoAddContentFileIndexes )
414  {
415  fnc_addIfInContent( "content" );
416  }
417  }
418 
420  const OnMediaLocation &resource,
421  filesystem::DirContent &content )
422  {
423  if ( _dircontent.find(resource.filename().asString())
424  != _dircontent.end() )
425  {
426  filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
427 
428  std::copy(filled.begin(), filled.end(), std::back_inserter(content));
429  }
430  else
431  {
432  filesystem::DirContent tofill;
433  media.dirInfo( tofill,
434  resource.filename(),
435  false /* dots */,
436  resource.medianr());
437  std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
438  _dircontent[resource.filename().asString()] = tofill;
439  }
440  }
441 
443  const OnMediaLocation &resource,
444  const Pathname &dest_dir, FetcherJob::Flags flags )
445  {
446  // first get the content of the directory so we can add
447  // individual transfer jobs
448  MIL << "Adding directory " << resource.filename() << endl;
449  filesystem::DirContent content;
450  try {
451  getDirectoryContent(media, resource, content);
452  }
453  catch ( media::MediaFileNotFoundException & exception )
454  {
455  ZYPP_CAUGHT( exception );
456  WAR << "Skiping subtree hidden at " << resource.filename() << endl;
457  return;
458  }
459 
460  // this method test for the option flags so indexes are added
461  // only if the options are enabled
462  autoaddIndexes(content, media, resource, dest_dir);
463 
464  for ( filesystem::DirContent::const_iterator it = content.begin();
465  it != content.end();
466  ++it )
467  {
468  // skip CHECKSUMS* as they were already retrieved
469  if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
470  continue;
471 
472  Pathname filename = resource.filename() + it->name;
473 
474  switch ( it->type )
475  {
476  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
477  case filesystem::FT_FILE:
478  {
479  CheckSum chksm(resource.checksum());
480  if ( _checksums.find(filename.asString()) != _checksums.end() )
481  {
482  // the checksum can be replaced with the one in the index.
483  chksm = _checksums[filename.asString()];
484  //MIL << "resource " << filename << " has checksum in the index file." << endl;
485  }
486  else
487  WAR << "Resource " << filename << " has no checksum in the index either." << endl;
488 
489  if ( flags & FetcherJob::AlwaysVerifyChecksum )
490  enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
491  else
492  enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
493  break;
494  }
495  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
496  if ( flags & FetcherJob::Recursive )
497  addDirJobs(media, filename, dest_dir, flags);
498  break;
499  default:
500  // don't provide devices, sockets, etc.
501  break;
502  }
503  }
504  }
505 
506  void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
507  {
508  const OnMediaLocation & resource( jobp_r->location );
509 
510  try
511  {
512  scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
513 
514  // get cached file (by checksum) or provide from media
515  Pathname tmpFile = locateInCache( resource, destDir_r );
516  if ( tmpFile.empty() )
517  {
518  MIL << "Not found in cache, retrieving..." << endl;
519  tmpFile = media_r.provideFile( resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, jobp_r->deltafile );
520  releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
521  }
522 
523  // The final destination: locateInCache also checks destFullPath!
524  // If we find a cache match (by checksum) at destFullPath, take
525  // care it gets deleted, in case the validation fails.
526  ManagedFile destFullPath( destDir_r / resource.filename() );
527  if ( tmpFile == destFullPath )
528  destFullPath.setDispose( filesystem::unlink );
529 
530  // validate the file (throws if not valid)
531  validate( tmpFile, jobp_r->checkers );
532 
533  // move it to the final destination
534  if ( tmpFile == destFullPath )
535  destFullPath.resetDispose(); // keep it!
536  else
537  {
538  if ( assert_dir( destFullPath->dirname() ) != 0 )
539  ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
540 
541  if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
542  ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
543  }
544  }
545  catch ( Exception & excpt )
546  {
547  if ( resource.optional() )
548  {
549  ZYPP_CAUGHT( excpt );
550  WAR << "optional resource " << resource << " could not be transferred." << endl;
551  return;
552  }
553  else
554  {
555  excpt.remember( "Can't provide " + resource.filename().asString() );
556  ZYPP_RETHROW( excpt );
557  }
558  }
559  }
560 
561  // helper class to consume a content file
563  {
565  {
566  setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
567  }
568 
569  void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
570  { _repoindex = data_r; }
571 
572  parser::susetags::RepoIndex_Ptr _repoindex;
573  };
574 
575  // generic function for reading indexes
576  void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
577  {
578  if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
579  readChecksumsIndex(index, basedir);
580  else if ( index.basename() == "content" )
581  readContentFileIndex(index, basedir);
582  else
583  WAR << index << ": index file format not known" << endl;
584  }
585 
586  // reads a content file index
587  void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
588  {
589  ContentReaderHelper reader;
590  reader.parse(index);
591  MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
592  for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
593  {
594  // content file entries don't start with /
595  _checksums[(basedir + it->first).asString()] = it->second;
596  }
597  }
598 
599  // reads a CHECKSUMS (old SHA1SUMS) file index
600  void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
601  {
602  std::ifstream in( index.c_str() );
603  if ( ! in.fail() )
604  {
605  std::string buffer;
606  while ( getline( in, buffer ) )
607  {
608 
609  if ( buffer[0] == '#' )
610  continue; // simple comment
611 
612  CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
613  if ( checksum.empty() )
614  continue; // empty line | unknown cheksum format
615 
616  if ( buffer.empty() )
617  {
618  WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
619  continue;
620  }
621 
622  _checksums[(basedir/buffer).asString()] = checksum;
623  }
624  }
625  else
626  ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
627  }
628 
629  void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
630  {
631  MIL << "downloading index " << resource << endl;
632  // create a new fetcher with a different state to transfer the
633  // file containing checksums and its signature
634  Fetcher fetcher;
635  // signature checker for index. We havent got the signature from
636  // the nextwork yet.
637  SignatureFileChecker sigchecker;
638 
639  // build the name of the index and the signature
640  OnMediaLocation idxloc(resource);
641  OnMediaLocation sigloc(resource);
642  OnMediaLocation keyloc(resource);
643 
644  // we should not fail the download if those don't exists
645  // the checking will warn later
646  sigloc.setOptional(true);
647  keyloc.setOptional(true);
648 
649  // calculate signature and key name
650  sigloc.changeFilename( sigloc.filename().extend(".asc") );
651  keyloc.changeFilename( keyloc.filename().extend(".key") );
652 
653  //assert_dir(dest_dir + idxloc.filename().dirname());
654 
655  // transfer the signature
656  fetcher.enqueue(sigloc);
657  fetcher.start( dest_dir, media );
658  // if we get the signature, update the checker
659  if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
660  sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
661 
662  fetcher.reset();
663 
664  // now the key
665  fetcher.enqueue(keyloc);
666  fetcher.start( dest_dir, media );
667  fetcher.reset();
668 
669  // try to import the key
670  if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
671  getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
672  else
673  WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
674 
675  // now the index itself
676  fetcher.enqueue( idxloc, FileChecker(sigchecker) );
677  fetcher.start( dest_dir, media );
678  fetcher.reset();
679  }
680 
681  // this method takes all the user pointed indexes, gets them and also tries to
682  // download their signature, and verify them. After that, its parses each one
683  // to fill the checksum cache.
685  {
686  // if there is no indexes, then just return to avoid
687  // the directory listing
688  if ( _indexes.empty() )
689  {
690  MIL << "No indexes to read." << endl;
691  return;
692  }
693 
694  for_( it_idx, _indexes.begin(), _indexes.end() )
695  {
696  if ( (*it_idx)->read )
697  {
698  DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
699  }
700  else
701  {
702  // base::LogControl::TmpLineWriter shutUp;
703  downloadIndex( media, (*it_idx)->location, dest_dir );
704  // now we have the indexes in dest_dir
705  readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
706  // Take care we don't process it again
707  MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
708  (*it_idx)->read = true;
709  }
710  }
711  MIL << "done reading indexes" << endl;
712  }
713 
714  // start processing all fetcher jobs.
715  // it processes any user pointed index first
716  void Fetcher::Impl::start( const Pathname &dest_dir,
717  MediaSetAccess &media,
718  const ProgressData::ReceiverFnc & progress_receiver )
719  {
720  ProgressData progress(_resources.size());
721  progress.sendTo(progress_receiver);
722 
723  downloadAndReadIndexList(media, dest_dir);
724 
725  for ( const FetcherJob_Ptr & jobp : _resources )
726  {
727  if ( jobp->flags & FetcherJob::Directory )
728  {
729  const OnMediaLocation location(jobp->location);
730  addDirJobs(media, location, dest_dir, jobp->flags);
731  continue;
732  }
733 
734  // may be this code can be factored out
735  // together with the autodiscovery of indexes
736  // of addDirJobs
737  if ( ( _options & AutoAddChecksumsIndexes ) ||
738  ( _options & AutoAddContentFileIndexes ) )
739  {
740  // if auto indexing is enabled, then we need to read the
741  // index for each file. We look only in the directory
742  // where the file is. this is expensive of course.
743  filesystem::DirContent content;
744  getDirectoryContent(media, jobp->location.filename().dirname(), content);
745  // this method test for the option flags so indexes are added
746  // only if the options are enabled
747  MIL << "Autodiscovering signed indexes on '"
748  << jobp->location.filename().dirname() << "' for '"
749  << jobp->location.filename() << "'" << endl;
750 
751  autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
752 
753  // also look in the root of the media
754  content.clear();
755  getDirectoryContent(media, Pathname("/"), content);
756  // this method test for the option flags so indexes are added
757  // only if the options are enabled
758  MIL << "Autodiscovering signed indexes on '"
759  << "/" << "' for '"
760  << jobp->location.filename() << "'" << endl;
761 
762  autoaddIndexes(content, media, Pathname("/"), dest_dir);
763  }
764 
765  // if the checksum is empty, but the checksum is in one of the
766  // indexes checksum, then add a checker
767  if ( jobp->location.checksum().empty() )
768  {
769  if ( _checksums.find(jobp->location.filename().asString())
770  != _checksums.end() )
771  {
772  CheckSum chksm = _checksums[jobp->location.filename().asString()];
773  ChecksumFileChecker digest_check(chksm);
774  jobp->checkers.push_back(digest_check);
775  }
776  else
777  {
778  // if the index checksum is empty too, we only add the checker
779  // if the AlwaysVerifyChecksum option is set on
780  if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
781  {
782  // add the checker with the empty checksum
783  ChecksumFileChecker digest_check(jobp->location.checksum());
784  jobp->checkers.push_back(digest_check);
785  }
786  }
787  }
788  else
789  {
790  // checksum is not empty, so add a checksum checker
791  ChecksumFileChecker digest_check(jobp->location.checksum());
792  jobp->checkers.push_back(digest_check);
793  }
794 
795  // Provide and validate the file. If the file was not transferred
796  // and no exception was thrown, it was an optional file.
797  provideToDest( media, dest_dir, jobp );
798 
799  if ( ! progress.incr() )
800  ZYPP_THROW(AbortRequestException());
801  } // for each job
802  }
803 
805  inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
806  {
807  for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
808  {
809  str << *it_res;
810  }
811  return str;
812  }
813 
815  : _pimpl( new Impl() )
816  {}
817 
819  {}
820 
821  void Fetcher::setOptions( Fetcher::Options options )
822  {
824  }
825 
826  Fetcher::Options Fetcher::options() const
827  {
828  return _pimpl->options();
829  }
830 
831  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
832  {
833  _pimpl->enqueueDigested(resource, checker, deltafile);
834  }
835 
836  void Fetcher::enqueueDir( const OnMediaLocation &resource,
837  bool recursive,
838  const FileChecker &checker )
839  {
840  _pimpl->enqueueDir(resource, recursive, checker);
841  }
842 
844  bool recursive,
845  const FileChecker &checker )
846  {
847  _pimpl->enqueueDigestedDir(resource, recursive, checker);
848  }
849 
850 
851  void Fetcher::addIndex( const OnMediaLocation &resource )
852  {
853  _pimpl->addIndex(resource);
854  }
855 
856 
857  void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
858  {
859  _pimpl->enqueue(resource, checker);
860  }
861 
862  void Fetcher::addCachePath( const Pathname &cache_dir )
863  {
864  _pimpl->addCachePath(cache_dir);
865  }
866 
868  {
869  _pimpl->reset();
870  }
871 
872  void Fetcher::start( const Pathname &dest_dir,
873  MediaSetAccess &media,
874  const ProgressData::ReceiverFnc & progress_receiver )
875  {
876  _pimpl->start(dest_dir, media, progress_receiver);
877  }
878 
879  std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
880  {
881  return str << *obj._pimpl;
882  }
883 
885 } // namespace zypp
887 
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:320
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:569
#define MIL
Definition: Logger.h:64
Pathname deltafile
Definition: Fetcher.cc:107
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:587
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:53
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:821
Listentry returned by readdir.
Definition: PathInfo.h:532
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: ProgressData.h:226
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
Describes a path on a certain media amongs as the information required to download it...
Checks for the validity of a signature.
Definition: FileChecker.h:93
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:851
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:572
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:268
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory)...
Definition: Fetcher.cc:332
OnMediaLocation & setOptional(bool val)
Set the whether the resource is optional or not.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:169
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
const char * c_str() const
String representation.
Definition: Pathname.h:109
String related utilities and Regular expression matching.
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition: Fetcher.cc:506
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:285
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:862
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:27
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:127
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:843
void addCachePath(const Pathname &cache_dir)
Definition: Fetcher.cc:309
const bool optional() const
whether this is an optional resource.
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:48
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
std::list< FileChecker > checkers
Definition: Fetcher.cc:109
#define ERR
Definition: Logger.h:66
unsigned medianr() const
media number where the resource is located.
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:152
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:325
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:831
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:105
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:223
bool empty() const
Test for an empty path.
Definition: Pathname.h:113
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:400
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:261
Fetcher::Options _options
Definition: Fetcher.cc:230
std::ostream & operator<<(std::ostream &str, const Exception &obj)
Definition: Exception.cc:147
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:653
const std::string & asString() const
String representation.
Definition: Pathname.h:90
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:281
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:867
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:123
#define WAR
Definition: Logger.h:65
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:576
Parse repoindex part from a content file.
std::set ordering (less semantic)
Definition: Fetcher.cc:56
Fetcher::Options options() const
Definition: Fetcher.cc:250
Maintain [min,max] and counter (value) for progress counting.
Definition: ProgressData.h:130
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:253
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:50
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:836
std::map< std::string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:228
OnMediaLocation location
Definition: Fetcher.cc:106
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:629
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
Definition: Fetcher.cc:872
std::list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:222
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Definition: Fetcher.cc:234
Fetcher()
Default ctor.
Definition: Fetcher.cc:814
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:836
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition: Fetcher.cc:362
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:50
ZYPP_DECLARE_FLAGS(Flags, Flag)
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:419
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Definition: AutoDispose.h:158
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:396
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:78
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:247
const Pathname & filename() const
The path to the resource relatve to the url and path.
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:44
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition: PublicKey.h:269
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:600
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:684
Fetcher implementation.
Definition: Fetcher.cc:126
Base class for Exception.
Definition: Exception.h:145
Built in file checkers.
Definition: FileChecker.h:70
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
Definition: Fetcher.cc:93
bool empty() const
Definition: CheckSum.cc:173
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:1004
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:92
std::map< std::string, CheckSum > _checksums
Definition: Fetcher.cc:226
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
OnMediaLocation & changeFilename(const Pathname &val_r)
Individual manipulation of filename (prefer setLocation).
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:42
std::set< Pathname > _caches
Definition: Fetcher.cc:224
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
Definition: Fetcher.cc:716
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:28
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Definition: Fetcher.cc:805
Options options() const
Get current options.
Definition: Fetcher.cc:826
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:114
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:857
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:818
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs)
Definition: Fetcher.cc:58
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT, const Pathname &deltafile=Pathname())
Provides a file from a media location.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:391
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1011
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:105
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:294
#define DBG
Definition: Logger.h:63
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1016
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:442
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
const CheckSum & checksum() const
the checksum of the resource
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:219
The user is not asked anything, and the error exception is just propagated.