libzypp  15.28.6
Fetcher.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include <iostream>
13 #include <fstream>
14 #include <list>
15 #include <map>
16 
17 #include "zypp/base/Easy.h"
18 #include "zypp/base/LogControl.h"
19 #include "zypp/base/LogTools.h"
20 #include "zypp/base/PtrTypes.h"
22 #include "zypp/base/String.h"
23 #include "zypp/Fetcher.h"
24 #include "zypp/ZYppFactory.h"
25 #include "zypp/CheckSum.h"
29 
30 #undef ZYPP_BASE_LOGGER_LOGGROUP
31 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
32 
34 namespace zypp
35 {
36 
42  struct FetcherIndex
43  {
45  : location(loc)
46  {}
51  };
52 
53  typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
54 
57  {
58  bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs )
59  {
60  if ( lhs == rhs )
61  return false; // incl. NULL == NULL
62  if ( ! lhs )
63  return true; // NULL < nonNULL
64  if ( ! rhs )
65  return false; // nonNULL > NULL
66  // both nonNULL ==> compare medianr and path
67  if ( lhs->location.medianr() == rhs->location.medianr() )
68  return lhs->location.filename() < rhs->location.filename();
69  //else
70  return lhs->location.medianr() < rhs->location.medianr();
71  }
72  };
73 
78  struct FetcherJob
79  {
80  enum Flag
81  {
82  None = 0x0000,
83  Directory = 0x0001,
84  Recursive = 0x0002,
86  // check checksums even if there is no such
87  // checksum (warns of no checksum)
89  };
90  ZYPP_DECLARE_FLAGS(Flags, Flag);
91 
92 
93  FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname())
94  : location(loc)
95  , deltafile(dfile)
96  , flags(None)
97  {
98  //MIL << location << endl;
99  }
100 
102  {
103  //MIL << location << " | * " << checkers.size() << endl;
104  }
105 
107  Pathname deltafile;
108  //CompositeFileChecker checkers;
109  std::list<FileChecker> checkers;
110  Flags flags;
111  };
112 
113  ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
114  typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
115 
116  std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
117  {
118  return str << obj->location;
119  }
120 
122  //
123  // CLASS NAME : Fetcher::Impl
124  //
127  {
128  friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
129 
130  public:
131  Impl();
132 
133  ~Impl() {}
134 
135  void setOptions( Fetcher::Options options );
136  Fetcher::Options options() const;
137 
138  void addIndex( const OnMediaLocation &resource );
139 
140  void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
141  void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
142 
143  void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
144  void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() );
145  void addCachePath( const Pathname &cache_dir );
146  void reset();
147  void start( const Pathname &dest_dir,
148  MediaSetAccess &media,
149  const ProgressData::ReceiverFnc & progress_receiver );
150 
152  static shared_ptr<Impl> nullimpl()
153  {
154  static shared_ptr<Impl> _nullimpl( new Impl );
155  return _nullimpl;
156  }
157  private:
161  void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
162 
166  void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
167 
175  void readIndex( const Pathname &index, const Pathname &basedir );
176 
178  void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
179 
181  void readContentFileIndex( const Pathname &index, const Pathname &basedir );
182 
184  void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
185 
191  Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
196  void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
197 
201  void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
202  const Pathname &dest_dir, FetcherJob::Flags flags );
203 
207  void autoaddIndexes( const filesystem::DirContent &content,
208  MediaSetAccess &media,
209  const OnMediaLocation &resource,
210  const Pathname &dest_dir );
214  void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
215 
216  private:
217  friend Impl * rwcowClone<Impl>( const Impl * rhs );
219  Impl * clone() const
220  { return new Impl( *this ); }
221 
222  std::list<FetcherJob_Ptr> _resources;
223  std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
224  std::set<Pathname> _caches;
225  // checksums read from the indexes
226  std::map<std::string, CheckSum> _checksums;
227  // cache of dir contents
228  std::map<std::string, filesystem::DirContent> _dircontent;
229 
230  Fetcher::Options _options;
231  };
233 
234  void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
235  {
236  FetcherJob_Ptr job;
237  job.reset(new FetcherJob(resource, deltafile));
238  job->flags |= FetcherJob:: AlwaysVerifyChecksum;
239  _resources.push_back(job);
240  }
241 
243  : _options(0)
244  {
245  }
246 
247  void Fetcher::Impl::setOptions( Fetcher::Options options )
248  { _options = options; }
249 
250  Fetcher::Options Fetcher::Impl::options() const
251  { return _options; }
252 
254  bool recursive,
255  const FileChecker &checker )
256  {
257  FetcherJob_Ptr job;
258  job.reset(new FetcherJob(resource));
259  if ( checker )
260  job->checkers.push_back(checker);
261  if ( recursive )
262  job->flags |= FetcherJob::Recursive;
263  job->flags |= FetcherJob::Directory;
264 
265  _resources.push_back(job);
266  }
267 
269  bool recursive,
270  const FileChecker &checker )
271  {
272  FetcherJob_Ptr job;
273  job.reset(new FetcherJob(resource));
274  if ( checker )
275  job->checkers.push_back(checker);
276  if ( recursive )
277  job->flags |= FetcherJob::Recursive;
278  job->flags |= FetcherJob::Directory;
279  job->flags |= FetcherJob::AlwaysVerifyChecksum;
280 
281  _resources.push_back(job);
282 
283  }
284 
285  void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
286  {
287  FetcherJob_Ptr job;
288  job.reset(new FetcherJob(resource));
289  if ( checker )
290  job->checkers.push_back(checker);
291  _resources.push_back(job);
292  }
293 
294  void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
295  {
296  MIL << "adding index " << resource << endl;
297  _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
298  }
299 
300 
302  {
303  _resources.clear();
304  _indexes.clear();
305  _checksums.clear();
306  _dircontent.clear();
307  }
308 
309  void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
310  {
311  PathInfo info(cache_dir);
312  if ( info.isExist() )
313  {
314  if ( info.isDir() )
315  {
316  DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
317  _caches.insert(cache_dir);
318  }
319  else
320  {
321  // don't add bad cache directory, just log the error
322  ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
323  }
324  }
325  else
326  {
327  ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
328  }
329 
330  }
331 
332  Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
333  {
334  Pathname ret;
335  // No checksum - no match
336  if ( resource_r.checksum().empty() )
337  return ret;
338 
339  // first check in the destination directory
340  Pathname cacheLocation = destDir_r / resource_r.filename();
341  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
342  {
343  swap( ret, cacheLocation );
344  return ret;
345  }
346 
347  MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
348  for( const Pathname & cacheDir : _caches )
349  {
350  cacheLocation = cacheDir / resource_r.filename();
351  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
352  {
353  MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl;
354  swap( ret, cacheLocation );
355  return ret;
356  }
357  }
358 
359  return ret;
360  }
361 
362  void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
363  {
364  try
365  {
366  MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
367 
368  for ( const FileChecker & chkfnc : checkers_r )
369  {
370  if ( chkfnc )
371  chkfnc( localfile_r );
372  else
373  ERR << "Invalid checker for '" << localfile_r << "'" << endl;
374  }
375 
376  }
377  catch ( const FileCheckException &e )
378  {
379  ZYPP_RETHROW(e);
380  }
381  catch ( const Exception &e )
382  {
383  ZYPP_RETHROW(e);
384  }
385  catch (...)
386  {
387  ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
388  }
389  }
390 
392  MediaSetAccess &media,
393  const OnMediaLocation &resource,
394  const Pathname &dest_dir )
395  {
396  auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
397  {
398  if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
399  return false;
400  // add the index of this directory
401  OnMediaLocation indexloc( resource );
402  indexloc.changeFilename( resource.filename() + index_r );
403  addIndex( indexloc );
404  // we need to read it now
405  downloadAndReadIndexList( media, dest_dir );
406  return true;
407  } );
408 
409  if ( _options & AutoAddChecksumsIndexes )
410  {
411  fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
412  }
413  if ( _options & AutoAddContentFileIndexes )
414  {
415  fnc_addIfInContent( "content" );
416  }
417  }
418 
420  const OnMediaLocation &resource,
421  filesystem::DirContent &content )
422  {
423  if ( _dircontent.find(resource.filename().asString())
424  != _dircontent.end() )
425  {
426  filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
427 
428  std::copy(filled.begin(), filled.end(), std::back_inserter(content));
429  }
430  else
431  {
432  filesystem::DirContent tofill;
433  media.dirInfo( tofill,
434  resource.filename(),
435  false /* dots */,
436  resource.medianr());
437  std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
438  _dircontent[resource.filename().asString()] = tofill;
439  }
440  }
441 
443  const OnMediaLocation &resource,
444  const Pathname &dest_dir, FetcherJob::Flags flags )
445  {
446  // first get the content of the directory so we can add
447  // individual transfer jobs
448  MIL << "Adding directory " << resource.filename() << endl;
449  filesystem::DirContent content;
450  try {
451  getDirectoryContent(media, resource, content);
452  }
453  catch ( media::MediaFileNotFoundException & exception )
454  {
455  ZYPP_CAUGHT( exception );
456  WAR << "Skiping subtree hidden at " << resource.filename() << endl;
457  return;
458  }
459 
460  // this method test for the option flags so indexes are added
461  // only if the options are enabled
462  autoaddIndexes(content, media, resource, dest_dir);
463 
464  for ( filesystem::DirContent::const_iterator it = content.begin();
465  it != content.end();
466  ++it )
467  {
468  // skip CHECKSUMS* as they were already retrieved
469  if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
470  continue;
471 
472  Pathname filename = resource.filename() + it->name;
473 
474  switch ( it->type )
475  {
476  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
477  case filesystem::FT_FILE:
478  {
479  CheckSum chksm(resource.checksum());
480  if ( _checksums.find(filename.asString()) != _checksums.end() )
481  {
482  // the checksum can be replaced with the one in the index.
483  chksm = _checksums[filename.asString()];
484  //MIL << "resource " << filename << " has checksum in the index file." << endl;
485  }
486  else
487  WAR << "Resource " << filename << " has no checksum in the index either." << endl;
488 
489  if ( flags & FetcherJob::AlwaysVerifyChecksum )
490  enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
491  else
492  enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
493  break;
494  }
495  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
496  if ( flags & FetcherJob::Recursive )
497  addDirJobs(media, filename, dest_dir, flags);
498  break;
499  default:
500  // don't provide devices, sockets, etc.
501  break;
502  }
503  }
504  }
505 
506  void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
507  {
508  const OnMediaLocation & resource( jobp_r->location );
509 
510  try
511  {
512  scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
513 
514  // get cached file (by checksum) or provide from media
515  Pathname tmpFile = locateInCache( resource, destDir_r );
516  if ( tmpFile.empty() )
517  {
518  MIL << "Not found in cache, retrieving..." << endl;
519  tmpFile = media_r.provideFile( resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, jobp_r->deltafile );
520  releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
521  }
522 
523  // The final destination: locateInCache also checks destFullPath!
524  // If we find a cache match (by checksum) at destFullPath, take
525  // care it gets deleted, in case the validation fails.
526  ManagedFile destFullPath( destDir_r / resource.filename() );
527  if ( tmpFile == destFullPath )
528  destFullPath.setDispose( filesystem::unlink );
529 
530  // validate the file (throws if not valid)
531  validate( tmpFile, jobp_r->checkers );
532 
533  // move it to the final destination
534  if ( tmpFile == destFullPath )
535  destFullPath.resetDispose(); // keep it!
536  else
537  {
538  if ( assert_dir( destFullPath->dirname() ) != 0 )
539  ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
540 
541  if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
542  ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
543  }
544  }
545  catch ( Exception & excpt )
546  {
547  if ( resource.optional() )
548  {
549  ZYPP_CAUGHT( excpt );
550  WAR << "optional resource " << resource << " could not be transferred." << endl;
551  return;
552  }
553  else
554  {
555  excpt.remember( "Can't provide " + resource.filename().asString() );
556  ZYPP_RETHROW( excpt );
557  }
558  }
559  }
560 
561  // helper class to consume a content file
563  {
565  {
566  setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
567  }
568 
569  void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
570  { _repoindex = data_r; }
571 
572  parser::susetags::RepoIndex_Ptr _repoindex;
573  };
574 
575  // generic function for reading indexes
576  void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
577  {
578  if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
579  readChecksumsIndex(index, basedir);
580  else if ( index.basename() == "content" )
581  readContentFileIndex(index, basedir);
582  else
583  WAR << index << ": index file format not known" << endl;
584  }
585 
586  // reads a content file index
587  void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
588  {
589  ContentReaderHelper reader;
590  reader.parse(index);
591  MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
592  for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
593  {
594  // content file entries don't start with /
595  _checksums[(basedir + it->first).asString()] = it->second;
596  }
597  }
598 
599  // reads a CHECKSUMS (old SHA1SUMS) file index
600  void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
601  {
602  std::ifstream in( index.c_str() );
603  if ( ! in.fail() )
604  {
605  std::string buffer;
606  while ( getline( in, buffer ) )
607  {
608 
609  if ( buffer[0] == '#' )
610  continue; // simple comment
611 
612  CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
613  if ( checksum.empty() )
614  continue; // empty line | unknown cheksum format
615 
616  if ( buffer.empty() )
617  {
618  WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
619  continue;
620  }
621 
622  _checksums[(basedir/buffer).asString()] = checksum;
623  }
624  }
625  else
626  ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
627  }
628 
629  void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
630  {
631  MIL << "downloading index " << resource << endl;
632  // create a new fetcher with a different state to transfer the
633  // file containing checksums and its signature
634  Fetcher fetcher;
635  // signature checker for index. We havent got the signature from
636  // the nextwork yet.
637  SignatureFileChecker sigchecker;
638 
639  // build the name of the index and the signature
640  OnMediaLocation idxloc(resource);
641  OnMediaLocation sigloc(resource);
642  OnMediaLocation keyloc(resource);
643 
644  // we should not fail the download if those don't exists
645  // the checking will warn later
646  sigloc.setOptional(true);
647  keyloc.setOptional(true);
648 
649  // calculate signature and key name
650  sigloc.changeFilename( sigloc.filename().extend(".asc") );
651  keyloc.changeFilename( keyloc.filename().extend(".key") );
652 
653  //assert_dir(dest_dir + idxloc.filename().dirname());
654 
655  // transfer the signature
656  fetcher.enqueue(sigloc);
657  fetcher.start( dest_dir, media );
658  // if we get the signature, update the checker
659  if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
660  sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
661 
662  fetcher.reset();
663 
664  // now the key
665  fetcher.enqueue(keyloc);
666  fetcher.start( dest_dir, media );
667  fetcher.reset();
668 
669  // try to import the key
670  if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
671  getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
672  else
673  WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
674 
675  // now the index itself
676  fetcher.enqueue( idxloc, FileChecker(sigchecker) );
677  fetcher.start( dest_dir, media );
678  fetcher.reset();
679  }
680 
681  // this method takes all the user pointed indexes, gets them and also tries to
682  // download their signature, and verify them. After that, its parses each one
683  // to fill the checksum cache.
684  void Fetcher::Impl::downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir)
685  {
686  // if there is no indexes, then just return to avoid
687  // the directory listing
688  if ( _indexes.empty() )
689  {
690  MIL << "No indexes to read." << endl;
691  return;
692  }
693 
694  for_( it_idx, _indexes.begin(), _indexes.end() )
695  {
696  if ( (*it_idx)->read )
697  {
698  DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
699  }
700  else
701  {
702  // base::LogControl::TmpLineWriter shutUp;
703  downloadIndex( media, (*it_idx)->location, dest_dir );
704  // now we have the indexes in dest_dir
705  readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
706  // Take care we don't process it again
707  MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
708  (*it_idx)->read = true;
709  }
710  }
711  MIL << "done reading indexes" << endl;
712  }
713 
714  // start processing all fetcher jobs.
715  // it processes any user pointed index first
716  void Fetcher::Impl::start( const Pathname &dest_dir,
717  MediaSetAccess &media,
718  const ProgressData::ReceiverFnc & progress_receiver )
719  {
720  ProgressData progress(_resources.size());
721  progress.sendTo(progress_receiver);
722 
723  downloadAndReadIndexList(media, dest_dir);
724 
725  for ( const FetcherJob_Ptr & jobp : _resources )
726  {
727  if ( jobp->flags & FetcherJob::Directory )
728  {
729  const OnMediaLocation location(jobp->location);
730  addDirJobs(media, location, dest_dir, jobp->flags);
731  continue;
732  }
733 
734  // may be this code can be factored out
735  // together with the autodiscovery of indexes
736  // of addDirJobs
737  if ( ( _options & AutoAddChecksumsIndexes ) ||
738  ( _options & AutoAddContentFileIndexes ) )
739  {
740  // if auto indexing is enabled, then we need to read the
741  // index for each file. We look only in the directory
742  // where the file is. this is expensive of course.
743  filesystem::DirContent content;
744  getDirectoryContent(media, jobp->location.filename().dirname(), content);
745  // this method test for the option flags so indexes are added
746  // only if the options are enabled
747  MIL << "Autodiscovering signed indexes on '"
748  << jobp->location.filename().dirname() << "' for '"
749  << jobp->location.filename() << "'" << endl;
750 
751  autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
752 
753  // also look in the root of the media
754  content.clear();
755  getDirectoryContent(media, Pathname("/"), content);
756  // this method test for the option flags so indexes are added
757  // only if the options are enabled
758  MIL << "Autodiscovering signed indexes on '"
759  << "/" << "' for '"
760  << jobp->location.filename() << "'" << endl;
761 
762  autoaddIndexes(content, media, Pathname("/"), dest_dir);
763  }
764 
765  // if the checksum is empty, but the checksum is in one of the
766  // indexes checksum, then add a checker
767  if ( jobp->location.checksum().empty() )
768  {
769  if ( _checksums.find(jobp->location.filename().asString())
770  != _checksums.end() )
771  {
772  CheckSum chksm = _checksums[jobp->location.filename().asString()];
773  ChecksumFileChecker digest_check(chksm);
774  jobp->checkers.push_back(digest_check);
775  }
776  else
777  {
778  // if the index checksum is empty too, we only add the checker
779  // if the AlwaysVerifyChecksum option is set on
780  if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
781  {
782  // add the checker with the empty checksum
783  ChecksumFileChecker digest_check(jobp->location.checksum());
784  jobp->checkers.push_back(digest_check);
785  }
786  }
787  }
788  else
789  {
790  // checksum is not empty, so add a checksum checker
791  ChecksumFileChecker digest_check(jobp->location.checksum());
792  jobp->checkers.push_back(digest_check);
793  }
794 
795  // Provide and validate the file. If the file was not transferred
796  // and no exception was thrown, it was an optional file.
797  provideToDest( media, dest_dir, jobp );
798 
799  if ( ! progress.incr() )
800  ZYPP_THROW(AbortRequestException());
801  } // for each job
802  }
803 
805  inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
806  {
807  for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
808  {
809  str << *it_res;
810  }
811  return str;
812  }
813 
815  : _pimpl( new Impl() )
816  {}
817 
819  {}
820 
821  void Fetcher::setOptions( Fetcher::Options options )
822  {
823  _pimpl->setOptions(options);
824  }
825 
826  Fetcher::Options Fetcher::options() const
827  {
828  return _pimpl->options();
829  }
830 
831  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
832  {
833  _pimpl->enqueueDigested(resource, checker, deltafile);
834  }
835 
836  void Fetcher::enqueueDir( const OnMediaLocation &resource,
837  bool recursive,
838  const FileChecker &checker )
839  {
840  _pimpl->enqueueDir(resource, recursive, checker);
841  }
842 
844  bool recursive,
845  const FileChecker &checker )
846  {
847  _pimpl->enqueueDigestedDir(resource, recursive, checker);
848  }
849 
850 
851  void Fetcher::addIndex( const OnMediaLocation &resource )
852  {
853  _pimpl->addIndex(resource);
854  }
855 
856 
857  void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
858  {
859  _pimpl->enqueue(resource, checker);
860  }
861 
862  void Fetcher::addCachePath( const Pathname &cache_dir )
863  {
864  _pimpl->addCachePath(cache_dir);
865  }
866 
868  {
869  _pimpl->reset();
870  }
871 
872  void Fetcher::start( const Pathname &dest_dir,
873  MediaSetAccess &media,
874  const ProgressData::ReceiverFnc & progress_receiver )
875  {
876  _pimpl->start(dest_dir, media, progress_receiver);
877  }
878 
879  std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
880  {
881  return str << *obj._pimpl;
882  }
883 
885 } // namespace zypp
887 
std::string asString(const Patch::Category &obj)
Definition: Patch.cc:117
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:320
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:569
#define MIL
Definition: Logger.h:64
Pathname deltafile
Definition: Fetcher.cc:107
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:587
Fetcher::Options options() const
Definition: Fetcher.cc:250
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:53
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:821
Listentry returned by readdir.
Definition: PathInfo.h:532
bool empty() const
Definition: CheckSum.cc:173
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: ProgressData.h:226
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:321
Describes a path on a certain media amongs as the information required to download it...
Checks for the validity of a signature.
Definition: FileChecker.h:93
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:851
Options options() const
Get current options.
Definition: Fetcher.cc:826
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:572
unsigned medianr() const
media number where the resource is located.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:268
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory)...
Definition: Fetcher.cc:332
OnMediaLocation & setOptional(bool val)
Set the whether the resource is optional or not.
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition: Fetcher.cc:506
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:285
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:862
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:27
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:843
void addCachePath(const Pathname &cache_dir)
Definition: Fetcher.cc:309
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:48
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
std::list< FileChecker > checkers
Definition: Fetcher.cc:109
#define ERR
Definition: Logger.h:66
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:152
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:325
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:831
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:89
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:223
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:329
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:261
Fetcher::Options _options
Definition: Fetcher.cc:230
std::ostream & operator<<(std::ostream &str, const Exception &obj)
Definition: Exception.cc:120
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:662
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:867
#define WAR
Definition: Logger.h:65
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:576
Parse repoindex part from a content file.
std::set ordering (less semantic)
Definition: Fetcher.cc:56
Maintain [min,max] and counter (value) for progress counting.
Definition: ProgressData.h:130
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:253
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:50
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:817
std::map< std::string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:228
OnMediaLocation location
Definition: Fetcher.cc:106
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:629
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
Definition: Fetcher.cc:872
std::list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:222
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Definition: Fetcher.cc:234
Fetcher()
Default ctor.
Definition: Fetcher.cc:814
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:836
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition: Fetcher.cc:362
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:50
ZYPP_DECLARE_FLAGS(Flags, Flag)
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:419
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Definition: AutoDispose.h:158
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:325
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:78
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:247
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:44
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition: PublicKey.h:277
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:600
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:684
Fetcher implementation.
Definition: Fetcher.cc:126
Base class for Exception.
Definition: Exception.h:143
Built in file checkers.
Definition: FileChecker.h:70
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
Definition: Fetcher.cc:93
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:983
const Pathname & filename() const
The path to the resource relatve to the url and path.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:92
std::map< std::string, CheckSum > _checksums
Definition: Fetcher.cc:226
OnMediaLocation & changeFilename(const Pathname &val_r)
Individual manipulation of filename (prefer setLocation).
const bool optional() const
whether this is an optional resource.
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:42
std::set< Pathname > _caches
Definition: Fetcher.cc:224
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
Definition: Fetcher.cc:716
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:28
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Definition: Fetcher.cc:805
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:114
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:857
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:818
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs)
Definition: Fetcher.cc:58
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT, const Pathname &deltafile=Pathname())
Provides a file from a media location.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:391
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1036
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:219
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:105
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:294
Media access layer responsible for handling files distributed on a set of media with media change and...
#define DBG
Definition: Logger.h:63
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:995
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:442
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
const CheckSum & checksum() const
the checksum of the resource
The user is not asked anything, and the error exception is just propagated.