libzypp 17.31.0
Fetcher.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
12#include <iostream>
13#include <fstream>
14#include <list>
15#include <map>
16
17#include <zypp/base/Easy.h>
18#include <zypp/base/LogControl.h>
19#include <zypp/base/LogTools.h>
20#include <zypp/base/PtrTypes.h>
21#include <zypp-core/base/DefaultIntegral>
22#include <zypp/base/String.h>
23#include <zypp-media/MediaException>
24#include <zypp/Fetcher.h>
25#include <zypp/ZYppFactory.h>
26#include <zypp/CheckSum.h>
27#include <zypp-core/base/UserRequestException>
30
31#undef ZYPP_BASE_LOGGER_LOGGROUP
32#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher"
33
35namespace zypp
36{
37
44 {
46 : location(loc)
47 {}
52 };
53
54 typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
55
58 {
59 bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const
60 {
61 if ( lhs == rhs )
62 return false; // incl. NULL == NULL
63 if ( ! lhs )
64 return true; // NULL < nonNULL
65 if ( ! rhs )
66 return false; // nonNULL > NULL
67 // both nonNULL ==> compare medianr and path
68 if ( lhs->location.medianr() == rhs->location.medianr() )
69 return lhs->location.filename() < rhs->location.filename();
70 //else
71 return lhs->location.medianr() < rhs->location.medianr();
72 }
73 };
74
80 {
81 enum Flag
82 {
83 None = 0x0000,
84 Directory = 0x0001,
85 Recursive = 0x0002,
87 // check checksums even if there is no such
88 // checksum (warns of no checksum)
90 };
92
93
95 : location(loc)
96 , flags(None)
97 {
98 //MIL << location << endl;
99 }
100
102 {
103 //MIL << location << " | * " << checkers.size() << endl;
104 }
105
107 //CompositeFileChecker checkers;
108 std::list<FileChecker> checkers;
109 Flags flags;
110 };
111
113 typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
114
115 std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
116 {
117 return str << obj->location;
118 }
119
121 //
122 // CLASS NAME : Fetcher::Impl
123 //
126 {
127 friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
128
129 public:
130 Impl();
131
132 ~Impl() {}
133
134 void setOptions( Fetcher::Options options );
135 Fetcher::Options options() const;
136
137 void addIndex( const OnMediaLocation &resource );
138
139 void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
140 void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
141
142 void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
143 void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
144 void addCachePath( const Pathname &cache_dir );
145 void reset();
146 void setMediaSetAccess ( MediaSetAccess &media );
147 void start( const Pathname &dest_dir,
148 const ProgressData::ReceiverFnc &progress );
149 void start( const Pathname &dest_dir,
150 MediaSetAccess &media,
151 const ProgressData::ReceiverFnc & progress_receiver );
152
154 static shared_ptr<Impl> nullimpl()
155 {
156 static shared_ptr<Impl> _nullimpl( new Impl );
157 return _nullimpl;
158 }
159 private:
163 void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
164
168 void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
169
177 void readIndex( const Pathname &index, const Pathname &basedir );
178
180 void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
181
183 void readContentFileIndex( const Pathname &index, const Pathname &basedir );
184
186 void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
187
193 Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
198 void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
199
203 void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
204 const Pathname &dest_dir, FetcherJob::Flags flags );
205
209 void autoaddIndexes( const filesystem::DirContent &content,
210 MediaSetAccess &media,
211 const OnMediaLocation &resource,
212 const Pathname &dest_dir );
216 void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
217
218 private:
219 friend Impl * rwcowClone<Impl>( const Impl * rhs );
221 Impl * clone() const
222 { return new Impl( *this ); }
223
224 std::list<FetcherJob_Ptr> _resources;
225 std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
226 std::set<Pathname> _caches;
227 // checksums read from the indexes
228 std::map<std::string, CheckSum> _checksums;
229 // cache of dir contents
230 std::map<std::string, filesystem::DirContent> _dircontent;
231
233
234 Fetcher::Options _options;
235 };
237
239 {
240 if ( _mediaSetAccess )
241 _mediaSetAccess->precacheFiles( {resource} );
242
243 FetcherJob_Ptr job;
244 job.reset(new FetcherJob(resource));
246 _resources.push_back(job);
247 }
248
250 : _options(0)
251 {
252 }
253
254 void Fetcher::Impl::setOptions( Fetcher::Options options )
255 { _options = options; }
256
257 Fetcher::Options Fetcher::Impl::options() const
258 { return _options; }
259
261 bool recursive,
262 const FileChecker &checker )
263 {
264 FetcherJob_Ptr job;
265 job.reset(new FetcherJob(resource));
266 if ( checker )
267 job->checkers.push_back(checker);
268 if ( recursive )
269 job->flags |= FetcherJob::Recursive;
270 job->flags |= FetcherJob::Directory;
271
272 _resources.push_back(job);
273 }
274
276 bool recursive,
277 const FileChecker &checker )
278 {
279 FetcherJob_Ptr job;
280 job.reset(new FetcherJob(resource));
281 if ( checker )
282 job->checkers.push_back(checker);
283 if ( recursive )
284 job->flags |= FetcherJob::Recursive;
285 job->flags |= FetcherJob::Directory;
287
288 _resources.push_back(job);
289
290 }
291
292 void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
293 {
294 if ( _mediaSetAccess )
295 _mediaSetAccess->precacheFiles( {resource} );
296
297 FetcherJob_Ptr job;
298 job.reset(new FetcherJob(resource));
299 if ( checker )
300 job->checkers.push_back(checker);
301 _resources.push_back(job);
302 }
303
305 {
306 MIL << "adding index " << resource << endl;
307 _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
308 }
309
310
312 {
313 _resources.clear();
314 _indexes.clear();
315 _checksums.clear();
316 _dircontent.clear();
317 }
318
320 {
321 _mediaSetAccess = &media;
322 }
323
324 void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
325 {
326 PathInfo info(cache_dir);
327 if ( info.isExist() )
328 {
329 if ( info.isDir() )
330 {
331 DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
332 _caches.insert(cache_dir);
333 }
334 else
335 {
336 // don't add bad cache directory, just log the error
337 ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
338 }
339 }
340 else
341 {
342 ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
343 }
344
345 }
346
347 Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
348 {
349 Pathname ret;
350 // No checksum - no match
351 if ( resource_r.checksum().empty() )
352 return ret;
353
354 // first check in the destination directory
355 Pathname cacheLocation = destDir_r / resource_r.filename();
356 if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
357 {
358 swap( ret, cacheLocation );
359 return ret;
360 }
361
362 MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
363 for( const Pathname & cacheDir : _caches )
364 {
365 cacheLocation = cacheDir / resource_r.filename();
366 if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
367 {
368 MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl;
369 swap( ret, cacheLocation );
370 return ret;
371 }
372 }
373
374 return ret;
375 }
376
377 void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
378 {
379 try
380 {
381 MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
382
383 for ( const FileChecker & chkfnc : checkers_r )
384 {
385 if ( chkfnc )
386 chkfnc( localfile_r );
387 else
388 ERR << "Invalid checker for '" << localfile_r << "'" << endl;
389 }
390
391 }
392 catch ( const FileCheckException &e )
393 {
394 throw; // let known exceptions bypass silently
395 }
396 catch ( const Exception &e )
397 {
398 throw; // slet known exceptions bypass silently
399 }
400 catch (...)
401 {
402 ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
403 }
404 }
405
407 MediaSetAccess &media,
408 const OnMediaLocation &resource,
409 const Pathname &dest_dir )
410 {
411 auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
412 {
413 if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
414 return false;
415 // add the index of this directory
416 OnMediaLocation indexloc( resource );
417 indexloc.changeFilename( resource.filename() + index_r );
418 addIndex( indexloc );
419 // we need to read it now
420 downloadAndReadIndexList( media, dest_dir );
421 return true;
422 } );
423
424 if ( _options & AutoAddChecksumsIndexes )
425 {
426 fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
427 }
428 if ( _options & AutoAddContentFileIndexes )
429 {
430 fnc_addIfInContent( "content" );
431 }
432 }
433
435 const OnMediaLocation &resource,
436 filesystem::DirContent &content )
437 {
438 if ( _dircontent.find(resource.filename().asString())
439 != _dircontent.end() )
440 {
441 filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
442
443 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
444 }
445 else
446 {
448 media.dirInfo( tofill,
449 resource.filename(),
450 false /* dots */,
451 resource.medianr());
452 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
453 _dircontent[resource.filename().asString()] = tofill;
454 }
455 }
456
458 const OnMediaLocation &resource,
459 const Pathname &dest_dir, FetcherJob::Flags flags )
460 {
461 // first get the content of the directory so we can add
462 // individual transfer jobs
463 MIL << "Adding directory " << resource.filename() << endl;
465 try {
466 getDirectoryContent(media, resource, content);
467 }
468 catch ( media::MediaFileNotFoundException & exception )
469 {
470 ZYPP_CAUGHT( exception );
471 WAR << "Skipping subtree hidden at " << resource.filename() << endl;
472 return;
473 }
474
475 // this method test for the option flags so indexes are added
476 // only if the options are enabled
477 autoaddIndexes(content, media, resource, dest_dir);
478
479 for ( filesystem::DirContent::const_iterator it = content.begin();
480 it != content.end();
481 ++it )
482 {
483 // skip CHECKSUMS* as they were already retrieved
484 if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
485 continue;
486
487 Pathname filename = resource.filename() + it->name;
488
489 switch ( it->type )
490 {
491 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
493 {
494 CheckSum chksm(resource.checksum());
495 if ( _checksums.find(filename.asString()) != _checksums.end() )
496 {
497 // the checksum can be replaced with the one in the index.
498 chksm = _checksums[filename.asString()];
499 //MIL << "resource " << filename << " has checksum in the index file." << endl;
500 }
501 else
502 WAR << "Resource " << filename << " has no checksum in the index either." << endl;
503
505 enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
506 else
507 enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
508 break;
509 }
510 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
511 if ( flags & FetcherJob::Recursive )
512 addDirJobs(media, filename, dest_dir, flags);
513 break;
514 default:
515 // don't provide devices, sockets, etc.
516 break;
517 }
518 }
519 }
520
521 void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
522 {
523 const OnMediaLocation & resource( jobp_r->location );
524
525 try
526 {
527 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
528
529 // get cached file (by checksum) or provide from media
530 Pathname tmpFile = locateInCache( resource, destDir_r );
531 if ( tmpFile.empty() )
532 {
533 MIL << "Not found in cache, retrieving..." << endl;
535 releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
536 }
537
538 // The final destination: locateInCache also checks destFullPath!
539 // If we find a cache match (by checksum) at destFullPath, take
540 // care it gets deleted, in case the validation fails.
541 ManagedFile destFullPath( destDir_r / resource.filename() );
542 if ( tmpFile == destFullPath )
543 destFullPath.setDispose( filesystem::unlink );
544
545 // validate the file (throws if not valid)
546 validate( tmpFile, jobp_r->checkers );
547
548 // move it to the final destination
549 if ( tmpFile == destFullPath )
550 destFullPath.resetDispose(); // keep it!
551 else
552 {
553 if ( assert_dir( destFullPath->dirname() ) != 0 )
554 ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
555
556 if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
557 ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
558 }
559 }
560 catch ( Exception & excpt )
561 {
562 if ( resource.optional() )
563 {
564 ZYPP_CAUGHT( excpt );
565 WAR << "optional resource " << resource << " could not be transferred." << endl;
566 return;
567 }
568 else
569 {
570 excpt.remember( "Can't provide " + resource.filename().asString() );
571 ZYPP_RETHROW( excpt );
572 }
573 }
574 }
575
576 // helper class to consume a content file
578 {
580 {
581 setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
582 }
583
584 void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
585 { _repoindex = data_r; }
586
587 parser::susetags::RepoIndex_Ptr _repoindex;
588 };
589
590 // generic function for reading indexes
591 void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
592 {
593 if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
594 readChecksumsIndex(index, basedir);
595 else if ( index.basename() == "content" )
596 readContentFileIndex(index, basedir);
597 else
598 WAR << index << ": index file format not known" << endl;
599 }
600
601 // reads a content file index
602 void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
603 {
604 ContentReaderHelper reader;
605 reader.parse(index);
606 MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
607 for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
608 {
609 // content file entries don't start with /
610 _checksums[(basedir + it->first).asString()] = it->second;
611 }
612 }
613
614 // reads a CHECKSUMS (old SHA1SUMS) file index
615 void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
616 {
617 std::ifstream in( index.c_str() );
618 if ( ! in.fail() )
619 {
620 std::string buffer;
621 while ( getline( in, buffer ) )
622 {
623
624 if ( buffer[0] == '#' )
625 continue; // simple comment
626
627 CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
628 if ( checksum.empty() )
629 continue; // empty line | unknown cheksum format
630
631 if ( buffer.empty() )
632 {
633 WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
634 continue;
635 }
636
637 _checksums[(basedir/buffer).asString()] = checksum;
638 }
639 }
640 else
641 ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
642 }
643
644 void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
645 {
646 MIL << "downloading index " << resource << endl;
647 // create a new fetcher with a different state to transfer the
648 // file containing checksums and its signature
649 Fetcher fetcher;
650 // signature checker for index. We havent got the signature from
651 // the nextwork yet.
652 SignatureFileChecker sigchecker;
653
654 // build the name of the index and the signature
655 OnMediaLocation idxloc(resource);
656 OnMediaLocation sigloc(resource);
657 OnMediaLocation keyloc(resource);
658
659 // we should not fail the download if those don't exists
660 // the checking will warn later
661 sigloc.setOptional(true);
662 keyloc.setOptional(true);
663
664 // calculate signature and key name
665 sigloc.changeFilename( sigloc.filename().extend(".asc") );
666 keyloc.changeFilename( keyloc.filename().extend(".key") );
667
668 //assert_dir(dest_dir + idxloc.filename().dirname());
669
670 // transfer the signature
671 fetcher.enqueue(sigloc);
672 fetcher.start( dest_dir, media );
673 // if we get the signature, update the checker
674 if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
675 sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
676
677 fetcher.reset();
678
679 // now the key
680 fetcher.enqueue(keyloc);
681 fetcher.start( dest_dir, media );
682 fetcher.reset();
683
684 // try to import the key
685 if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
686 getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
687 else
688 WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
689
690 // now the index itself
691 fetcher.enqueue( idxloc, FileChecker(sigchecker) );
692 fetcher.start( dest_dir, media );
693 fetcher.reset();
694 }
695
696 // this method takes all the user pointed indexes, gets them and also tries to
697 // download their signature, and verify them. After that, it parses each one
698 // to fill the checksum cache.
700 {
701 // if there is no indexes, then just return to avoid
702 // the directory listing
703 if ( _indexes.empty() )
704 {
705 MIL << "No indexes to read." << endl;
706 return;
707 }
708
709 for_( it_idx, _indexes.begin(), _indexes.end() )
710 {
711 if ( (*it_idx)->read )
712 {
713 DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
714 }
715 else
716 {
717 // base::LogControl::TmpLineWriter shutUp;
718 downloadIndex( media, (*it_idx)->location, dest_dir );
719 // now we have the indexes in dest_dir
720 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
721 // Take care we don't process it again
722 MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
723 (*it_idx)->read = true;
724 }
725 }
726 MIL << "done reading indexes" << endl;
727 }
728
729 void Fetcher::Impl::start( const Pathname &dest_dir,
730 const ProgressData::ReceiverFnc & progress )
731 {
732 if ( !_mediaSetAccess )
733 ZYPP_THROW( zypp::Exception("Called Fetcher::start without setting MediaSetAccess before.") );
734 start( dest_dir, *_mediaSetAccess, progress );
735 }
736
737 // start processing all fetcher jobs.
738 // it processes any user pointed index first
739 void Fetcher::Impl::start( const Pathname &dest_dir,
740 MediaSetAccess &media,
741 const ProgressData::ReceiverFnc & progress_receiver )
742 {
743 _mediaSetAccess = nullptr; //reset the internally stored MediaSetAccess
744
745 ProgressData progress(_resources.size());
746 progress.sendTo(progress_receiver);
747
748 downloadAndReadIndexList(media, dest_dir);
749
750 for ( const FetcherJob_Ptr & jobp : _resources )
751 {
752 if ( jobp->flags & FetcherJob::Directory )
753 {
754 const OnMediaLocation location(jobp->location);
755 addDirJobs(media, location, dest_dir, jobp->flags);
756 continue;
757 }
758
759 // may be this code can be factored out
760 // together with the autodiscovery of indexes
761 // of addDirJobs
762 if ( ( _options & AutoAddChecksumsIndexes ) ||
763 ( _options & AutoAddContentFileIndexes ) )
764 {
765 // if auto indexing is enabled, then we need to read the
766 // index for each file. We look only in the directory
767 // where the file is. this is expensive of course.
769 getDirectoryContent(media, jobp->location.filename().dirname(), content);
770 // this method test for the option flags so indexes are added
771 // only if the options are enabled
772 MIL << "Autodiscovering signed indexes on '"
773 << jobp->location.filename().dirname() << "' for '"
774 << jobp->location.filename() << "'" << endl;
775
776 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
777
778 // also look in the root of the media
779 content.clear();
780 getDirectoryContent(media, Pathname("/"), content);
781 // this method test for the option flags so indexes are added
782 // only if the options are enabled
783 MIL << "Autodiscovering signed indexes on '"
784 << "/" << "' for '"
785 << jobp->location.filename() << "'" << endl;
786
787 autoaddIndexes(content, media, Pathname("/"), dest_dir);
788 }
789
790 // if the checksum is empty, but the checksum is in one of the
791 // indexes checksum, then add a checker
792 if ( jobp->location.checksum().empty() )
793 {
794 if ( _checksums.find(jobp->location.filename().asString())
795 != _checksums.end() )
796 {
797 CheckSum chksm = _checksums[jobp->location.filename().asString()];
798 ChecksumFileChecker digest_check(chksm);
799 jobp->checkers.push_back(digest_check);
800 }
801 else
802 {
803 // if the index checksum is empty too, we only add the checker
804 // if the AlwaysVerifyChecksum option is set on
805 if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
806 {
807 // add the checker with the empty checksum
808 ChecksumFileChecker digest_check(jobp->location.checksum());
809 jobp->checkers.push_back(digest_check);
810 }
811 }
812 }
813 else
814 {
815 // checksum is not empty, so add a checksum checker
816 ChecksumFileChecker digest_check(jobp->location.checksum());
817 jobp->checkers.push_back(digest_check);
818 }
819
820 // Provide and validate the file. If the file was not transferred
821 // and no exception was thrown, it was an optional file.
822 provideToDest( media, dest_dir, jobp );
823
824 if ( ! progress.incr() )
825 ZYPP_THROW(AbortRequestException());
826 } // for each job
827 }
828
830 inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
831 {
832 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
833 {
834 str << *it_res;
835 }
836 return str;
837 }
838
840 : _pimpl( new Impl() )
841 {}
842
844 {}
845
846 void Fetcher::setOptions( Fetcher::Options options )
847 {
848 _pimpl->setOptions(options);
849 }
850
851 Fetcher::Options Fetcher::options() const
852 {
853 return _pimpl->options();
854 }
855
856 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
857 {
858 enqueueDigested( OnMediaLocation(resource).setDeltafile(deltafile), checker );
859 }
860
861 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
862 {
863 _pimpl->enqueueDigested( resource, checker );
864 }
865
867 bool recursive,
868 const FileChecker &checker )
869 {
870 _pimpl->enqueueDir(resource, recursive, checker);
871 }
872
874 bool recursive,
875 const FileChecker &checker )
876 {
877 _pimpl->enqueueDigestedDir(resource, recursive, checker);
878 }
879
880
881 void Fetcher::addIndex( const OnMediaLocation &resource )
882 {
883 _pimpl->addIndex(resource);
884 }
885
886
887 void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
888 {
889 _pimpl->enqueue(resource, checker);
890 }
891
892
893 void Fetcher::addCachePath( const Pathname &cache_dir )
894 {
895 _pimpl->addCachePath(cache_dir);
896 }
897
899 {
900 _pimpl->reset();
901 }
902
904 {
905 _pimpl->setMediaSetAccess( media );
906 }
907
909 {
910 _pimpl->start( dest_dir, progress );
911 }
912
913 void Fetcher::start( const Pathname &dest_dir,
914 MediaSetAccess &media,
915 const ProgressData::ReceiverFnc & progress_receiver )
916 {
917 _pimpl->start(dest_dir, media, progress_receiver);
918 }
919
920 std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
921 {
922 return str << *obj._pimpl;
923 }
924
926} // namespace zypp
928
Pathname deltafile
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:94
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Definition: AutoDispose.h:176
bool empty() const
Definition: CheckSum.cc:173
Built in file checkers.
Definition: FileChecker.h:48
Base class for Exception.
Definition: Exception.h:146
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:105
Fetcher implementation.
Definition: Fetcher.cc:126
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:238
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:406
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:644
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:602
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:591
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:225
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
std::map< std::string, CheckSum > _checksums
Definition: Fetcher.cc:228
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:699
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:457
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:221
void addCachePath(const Pathname &cache_dir)
Definition: Fetcher.cc:324
std::list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:224
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:615
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:260
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:304
std::map< std::string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:230
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition: Fetcher.cc:521
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:292
MediaSetAccess * _mediaSetAccess
Definition: Fetcher.cc:232
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
Definition: Fetcher.cc:347
std::set< Pathname > _caches
Definition: Fetcher.cc:226
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:275
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition: Fetcher.cc:377
Fetcher::Options options() const
Definition: Fetcher.cc:257
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:434
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:154
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
Definition: Fetcher.cc:729
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:254
Fetcher::Options _options
Definition: Fetcher.cc:234
void setMediaSetAccess(MediaSetAccess &media)
Definition: Fetcher.cc:319
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:106
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:346
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:881
Options options() const
Get current options.
Definition: Fetcher.cc:851
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:898
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:887
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:866
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:846
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
Fetcher()
Default ctor.
Definition: Fetcher.cc:839
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:873
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:861
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:893
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:903
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:843
Media access layer responsible for handling files distributed on a set of media with media change and...
void precacheFiles(const std::vector< OnMediaLocation > &files)
Tries to fetch the given files and precaches them.
@ PROVIDE_DEFAULT
The user is not asked anything, and the error exception is just propagated.
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT)
Provides a file from a media location.
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
Describes a resource file located on a medium.
bool optional() const
Whether this is an optional resource.
const Pathname & filename() const
The path to the resource on the medium.
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
unsigned medianr() const
The media number the resource is located on.
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
const CheckSum & checksum() const
The checksum of the resource on the server.
OnMediaLocation & setOptional(bool val)
Set whether the resource is optional.
Maintain [min,max] and counter (value) for progress counting.
Definition: progressdata.h:131
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: progressdata.h:226
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:139
bool incr(value_type val_r=1)
Increment counter value (default by 1).
Definition: progressdata.h:261
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition: PublicKey.h:359
Checks for the validity of a signature.
Definition: FileChecker.h:71
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:281
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const char * c_str() const
String representation.
Definition: Pathname.h:110
const std::string & asString() const
String representation.
Definition: Pathname.h:91
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
String related utilities and Regular expression matching.
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:1051
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:700
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1063
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:319
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:883
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:518
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:50
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:263
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:54
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:113
std::ostream & operator<<(std::ostream &str, const SerialNumber &obj)
Definition: SerialNumber.cc:52
std::string asString(const Patch::Category &obj)
Definition: Patch.cc:122
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:29
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:587
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:584
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:44
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:49
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:45
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:51
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:80
std::list< FileChecker > checkers
Definition: Fetcher.cc:108
@ AlwaysVerifyChecksum
Definition: Fetcher.cc:89
OnMediaLocation location
Definition: Fetcher.cc:106
ZYPP_DECLARE_FLAGS(Flags, Flag)
FetcherJob(const OnMediaLocation &loc)
Definition: Fetcher.cc:94
std::set ordering (less semantic)
Definition: Fetcher.cc:58
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Definition: Fetcher.cc:59
Listentry returned by readdir.
Definition: PathInfo.h:501
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:28
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
#define DBG
Definition: Logger.h:95
#define MIL
Definition: Logger.h:96
#define ERR
Definition: Logger.h:98
#define WAR
Definition: Logger.h:97