libzypp 17.31.7
Downloader.cc
Go to the documentation of this file.
1
2#include <iostream>
3#include <fstream>
4
5#include <zypp/base/LogTools.h>
6#include <zypp/base/Gettext.h>
7#include <zypp/base/String.h>
8#include <zypp/base/Regex.h>
9#include <zypp-core/OnMediaLocation>
10#include <zypp/MediaSetAccess.h>
11#include <zypp/Fetcher.h>
12#include <zypp/Locale.h>
13#include <zypp/ZConfig.h>
15#include <zypp/repo/susetags/Downloader.h>
16#include <zypp-core/parser/ParseException>
18#include <zypp-core/base/UserRequestException>
19
20using namespace zypp::parser;
21using namespace zypp::parser::susetags;
22
23namespace zypp
24{
25namespace repo
26{
27namespace susetags
28{
29
30Downloader::Downloader( const RepoInfo &repoinfo, const Pathname &delta_dir )
31 : repo::Downloader(repoinfo), _delta_dir(delta_dir)
32{
33}
34
36{
37 RepoStatus ret( media.provideOptionalFile( repoInfo().path() + "/content") );
38 if ( !ret.empty() ) // else: mandatory master index is missing
39 ret = ret && RepoStatus( media.provideOptionalFile( "/media.1/media" ) );
40 // else: mandatory master index is missing -> stay empty
41 return ret;
42}
43
44// search old repository file file to run the delta algorithm on
45static Pathname search_deltafile( const Pathname &dir, const Pathname &file )
46{
47 Pathname deltafile(dir + file.basename());
48 if (PathInfo(deltafile).isExist())
49 return deltafile;
50 return Pathname();
51}
52
53
56 const Pathname &dest_dir,
57 const ProgressData::ReceiverFnc & progress )
58{
59 downloadMediaInfo( dest_dir, media );
60
61 Pathname masterIndex( repoInfo().path() / "/content" );
62 defaultDownloadMasterIndex( media, dest_dir, masterIndex );
63
64 // Content file first to get the repoindex
65 {
66 Pathname inputfile( dest_dir / masterIndex );
67 ContentFileReader content;
68 content.setRepoIndexConsumer( bind( &Downloader::consumeIndex, this, _1 ) );
69 content.parse( inputfile );
70 }
71 if ( ! _repoindex )
72 {
73 ZYPP_THROW( ParseException( (dest_dir+repoInfo().path()).asString() + ": " + "No repository index in content file." ) );
74 }
75 MIL << "RepoIndex: " << _repoindex << endl;
76 if ( _repoindex->metaFileChecksums.empty() )
77 {
78 ZYPP_THROW( ParseException( (dest_dir+repoInfo().path()).asString() + ": " + "No metadata checksums in content file." ) );
79 }
80 if ( _repoindex->signingKeys.empty() )
81 {
82 WAR << "No signing keys defined." << endl;
83 }
84
85 // Prepare parsing
86 Pathname descr_dir = _repoindex->descrdir; // path below reporoot
87 //_datadir = _repoIndex->datadir; // path below reporoot
88
89 std::map<std::string,RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
90
91 for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
92 {
93 // omit unwanted translations
94 if ( str::hasPrefix( it->first, "packages" ) )
95 {
96 static const str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
97 str::smatch what;
98 if ( str::regex_match( it->first, what, rx_packages ) )
99 {
100 if ( what[4].empty() // packages(.gz)?
101 || what[4] == "DU"
102 || what[4] == "en" )
103 { ; /* always downloaded */ }
104 else if ( what[4] == "FL" )
105 { continue; /* never downloaded */ }
106 else
107 {
108 // remember and decide later
109 availablePackageTranslations[what[4]] = it;
110 continue;
111 }
112 }
113 else
114 continue; // discard
115 }
116 else if ( it->first == "patterns.pat"
117 || it->first == "patterns.pat.gz" )
118 {
119 // take all patterns in one go
120 }
121 else if ( str::endsWith( it->first, ".pat" )
122 || str::endsWith( it->first, ".pat.gz" ) )
123 {
124
125 // *** see also zypp/parser/susetags/RepoParser.cc ***
126
127 // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
128 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
129 // split at dots, take .pat or .pat.gz into account
130
131 std::vector<std::string> patparts;
132 unsigned archpos = 2;
133 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
134 unsigned count = str::split( it->first, std::back_inserter(patparts), "." );
135 if ( patparts[count-1] == "gz" )
136 archpos++;
137
138 if ( count > archpos )
139 {
140 try // might by an invalid architecture
141 {
142 Arch patarch( patparts[count-archpos] );
144 {
145 // discard, if not compatible
146 MIL << "Discarding pattern " << it->first << endl;
147 continue;
148 }
149 }
150 catch ( const Exception & excpt )
151 {
152 WAR << "Pattern file name does not contain recognizable architecture: " << it->first << endl;
153 // keep .pat file if it doesn't contain an recognizable arch
154 }
155 }
156 }
157 MIL << "adding job " << it->first << endl;
158 auto location = std::move( OnMediaLocation( repoInfo().path() + descr_dir + it->first, 1 )
159 .setChecksum( it->second )
160 .setDeltafile( search_deltafile(_delta_dir + descr_dir, it->first) ));
161 enqueueDigested( location, FileChecker() );
162 }
163
164 // check whether to download more package translations:
165 {
166 auto fnc_checkTransaltions( [&]( const Locale & locale_r ) {
167 for ( Locale toGet( locale_r ); toGet; toGet = toGet.fallback() )
168 {
169 auto it( availablePackageTranslations.find( toGet.code() ) );
170 if ( it != availablePackageTranslations.end() )
171 {
172 auto mit( it->second );
173 MIL << "adding job " << mit->first << endl;
174 auto location = std::move( OnMediaLocation( repoInfo().path() + descr_dir + mit->first, 1 )
175 .setChecksum( mit->second )
176 .setDeltafile( search_deltafile(_delta_dir + descr_dir, mit->first) ));
177 enqueueDigested(location, FileChecker());
178 break;
179 }
180 }
181 });
182 for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
183 {
184 fnc_checkTransaltions( it );
185 }
186 fnc_checkTransaltions( ZConfig::instance().textLocale() );
187 }
188
189 for_( it, _repoindex->mediaFileChecksums.begin(), _repoindex->mediaFileChecksums.end() )
190 {
191 // Repo adopts license files listed in HASH
192 if ( it->first != "license.tar.gz" )
193 continue;
194
195 MIL << "adding job " << it->first << endl;
196 auto location = std::move( OnMediaLocation ( repoInfo().path() + it->first, 1 )
197 .setChecksum( it->second )
198 .setDeltafile( search_deltafile( _delta_dir, it->first ) ));
199 enqueueDigested(location, FileChecker() );
200 }
201
202 for_( it, _repoindex->signingKeys.begin(),_repoindex->signingKeys.end() )
203 {
204 MIL << "adding job " << it->first << endl;
205 OnMediaLocation location( repoInfo().path() + it->first, 1 );
206 location.setChecksum( it->second );
207 enqueueDigested(location);
208 }
209
210 start( dest_dir, media );
211}
212
213void Downloader::consumeIndex( const RepoIndex_Ptr & data_r )
214{
215 MIL << "Consuming repo index" << endl;
216 _repoindex = data_r;
217}
218
219}// ns susetags
220}// ns source
221} // ns zypp
Pathname deltafile
Architecture.
Definition: Arch.h:37
bool compatibleWith(const Arch &targetArch_r) const
Compatibility relation.
Definition: Arch.cc:505
Base class for Exception.
Definition: Exception.h:146
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:861
'Language[_Country]' codes.
Definition: Locale.h:50
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
Media access layer responsible for handling files distributed on a set of media with media change and...
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
Describes a resource file located on a medium.
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
OnMediaLocation & setDeltafile(Pathname path)
Set the deltafile.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:140
What is known about a repository.
Definition: RepoInfo.h:72
Track changing files or directories.
Definition: RepoStatus.h:41
bool empty() const
Whether the status is empty (empty checksum)
Definition: RepoStatus.cc:222
Arch systemArchitecture() const
The system architecture zypp uses.
Definition: ZConfig.cc:884
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:832
LocaleSet repoRefreshLocales() const
List of locales for which translated package descriptions should be downloaded.
Definition: ZConfig.cc:1074
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
const RepoInfo & repoInfo() const
Definition: Downloader.h:63
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:141
Downloader()
Constructor.
Definition: Downloader.cc:118
Downloader for SUSETags (YaST2) repositories Encapsulates all the knowledge of which files have to be...
Definition: Downloader.h:35
void download(MediaSetAccess &media, const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:55
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Content file parser consumer.
Definition: Downloader.cc:213
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Downloader.h:68
RepoStatus status(MediaSetAccess &media) override
Status of the remote repository.
Definition: Downloader.cc:35
Regular expression.
Definition: Regex.h:95
Regular expression match result.
Definition: Regex.h:168
static Pathname search_deltafile(const Pathname &dir, const Pathname &file)
Definition: Downloader.cc:45
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
\relates regex \ingroup ZYPP_STR_REGEX \relates regex \ingroup ZYPP_STR_REGEX
Definition: Regex.h:70
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t", const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
std::string asString(const Patch::Category &obj)
Definition: Patch.cc:122
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:29
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:28
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
#define MIL
Definition: Logger.h:96
#define WAR
Definition: Logger.h:97