00001
00002
00003
00004
00005
00006
00007
00008
00013 #include <iostream>
00014 #include <list>
00015 #include <vector>
00016 #include <fstream>
00017 #include <boost/lexical_cast.hpp>
00018
00019 #include "zypp/base/Logger.h"
00020 #include "zypp/base/Regex.h"
00021 #include "zypp/ExternalProgram.h"
00022 #include "zypp/ProgressData.h"
00023 #include "zypp/base/String.h"
00024 #include "zypp/base/Gettext.h"
00025 #include "zypp/base/Sysconfig.h"
00026 #include "zypp/base/Gettext.h"
00027 #include "zypp/ZYppCallbacks.h"
00028
00029 #include "zypp/Edition.h"
00030 #include "zypp/Target.h"
00031 #include "zypp/ZYppFactory.h"
00032 #include "zypp/ZConfig.h"
00033
00034 #include "zypp/TmpPath.h"
00035
00036 #include "zypp/media/MediaAria2c.h"
00037 #include "zypp/media/proxyinfo/ProxyInfos.h"
00038 #include "zypp/media/ProxyInfo.h"
00039 #include "zypp/media/MediaUserAuth.h"
00040 #include "zypp/media/MediaCurl.h"
00041 #include "zypp/thread/Once.h"
00042 #include <cstdlib>
00043 #include <sys/types.h>
00044 #include <sys/stat.h>
00045 #include <sys/mount.h>
00046 #include <errno.h>
00047 #include <dirent.h>
00048 #include <unistd.h>
00049 #include <boost/format.hpp>
00050
00051 #define DETECT_DIR_INDEX 0
00052 #define CONNECT_TIMEOUT 60
00053 #define TRANSFER_TIMEOUT 60 * 3
00054 #define TRANSFER_TIMEOUT_MAX 60 * 60
00055
00056 #define ARIA_BINARY "aria2c"
00057
00058 using namespace std;
00059 using namespace zypp::base;
00060
00061 namespace zypp
00062 {
00063 namespace media
00064 {
00065
00066 Pathname MediaAria2c::_cookieFile = "/var/lib/YaST2/cookies";
00067 std::string MediaAria2c::_aria2cVersion = "WE DON'T KNOW ARIA2C VERSION";
00068
00069
00070 bool
00071 MediaAria2c::existsAria2cmd()
00072 {
00073 static const char* argv[] =
00074 {
00075 ARIA_BINARY,
00076 "--version",
00077 NULL
00078 };
00079 ExternalProgram aria( argv, ExternalProgram::Stderr_To_Stdout );
00080 return( aria.close() == 0 );
00081 }
00082
00088 void fillAriaCmdLine( const string &ariaver,
00089 const TransferSettings &s,
00090 filesystem::TmpPath &credentials,
00091 const Url &url,
00092 const Pathname &destination,
00093 ExternalProgram::Arguments &args )
00094 {
00095
00096
00097
00098
00099 list<string> file_options;
00100
00101 args.push_back(ARIA_BINARY);
00102 args.push_back(str::form("--user-agent=%s", s.userAgentString().c_str()));
00103 args.push_back("--summary-interval=1");
00104 args.push_back("--follow-metalink=mem");
00105 args.push_back("--check-integrity=true");
00106 args.push_back("--file-allocation=none");
00107
00108
00109 Pathname statsFile = ZConfig::instance().repoCachePath() / "aria2.stats";
00110 args.push_back(str::form("--server-stat-of=%s", statsFile.c_str()));
00111 args.push_back(str::form("--server-stat-if=%s", statsFile.c_str()));
00112 args.push_back("--uri-selector=adaptive");
00113
00114
00115 vector<string> fields;
00116
00117 str::split( ariaver, std::back_inserter(fields));
00118 if ( fields.size() == 3 )
00119 {
00120 if ( Edition(fields[2]) >= Edition("1.1.2") )
00121 args.push_back( "--use-head=false");
00122 }
00123
00124 if ( s.maxDownloadSpeed() > 0 )
00125 args.push_back(str::form("--max-download-limit=%ld", s.maxDownloadSpeed()));
00126 if ( s.minDownloadSpeed() > 0 )
00127 args.push_back(str::form("--lowest-speed-limit=%ld", s.minDownloadSpeed()));
00128
00129 args.push_back(str::form("--max-tries=%ld", s.maxSilentTries()));
00130
00131 if ( Edition(fields[2]) < Edition("1.2.0") )
00132 WAR << "aria2c is older than 1.2.0, some features may be disabled" << endl;
00133
00134
00135 args.push_back(str::form("--max-concurrent-downloads=%ld", s.maxConcurrentConnections()));
00136
00137
00138 for ( TransferSettings::Headers::const_iterator it = s.headersBegin();
00139 it != s.headersEnd();
00140 ++it )
00141 args.push_back(str::form("--header=%s", it->c_str() ));
00142
00143 args.push_back( str::form("--connect-timeout=%ld", s.timeout()));
00144
00145 if ( s.username().empty() )
00146 {
00147 if ( url.getScheme() == "ftp" )
00148 {
00149
00150 args.push_back(str::form("--ftp-user=%s", "suseuser" ));
00151 args.push_back(str::form("--ftp-passwd=%s", VERSION ));
00152
00153 string id = "yast2";
00154 id += VERSION;
00155 DBG << "Anonymous FTP identification: '" << id << "'" << endl;
00156 }
00157 }
00158 else
00159 {
00160 MIL << "Passing " << url.getScheme() << " credentials '" << s.username() << ':' << (s.password().empty() ? "" : "PASSWORD")<< "'" << endl;
00161 if ( url.getScheme() == "ftp" )
00162 file_options.push_back(str::form("ftp-user=%s", s.username().c_str() ));
00163 else if ( url.getScheme() == "http" ||
00164 url.getScheme() == "https" )
00165 file_options.push_back(str::form("http-user=%s", s.username().c_str() ));
00166
00167 if ( s.password().size() )
00168 {
00169 if ( url.getScheme() == "ftp" )
00170 file_options.push_back(str::form("ftp-passwd=%s", s.password().c_str() ));
00171 else if ( url.getScheme() == "http" ||
00172 url.getScheme() == "https" )
00173 file_options.push_back(str::form("http-passwd=%s", s.password().c_str() ));
00174 }
00175 }
00176
00177 if ( s.proxyEnabled() )
00178 {
00179 args.push_back(str::form("--http-proxy=%s", s.proxy().c_str() ));
00180 if ( ! s.proxyUsername().empty() )
00181 {
00182 MIL << "Passing " << "http" << "-proxy credentials '" << s.proxyUsername() << ':' << (s.proxyPassword().empty() ? "" : "PASSWORD")<< "'" << endl;
00183 file_options.push_back(str::form("http-proxy-user=%s", s.proxyUsername().c_str() ));
00184 if ( ! s.proxyPassword().empty() )
00185 file_options.push_back(str::form("http-proxy-passwd=%s", s.proxyPassword().c_str() ));
00186 }
00187 }
00188
00189 if ( ! destination.empty() )
00190 args.push_back(str::form("--dir=%s", destination.c_str()));
00191
00192
00193 if ( ! file_options.empty() )
00194 {
00195 filesystem::TmpFile tmp;
00196 ofstream outs( tmp.path().c_str() );
00197 for_( it, file_options.begin(), file_options.end() )
00198 outs << *it << endl;
00199 outs.close();
00200
00201 credentials = tmp;
00202 args.push_back(str::form("--conf-path=%s", credentials.path().c_str()));
00203 }
00204
00205
00206
00207 args.push_back(url.asString( url.getViewOptions()
00208 - url::ViewOptions::WITH_USERNAME
00209 - url::ViewOptions::WITH_PASSWORD ).c_str());
00210 }
00211
00212 const char *const MediaAria2c::agentString()
00213 {
00214
00215
00216
00217
00218 Target_Ptr target = zypp::getZYpp()->getTarget();
00219
00220 static const std::string _value(
00221 str::form(
00222 "ZYpp %s (%s) %s"
00223 , VERSION
00224 , MediaAria2c::_aria2cVersion.c_str()
00225 , Target::targetDistribution( Pathname() ).c_str()
00226 )
00227 );
00228 return _value.c_str();
00229 }
00230
00231
00232
00233 MediaAria2c::MediaAria2c( const Url & url_r,
00234 const Pathname & attach_point_hint_r )
00235 : MediaCurl( url_r, attach_point_hint_r )
00236 {
00237 MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl;
00238
00239 _aria2cVersion = getAria2cVersion();
00240 }
00241
00242 void MediaAria2c::attachTo (bool next)
00243 {
00244 MediaCurl::attachTo(next);
00245 _settings.setUserAgentString(agentString());
00246 }
00247
00248 bool
00249 MediaAria2c::checkAttachPoint(const Pathname &apoint) const
00250 {
00251 return MediaCurl::checkAttachPoint( apoint );
00252 }
00253
00254 void MediaAria2c::disconnectFrom()
00255 {
00256 MediaCurl::disconnectFrom();
00257 }
00258
00259 void MediaAria2c::releaseFrom( const std::string & ejectDev )
00260 {
00261 MediaCurl::releaseFrom(ejectDev);
00262 }
00263
00264 void MediaAria2c::getFile( const Pathname & filename ) const
00265 {
00266
00267
00268 getFileCopy(filename, localPath(filename).absolutename());
00269 }
00270
00271 void MediaAria2c::getFileCopy( const Pathname & filename , const Pathname & target) const
00272 {
00273 callback::SendReport<DownloadProgressReport> report;
00274
00275 Url fileurl(getFileUrl(filename));
00276
00277 bool retry = false;
00278
00279 ExternalProgram::Arguments args;
00280
00281 filesystem::TmpPath credentials;
00282 fillAriaCmdLine(_aria2cVersion, _settings, credentials, fileurl, target.dirname(), args);
00283
00284 do
00285 {
00286 try
00287 {
00288 report->start(fileurl, target.asString() );
00289
00290 ExternalProgram aria(args, ExternalProgram::Stderr_To_Stdout);
00291
00292
00293
00294
00295
00296
00297
00298 static str::regex rxProgress(
00299 "^\\[#[0-9]+ SIZE:[0-9\\.]+(|Ki|Mi|Ti)B/[0-9\\.]+(|Ki|Mi|Ti)B\\(?([0-9]+)?%?\\)? CN:[0-9]+ SPD:([0-9\\.]+)(|Ki|Mi|Ti)Bs.*\\]$");
00300
00301
00302 bool gotProgress = false;
00303
00304 int progress = 0;
00305
00306 double current_speed = 0;
00307
00308 double average_speed = 0;
00309
00310 long average_speed_count = 0;
00311
00312
00313 vector<string> ariaExceptions;
00314
00315
00316 bool partialDownload = false;
00317
00318 bool userAbort = false;
00319
00320
00321 for(std::string ariaResponse( aria.receiveLine());
00322 ariaResponse.length();
00323 ariaResponse = aria.receiveLine())
00324 {
00325 string line = str::trim(ariaResponse);
00326
00327
00328
00329
00330 if ( str::hasPrefix(line, "[#") )
00331 {
00332 str::smatch progressValues;
00333 if (( gotProgress = str::regex_match(line, progressValues, rxProgress) ))
00334 {
00335
00336
00337
00338
00339
00340 progress = std::atoi(progressValues[3].c_str());
00341
00342
00343
00344 int factor = 1;
00345 if (progressValues[5] == "Ki")
00346 factor = 1024;
00347 else if (progressValues[5] == "Mi")
00348 factor = 0x100000;
00349 else if (progressValues[5] == "Ti")
00350 factor = 0x40000000;
00351
00352 try {
00353 current_speed = boost::lexical_cast<double>(progressValues[4]);
00354
00355 current_speed *= factor;
00356 }
00357 catch (const std::exception&) {
00358 ERR << "Can't parse speed from '" << progressValues[4] << "'" << endl;
00359 current_speed = 0;
00360 }
00361 }
00362 else
00363 ERR << "Can't parse progress line '" << line << "'" << endl;
00364 }
00365
00366 else if ( str::hasPrefix(line, "Exception: ") )
00367 {
00368
00369 if (!line.substr(0,31).compare("Exception: Authorization failed") )
00370 {
00371 ZYPP_THROW(MediaUnauthorizedException(
00372 _url, "Login failed.", "Login failed", "auth hint"
00373 ));
00374 }
00375
00376 string excpMsg = line.substr(10, line.size());
00377 DBG << "aria2c reported: '" << excpMsg << "'" << endl;
00378 ariaExceptions.push_back(excpMsg);
00379 }
00380
00381
00382 else if ( str::hasPrefix(line, "FILE: ") )
00383 {
00384
00385 string theFile(line.substr(6, line.size()));
00386
00387
00388
00389
00390
00391 if ( Pathname(theFile) == target || theFile == fileurl.asCompleteString() )
00392 {
00393
00394
00395 if ( gotProgress )
00396 {
00397
00398 average_speed_count++;
00399
00400
00401
00402 average_speed =
00403 (((average_speed_count - 1)*average_speed) + current_speed)
00404 / average_speed_count;
00405
00406 if (!partialDownload && progress > 0)
00407 partialDownload = true;
00408
00409 if ( ! report->progress ( progress, fileurl, average_speed, current_speed ) )
00410 userAbort = true;
00411
00412
00413 gotProgress = false;
00414 }
00415 else
00416 {
00417 WAR << "aria2c reported a file, but no progress data available" << endl;
00418 }
00419 }
00420 else
00421 {
00422 DBG << "Progress is not about '" << target << "' but '" << theFile << "'" << endl;
00423 }
00424 }
00425 else
00426 {
00427
00428 }
00429 }
00430
00431 int code;
00432 if (userAbort)
00433 {
00434 aria.kill();
00435 code = 7;
00436 }
00437 else
00438 code = aria.close();
00439
00440 switch ( code )
00441 {
00442 case 0:
00443 if ( ! PathInfo( target ).isExist() )
00444 {
00445
00446
00447
00448
00449 std::string msg( str::form(_("Failed to download %s from %s"),
00450 filename.c_str(), _url.asString().c_str() ) );
00451
00452 MediaException e( msg );
00453 for_( it, ariaExceptions.begin(), ariaExceptions.end() )
00454 e.addHistory( *it );
00455
00456 ZYPP_THROW( e );
00457 }
00458 break;
00459
00460 case 2:
00461 {
00462 MediaTimeoutException e(_url);
00463 for_(it, ariaExceptions.begin(), ariaExceptions.end())
00464 e.addHistory(*it);
00465 ZYPP_THROW(e);
00466 }
00467 break;
00468
00469 case 3:
00470 case 4:
00471 {
00472 MediaFileNotFoundException e(_url, filename);
00473 for_(it, ariaExceptions.begin(), ariaExceptions.end())
00474 e.addHistory(*it);
00475 ZYPP_THROW(e);
00476 }
00477 break;
00478
00479 case 5:
00480 case 6:
00481 case 7:
00482 case 1:
00483 default:
00484 {
00485 if ( partialDownload )
00486 {
00487
00488
00489 MediaException e(str::form(_("Download interrupted at %d%%"), progress ));
00490 for_(it, ariaExceptions.begin(), ariaExceptions.end())
00491 e.addHistory(*it);
00492
00493 DownloadProgressReport::Action action = report->problem( _url, DownloadProgressReport::ERROR, e.asUserHistory() );
00494 if ( action == DownloadProgressReport::RETRY )
00495 {
00496 retry = true;
00497 continue;
00498 }
00499 }
00500
00501 string msg;
00502 if (userAbort)
00503 msg = _("Download interrupted by user");
00504 else
00505
00506 msg = str::form(_("Failed to download %s from %s"),
00507 filename.c_str(), _url.asString().c_str());
00508
00509 MediaException e(msg);
00510 for_(it, ariaExceptions.begin(), ariaExceptions.end())
00511 e.addHistory(*it);
00512
00513 ZYPP_THROW(e);
00514 }
00515 break;
00516 }
00517
00518 retry = false;
00519 }
00520
00521 catch (MediaUnauthorizedException & ex_r)
00522 {
00523 if(authenticate(ex_r.hint(), !retry))
00524 retry = true;
00525 else
00526 {
00527 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
00528 ZYPP_RETHROW(ex_r);
00529 }
00530
00531 }
00532
00533 catch (MediaException & excpt_r)
00534 {
00535
00536 report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
00537 ZYPP_RETHROW(excpt_r);
00538 }
00539 }
00540 while (retry);
00541
00542 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
00543 }
00544
00545 bool MediaAria2c::getDoesFileExist( const Pathname & filename ) const
00546 {
00547 return MediaCurl::getDoesFileExist(filename);
00548 }
00549
00550 bool MediaAria2c::doGetDoesFileExist( const Pathname & filename ) const
00551 {
00552 return MediaCurl::doGetDoesFileExist(filename);
00553 }
00554
00555 void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const
00556 {
00557 MediaCurl::getDir(dirname, recurse_r);
00558 }
00559
00560 bool MediaAria2c::authenticate(const std::string & availAuthTypes, bool firstTry) const
00561 {
00562 return false;
00563 }
00564
00565 void MediaAria2c::getDirInfo( std::list<std::string> & retlist,
00566 const Pathname & dirname, bool dots ) const
00567 {
00568 getDirectoryYast( retlist, dirname, dots );
00569 }
00570
00571 void MediaAria2c::getDirInfo( filesystem::DirContent & retlist,
00572 const Pathname & dirname, bool dots ) const
00573 {
00574 getDirectoryYast( retlist, dirname, dots );
00575 }
00576
00577 std::string MediaAria2c::getAria2cVersion()
00578 {
00579 static const char* argv[] =
00580 {
00581 ARIA_BINARY,
00582 "--version",
00583 NULL
00584 };
00585 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
00586 std::string vResponse( str::trim( aria.receiveLine() ) );
00587 aria.close();
00588 return vResponse;
00589 }
00590 }
00591 }
00592