libzypp  17.38.17
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp-core/base/Logger.h>
19 #include <zypp-core/base/String.h>
20 #include <zypp-core/base/Gettext.h>
21 #include <utility>
22 #include <zypp-core/parser/Sysconfig>
23 #include <zypp-core/base/Gettext.h>
24 
25 #include <zypp/media/MediaCurl.h>
28 #include <zypp-curl/ProxyInfo>
29 #include <zypp-curl/auth/CurlAuthData>
30 #include <zypp-media/auth/CredentialManager>
31 #include <zypp-curl/CurlConfig>
32 #include <zypp/Target.h>
33 #include <zypp/ZYppFactory.h>
34 #include <zypp/ZConfig.h>
35 #include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
36 
37 #include <cstdlib>
38 #include <sys/types.h>
39 #include <sys/stat.h>
40 #include <sys/mount.h>
41 #include <dirent.h>
42 #include <unistd.h>
43 #include <glib.h>
44 
46 
47 using std::endl;
48 
49 namespace internal {
50  using namespace zypp;
51  struct ProgressData
52  {
53  ProgressData( AutoFILE file, CURL *curl, time_t timeout = 0, zypp::Url url = zypp::Url(),
54  zypp::ByteCount expectedFileSize_r = 0,
56 
57  void updateStats( curl_off_t dltotal = 0.0, curl_off_t dlnow = 0.0 );
58 
59  int reportProgress() const;
60 
61  CURL * curl()
62  { return _curl; }
63 
64  bool timeoutReached() const
65  { return _timeoutReached; }
66 
67  bool fileSizeExceeded() const
68  { return _fileSizeExceeded; }
69 
71  { return _expectedFileSize; }
72 
73  void expectedFileSize( ByteCount newval_r )
74  { _expectedFileSize = newval_r; }
75 
76  zypp::Url url() const
77  { return _url; }
78 
80  { return _file.value(); }
81 
82  size_t writeBytes( char *ptr, ByteCount bytes );
83 
85  return _bytesWritten;
86  }
87 
88  private:
89  CURL * _curl;
92  time_t _timeout;
97 
98  time_t _timeStart = 0;
99  time_t _timeLast = 0;
100  time_t _timeRcv = 0;
101  time_t _timeNow = 0;
102 
103  curl_off_t _dnlTotal = 0.0;
104  curl_off_t _dnlLast = 0.0;
105  curl_off_t _dnlNow = 0.0;
106 
107  ByteCount _bytesWritten = 0;
108 
109  int _dnlPercent= 0;
110 
111  double _drateTotal= 0.0;
112  double _drateLast = 0.0;
113  };
114 
115 
116 
118  : _curl( curl )
119  , _file( std::move(file) )
120  , _url(std::move( url ))
121  , _timeout( timeout )
122  , _timeoutReached( false )
123  , _fileSizeExceeded ( false )
124  , _expectedFileSize( expectedFileSize_r )
125  , report( _report )
126  {}
127 
128  void ProgressData::updateStats( curl_off_t dltotal, curl_off_t dlnow )
129  {
130  time_t now = _timeNow = time(0);
131 
132  // If called without args (0.0), recompute based on the last values seen
133  if ( dltotal && dltotal != _dnlTotal )
134  _dnlTotal = dltotal;
135 
136  if ( dlnow && dlnow != _dnlNow )
137  {
138  _timeRcv = now;
139  _dnlNow = dlnow;
140  }
141 
142  // init or reset if time jumps back
143  if ( !_timeStart || _timeStart > now )
144  _timeStart = _timeLast = _timeRcv = now;
145 
146  // timeout condition
147  if ( _timeout )
148  _timeoutReached = ( (now - _timeRcv) > _timeout );
149 
150  // percentage:
151  if ( _dnlTotal )
152  _dnlPercent = int( _dnlNow * 100 / _dnlTotal );
153 
154  // download rates:
155  _drateTotal = double(_dnlNow) / std::max( int(now - _timeStart), 1 );
156 
157  if ( _timeLast < now )
158  {
159  _drateLast = double(_dnlNow - _dnlLast) / int(now - _timeLast);
160  // start new period
161  _timeLast = now;
162  _dnlLast = _dnlNow;
163  }
164  else if ( _timeStart == _timeLast )
166  }
167 
169  {
170  if ( _fileSizeExceeded )
171  return 1;
172  if ( _timeoutReached )
173  return 1; // no-data timeout
174  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
175  return 1; // user requested abort
176  return 0;
177  }
178 
179  size_t ProgressData::writeBytes(char *ptr, ByteCount bytes)
180  {
181  // check if the downloaded data is already bigger than what we expected
183  if ( _fileSizeExceeded )
184  return 0;
185 
186  auto written = fwrite( ptr, 1, bytes, _file );
187  _bytesWritten += written;
188  return written;
189  }
190 
195  {
196  public:
198  const std::string & err_r,
199  const std::string & msg_r )
200  : media::MediaCurlException( url_r, err_r, msg_r )
201  {}
202  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
203  };
204 
205 }
206 
207 
208 using namespace internal;
209 using namespace zypp::base;
210 
211 namespace zypp {
212 
213  namespace media {
214 
215 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
216 
217 // we use this define to unbloat code as this C setting option
218 // and catching exception is done frequently.
220 #define SET_OPTION(opt,val) do { \
221  ret = curl_easy_setopt ( curl, opt, val ); \
222  if ( ret != 0) { \
223  ZYPP_THROW(MediaCurlSetOptException(_origin.at(rData.mirror).url(), _curlError)); \
224  } \
225  } while ( false )
226 
227 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
228 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
229 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
230 
231 MediaCurl::MediaCurl(const MirroredOrigin &origin_r,
232  const Pathname & attach_point_hint_r )
233  : MediaNetworkCommonHandler( origin_r, attach_point_hint_r,
234  "/", // urlpath at attachpoint
235  true ), // does_download
236  _customHeaders(0L)
237 {
238  _multi = curl_multi_init();
239 
240  _curlError[0] = '\0';
241 
242  MIL << "MediaCurl::MediaCurl(" << origin_r.authority().url() << ", " << attach_point_hint_r << ")" << endl;
243 
245 
246  if( !attachPoint().empty())
247  {
248  PathInfo ainfo(attachPoint());
249  Pathname apath(attachPoint() + "XXXXXX");
250  char *atemp = ::strdup( apath.asString().c_str());
251  char *atest = NULL;
252  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
253  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
254  {
255  WAR << "attach point " << ainfo.path()
256  << " is not useable for " << origin_r.authority().url().getScheme() << endl;
257  setAttachPoint("", true);
258  }
259  else if( atest != NULL)
260  ::rmdir(atest);
261 
262  if( atemp != NULL)
263  ::free(atemp);
264  }
265 }
266 
268 {
269  try { release(); } catch(...) {}
270  if (_multi)
271  curl_multi_cleanup(_multi);
272 }
273 
274 void MediaCurl::setCookieFile( const Pathname &fileName )
275 {
276  _cookieFile = fileName;
277 }
278 
279 void MediaCurl::setCurlError(const char* error)
280 {
281  // FIXME(dmllr): Use strlcpy if available for better performance
282  strncpy(_curlError, error, sizeof(_curlError)-1);
283  _curlError[sizeof(_curlError)-1] = '\0';
284 }
285 
287 
288 void MediaCurl::checkProtocol(const Url &url) const
289 {
290  curl_version_info_data *curl_info = NULL;
291  curl_info = curl_version_info(CURLVERSION_NOW);
292  // curl_info does not need any free (is static)
293  if (curl_info->protocols)
294  {
295  const char * const *proto = nullptr;
296  std::string scheme( url.getScheme());
297  bool found = false;
298  for(proto=curl_info->protocols; !found && *proto; ++proto)
299  {
300  if( scheme == std::string((const char *)*proto))
301  found = true;
302  }
303  if( !found)
304  {
305  std::string msg("Unsupported protocol '");
306  msg += scheme;
307  msg += "'";
309  }
310  }
311 }
312 
314 {
315  CURL *curl = rData.curl;
316 
317  // kill old settings
318  curl_easy_reset ( curl );
319 
321  curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
322  curl_easy_setopt(curl, CURLOPT_HEADERDATA, &_lastRedirect);
323  CURLcode ret = curl_easy_setopt( curl, CURLOPT_ERRORBUFFER, _curlError );
324  if ( ret != 0 ) {
325  ZYPP_THROW(MediaCurlSetOptException( _origin.at(rData.mirror).url(), "Error setting error buffer"));
326  }
327 
328  SET_OPTION(CURLOPT_FAILONERROR, 1L);
329  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
330 
332  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
333  {
334  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
335  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
336  }
337 
341  SET_OPTION(CURLOPT_CONNECTTIMEOUT, settings.connectTimeout());
342  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
343  // just in case curl does not trigger its progress callback frequently
344  // enough.
345  if ( settings.timeout() )
346  {
347  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
348  }
349 
350  // follow any Location: header that the server sends as part of
351  // an HTTP header (#113275)
352  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
353  // 3 redirects seem to be too few in some cases (bnc #465532)
354  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
355 
356  if ( _origin.at(rData.mirror).url().getScheme() == "https" )
357  {
358  if ( :: internal::setCurlRedirProtocols ( curl ) != CURLE_OK ) {
360  }
361 
362  if( settings.verifyPeerEnabled() ||
363  settings.verifyHostEnabled() )
364  {
365  SET_OPTION(CURLOPT_CAPATH, settings.certificateAuthoritiesPath().c_str());
366  }
367 
368  if( ! settings.clientCertificatePath().empty() )
369  {
370  SET_OPTION(CURLOPT_SSLCERT, settings.clientCertificatePath().c_str());
371  }
372  if( ! settings.clientKeyPath().empty() )
373  {
374  SET_OPTION(CURLOPT_SSLKEY, settings.clientKeyPath().c_str());
375  }
376 
377 #ifdef CURLSSLOPT_ALLOW_BEAST
378  // see bnc#779177
379  ret = curl_easy_setopt( curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
380  if ( ret != 0 ) {
381  disconnectFrom();
383  }
384 #endif
385  SET_OPTION(CURLOPT_SSL_VERIFYPEER, settings.verifyPeerEnabled() ? 1L : 0L);
386  SET_OPTION(CURLOPT_SSL_VERIFYHOST, settings.verifyHostEnabled() ? 2L : 0L);
387  // bnc#903405 - POODLE: libzypp should only talk TLS
388  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
389  }
390 
391  SET_OPTION(CURLOPT_USERAGENT, settings.userAgentString().c_str() );
392 
393  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
394  * We should proactively add the password to the request if basic auth is configured
395  * and a password is available in the credentials but not in the URL.
396  *
397  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
398  * and ask the server first about the auth method
399  */
400  if ( settings.authType() == "basic" && not settings.username().empty() && settings.password().empty() ) {
401  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
402  const auto cred = cm.getCred( _origin.at(rData.mirror).url() );
403  if ( cred && cred->valid() ) {
404  settings.setPassword(cred->password());
405  }
406  }
407 
408  /*---------------------------------------------------------------*
409  CURLOPT_USERPWD: [user name]:[password]
410 
411  Url::username/password -> CURLOPT_USERPWD
412  If not provided, anonymous FTP identification
413  *---------------------------------------------------------------*/
414 
415  if ( settings.hasCredentials() )
416  {
417  settings.logCredentials( DBG << "Credentials: " ) << endl;
418  SET_OPTION(CURLOPT_USERNAME, settings.username().c_str());
419  SET_OPTION(CURLOPT_PASSWORD, settings.password().c_str());
420  std::string use_auth = settings.authType();
421  if (use_auth.empty())
422  use_auth = "digest,basic"; // our default
423  long auth = CurlAuthData::auth_type_str2long(use_auth);
424  if( auth != CURLAUTH_NONE)
425  {
426  DBG << "Enabling HTTP authentication methods: " << use_auth
427  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
428  SET_OPTION(CURLOPT_HTTPAUTH, auth);
429  }
430  }
431 
432  if ( settings.proxyEnabled() && ! settings.proxy().empty() )
433  {
434  DBG << "Proxy: '" << settings.proxy() << "'" << endl;
435  SET_OPTION(CURLOPT_PROXY, settings.proxy().c_str());
436  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
437  /*---------------------------------------------------------------*
438  * CURLOPT_PROXYUSERPWD: [user name]:[password]
439  *
440  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
441  * If not provided, $HOME/.curlrc is evaluated
442  *---------------------------------------------------------------*/
443 
444  std::string proxyuserpwd = settings.proxyUserPassword();
445 
446  if ( proxyuserpwd.empty() )
447  {
448  CurlConfig curlconf;
449  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
450  if ( curlconf.proxyuserpwd.empty() )
451  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
452  else
453  {
454  proxyuserpwd = curlconf.proxyuserpwd;
455  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
456  }
457  }
458  else
459  {
460  DBG << "Proxy: using provided proxy-user '" << settings.proxyUsername() << "'" << endl;
461  }
462 
463  if ( ! proxyuserpwd.empty() )
464  {
465  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
466  }
467  }
468 #if CURLVERSION_AT_LEAST(7,19,4)
469  else if ( settings.proxy() == EXPLICITLY_NO_PROXY )
470  {
471  // Explicitly disabled in URL (see fillSettingsFromUrl()).
472  // This should also prevent libcurl from looking into the environment.
473  DBG << "Proxy: explicitly NOPROXY" << endl;
474  SET_OPTION(CURLOPT_NOPROXY, "*");
475  }
476 #endif
477  else
478  {
479  DBG << "Proxy: not explicitly set, libcurl may look into the environment" << endl;
480  }
481 
483  if ( settings.minDownloadSpeed() != 0 )
484  {
485  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, settings.minDownloadSpeed());
486  SET_OPTION(CURLOPT_LOW_SPEED_TIME, settings.minDownloadSpeedTime());
487  }
488 
489 #if CURLVERSION_AT_LEAST(7,15,5)
490  if ( settings.maxDownloadSpeed() != 0 )
491  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, settings.maxDownloadSpeed());
492 #endif
493 
494  /*---------------------------------------------------------------*
495  *---------------------------------------------------------------*/
496 
498  if ( ::geteuid() == 0 || PathInfo(_currentCookieFile).owner() == ::geteuid() )
500 
501  const auto &cookieFileParam = _origin.at(rData.mirror).url().getQueryParam( "cookies" );
502  if ( !cookieFileParam.empty() && str::strToBool( cookieFileParam, true ) )
503  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
504  else
505  MIL << "No cookies requested" << endl;
506  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
507  SET_OPTION(CURLOPT_XFERINFOFUNCTION, &progressCallback );
508  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
509 
510 #if CURLVERSION_AT_LEAST(7,18,0)
511  // bnc #306272
512  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
513 #endif
514  // Append settings custom headers to curl.
515  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
516  if ( _customHeaders ) {
517  curl_slist_free_all(_customHeaders);
518  _customHeaders = 0L;
519  }
520  for ( const auto &header : settings.headers() ) {
521  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
522  if ( !_customHeaders )
524  }
525  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
526 }
527 
529 
531 {
532  if ( _customHeaders ) {
533  curl_slist_free_all(_customHeaders);
534  _customHeaders = 0L;
535  }
536 
537  // clear effective settings
539 }
540 
542 
543 void MediaCurl::releaseFrom( const std::string & ejectDev )
544 {
545  disconnect();
546 }
547 
549 
550 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
551 {
552  // we need a non const pointer to work around the current API
553  auto that = const_cast<MediaCurl *>(this);
554  std::exception_ptr lastErr;
555  const auto &mirrOrder = mirrorOrder (srcFile);
556  for ( unsigned mirr : mirrOrder ) {
557  try {
558  return that->getFileCopyFromMirror ( mirr, srcFile, target );
559 
560  } catch (MediaException & excpt_r) {
561  if ( !canTryNextMirror ( excpt_r ) )
562  ZYPP_RETHROW(excpt_r);
563 
564  that->deprioritizeMirror( mirr );
565 
566  lastErr = ZYPP_FWD_CURRENT_EXCPT();
567  }
568  }
569  if ( lastErr ) {
570  ZYPP_RETHROW( lastErr );
571  }
572 
573  // should not happen
574  ZYPP_THROW( MediaException("No usable mirror available.") );
575 
576 }
577 
578 void MediaCurl::getFileCopyFromMirror(const int mirror, const OnMediaLocation &srcFile, const Pathname &target)
579 {
580  const auto &filename = srcFile.filename();
581 
582  // Optional files will send no report until data are actually received (we know it exists).
583  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
585 
586  auto &endpoint = _origin[mirror];
587  auto &settings = endpoint.getConfig<TransferSettings>(MIRR_SETTINGS_KEY.data());
588 
589  AutoDispose<CURL*> curl( curl_easy_init(), []( CURL *hdl ) { if ( hdl ) { curl_easy_cleanup(hdl); } } );
590 
591  RequestData rData;
592  rData.mirror = mirror;
593  rData.curl = curl.value ();
594 
595  if( !endpoint.url().isValid() )
596  ZYPP_THROW(MediaBadUrlException(endpoint.url()));
597 
598  if( endpoint.url().getHost().empty() )
599  ZYPP_THROW(MediaBadUrlEmptyHostException(endpoint.url()));
600 
601  Url fileurl( getFileUrl(mirror, filename) );
602 
603  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
604  unsigned internalTry = 0;
605  static constexpr unsigned maxInternalTry = 3;
606 
607  do
608  {
609  try
610  {
611  Pathname dest = target.absolutename();
612  if( assert_dir( dest.dirname() ) )
613  {
614  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
615  ZYPP_THROW( MediaSystemException(fileurl, "System error on " + dest.dirname().asString()) );
616  }
617 
618  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
619  AutoFILE file;
620  {
621  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
622  if( ! buf )
623  {
624  ERR << "out of memory for temp file name" << endl;
625  ZYPP_THROW(MediaSystemException(fileurl, "out of memory for temp file name"));
626  }
627 
628  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
629  if( tmp_fd == -1 )
630  {
631  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
633  }
634  destNew = ManagedFile( (*buf), filesystem::unlink );
635 
636  file = ::fdopen( tmp_fd, "we" );
637  if ( ! file )
638  {
639  ERR << "fopen failed for file '" << destNew << "'" << endl;
641  }
642  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
643  }
644 
645  DBG << "dest: " << dest << endl;
646  DBG << "temp: " << destNew << endl;
647 
648  setupEasy( rData, settings );
649 
650  // set IFMODSINCE time condition (no download if not modified)
651  if( PathInfo(target).isExist() )
652  {
653  curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
654  curl_easy_setopt(curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
655  }
656  else
657  {
658  curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
659  curl_easy_setopt(curl, CURLOPT_TIMEVALUE, 0L);
660  }
661 
662  zypp_defer{
663  curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
664  curl_easy_setopt(curl, CURLOPT_TIMEVALUE, 0L);
665  };
666 
667  DBG << srcFile.filename().asString() << endl;
668 
669  DBG << "URL: " << fileurl.asString() << endl;
670  // Use URL without options and without username and passwd
671  // (some proxies dislike them in the URL).
672  // Curl seems to need the just scheme, hostname and a path;
673  // the rest was already passed as curl options (in attachTo).
674  Url curlUrl( clearQueryString(fileurl) );
675 
676  //
677  // See also Bug #154197 and ftp url definition in RFC 1738:
678  // The url "ftp://user@host/foo/bar/file" contains a path,
679  // that is relative to the user's home.
680  // The url "ftp://user@host//foo/bar/file" (or also with
681  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
682  // contains an absolute path.
683  //
684  _lastRedirect.clear();
685  std::string urlBuffer( curlUrl.asString());
686  CURLcode ret = curl_easy_setopt( curl, CURLOPT_URL,
687  urlBuffer.c_str() );
688  if ( ret != 0 ) {
690  }
691 
692  // Set callback and perform.
693  internal::ProgressData progressData( file, curl, settings.timeout(), fileurl, srcFile.downloadSize(), &report );
694 
695  ret = curl_easy_setopt( curl, CURLOPT_WRITEDATA, &progressData );
696  if ( ret != 0 ) {
698  }
699 
700  ret = curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, &MediaCurl::writeCallback );
701  if ( ret != 0 ) {
703  }
704 
705  report->start(fileurl, dest);
706 
707  if ( curl_easy_setopt( curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
708  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
709  }
710 
711  ret = executeCurl( rData );
712 
713  // flush buffers
714  fflush ( file );
715 
716  #if CURLVERSION_AT_LEAST(7,19,4)
717  // bnc#692260: If the client sends a request with an If-Modified-Since header
718  // with a future date for the server, the server may respond 200 sending a
719  // zero size file.
720  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
721  if ( ftell(file) == 0 && ret == 0 )
722  {
723  long httpReturnCode = 33;
724  if ( curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
725  {
726  long conditionUnmet = 33;
727  if ( curl_easy_getinfo( curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
728  {
729  WAR << "TIMECONDITION unmet - retry without." << endl;
730  curl_easy_setopt( curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
731  curl_easy_setopt( curl, CURLOPT_TIMEVALUE, 0L);
732  ret = executeCurl( rData );
733  }
734  }
735  }
736  #endif
737 
738  if ( curl_easy_setopt( curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
739  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
740  }
741 
742  if ( ret != 0 ) {
743  ERR << "curl error: " << ret << ": " << _curlError
744  << ", temp file size " << ftell(file)
745  << " bytes." << endl;
746 
747  // the timeout is determined by the progress data object
748  // which holds whether the timeout was reached or not,
749  // otherwise it would be a user cancel
750 
751  if ( progressData.fileSizeExceeded() )
752  ZYPP_THROW(MediaFileSizeExceededException(fileurl, progressData.expectedFileSize()));
753 
754  evaluateCurlCode( rData, srcFile.filename(), ret, progressData.timeoutReached() );
755  }
756 
757  long httpReturnCode = 0;
758  CURLcode infoRet = curl_easy_getinfo(curl,
759  CURLINFO_RESPONSE_CODE,
760  &httpReturnCode);
761  bool modified = true;
762  if (infoRet == CURLE_OK)
763  {
764  DBG << "HTTP response: " + str::numstring(httpReturnCode);
765  if ( httpReturnCode == 304
766  || ( httpReturnCode == 213 && (endpoint.url().getScheme() == "ftp" || endpoint.url().getScheme() == "tftp") ) ) // not modified
767  {
768  DBG << " Not modified.";
769  modified = false;
770  }
771  DBG << endl;
772  }
773  else
774  {
775  WAR << "Could not get the response code." << endl;
776  }
777 
778  if (modified || infoRet != CURLE_OK)
779  {
780  // apply umask
781  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
782  {
783  ERR << "Failed to chmod file " << destNew << endl;
784  }
785 
786  file.resetDispose(); // we're going to close it manually here
787  if ( ::fclose( file ) )
788  {
789  ERR << "Fclose failed for file '" << destNew << "'" << endl;
791  }
792 
793  // move the temp file into dest
794  if ( rename( destNew, dest ) != 0 ) {
795  ERR << "Rename failed" << endl;
797  }
798  destNew.resetDispose(); // no more need to unlink it
799  }
800 
801  DBG << "done: " << PathInfo(dest) << endl;
802  break; // success!
803  }
804  catch (MediaUnauthorizedException & ex_r)
805  {
806  if ( authenticate( endpoint.url(), settings, ex_r.hint(), firstAuth) ) {
807  firstAuth = false; // must not return stored credentials again
808  continue; // retry
809  }
810 
811  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
812  ZYPP_RETHROW(ex_r);
813  }
814  // unexpected exception
815  catch (MediaException & excpt_r)
816  {
817  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
818  ++internalTry;
819  if ( internalTry < maxInternalTry ) {
820  // just report (NO_ERROR); no interactive request to the user
821  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
822  continue; // retry
823  }
824  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
825  }
826 
828  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
829  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
830  {
832  }
833  report->finish(fileurl, reason, excpt_r.asUserHistory());
834  ZYPP_RETHROW(excpt_r);
835  }
836  } while ( true );
837 
838  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
839 }
840 
842 
843 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
844 {
845  // we need a non const pointer to work around the current API
846  auto that = const_cast<MediaCurl *>(this);
847 
848  std::exception_ptr lastErr;
849  for ( int i : mirrorOrder( OnMediaLocation(filename).setMirrorsAllowed(false) )) {
850  try {
851  return that->doGetDoesFileExist( i, filename );
852 
853  } catch (MediaException & excpt_r) {
854  if ( !canTryNextMirror ( excpt_r ) )
855  ZYPP_RETHROW(excpt_r);
856 
857  that->deprioritizeMirror( i );
858 
859  lastErr = ZYPP_FWD_CURRENT_EXCPT();
860  }
861  }
862  if ( lastErr ) {
863  try {
864  ZYPP_RETHROW( lastErr );
865  } catch ( const MediaFileNotFoundException &e ) {
866  // on file not found we return false
867  ZYPP_CAUGHT(e);
868  return false;
869  }
870  }
871  return false;
872 }
873 
875 
877  const Pathname &filename,
878  CURLcode code,
879  bool timeout_reached) const
880 {
881  if ( code != 0 )
882  {
883  const auto &baseMirr = _origin[rData.mirror];
884  Url url;
885  if (filename.empty())
886  url = baseMirr.url();
887  else
888  url = getFileUrl(rData.mirror, filename);
889 
890  std::string err;
891  {
892  switch ( code )
893  {
894  case CURLE_UNSUPPORTED_PROTOCOL:
895  err = " Unsupported protocol";
896  if ( !_lastRedirect.empty() )
897  {
898  err += " or redirect (";
899  err += _lastRedirect;
900  err += ")";
901  }
902  break;
903  case CURLE_URL_MALFORMAT:
904  case CURLE_URL_MALFORMAT_USER:
905  err = " Bad URL";
906  break;
907  case CURLE_LOGIN_DENIED:
908  ZYPP_THROW(
909  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
910  break;
911  case CURLE_HTTP_RETURNED_ERROR:
912  {
913  long httpReturnCode = 0;
914  CURLcode infoRet = curl_easy_getinfo( rData.curl,
915  CURLINFO_RESPONSE_CODE,
916  &httpReturnCode );
917  if ( infoRet == CURLE_OK )
918  {
919  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
920  switch ( httpReturnCode )
921  {
922  case 401:
923  {
924  std::string auth_hint = getAuthHint( rData.curl );
925 
926  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
927  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
928 
930  url, "Login failed.", _curlError, auth_hint
931  ));
932  }
933 
934  case 502: // bad gateway (bnc #1070851)
935  case 503: // service temporarily unavailable (bnc #462545)
937  case 504: // gateway timeout
939  case 403:
940  {
941  std::string msg403;
942  if ( url.getHost().find(".suse.com") != std::string::npos )
943  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
944  else if (url.asString().find("novell.com") != std::string::npos)
945  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
947  }
948  case 404:
949  case 410:
950  ZYPP_THROW(MediaFileNotFoundException(baseMirr.url(), filename));
951  }
952 
953  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
955  }
956  else
957  {
958  std::string msg = "Unable to retrieve HTTP response:";
959  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
961  }
962  }
963  break;
964  case CURLE_FTP_COULDNT_RETR_FILE:
965 #if CURLVERSION_AT_LEAST(7,16,0)
966  case CURLE_REMOTE_FILE_NOT_FOUND:
967 #endif
968  case CURLE_FTP_ACCESS_DENIED:
969  case CURLE_TFTP_NOTFOUND:
970  err = "File not found";
971  ZYPP_THROW(MediaFileNotFoundException(baseMirr.url(), filename));
972  break;
973  case CURLE_BAD_PASSWORD_ENTERED:
974  case CURLE_FTP_USER_PASSWORD_INCORRECT:
975  err = "Login failed";
976  break;
977  case CURLE_COULDNT_RESOLVE_PROXY:
978  case CURLE_COULDNT_RESOLVE_HOST:
979  case CURLE_COULDNT_CONNECT:
980  case CURLE_FTP_CANT_GET_HOST:
981  err = "Connection failed";
982  break;
983  case CURLE_WRITE_ERROR:
984  err = "Write error";
985  break;
986  case CURLE_PARTIAL_FILE:
987  case CURLE_OPERATION_TIMEDOUT:
988  timeout_reached = true; // fall though to TimeoutException
989  // fall though...
990  case CURLE_ABORTED_BY_CALLBACK:
991  if( timeout_reached )
992  {
993  err = "Timeout reached";
995  }
996  else
997  {
998  err = "User abort";
999  }
1000  break;
1001 
1002  default:
1003  err = "Curl error " + str::numstring( code );
1004  break;
1005  }
1006 
1007  // uhm, no 0 code but unknown curl exception
1009  }
1010  }
1011  else
1012  {
1013  // actually the code is 0, nothing happened
1014  }
1015 }
1016 
1018 
1019 bool MediaCurl::doGetDoesFileExist( const int mirror, const Pathname & filename )
1020 {
1021  DBG << filename.asString() << endl;
1022 
1023  AutoDispose<CURL*> curl( curl_easy_init(), []( CURL *hdl ) { if ( hdl ) { curl_easy_cleanup(hdl); } } );
1024  RequestData rData;
1025  rData.mirror = mirror;
1026  rData.curl = curl.value ();
1027 
1028  auto &endpoint = _origin[mirror];
1029 
1030  if( !endpoint.url().isValid() )
1031  ZYPP_THROW(MediaBadUrlException(endpoint.url()));
1032 
1033  if( endpoint.url().getHost().empty() )
1034  ZYPP_THROW(MediaBadUrlEmptyHostException(endpoint.url()));
1035 
1036  Url url(getFileUrl(mirror, filename));
1037 
1038  DBG << "URL: " << url.asString() << endl;
1039  // Use URL without options and without username and passwd
1040  // (some proxies dislike them in the URL).
1041  // Curl seems to need the just scheme, hostname and a path;
1042  // the rest was already passed as curl options (in attachTo).
1043  Url curlUrl( clearQueryString(url) );
1044 
1045  // See also Bug #154197 and ftp url definition in RFC 1738:
1046  // The url "ftp://user@host/foo/bar/file" contains a path,
1047  // that is relative to the user's home.
1048  // The url "ftp://user@host//foo/bar/file" (or also with
1049  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1050  // contains an absolute path.
1051  //
1052  _lastRedirect.clear();
1053  std::string urlBuffer( curlUrl.asString());
1054 
1055  CURLcode ok;
1056  bool canRetry = true;
1057  bool firstAuth = true;
1058  auto &settings = endpoint.getConfig<TransferSettings>( MIRR_SETTINGS_KEY.data() );
1059 
1060  while ( canRetry ) {
1061  canRetry = false;
1062  setupEasy( rData, settings );
1063 
1064  CURLcode ret = curl_easy_setopt( curl, CURLOPT_URL,
1065  urlBuffer.c_str() );
1066  if ( ret != 0 ) {
1068  }
1069 
1070  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1071  if ( !file ) {
1072  ERR << "fopen failed for /dev/null" << endl;
1073  ZYPP_THROW(MediaWriteException("/dev/null"));
1074  }
1075 
1076  ret = curl_easy_setopt( curl, CURLOPT_WRITEDATA, (*file) );
1077  if ( ret != 0 ) {
1079  }
1080 
1081  // If no head requests allowed (?head_requests=no):
1082  // Instead of returning no data with NOBODY, we return
1083  // little data, that works with broken servers, and
1084  // works for ftp as well, because retrieving only headers
1085  // ftp will return always OK code ?
1086  // See http://curl.haxx.se/docs/knownbugs.html #58
1087  const bool doHeadRequest = (endpoint.url().getScheme() == "http" || endpoint.url().getScheme() == "https") && settings.headRequestsAllowed();
1088  if ( doHeadRequest ) {
1089  curl_easy_setopt( curl, CURLOPT_NOBODY, 1L );
1090  } else {
1091  curl_easy_setopt( curl, CURLOPT_RANGE, "0-1" );
1092  }
1093 
1094  try {
1095  ok = const_cast<MediaCurl *>(this)->executeCurl( rData );
1096  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1097 
1098  // as we are not having user interaction, the user can't cancel
1099  // the file existence checking, a callback or timeout return code
1100  // will be always a timeout.
1101  evaluateCurlCode( rData, filename, ok, true /* timeout */);
1102  }
1103  catch ( const MediaFileNotFoundException &e ) {
1104  // if the file did not exist then we can return false
1105  return false;
1106  }
1107  catch ( const MediaUnauthorizedException &e ) {
1108  if ( authenticate( endpoint.url(), settings, e.hint(), firstAuth ) ) {
1109  firstAuth = false;
1110  canRetry = true;
1111  continue;
1112  }
1113  }
1114 
1115  // exists
1116  return ( ok == CURLE_OK );
1117  }
1118 
1119  return false;
1120 }
1121 
1123 //
1124 int MediaCurl::aliveCallback( void *clientp, curl_off_t /*dltotal*/, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/ )
1125 {
1126  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1127  if( pdata )
1128  {
1129  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1130  // prevent a percentage raise while downloading a metalink file. Download
1131  // activity however is indicated by propagating the download rate (via dlnow).
1132  pdata->updateStats( 0.0, dlnow );
1133  return pdata->reportProgress();
1134  }
1135  return 0;
1136 }
1137 
1138 int MediaCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
1139 {
1140  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1141  if( pdata )
1142  {
1143  // work around curl bug that gives us old data
1144  long httpReturnCode = 0;
1145  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 ) {
1146  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1147  }
1148  pdata->updateStats( dltotal, dlnow );
1149  return pdata->reportProgress();
1150  }
1151  return 0;
1152 }
1153 
1154 size_t MediaCurl::writeCallback( char *ptr, size_t size, size_t nmemb, void *userdata )
1155 {
1156  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( userdata );
1157  if( pdata ) {
1158  return pdata->writeBytes ( ptr, size * nmemb );
1159  }
1160  return 0;
1161 }
1162 
1164 
1165 std::string MediaCurl::getAuthHint( CURL *curl ) const
1166 {
1167  long auth_info = CURLAUTH_NONE;
1168 
1169  CURLcode infoRet =
1170  curl_easy_getinfo(curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1171 
1172  if(infoRet == CURLE_OK)
1173  {
1174  return CurlAuthData::auth_type_long2str(auth_info);
1175  }
1176 
1177  return "";
1178 }
1179 
1184 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1185 {
1186  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1187  if ( data ) {
1188  data->expectedFileSize( expectedFileSize );
1189  }
1190 }
1191 
1198 {
1199  CURL *curl = rData.curl;
1200  const auto &baseUrl = _origin.at(rData.mirror);
1201 
1202  if (!_multi)
1203  ZYPP_THROW(MediaCurlInitException(baseUrl.url()));
1204 
1205  internal::CurlPollHelper _curlHelper(*this);
1206 
1207  // add the easy handle to the multi instance
1208  if ( curl_multi_add_handle( _multi, curl ) != CURLM_OK )
1209  ZYPP_THROW(MediaCurlException( baseUrl.url(), "curl_multi_add_handle", "unknown error"));
1210 
1211  // make sure the handle is cleanly removed from the multi handle
1212  OnScopeExit autoRemove([&](){ curl_multi_remove_handle( _multi, curl ); });
1213 
1214  // kickstart curl, this will cause libcurl to go over the added handles and register sockets and timeouts
1215  CURLMcode mcode = _curlHelper.handleTimout();
1216  if (mcode != CURLM_OK)
1217  ZYPP_THROW(MediaCurlException( baseUrl.url(), "curl_multi_socket_action", "unknown error"));
1218 
1219  bool canContinue = true;
1220  while ( canContinue ) {
1221 
1222  CURLMsg *msg = nullptr;
1223  int nqueue = 0;
1224  while ((msg = curl_multi_info_read( _multi, &nqueue)) != 0) {
1225  if ( msg->msg != CURLMSG_DONE ) continue;
1226  if ( msg->easy_handle != curl ) continue;
1227 
1228  return msg->data.result;
1229  }
1230 
1231  // copy watched sockets in case curl changes the vector as we go over the events later
1232  std::vector<GPollFD> requestedFds = _curlHelper.socks;
1233 
1234  int r = zypp_detail::zypp_poll( requestedFds, _curlHelper.timeout_ms.value_or( -1 ) );
1235  if ( r == -1 )
1236  ZYPP_THROW( MediaCurlException(baseUrl.url(), "zypp_poll() failed", "unknown error") );
1237 
1238  // run curl
1239  if ( r == 0 ) {
1240  CURLMcode mcode = _curlHelper.handleTimout();
1241  if (mcode != CURLM_OK)
1242  ZYPP_THROW(MediaCurlException(baseUrl.url(), "curl_multi_socket_action", "unknown error"));
1243  } else {
1244  CURLMcode mcode = _curlHelper.handleSocketActions( requestedFds );
1245  if (mcode != CURLM_OK)
1246  ZYPP_THROW(MediaCurlException(baseUrl.url(), "curl_multi_socket_action", "unknown error"));
1247  }
1248  }
1249  return CURLE_OK;
1250 }
1251 
1252 
1253  } // namespace media
1254 } // namespace zypp
1255 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:560
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:197
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:227
#define MIL
Definition: Logger.h:130
std::string curlUnEscape(const std::string &text_r)
Definition: curlhelper.cc:394
#define zypp_defer
Definition: AutoDispose.h:293
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::vector< unsigned > mirrorOrder(const OnMediaLocation &loc) const
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:338
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:143
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:251
ByteCount _bytesWritten
Bytes actually written into the file.
Definition: MediaCurl.cc:107
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:459
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:794
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:32
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1224
void setPassword(const std::string &val_r)
sets the auth password
CURLcode executeCurl(RequestData &rData)
Definition: MediaCurl.cc:1197
ByteCount _expectedFileSize
Definition: MediaCurl.cc:95
static constexpr std::string_view MIRR_SETTINGS_KEY
Store and operate with byte count.
Definition: ByteCount.h:31
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
Holds transfer setting.
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:192
static bool canTryNextMirror(const Excpt &excpt_r)
int reportProgress() const
Definition: MediaCurl.cc:168
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
size_t writeBytes(char *ptr, ByteCount bytes)
Definition: MediaCurl.cc:179
const char * c_str() const
String representation.
Definition: Pathname.h:113
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:189
MediaCurlException(const Url &url_r, std::string err_r, std::string msg_r)
time_t _timeNow
Now.
Definition: MediaCurl.cc:101
Definition: ansi.h:854
const zypp::Url & url() const
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
bool timeoutReached() const
Definition: MediaCurl.cc:64
Convenient building of std::string with boost::format.
Definition: String.h:253
Structure holding values of curlrc options.
Definition: curlconfig.h:26
bool doGetDoesFileExist(const int mirror, const Pathname &filename)
Definition: MediaCurl.cc:1019
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
AutoDispose<int> calling ::close
Definition: AutoDispose.h:309
void setupEasy(RequestData &rData, TransferSettings &settings)
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:313
std::string _currentCookieFile
Definition: MediaCurl.h:126
const std::string & password() const
auth password
CURLMcode handleSocketActions(const std::vector< GPollFD > &actionsFds, int first=0)
Definition: curlhelper.cc:501
#define ERR
Definition: Logger.h:132
void evaluateCurlCode(RequestData &rData, const zypp::Pathname &fileName, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:876
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:67
void checkProtocol(const Url &url) const override
check the url is supported by the curl library
Definition: MediaCurl.cc:288
const Headers & headers() const
returns a list of all added headers (trimmed)
zypp::Url _url
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static size_t writeCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
Callback writing the data into our file.
Definition: MediaCurl.cc:1154
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:274
void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:543
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1184
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:117
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:479
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
long minDownloadSpeedTime() const
The time in seconds that the transfer speed must remain below minDownloadSpeed before the transfer is...
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:524
const OriginEndpoint & at(uint index) const
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:687
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
bool hasCredentials() const
has a non-empty username and a non-empty password
MirroredOrigin _origin
Contains the authority URL and mirrors.
Definition: MediaHandler.h:112
Manages a data source characterized by an authoritative URL and a list of mirror URLs.
std::vector< GPollFD > socks
Definition: curlhelper_p.h:109
const std::string & asString() const
String representation.
Definition: Pathname.h:94
Just inherits Exception to separate media exceptions.
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:140
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:133
#define WAR
Definition: Logger.h:131
std::unique_ptr< ReportType > _report
Definition: TargetImpl.cc:269
void getFileCopyFromMirror(const int mirror, const OnMediaLocation &srcFile, const Pathname &target)
Definition: MediaCurl.cc:578
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1138
#define _(MSG)
Definition: Gettext.h:39
std::string proxyuserpwd
Definition: curlconfig.h:49
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:73
const Pathname & filename() const
The path to the resource on the medium.
std::string numstring(char n, int w=0)
Definition: String.h:290
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:719
bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:843
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:171
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:109
curl_slist * _customHeaders
Definition: MediaCurl.h:131
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:475
bool proxyEnabled() const
proxy is enabled
constexpr std::string_view FILE("file")
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:220
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:148
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:130
reference value() const
Reference to the Tp object.
Definition: AutoDispose.h:138
curl_off_t _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:105
Url url() const
Primary Url used.
Definition: MediaHandler.h:507
Url getFileUrl(int mirrorIdx, const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
curl_off_t _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:103
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:112
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:615
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:98
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:130
std::ostream & logCredentials(std::ostream &str) const
log credentials to stream hiding the password.
void disconnectFrom() override
Definition: MediaCurl.cc:530
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:500
void setCurlError(const char *error)
Definition: MediaCurl.cc:279
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
zypp::Url url() const
Definition: MediaCurl.cc:76
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:94
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
curl_off_t _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:104
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:320
static Pathname _cookieFile
Definition: MediaCurl.h:127
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:111
std::string getAuthHint(CURL *curl) const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1165
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:814
std::optional< long > timeout_ms
Definition: curlhelper_p.h:110
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:100
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition: Exception.h:471
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:23
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:761
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Definition: ZYppImpl.cc:323
void updateStats(curl_off_t dltotal=0.0, curl_off_t dlnow=0.0)
Definition: MediaCurl.cc:128
void getFileCopy(const OnMediaLocation &srcFile, const Pathname &target) const override
Definition: MediaCurl.cc:550
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
const OriginEndpoint & authority() const
void release(const std::string &ejectDev="")
Use concrete handler to release the media.
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:128
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:70
CURLcode setCurlRedirProtocols(CURL *curl)
Definition: curlhelper.cc:544
zypp::Pathname _file
Definition: downloadwf.cc:185
const std::string & proxy() const
proxy host
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:385
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:99
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1124
bool userMayRWX() const
Definition: PathInfo.h:361
const std::string & userAgentString() const
user agent string (trimmed)
Url manipulation class.
Definition: Url.h:92
#define DBG
Definition: Logger.h:129
ByteCount bytesWritten() const
Definition: MediaCurl.cc:84
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:194
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:96