libzypp  17.38.17
RepoMirrorList.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <fstream>
15 #include <utility>
16 #include <vector>
17 #include <time.h>
19 #include <zypp-curl/parser/MetaLinkParser>
20 #include <zypp/MediaSetAccess.h>
22 #include <zypp/ZConfig.h>
23 #include <zypp/PathInfo.h>
25 
26 #include <zypp-core/fs/TmpPath.h>
32 
34 #include <zypp/media/MediaNetworkCommonHandler.h> // for the authentication workflow
35 
36 #include <zypp-core/parser/json.h>
37 
38 
40 namespace zypp
41 {
42  namespace repo
44  {
45 
47  namespace
48  {
56  struct RepoMirrorListTempProvider
57  {
58  RepoMirrorListTempProvider()
59  {}
60 
61  RepoMirrorListTempProvider( Pathname localfile_r )
62  : _localfile(std::move( localfile_r ))
63  {}
64 
65  RepoMirrorListTempProvider( const Url & url_r )
66  {
67  if ( url_r.schemeIsDownloading()
69  && url_r.getQueryStringMap().count("mirrorlist") > 0 ) {
70 
71  // Auth will probably never be triggered, but add it for completeness
72  const auto &authCb = [&]( const zypp::Url &, media::TransferSettings &settings, const std::string & availAuthTypes, bool firstTry, bool &canContinue ) {
73  media::CredentialManager cm(media::CredManagerOptions(ZConfig::instance().repoManagerRoot()));
74  if ( media::MediaNetworkCommonHandler::authenticate( url_r, cm, settings, availAuthTypes, firstTry ) ) {
75  canContinue = true;
76  return;
77  }
78  canContinue = false;
79  };
80 
81  internal::MediaNetworkRequestExecutor executor;
82  executor.sigAuthRequired ().connect(authCb);
83 
84  _tmpfile = filesystem::TmpFile();
85  _localfile = _tmpfile->path();
86 
87  // prepare Url and Settings
88  auto url = url_r;
89  auto tSettings = media::TransferSettings();
90  ::internal::prepareSettingsAndUrl( url, tSettings );
91 
92  auto req = std::make_shared<zyppng::NetworkRequest>( url_r, _localfile );
93  req->transferSettings () = tSettings;
94  executor.executeRequest ( req, nullptr );
95 
96  // apply umask
97  if ( ::chmod( _localfile.c_str(), filesystem::applyUmaskTo( 0644 ) ) )
98  {
99  ERR << "Failed to chmod file " << _localfile << endl;
100  }
101 
102  return;
103  }
104 
105  // this will handle traditional media including URL resolver plugins
106  Url abs_url( url_r );
107  abs_url.setPathName( "/" );
108  _access.reset( new MediaSetAccess( zypp::MirroredOrigin{abs_url} ) );
109  _localfile = _access->provideFile( url_r.getPathName() );
110 
111  }
112 
113  const Pathname & localfile() const
114  { return _localfile; }
115  private:
117  Pathname _localfile;
118  std::optional<filesystem::TmpFile> _tmpfile;
119  };
120 
121  enum class RepoMirrorListFormat {
122  Error,
123  Empty,
124  MirrorListTxt,
125  MirrorListJson,
126  MetaLink
127  };
128 
129  static RepoMirrorListFormat detectRepoMirrorListFormat( const Pathname &localfile ) {
130  // a file starting with < is most likely a metalink file,
131  // a file starting with [ is most likely a json file,
132  // else we go for txt
133  MIL << "Detecting RepoMirrorlist Format based on file content" << std::endl;
134 
135  if ( localfile.empty () )
136  return RepoMirrorListFormat::Empty;
137 
138  InputStream tmpfstream (localfile);
139  auto &str = tmpfstream.stream();
140  auto c = str.get ();
141 
142  // skip preceding whitespaces
143  while ( !str.eof () && !str.bad() && ( c == ' ' || c == '\t' || c == '\n' || c == '\r') )
144  c = str.get ();
145 
146  if ( str.eof() ) {
147  ERR << "Failed to read RepoMirrorList file, stream hit EOF early." << std::endl;
148  return RepoMirrorListFormat::Empty;
149  }
150 
151  if ( str.bad() ) {
152  ERR << "Failed to read RepoMirrorList file, stream became bad." << std::endl;
154  }
155 
156  switch ( c ) {
157  case '<': {
158  MIL << "Detected Metalink, file starts with <" << std::endl;
159  return RepoMirrorListFormat::MetaLink;
160  }
161  case '[': {
162  MIL << "Detected JSON, file starts with [" << std::endl;
163  return RepoMirrorListFormat::MirrorListJson;
164  }
165  default: {
166  MIL << "Detected TXT, file starts with " << c << std::endl;
167  return RepoMirrorListFormat::MirrorListTxt;
168  }
169  }
170  }
171 
172  inline std::vector<Url> RepoMirrorListParseXML( const Pathname &tmpfile )
173  {
174  try {
175  media::MetaLinkParser metalink;
176  metalink.parse(tmpfile);
177  return metalink.getUrls();
178  } catch (...) {
179  ZYPP_CAUGHT( std::current_exception() );
180  zypp::parser::ParseException ex("Invalid repo metalink format.");
181  ex.remember ( std::current_exception () );
182  ZYPP_THROW(ex);
183  }
184  }
185 
186  inline std::vector<Url> RepoMirrorListParseJSON( const Pathname &tmpfile )
187  {
188  InputStream tmpfstream (tmpfile);
189 
190  try {
191  using namespace zyppng::operators;
192  using zyppng::operators::operator|;
193 
194  auto res = json::parseDocumentExpected( tmpfstream )
195  | and_then([&]( json::Value data ) {
196 
197  std::vector<Url> urls;
198  if ( data.isNull () ) {
199  MIL << "Empty mirrorlist received, no mirrors available." << std::endl;
200  return zyppng::make_expected_success(urls);
201  }
202 
203  if ( data.type() != json::Value::ArrayType ) {
204  MIL << "Unexpected JSON format, top level element must be an array." << std::endl;
205  return zyppng::expected<std::vector<Url>>::error( ZYPP_EXCPT_PTR( zypp::Exception("Unexpected JSON format, top level element must be an array.") ));
206  }
207  const auto &topArray = data.asArray ();
208  for ( const auto &val : topArray ) {
209  if ( val.type () != json::Value::ObjectType ) {
210  MIL << "Unexpected JSON element, array must contain only objects. Ignoring current element" << std::endl;
211  continue;
212  }
213 
214  const auto &obj = val.asObject();
215  for ( const auto &key : obj ) {
216  if ( key.first == "url" ) {
217  const auto &elemValue = key.second;
218  if ( elemValue.type() != json::Value::StringType ) {
219  MIL << "Unexpected JSON element, element \"url\" must contain a string. Ignoring current element" << std::endl;
220  break;
221  }
222  try {
223  MIL << "Trying to parse URL: " << std::string(elemValue.asString()) << std::endl;
224  urls.push_back ( Url( elemValue.asString() ) );
225  } catch ( const url::UrlException &e ) {
226  ZYPP_CAUGHT(e);
227  MIL << "Invalid URL in mirrors file: "<< elemValue.asString() << ", ignoring" << std::endl;
228  }
229  }
230  }
231  }
232  return zyppng::make_expected_success(urls);
233  });
234 
235  if ( !res ) {
236  using zypp::operator<<;
237  MIL << "Error while parsing mirrorlist: (" << res.error() << "), no mirrors available" << std::endl;
238  ZYPP_RETHROW( res.error () );
239  }
240 
241  return *res;
242 
243  } catch (...) {
244  ZYPP_CAUGHT( std::current_exception() );
245  MIL << "Caught exception while parsing json" << std::endl;
246 
247  zypp::parser::ParseException ex("Invalid repo mirror list format, valid JSON was expected.");
248  ex.remember ( std::current_exception () );
249  ZYPP_THROW(ex);
250  }
251  return {};
252  }
253 
254  inline std::vector<Url> RepoMirrorListParseTXT( const Pathname &tmpfile )
255  {
256  InputStream tmpfstream (tmpfile);
257  std::vector<Url> my_urls;
258  std::string tmpurl;
259  while (getline(tmpfstream.stream(), tmpurl))
260  {
261  if ( tmpurl[0] == '#' )
262  continue;
263  try {
264  Url mirrUrl( tmpurl );
265  if ( !mirrUrl.schemeIsDownloading( ) ) {
266  MIL << "Ignoring non downloading URL " << tmpurl << std::endl;
267  }
268  my_urls.push_back(Url(tmpurl));
269  }
270  catch (...)
271  {
272  ZYPP_CAUGHT( std::current_exception() );
273 
274  // fail on invalid URLs
275  ERR << "Invalid URL in mirrorlist file." << std::endl;
276 
277  zypp::parser::ParseException ex("Invalid repo mirror list format, all Urls must be valid in a mirrorlist txt file.");
278  ex.remember ( std::current_exception () );
279  ZYPP_THROW(ex);
280  }
281  }
282  return my_urls;
283  }
284 
286  inline std::vector<Url> RepoMirrorListParse( const Url & url_r, const Pathname & listfile_r )
287  {
288  MIL << "Parsing mirrorlist file: " << listfile_r << " originally received from " << url_r << endl;
289 
290  std::vector<Url> mirrorurls;
291  switch( detectRepoMirrorListFormat (listfile_r) ) {
293  // should not happen, except when the instr goes bad
294  ZYPP_THROW( zypp::parser::ParseException( str::Format("Unable to detect metalink file format for: %1%") % listfile_r ));
295  case RepoMirrorListFormat::Empty:
296  mirrorurls = {};
297  break;
298  case RepoMirrorListFormat::MetaLink:
299  mirrorurls = RepoMirrorListParseXML( listfile_r );
300  break;
301  case RepoMirrorListFormat::MirrorListJson:
302  mirrorurls = RepoMirrorListParseJSON( listfile_r );
303  break;
304  case RepoMirrorListFormat::MirrorListTxt:
305  mirrorurls = RepoMirrorListParseTXT( listfile_r );
306  break;
307  }
308 
309  std::vector<Url> ret;
310  for ( auto & murl : mirrorurls )
311  {
312  if ( murl.getScheme() != "rsync" )
313  {
314  std::string pName = murl.getPathName();
315  size_t delpos = pName.find("repodata/repomd.xml");
316  if( delpos != std::string::npos )
317  {
318  murl.setPathName( pName.erase(delpos) );
319  }
320  ret.push_back( murl );
321  }
322  }
323  return ret;
324  }
325 
326  } // namespace
328 
329  RepoMirrorList::RepoMirrorList( const Url & url_r, const Pathname & metadatapath_r )
330  {
331  PathInfo metaPathInfo( metadatapath_r);
332  std::exception_ptr errors; // we collect errors here
333  try {
334  if ( url_r.getScheme() == "file" )
335  {
336  // never cache for local mirrorlist
337  _urls = RepoMirrorListParse( url_r, url_r.getPathName() );
338  }
339  else if ( !metaPathInfo.isDir() )
340  {
341  // no cachedir or no access
342  RepoMirrorListTempProvider provider( url_r ); // RAII: lifetime of any downloaded files
343  _urls = RepoMirrorListParse( url_r, provider.localfile() );
344  }
345  else
346  {
347  // have cachedir
348  const Pathname cachefile = metadatapath_r / cacheFileName();
349  const Pathname cookiefile = metadatapath_r / cookieFileName();
350  zypp::filesystem::PathInfo cacheinfo( cachefile );
351 
352  bool needRefresh = ( !cacheinfo.isFile()
353  // force a update on a old cache ONLY if the user can write the cache, otherwise we use an already existing cachefile
354  // it makes no sense to continously download the mirrors file if we can't store it
355  || ( cacheinfo.mtime() < time(NULL) - (long) ZConfig::instance().repo_refresh_delay() * 60 && metaPathInfo.userMayRWX () ) )
356  || ( makeCookie( url_r ) != readCookieFile( cookiefile ) );
357 
358  // up to date: try to parse and use the URLs if sucessful
359  // otherwise fetch the URL again
360  if ( !needRefresh ) {
361  MIL << "Mirror cachefile cookie valid and cache is not too old, skipping download (" << cachefile << ")" << std::endl;
362  try {
363  _urls = RepoMirrorListParse( url_r, cachefile );
364  return;
365  } catch ( const zypp::Exception & e ) {
366  ZYPP_CAUGHT(e);
367  auto ex = e;
368  if ( errors )
369  ex.remember(errors);
370  errors = std::make_exception_ptr(ex);
371  MIL << "Invalid mirrorlist cachefile, deleting it and trying to fetch a new one" << std::endl;
372  }
373  }
374 
375  // remove the old cache and its cookie, it's either broken, empty or outdated
376  if( cacheinfo.isFile() ) {
377  filesystem::unlink(cachefile);
378  }
379 
380  if ( zypp::filesystem::PathInfo(cookiefile).isFile() ) {
381  filesystem::unlink(cookiefile);
382  }
383 
384  MIL << "Getting MirrorList from URL: " << url_r << endl;
385  RepoMirrorListTempProvider provider( url_r ); // RAII: lifetime of downloaded file
386  _urls = RepoMirrorListParse( url_r, provider.localfile() );
387 
388  // removed the && !_urls.empty() condition , we need to remember "no URLs" as well
389  // otherwise RepoInfo keeps spamming the server with requests
390  if ( metaPathInfo.userMayRWX() ) {
391  // Create directory, if not existing
392  DBG << "Copy MirrorList file to " << cachefile << endl;
393  zypp::filesystem::assert_dir( metadatapath_r );
394  if( zypp::filesystem::hardlinkCopy( provider.localfile(), cachefile ) != 0 ) {
395  // remember empty file
396  zypp::filesystem::assert_file( cachefile );
397  }
398  saveToCookieFile ( cookiefile, url_r );
399  // NOTE: Now we copied the mirrorlist into the metadata directory, but
400  // in case of refresh going on, new metadata are prepared in a sibling
401  // temp dir. Upon success RefreshContext<>::saveToRawCache() exchanges
402  // temp and metadata dirs. There we move an existing mirrorlist file into
403  // the new metadata dir.
404  }
405  }
406  } catch ( const zypp::Exception &e ) {
407  // Make a more user readable exception
408  ZYPP_CAUGHT(e);
409  parser::ParseException ex( str::Format("Failed to parse/receive mirror information for URL: %1%") % url_r );
410  ex.remember(e);
411  if ( errors ) ex.remember(errors);
412  ZYPP_THROW(ex);
413  }
414  }
415 
417  {
418  static const std::vector<std::string> hosts{
419  "download.opensuse.org",
420  "cdn.opensuse.org"
421  };
422  return ( std::find( hosts.begin(), hosts.end(), str::toLower( url.getHost() )) != hosts.end() );
423  }
424 
425  std::string RepoMirrorList::readCookieFile(const Pathname &path_r)
426  {
427  std::ifstream file( path_r.c_str() );
428  if ( not file ) {
429  WAR << "No cookie file " << path_r << endl;
430  return {};
431  }
432 
433  return str::getline( file );
434  }
435 
439  std::string RepoMirrorList::makeCookie( const Url &url_r )
440  {
442  }
443 
444  void RepoMirrorList::saveToCookieFile(const Pathname &path_r, const Url &url_r )
445  {
446  std::ofstream file(path_r.c_str());
447  if (!file) {
448  ERR << str::Str() << "Can't open " << path_r.asString() << std::endl;
449  return;
450  }
451  MIL << "Saving mirrorlist cookie file " << path_r << std::endl;
452  file << makeCookie(url_r);
453  file.close();
454  }
455 
457  } // namespace repo
460 } // namespace zypp
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:560
std::string toLower(const std::string &s)
Return lowercase version of s.
Definition: String.cc:180
#define MIL
Definition: Logger.h:130
static std::string readCookieFile(const Pathname &path_r)
int assert_file(const Pathname &path, unsigned mode)
Create an empty file if it does not yet exist.
Definition: PathInfo.cc:1205
static std::string makeCookie(const zypp::Url &url_r)
Generates the cookie value, currently this is only derived from the Url.
std::string getline(std::istream &str, const Trim trim_r)
Return stream content up to (but not returning) the next newline.
Definition: String.cc:481
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:338
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:459
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:794
int chmod(const Pathname &path, mode_t mode)
Like &#39;chmod&#39;.
Definition: PathInfo.cc:1111
time_t mtime() const
Definition: PathInfo.h:384
const char * c_str() const
String representation.
Definition: Pathname.h:113
String related utilities and Regular expression matching.
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
Definition: ansi.h:854
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition: expected.h:470
Helper to create and pass std::istream.
Definition: inputstream.h:56
Pathname _localfile
std::vector< Url > _urls
Convenient building of std::string with boost::format.
Definition: String.h:253
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
#define ZYPP_EXCPT_PTR(EXCPT)
Drops a logline and returns Exception as a std::exception_ptr.
Definition: Exception.h:463
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:902
#define ERR
Definition: Logger.h:132
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
Definition: curlhelper.cc:180
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:154
bool empty() const
Test for an empty path.
Definition: Pathname.h:117
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:479
static CheckSum sha256FromString(const std::string &input_r)
Definition: CheckSum.h:107
Public JSON API.
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:212
Manages a data source characterized by an authoritative URL and a list of mirror URLs.
const std::string & asString() const
String representation.
Definition: Pathname.h:94
unsigned repo_refresh_delay() const
Amount of time in minutes that must pass before another refresh.
Definition: ZConfig.cc:1044
static constexpr const char * cacheFileName()
#define WAR
Definition: Logger.h:131
std::string asCompleteString() const
Returns a complete string representation of the Url object.
Definition: Url.cc:532
RepoMirrorList(const Url &url_r, const Pathname &metadatapath_r)
zypp::media::TransferSettings TransferSettings
static void saveToCookieFile(const Pathname &path_r, const zypp::Url &url_r)
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:719
std::optional< filesystem::TmpFile > _tmpfile
zypp::Url Url
Definition: url.h:15
static bool urlSupportsMirrorLink(const zypp::Url &url)
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:475
Base class for Exception.
Definition: Exception.h:152
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:1070
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
Definition: Url.cc:631
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:615
shared_ptr< MediaSetAccess > _access
static constexpr const char * cookieFileName()
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
ResultType and_then(const expected< T, E > &exp, Function &&f)
Definition: expected.h:520
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:814
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
bool userMayRWX() const
Definition: PathInfo.h:361
Url manipulation class.
Definition: Url.h:92
#define DBG
Definition: Logger.h:129