libzypp  17.38.17
MediaNetworkCommonHandler.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
15 
16 #include <zypp/ZConfig.h>
17 #include <zypp-core/fs/PathInfo.h>
18 #include <zypp-core/base/Regex.h>
19 #include <zypp-core/base/Logger.h>
20 #include <zypp-core/base/Gettext.h>
21 #include <zypp/Target.h>
23 #include <zypp-media/auth/CredentialManager>
24 #include <zypp-curl/auth/CurlAuthData>
26 #include <zypp-curl/ng/network/HttpHeader>
27 
28 #include <zypp/ZYppCallbacks.h>
29 
30 #include <fstream>
31 #include <curl/curl.h>
32 
33 using std::endl;
34 
35 namespace zypp::media
36 {
37  MediaNetworkCommonHandler::MediaNetworkCommonHandler( const MirroredOrigin &origin_r, const Pathname &attach_point_r, const Pathname &urlpath_below_attachpoint_r, const bool does_download_r)
38  : MediaHandler( origin_r, attach_point_r, urlpath_below_attachpoint_r, does_download_r )
39  {
40  _redirTargets.clear ();
41  std::transform ( _origin.begin(), _origin.end(), std::back_inserter(_redirTargets), []( const OriginEndpoint &url_r ) { return findGeoIPRedirect(url_r.url()); } );
42 
43  // initialize default mirror order: mirrors first, then authorities
44  _mirrOrder.reserve( _origin.endpointCount() );
45  uint authCount = _origin.authorityCount();
46  for( unsigned i = authCount; i < _origin.endpointCount () ; i++ ) { _mirrOrder.push_back(i); }
47  for( unsigned i = 0; i < authCount ; i++ ) { _mirrOrder.push_back(i); }
48  }
49 
51  {
52  if ( next )
54 
55  if( !isUseableAttachPoint( attachPoint() ) ) {
57  }
58 
59  disconnectFrom(); // clean state if needed
60 
61  // here : setup TransferSettings
63 
64  // FIXME: need a derived class to propelly compare url's
65  MediaSourceRef media( new MediaSource( url().getScheme(), url().asString()) );
66  setMediaSource(media);
67  }
68 
70  {
71  return MediaHandler::checkAttachPoint( apoint, true, true);
72  }
73 
75  {
77  }
78 
80  {
81  try {
82  const auto &conf = ZConfig::instance();
83  if ( !conf.geoipEnabled() ) {
84  MIL << "GeoIp rewrites disabled via ZConfig." << std::endl;
85  return Url();
86  }
87 
88  if ( !( url.getQueryParam("COUNTRY").empty() && url.getQueryParam("AVOID_COUNTRY").empty() )) {
89  MIL << "GeoIp rewrites disabled since the baseurl " << url << " uses an explicit country setting." << std::endl;
90  return Url();
91  }
92 
93  const auto &hostname = url.getHost();
94  auto geoipFile = conf.geoipCachePath() / hostname ;
95  if ( PathInfo( geoipFile ).isFile() ) {
96 
97  MIL << "Found GeoIP file for host: " << hostname << std::endl;
98 
99  std::ifstream in( geoipFile.asString() );
100  if (!in.is_open()) {
101  MIL << "Failed to open GeoIP for host: " << hostname << std::endl;
102  return Url();
103  }
104 
105  try {
106  std::string newHost;
107  in >> newHost;
108 
109  Url newUrl = url;
110  newUrl.setHost( newHost );
111 
112  MIL << "Found GeoIP rewrite: " << hostname << " -> " << newHost << std::endl;
113 
114  return newUrl;
115 
116  } catch ( const zypp::Exception &e ) {
117  ZYPP_CAUGHT(e);
118  MIL << "No valid GeoIP rewrite target found for " << url << std::endl;
119  }
120  }
121  } catch ( const zypp::Exception &e ) {
122  ZYPP_CAUGHT(e);
123  MIL << "Failed to query GeoIP data, url rewriting disabled." << std::endl;
124  }
125 
126  // no rewrite
127  return Url();
128  }
129 
131 
133  {
134  // Use absolute file name to prevent access of files outside of the
135  // hierarchy below the attach point.
136  getFileCopy( file, localPath(file.filename()).absolutename() );
137  }
138 
139  void MediaNetworkCommonHandler::getDir( const Pathname & dirname, bool recurse_r ) const
140  {
141  filesystem::DirContent content;
142  getDirInfo( content, dirname, /*dots*/false );
143 
144  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
145  Pathname filename = dirname + it->name;
146  int res = 0;
147 
148  switch ( it->type ) {
149  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
150  case filesystem::FT_FILE:
151  getFile( OnMediaLocation( filename ) );
152  break;
153  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
154  if ( recurse_r ) {
155  getDir( filename, recurse_r );
156  } else {
157  res = assert_dir( localPath( filename ) );
158  if ( res ) {
159  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
160  }
161  }
162  break;
163  default:
164  // don't provide devices, sockets, etc.
165  break;
166  }
167  }
168  }
169 
170  void MediaNetworkCommonHandler::getDirInfo( std::list<std::string> & retlist,
171  const Pathname & dirname, bool dots ) const
172  {
173  getDirectoryYast( retlist, dirname, dots );
174  }
175 
177  const Pathname & dirname, bool dots ) const
178  {
179  getDirectoryYast( retlist, dirname, dots );
180  }
181 
183  {
184  static const HttpHeader _value { "X-ZYpp-AnonymousId", Target::anonymousUniqueId( Pathname()/*guess root*/ ) };
185  return _value;
186  }
187 
189  {
190  static const HttpHeader _value { "X-ZYpp-DistributionFlavor", Target::distributionFlavor( Pathname()/*guess root*/ ) };
191  return _value;
192  }
193 
194  Url MediaNetworkCommonHandler::getFileUrl( int mirrorIdx, const Pathname & filename_r ) const
195  {
196  static const zypp::str::regex invalidRewrites("^.*\\/repomd.xml(.asc|.key)?$|^\\/geoip$");
197 
198  if ( mirrorIdx < 0 || mirrorIdx > _redirTargets.size() )
199  return {};
200 
201  const bool canRedir = _redirTargets[mirrorIdx].isValid() && !invalidRewrites.matches(filename_r.asString());
202  const auto &baseUrl = ( canRedir ) ? _redirTargets[mirrorIdx] : _origin[mirrorIdx].url();
203 
204  if ( canRedir )
205  MIL << "Redirecting " << filename_r << " request to geoip location." << std::endl;
206 
207  // Simply extend the URLs pathname:
208  Url newurl { baseUrl };
209  newurl.appendPathName( filename_r );
210  return newurl;
211  }
212 
214  // we need to add the release and identifier to the
215  // agent string.
216  // The target could be not initialized, and then this information
217  // is guessed.
218  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
219  static const std::string _value(str::trim(str::form(
220  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s",
221  curl_version_info(CURLVERSION_NOW)->version,
222  Target::targetDistribution(Pathname() /*guess root*/).c_str())));
223  return _value.c_str();
224  }
225 
227  {
228  // fill some settings from url query parameters
229  try
230  {
232 
233  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
234  for( OriginEndpoint &u : _origin ) {
235 
236  u.setConfig( MIRR_SETTINGS_KEY.data() , std::make_any<TransferSettings>()); // init or reset to default
237 
238  if ( !u.isValid() )
240 
241  TransferSettings &set = u.getConfig<TransferSettings>( MIRR_SETTINGS_KEY.data() );
242 
243  checkProtocol(u.url());
244 
245  set.setUserAgentString(agentString());
246 
247  // apply MediaUrl settings
248  if ( u.hasConfig ("http-headers") ) {
249  // Set up the handler
250  for ( const auto & el : u.getConfig<UrlResolverPlugin::HeaderList>("http-headers") ) {
251  std::string header { el.first };
252  header += ": ";
253  header += el.second;
254  MIL << "Added custom header -> " << header << std::endl;
255  set.addHeader( std::move(header) );
256  }
257  }
258 
259  // add custom headers for download.opensuse.org (bsc#955801)
260  if ( u.url().getHost() == "download.opensuse.org" )
261  {
262  set.addHeader(anonymousIdHeader().asString());
263  set.addHeader(distributionFlavorHeader().asString());
264  }
265  set.addHeader("Pragma:");
266 
267  // apply legacy Url encoded settings
268  ::internal::fillSettingsFromUrl( u.url(), set);
269 
270  // if the proxy was not set (or explicitly unset) by url, then look...
271  if ( set.proxy().empty() )
272  {
273  // ...at the system proxy settings
275  }
276 
277  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
278  * We should proactively add the password to the request if basic auth is configured
279  * and a password is available in the credentials but not in the URL.
280  *
281  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
282  * and ask the server first about the auth method
283  */
284  if ( set.authType() == "basic" && not set.username().empty() && set.password().empty() ) {
285  const auto cred = cm.getCred( u.url() );
286  if ( cred && cred->valid() ) {
287  set.setPassword(cred->password());
288  }
289  }
290  }
291  }
292  catch ( const MediaException &e )
293  {
294  disconnectFrom();
295  ZYPP_RETHROW(e);
296  }
297  }
298 
300  {
301  for( OriginEndpoint &u : _origin ) {
302  u.eraseConfigValue ( MIRR_SETTINGS_KEY.data() );
303  }
304  }
305 
306 
307  bool MediaNetworkCommonHandler::authenticate(const Url &url, TransferSettings &settings, const std::string & availAuthTypes, bool firstTry)
308  {
310  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
311  return authenticate( url, cm, settings, availAuthTypes, firstTry );
312  }
313 
314  std::vector<unsigned int> MediaNetworkCommonHandler::mirrorOrder(const OnMediaLocation &loc) const
315  {
316  uint authCount = _origin.authorityCount();
317 
318  if ( !loc.mirrorsAllowed () ) {
319  MIL << "Fetching file " << loc << " from authorities only ( " << authCount << " ): " << _origin << std::endl;
320  std::vector<unsigned> authOrder;
321  authOrder.reserve( authCount );
322  // Filter _mirrOrder to include only authority indices
323  for ( unsigned idx : _mirrOrder ) {
324  if ( idx < authCount )
325  authOrder.push_back( idx );
326  }
327  return authOrder;
328  }
329 
330  return _mirrOrder;
331  }
332 
334  {
335  auto it = std::find( _mirrOrder.begin(), _mirrOrder.end(), mirr );
336  if ( it != _mirrOrder.end() && _mirrOrder.size() > 1 )
337  {
338  // move found index to the end
339  std::rotate( it, it + 1, _mirrOrder.end() );
340  }
341  }
342 
343  bool MediaNetworkCommonHandler::authenticate( const Url &url, CredentialManager &cm, TransferSettings &settings, const std::string & availAuthTypes, bool firstTry )
344  {
345  CurlAuthData_Ptr credentials;
346 
347  // get stored credentials
348  AuthData_Ptr cmcred = cm.getCred(url);
349 
350  if (cmcred && firstTry)
351  {
352  credentials.reset(new CurlAuthData(*cmcred));
353  DBG << "got stored credentials:" << endl << *credentials << endl;
354  }
355  // if not found, ask user
356  else
357  {
358 
359  CurlAuthData_Ptr curlcred;
360  curlcred.reset(new CurlAuthData());
362 
363  // preset the username if present in current url
364  if (!url.getUsername().empty() && firstTry)
365  curlcred->setUsername(url.getUsername());
366  // if CM has found some credentials, preset the username from there
367  else if (cmcred)
368  curlcred->setUsername(cmcred->username());
369 
370  // indicate we have no good credentials from CM
371  cmcred.reset();
372 
373  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % url.asString();
374 
375  // set available authentication types from the exception
376  // might be needed in prompt
377  curlcred->setAuthType(availAuthTypes);
378 
379  // ask user
380  if (auth_report->prompt(url, prompt_msg, *curlcred))
381  {
382  DBG << "callback answer: retry" << endl
383  << "CurlAuthData: " << *curlcred << endl;
384 
385  if (curlcred->valid())
386  {
387  credentials = curlcred;
388  // if (credentials->username() != _url.url().getUsername())
389  // _url.url().setUsername(credentials->username());
397  }
398  }
399  else
400  {
401  DBG << "callback answer: cancel" << endl;
402  }
403  }
404 
405  // set username and password
406  if (credentials)
407  {
408  settings.setUsername(credentials->username());
409  settings.setPassword(credentials->password());
410 
411  // set available authentication types from the exception
412  if (credentials->authType() == CURLAUTH_NONE)
413  credentials->setAuthType(availAuthTypes);
414 
415  // set auth type (seems this must be set _after_ setting the userpwd)
416  if (credentials->authType() != CURLAUTH_NONE) {
417  settings.setAuthType(credentials->authTypeAsString());
418  }
419 
420  if (!cmcred)
421  {
422  credentials->setUrl(url);
423  cm.addCred(*credentials);
424  cm.save();
425  }
426 
427  return true;
428  }
429 
430  return false;
431  }
432 
433 
434 } // namespace zypp::media
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
#define MIL
Definition: Logger.h:130
std::vector< unsigned > mirrorOrder(const OnMediaLocation &loc) const
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:338
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:459
Describes a resource file located on a medium.
Regular expression.
Definition: Regex.h:94
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:794
void setPassword(const std::string &val_r)
sets the auth password
std::string asString(const Patch::Category &obj)
relates: Patch::Category string representation.
Definition: Patch.cc:122
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
void appendPathName(const Pathname &path_r, EEncoding eflag_r=zypp::url::E_DECODED)
Extend the path name.
Definition: Url.cc:813
static constexpr std::string_view MIRR_SETTINGS_KEY
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:401
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
virtual void disconnectFrom()
Call concrete handler to disconnect media.
Definition: MediaHandler.h:321
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
static const HttpHeader & distributionFlavorHeader()
void setHost(const std::string &host)
Set the hostname or IP in the URL authority.
Definition: Url.cc:775
Convenient building of std::string with boost::format.
Definition: String.h:253
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:39
uint authorityCount() const
Edition * _value
Definition: SysContent.cc:311
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
bool mirrorsAllowed() const
The requested file is allowed to be fetched via mirrors ( defaults to true )
endpoint_iterator end()
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:534
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
virtual void getFileCopy(const OnMediaLocation &file, const Pathname &targetFilename) const
Call concrete handler to provide a file under a different place in the file system (usually not under...
void attachTo(bool next) override
Call concrete handler to attach the media.
bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:479
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:524
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:687
Abstract base class for &#39;physical&#39; MediaHandler like MediaCD, etc.
Definition: MediaHandler.h:50
MirroredOrigin _origin
Contains the authority URL and mirrors.
Definition: MediaHandler.h:112
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:226
Manages a data source characterized by an authoritative URL and a list of mirror URLs.
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:94
Just inherits Exception to separate media exceptions.
#define WAR
Definition: Logger.h:131
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:81
void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:196
bool matches(const char *s, str::smatch &matches, int flags=none) const
Definition: Regex.cc:57
#define _(MSG)
Definition: Gettext.h:39
MediaNetworkCommonHandler(const MirroredOrigin &origin_r, const Pathname &attach_point_r, const Pathname &urlpath_below_attachpoint_r, const bool does_download_r)
const Pathname & filename() const
The path to the resource on the medium.
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:475
Base class for Exception.
Definition: Exception.h:152
Pathname attachPoint() const
Return the currently used attach point.
Url url() const
Primary Url used.
Definition: MediaHandler.h:507
Url getFileUrl(int mirrorIdx, const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:615
virtual void checkProtocol(const Url &url) const =0
check the url is supported by the curl library
void deprioritizeMirror(unsigned mirr) const
Move mirror index mirr to the end of the attempt list.
std::multimap< std::string, std::string > HeaderList
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
uint endpointCount() const
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:359
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
void setupTransferSettings()
initializes the curl easy handle with the data from the url
void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
static zypp::Url findGeoIPRedirect(const zypp::Url &url)
Rewrites the baseURL to the geoIP target if one is found in the metadata cache, otherwise simply retu...
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Represents a single, configurable network endpoint, combining a URL with specific access settings...
endpoint_iterator begin()
Url manipulation class.
Definition: Url.h:92
#define DBG
Definition: Logger.h:129
auto transform(Container< Msg, CArgs... > &&val, Transformation &&transformation)
Definition: transform.h:64
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:599