|
# http://pyrocko.org - GPLv3 # # The Pyrocko Developers, 21st Century # ---|P------/S----------~Lg----------
Low-level FDSN web service client.
This module provides basic functionality to download station metadata, time series data and event information from FDSN web services. Password and token authentication are supported. Query responses are returned as open file-like objects or can be parsed into Pyrocko's native data structures where appropriate.
.. _registered-site-names:
Registered site names .....................
A list of known FDSN site names is maintained within the module for quick selection by the user. This list currently contains the following sites:
%s
Any other site can be specified by providing its full URL. '''
except ImportError: certifi = None
urlencode, Request, build_opener, HTTPDigestAuthHandler, urlopen, HTTPError
else: g_cafile = None
'geofon': 'https://geofon.gfz-potsdam.de', 'iris': 'http://service.iris.edu', 'orfeus': 'http://www.orfeus-eu.org', 'bgr': 'http://eida.bgr.de', 'geonet': 'http://service.geonet.org.nz', 'knmi': 'http://rdsa.knmi.nl', 'ncedc': 'http://service.ncedc.org', 'scedc': 'http://service.scedc.caltech.edu', 'usgs': 'http://earthquake.usgs.gov', 'koeri': 'http://eida-service.koeri.boun.edu.tr', 'ethz': 'http://eida.ethz.ch', 'icgc': 'http://ws.icgc.cat', 'ipgp': 'http://eida.ipgp.fr', 'ingv': 'http://webservices.ingv.it', 'isc': 'http://www.isc.ac.uk', 'lmu': 'http://erde.geophysik.uni-muenchen.de', 'noa': 'http://eida.gein.noa.gr', 'resif': 'http://ws.resif.fr', 'usp': 'http://seisrequest.iag.usp.br', 'niep': 'http://eida-sc3.infp.ro' }
'station': { 'starttime', 'endtime', 'startbefore', 'startafter', 'endbefore', 'endafter', 'network', 'station', 'location', 'channel', 'minlatitude', 'maxlatitude', 'minlongitude', 'maxlongitude', 'latitude', 'longitude', 'minradius', 'maxradius', 'level', 'includerestricted', 'includeavailability', 'updatedafter', 'matchtimeseries', 'format', 'nodata'}, 'dataselect': { 'starttime', 'endtime', 'network', 'station', 'location', 'channel', 'quality', 'minimumlength', 'longestonly', 'format', 'nodata'}, 'event': { 'starttime', 'endtime', 'minlatitude', 'maxlatitude', 'minlongitude', 'maxlongitude', 'latitude', 'longitude', 'minradius', 'maxradius', 'mindepth', 'maxdepth', 'minmagnitude', 'maxmagnitude', 'eventtype', 'includeallorigins', 'includeallmagnitudes', 'includearrivals', 'eventid', 'limit', 'offset', 'orderby', 'catalog', 'contributor', 'updatedafter', 'format', 'nodata'}, 'availability': { 'starttime', 'endtime', 'network', 'station', 'location', 'channel', 'quality', 'merge', 'orderby', 'limit', 'includerestricted', 'format', 'nodata', 'mergegaps', 'show'}}
return ', '.join("``'%s'``" % s for s in li)
hr, '%s %s' % (khead.ljust(lk), vhead.ljust(lv)), hr]
''' Get sorted list of registered site names. ''' return sorted(g_site_abbr.keys())
else: g_timeout = config.config().fdsn_timeout
''' Raised when failing to parse server response during authentication. '''
''' Raised when failing to parse server response during token authentication. '''
realm = dict(re_realm_from_auth_header.findall( headers['WWW-Authenticate'])).get('realm', None)
if realm is None: raise CannotGetRealmFromAuthHeader('headers=%s' % str(headers))
return realm
''' Raised when an empty server response is retrieved. ''' DownloadError.__init__(self) self._url = url
def __str__(self): return 'No results for request %s' % self._url
''' Raised when the server indicates that too much data was requested. ''' DownloadError.__init__(self) self._url = url
def __str__(self): return 'Request entity too large: %s' % self._url
''' Raised when an invalid request would be sent / has been sent. '''
''' Raised when the server does not respond within the allowed timeout period. '''
url, post=False, user=None, passwd=None, allow_TLSv1=False, timeout=g_timeout, **kwargs):
'timeout': timeout }
url_args['context'] = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
post = post.encode('utf8')
del url_args['context'] # context not avail before 3.4.3
raise EmptyResult(url)
raise RequestEntityTooLarge(url)
headers = getattr(e, 'headers', e.hdrs)
realm = get_realm_from_auth_header(headers)
if itry == 1 and user is not None: auth_handler = HTTPDigestAuthHandler() auth_handler.add_password( realm=realm, uri=url, user=user, passwd=passwd or '')
opener = build_opener(auth_handler) continue else: raise DownloadError( 'Authentication failed for realm "%s" when accessing ' 'url "%s". Original error was: %s' % ( realm, url, str(e)))
else: raise DownloadError( 'Error content returned by server (HTML stripped):\n%s\n' ' Original error was: %s' % ( indent( strip_html(e.read()), ' ! '), str(e)))
except socket.timeout: raise Timeout( 'Timeout error. No response received within %i s. You ' 'may want to retry with a longer timeout setting.' % timeout)
break
site=g_site_abbr.get(site, site), service=service, majorversion=majorversion, method=method)
'endtime', 'startbefore', 'startafter', 'endbefore', 'endafter', 'updatedafter']:
params['location'] = '--'
stations, tmin, tmax, channel_prio=[['BHZ', 'HHZ'], ['BH1', 'BHN', 'HH1', 'HHN'], ['BH2', 'BHE', 'HH2', 'HHE']]):
selection = [] for station in stations: wanted = [] for group in channel_prio: gchannels = [] for channel in station.get_channels(): if channel.name in group: gchannels.append(channel) if gchannels: gchannels.sort(key=lambda a: group.index(a.name)) wanted.append(gchannels[0])
if wanted: for channel in wanted: selection.append((station.network, station.station, station.location, channel.name, tmin, tmax))
return selection
site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout, check=True, selection=None, parsed=True, **kwargs):
''' Query FDSN web service for station metadata.
:param site: :ref:`Registered site name <registered-site-names>` or full base URL of the service (e.g. ``'https://geofon.gfz-potsdam.de'``). :type site: str, optional :param url: URL template (default should work in 99% of cases). :type url: str, optional :param majorversion: Major version of the service to query (always ``1`` at the time of writing). :type majorversion: int, optional :param timeout: Network timeout in [s]. Global default timeout can be configured in Pyrocko's configuration file under ``fdsn_timeout``. :type timeout: float, optional :param check: If ``True`` arguments are checked against self-description (WADL) of the queried web service if available or FDSN specification. :type check: bool, optional :param selection: If given, selection to be queried as a list of tuples ``(network, station, location, channel, tmin, tmax)``. Useful for detailed queries. :type selection: list of tuples, optional :param parsed: If ``True`` parse received content into :py:class:`~pyrocko.io.stationxml.FDSNStationXML` object, otherwise return open file handle to raw data stream. :type parsed: bool, optional :param \\*\\*kwargs: Parameters passed to the server (see `FDSN web services specification <https://www.fdsn.org/webservices>`_).
:returns: See description of ``parsed`` argument above.
:raises: On failure, :py:exc:`~pyrocko.util.DownloadError` or one of its sub-types defined in the :py:mod:`~pyrocko.client.fdsn` module is raised. '''
location = '--'
sdatetime(tmin), sdatetime(tmax))))
stream=_request(url, timeout=timeout, **params)) else: raise InvalidRequest('if format="text" shall be parsed, ' 'level="channel" is required')
stream=_request(url, timeout=timeout, **params)) else: raise InvalidRequest('format must be "xml" or "text"') else: return _request(url, timeout=timeout, **params)
url = fillurl(service, site, url, majorversion, 'auth')
f = _request(url, timeout=timeout, post=token) s = f.read().decode() try: user, passwd = s.strip().split(':') except ValueError: raise CannotGetCredentialsFromAuthRequest('data="%s"' % s)
return user, passwd
site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout, check=True, user=None, passwd=None, token=None, selection=None, **kwargs):
''' Query FDSN web service for time series data in miniSEED format.
:param site: :ref:`Registered site name <registered-site-names>` or full base URL of the service (e.g. ``'https://geofon.gfz-potsdam.de'``). :type site: str, optional :param url: URL template (default should work in 99% of cases). :type url: str, optional :param majorversion: Major version of the service to query (always ``1`` at the time of writing). :type majorversion: int, optional :param timeout: Network timeout in [s]. Global default timeout can be configured in Pyrocko's configuration file under ``fdsn_timeout``. :type timeout: float, optional :param check: If ``True`` arguments are checked against self-description (WADL) of the queried web service if available or FDSN specification. :type check: bool, optional :param user: User name for user/password authentication. :type user: str, optional :param passwd: Password for user/password authentication. :type passwd: str, optional :param token: Token for `token authentication <https://geofon.gfz-potsdam.de/waveform/archive/auth/auth-overview.php>`_. :type token: str, optional :param selection: If given, selection to be queried as a list of tuples ``(network, station, location, channel, tmin, tmax)``. :type selection: list of tuples, optional :param \\*\\*kwargs: Parameters passed to the server (see `FDSN web services specification <https://www.fdsn.org/webservices>`_).
:returns: Open file-like object providing raw miniSEED data. '''
method = 'queryauth' else:
user, passwd = get_auth_credentials( service, site, url, majorversion, token, timeout)
lst.append('%s=%s' % (k, v))
location = '--'
sdatetime(tmin), sdatetime(tmax))))
url, user=user, passwd=passwd, post=post.encode(), timeout=timeout) else: url, user=user, passwd=passwd, timeout=timeout, **params)
site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout, check=True, user=None, passwd=None, token=None, parsed=False, **kwargs):
''' Query FDSN web service for events.
:param site: :ref:`Registered site name <registered-site-names>` or full base URL of the service (e.g. ``'https://geofon.gfz-potsdam.de'``). :type site: str, optional :param url: URL template (default should work in 99% of cases). :type url: str, optional :param majorversion: Major version of the service to query (always ``1`` at the time of writing). :type majorversion: int, optional :param timeout: Network timeout in [s]. Global default timeout can be configured in Pyrocko's configuration file under ``fdsn_timeout``. :type timeout: float, optional :param check: If ``True`` arguments are checked against self-description (WADL) of the queried web service if available or FDSN specification. :type check: bool, optional :param user: User name for user/password authentication. :type user: str, optional :param passwd: Password for user/password authentication. :type passwd: str, optional :param token: Token for `token authentication <https://geofon.gfz-potsdam.de/waveform/archive/auth/auth-overview.php>`_. :type token: str, optional :param parsed: If ``True`` parse received content into :py:class:`~pyrocko.io.quakeml.QuakeML` object, otherwise return open file handle to raw data stream. Note: by default unparsed data is retrieved, differently from the default behaviour of :py:func:`station` (for backward compatibility). :type parsed: bool, optional :param \\*\\*kwargs: Parameters passed to the server (see `FDSN web services specification <https://www.fdsn.org/webservices>`_).
:returns: See description of ``parsed`` argument above.
'''
method = 'queryauth' else:
user, passwd = get_auth_credentials( service, site, url, majorversion, token, timeout)
from pyrocko.io import quakeml format = kwargs.get('format', 'xml') if format != 'xml': raise InvalidRequest( 'If parsed=True is selected, format="xml" must be selected.')
return quakeml.load_xml(stream=fh)
else:
method='query', site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout, check=True, user=None, passwd=None, token=None, selection=None, **kwargs):
''' Query FDSN web service for time series data availablity.
:param method: Availablility method to call: ``'query'``, or ``'extent'``. :param site: :ref:`Registered site name <registered-site-names>` or full base URL of the service (e.g. ``'https://geofon.gfz-potsdam.de'``). :type site: str, optional :param url: URL template (default should work in 99% of cases). :type url: str, optional :param majorversion: Major version of the service to query (always ``1`` at the time of writing). :type majorversion: int, optional :param timeout: Network timeout in [s]. Global default timeout can be configured in Pyrocko's configuration file under ``fdsn_timeout``. :type timeout: float, optional :param check: If ``True`` arguments are checked against self-description (WADL) of the queried web service if available or FDSN specification. :type check: bool, optional :param user: User name for user/password authentication. :type user: str, optional :param passwd: Password for user/password authentication. :type passwd: str, optional :param token: Token for `token authentication <https://geofon.gfz-potsdam.de/waveform/archive/auth/auth-overview.php>`_. :type token: str, optional :param selection: If given, selection to be queried as a list of tuples ``(network, station, location, channel, tmin, tmax)``. :type selection: list of tuples, optional :param \\*\\*kwargs: Parameters passed to the server (see `FDSN web services specification <https://www.fdsn.org/webservices>`_).
:returns: Open file-like object providing raw response. '''
method += 'auth'
user, passwd = get_auth_credentials( service, site, url, majorversion, token, timeout)
lst.append('%s=%s' % (k, v))
location = '--'
sdatetime(tmin), sdatetime(tmax))))
url, user=user, passwd=passwd, post=post.encode(), timeout=timeout) else: return _request( url, user=user, passwd=passwd, timeout=timeout, **params)
service, site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout, method='query', **kwargs):
''' Check query parameters against self-description of web service.
Downloads WADL description of the given service and site and checks parameters if they are available. Queried WADLs are cached in memory.
:param service: ``'station'``, ``'dataselect'``, ``'event'`` or ``'availability'``. :param site: :ref:`Registered site name <registered-site-names>` or full base URL of the service (e.g. ``'https://geofon.gfz-potsdam.de'``). :type site: str, optional :param url: URL template (default should work in 99% of cases). :type url: str, optional :param majorversion: Major version of the service to query (always ``1`` at the time of writing). :type majorversion: int, optional :param timeout: Network timeout in [s]. Global default timeout can be configured in Pyrocko's configuration file under ``fdsn_timeout``. :type timeout: float, optional :param \\*\\*kwargs: Parameters that would be passed to the server (see `FDSN web services specification <https://www.fdsn.org/webservices>`_).
:raises: :py:exc:`ValueError` is raised if unsupported parameters are encountered. '''
service, site, url, majorversion, timeout, method)
raise ValueError( 'Unsupported parameter%s for service "%s" at site "%s": %s' % ( '' if len(unavail) == 1 else 's', service, site, ', '.join(unavail)))
service, site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout, method='query'):
''' Get query parameter names supported by a given FDSN site and service.
If no WADL is provided by the queried service, default parameter sets from the FDSN standard are returned. Queried WADLs are cached in memory.
:param service: ``'station'``, ``'dataselect'``, ``'event'`` or ``'availability'``. :param site: :ref:`Registered site name <registered-site-names>` or full base URL of the service (e.g. ``'https://geofon.gfz-potsdam.de'``). :type site: str, optional :param url: URL template (default should work in 99% of cases). :type url: str, optional :param majorversion: Major version of the service to query (always ``1`` at the time of writing). :type majorversion: int, optional :param timeout: Network timeout in [s]. Global default timeout can be configured in Pyrocko's configuration file under ``fdsn_timeout``. :type timeout: float, optional
:returns: Supported parameter names. :rtype: set of str '''
else: return g_default_query_args[service]
for r in wadl.resources_list: r.base = r.base.replace('1/station', 'station/1') r.base = r.base.replace('1/dataselect', 'dataselect/1') r.base = r.base.replace('1/event', 'event/1')
service, site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout):
''' Retrieve self-description of a specific FDSN service.
:param service: ``'station'``, ``'dataselect'``, ``'event'`` or ``'availability'``. :param site: :ref:`Registered site name <registered-site-names>` or full base URL of the service (e.g. ``'https://geofon.gfz-potsdam.de'``). :type site: str, optional :param url: URL template (default should work in 99% of cases). :type url: str, optional :param majorversion: Major version of the service to query (always ``1`` at the time of writing). :type majorversion: int, optional :param timeout: Network timeout in [s]. Global default timeout can be configured in Pyrocko's configuration file under ``fdsn_timeout``. :type timeout: float, optional '''
patch_geonet_wadl(wadl)
service, site=g_default_site, url=g_url, majorversion=1, timeout=g_timeout):
''' Get self-description of a specific FDSN service.
Same as :py:func:`wadl`, but results are cached in memory. '''
except Timeout: raise
except DownloadError: logger.info( 'No service description (WADL) found for "%s" at site "%s".' % (service, site))
g_wadls[k] = None
if __name__ == '__main__': import sys
util.setup_logging('pyrocko.client.fdsn', 'info')
if len(sys.argv) == 1: sites = get_sites() else: sites = sys.argv[1:]
for site in sites: print('=== %s (%s) ===' % (site, g_site_abbr[site]))
for service in ['station', 'dataselect', 'event']: try: app = wadl(service, site=site, timeout=2.0) print(indent(str(app)))
except Timeout as e: logger.error(str(e)) print('%s: timeout' % (service,))
except util.DownloadError as e: logger.error(str(e)) print('%s: no wadl' % (service,)) |