1# http://pyrocko.org - GPLv3
2#
3# The Pyrocko Developers, 21st Century
4# ---|P------/S----------~Lg----------
6'''
7Data model and content types handled by the Squirrel framework.
9Squirrel uses flat content types to represent waveform, station, channel,
10response, event, and a few other objects. A common subset of the information in
11these objects is indexed in the database, currently: kind, codes, time interval
12and sampling rate. The :py:class:`Nut` objects encapsulate this information
13together with information about where and how to get the associated bulk data.
15Further content types are defined here to handle waveform orders, waveform
16promises, data coverage and sensor information.
17'''
19from __future__ import absolute_import, print_function
21import re
22import fnmatch
23import hashlib
24import numpy as num
25from os import urandom
26from base64 import urlsafe_b64encode
27from collections import defaultdict, namedtuple
29from pyrocko import util
30from pyrocko.guts import Object, SObject, String, Timestamp, Float, Int, \
31 Unicode, Tuple, List, StringChoice, Any, Dict
32from pyrocko.model import squirrel_content, Location
33from pyrocko.response import FrequencyResponse, MultiplyResponse, \
34 IntegrationResponse, DifferentiationResponse, simplify_responses, \
35 FrequencyResponseCheckpoint
37from .error import ConversionError, SquirrelError
40guts_prefix = 'squirrel'
43g_codes_pool = {}
46class CodesError(SquirrelError):
47 pass
50class Codes(SObject):
51 pass
54def normalize_nslce(*args, **kwargs):
55 if args and kwargs:
56 raise ValueError('Either *args or **kwargs accepted, not both.')
58 if len(args) == 1:
59 if isinstance(args[0], str):
60 args = tuple(args[0].split('.'))
61 elif isinstance(args[0], tuple):
62 args = args[0]
63 else:
64 raise ValueError('Invalid argument type: %s' % type(args[0]))
66 nargs = len(args)
67 if nargs == 5:
68 t = args
70 elif nargs == 4:
71 t = args + ('',)
73 elif nargs == 0:
74 d = dict(
75 network='',
76 station='',
77 location='',
78 channel='',
79 extra='')
81 d.update(kwargs)
82 t = tuple(kwargs.get(k, '') for k in (
83 'network', 'station', 'location', 'channel', 'extra'))
85 else:
86 raise CodesError(
87 'Does not match NSLC or NSLCE codes pattern: %s' % '.'.join(args))
89 if '.'.join(t).count('.') != 4:
90 raise CodesError(
91 'Codes may not contain a ".": "%s", "%s", "%s", "%s", "%s"' % t)
93 return t
96CodesNSLCEBase = namedtuple(
97 'CodesNSLCEBase', [
98 'network', 'station', 'location', 'channel', 'extra'])
101class CodesNSLCE(CodesNSLCEBase, Codes):
102 '''
103 Codes denominating a seismic channel (NSLC or NSLCE).
105 FDSN/SEED style NET.STA.LOC.CHA is accepted or NET.STA.LOC.CHA.EXTRA, where
106 the EXTRA part in the latter form can be used to identify a custom
107 processing applied to a channel.
108 '''
110 __slots__ = ()
111 __hash__ = CodesNSLCEBase.__hash__
113 as_dict = CodesNSLCEBase._asdict
115 def __new__(cls, *args, safe_str=None, **kwargs):
116 nargs = len(args)
117 if nargs == 1 and isinstance(args[0], CodesNSLCE):
118 return args[0]
119 elif nargs == 1 and isinstance(args[0], CodesNSL):
120 t = (args[0].nsl) + ('*', '*')
121 elif nargs == 1 and isinstance(args[0], CodesX):
122 t = ('*', '*', '*', '*', '*')
123 elif safe_str is not None:
124 t = safe_str.split('.')
125 else:
126 t = normalize_nslce(*args, **kwargs)
128 x = CodesNSLCEBase.__new__(cls, *t)
129 return g_codes_pool.setdefault(x, x)
131 def __init__(self, *args, **kwargs):
132 Codes.__init__(self)
134 def __str__(self):
135 if self.extra == '':
136 return '.'.join(self[:-1])
137 else:
138 return '.'.join(self)
140 def __eq__(self, other):
141 if not isinstance(other, CodesNSLCE):
142 other = CodesNSLCE(other)
144 return CodesNSLCEBase.__eq__(self, other)
146 @property
147 def safe_str(self):
148 return '.'.join(self)
150 @property
151 def nslce(self):
152 return self[:5]
154 @property
155 def nslc(self):
156 return self[:4]
158 @property
159 def nsl(self):
160 return self[:3]
162 @property
163 def ns(self):
164 return self[:2]
166 @property
167 def codes_nsl(self):
168 return CodesNSL(self)
170 @property
171 def codes_nsl_star(self):
172 return CodesNSL(self.network, self.station, '*')
174 def as_tuple(self):
175 return tuple(self)
177 def replace(self, **kwargs):
178 x = CodesNSLCEBase._replace(self, **kwargs)
179 return g_codes_pool.setdefault(x, x)
182def normalize_nsl(*args, **kwargs):
183 if args and kwargs:
184 raise ValueError('Either *args or **kwargs accepted, not both.')
186 if len(args) == 1:
187 if isinstance(args[0], str):
188 args = tuple(args[0].split('.'))
189 elif isinstance(args[0], tuple):
190 args = args[0]
191 else:
192 raise ValueError('Invalid argument type: %s' % type(args[0]))
194 nargs = len(args)
195 if nargs == 3:
196 t = args
198 elif nargs == 0:
199 d = dict(
200 network='',
201 station='',
202 location='')
204 d.update(kwargs)
205 t = tuple(kwargs.get(k, '') for k in (
206 'network', 'station', 'location'))
208 else:
209 raise CodesError(
210 'Does not match NSL codes pattern: %s' % '.'.join(args))
212 if '.'.join(t).count('.') != 2:
213 raise CodesError(
214 'Codes may not contain a ".": "%s", "%s", "%s"' % t)
216 return t
219CodesNSLBase = namedtuple(
220 'CodesNSLBase', [
221 'network', 'station', 'location'])
224class CodesNSL(CodesNSLBase, Codes):
225 '''
226 Codes denominating a seismic station (NSL).
228 NET.STA.LOC is accepted, slightly different from SEED/StationXML, where
229 LOC is part of the channel. By setting location='*' is possible to get
230 compatible behaviour in most cases.
231 '''
233 __slots__ = ()
234 __hash__ = CodesNSLBase.__hash__
236 as_dict = CodesNSLBase._asdict
238 def __new__(cls, *args, safe_str=None, **kwargs):
239 nargs = len(args)
240 if nargs == 1 and isinstance(args[0], CodesNSL):
241 return args[0]
242 elif nargs == 1 and isinstance(args[0], CodesNSLCE):
243 t = args[0].nsl
244 elif nargs == 1 and isinstance(args[0], CodesX):
245 t = ('*', '*', '*')
246 elif safe_str is not None:
247 t = safe_str.split('.')
248 else:
249 t = normalize_nsl(*args, **kwargs)
251 x = CodesNSLBase.__new__(cls, *t)
252 return g_codes_pool.setdefault(x, x)
254 def __init__(self, *args, **kwargs):
255 Codes.__init__(self)
257 def __str__(self):
258 return '.'.join(self)
260 def __eq__(self, other):
261 if not isinstance(other, CodesNSL):
262 other = CodesNSL(other)
264 return CodesNSLBase.__eq__(self, other)
266 @property
267 def safe_str(self):
268 return '.'.join(self)
270 @property
271 def ns(self):
272 return self[:2]
274 @property
275 def nsl(self):
276 return self[:3]
278 def as_tuple(self):
279 return tuple(self)
281 def replace(self, **kwargs):
282 x = CodesNSLBase._replace(self, **kwargs)
283 return g_codes_pool.setdefault(x, x)
286CodesXBase = namedtuple(
287 'CodesXBase', [
288 'name'])
291class CodesX(CodesXBase, Codes):
292 '''
293 General purpose codes for anything other than channels or stations.
294 '''
296 __slots__ = ()
297 __hash__ = CodesXBase.__hash__
298 __eq__ = CodesXBase.__eq__
300 as_dict = CodesXBase._asdict
302 def __new__(cls, name='', safe_str=None):
303 if isinstance(name, CodesX):
304 return name
305 elif isinstance(name, (CodesNSLCE, CodesNSL)):
306 name = '*'
307 elif safe_str is not None:
308 name = safe_str
309 else:
310 if '.' in name:
311 raise CodesError('Code may not contain a ".": %s' % name)
313 x = CodesXBase.__new__(cls, name)
314 return g_codes_pool.setdefault(x, x)
316 def __init__(self, *args, **kwargs):
317 Codes.__init__(self)
319 def __str__(self):
320 return '.'.join(self)
322 @property
323 def safe_str(self):
324 return '.'.join(self)
326 @property
327 def ns(self):
328 return self[:2]
330 def as_tuple(self):
331 return tuple(self)
333 def replace(self, **kwargs):
334 x = CodesXBase._replace(self, **kwargs)
335 return g_codes_pool.setdefault(x, x)
338g_codes_patterns = {}
341def match_codes(pattern, codes):
342 spattern = pattern.safe_str
343 scodes = codes.safe_str
344 if spattern not in g_codes_patterns:
345 rpattern = re.compile(fnmatch.translate(spattern), re.I)
346 g_codes_patterns[spattern] = rpattern
348 rpattern = g_codes_patterns[spattern]
349 return bool(rpattern.match(scodes))
352g_content_kinds = [
353 'undefined',
354 'waveform',
355 'station',
356 'channel',
357 'response',
358 'event',
359 'waveform_promise']
362g_codes_classes = [
363 CodesX,
364 CodesNSLCE,
365 CodesNSL,
366 CodesNSLCE,
367 CodesNSLCE,
368 CodesX,
369 CodesNSLCE]
371g_codes_classes_ndot = {
372 0: CodesX,
373 2: CodesNSL,
374 3: CodesNSLCE,
375 4: CodesNSLCE}
378def to_codes_simple(kind_id, codes_safe_str):
379 return g_codes_classes[kind_id](safe_str=codes_safe_str)
382def to_codes(kind_id, obj):
383 return g_codes_classes[kind_id](obj)
386def to_codes_guess(s):
387 try:
388 return g_codes_classes_ndot[s.count('.')](s)
389 except KeyError:
390 raise CodesError('Cannot guess codes type: %s' % s)
393# derived list class to enable detection of already preprocessed codes patterns
394class codes_patterns_list(list):
395 pass
398def codes_patterns_for_kind(kind_id, codes):
399 if isinstance(codes, codes_patterns_list):
400 return codes
402 if isinstance(codes, list):
403 lcodes = codes_patterns_list()
404 for sc in codes:
405 lcodes.extend(codes_patterns_for_kind(kind_id, sc))
407 return lcodes
409 codes = to_codes(kind_id, codes)
411 lcodes = codes_patterns_list()
412 lcodes.append(codes)
413 if kind_id == STATION:
414 lcodes.append(codes.replace(location='[*]'))
416 return lcodes
419g_content_kind_ids = (
420 UNDEFINED, WAVEFORM, STATION, CHANNEL, RESPONSE, EVENT,
421 WAVEFORM_PROMISE) = range(len(g_content_kinds))
424g_tmin, g_tmax = util.get_working_system_time_range()[:2]
427try:
428 g_tmin_queries = max(g_tmin, util.str_to_time_fillup('1900-01-01'))
429except Exception:
430 g_tmin_queries = g_tmin
433def to_kind(kind_id):
434 return g_content_kinds[kind_id]
437def to_kinds(kind_ids):
438 return [g_content_kinds[kind_id] for kind_id in kind_ids]
441def to_kind_id(kind):
442 return g_content_kinds.index(kind)
445def to_kind_ids(kinds):
446 return [g_content_kinds.index(kind) for kind in kinds]
449g_kind_mask_all = 0xff
452def to_kind_mask(kinds):
453 if kinds:
454 return sum(1 << kind_id for kind_id in to_kind_ids(kinds))
455 else:
456 return g_kind_mask_all
459def str_or_none(x):
460 if x is None:
461 return None
462 else:
463 return str(x)
466def float_or_none(x):
467 if x is None:
468 return None
469 else:
470 return float(x)
473def int_or_none(x):
474 if x is None:
475 return None
476 else:
477 return int(x)
480def int_or_g_tmin(x):
481 if x is None:
482 return g_tmin
483 else:
484 return int(x)
487def int_or_g_tmax(x):
488 if x is None:
489 return g_tmax
490 else:
491 return int(x)
494def tmin_or_none(x):
495 if x == g_tmin:
496 return None
497 else:
498 return x
501def tmax_or_none(x):
502 if x == g_tmax:
503 return None
504 else:
505 return x
508def time_or_none_to_str(x):
509 if x is None:
510 return '...'.ljust(17)
511 else:
512 return util.time_to_str(x)
515def codes_to_str_abbreviated(codes, indent=' '):
516 codes = [str(x) for x in codes]
518 if len(codes) > 20:
519 scodes = '\n' + util.ewrap(codes[:10], indent=indent) \
520 + '\n%s[%i more]\n' % (indent, len(codes) - 20) \
521 + util.ewrap(codes[-10:], indent=' ')
522 else:
523 scodes = '\n' + util.ewrap(codes, indent=indent) \
524 if codes else '<none>'
526 return scodes
529g_offset_time_unit_inv = 1000000000
530g_offset_time_unit = 1.0 / g_offset_time_unit_inv
533def tsplit(t):
534 if t is None:
535 return None, 0
537 t = util.to_time_float(t)
538 if type(t) is float:
539 t = round(t, 5)
540 else:
541 t = round(t, 9)
543 seconds = num.floor(t)
544 offset = t - seconds
545 return int(seconds), int(round(offset * g_offset_time_unit_inv))
548def tjoin(seconds, offset):
549 if seconds is None:
550 return None
552 return util.to_time_float(seconds) \
553 + util.to_time_float(offset*g_offset_time_unit)
556tscale_min = 1
557tscale_max = 365 * 24 * 3600 # last edge is one above
558tscale_logbase = 20
560tscale_edges = [tscale_min]
561while True:
562 tscale_edges.append(tscale_edges[-1]*tscale_logbase)
563 if tscale_edges[-1] >= tscale_max:
564 break
567tscale_edges = num.array(tscale_edges)
570def tscale_to_kscale(tscale):
572 # 0 <= x < tscale_edges[1]: 0
573 # tscale_edges[1] <= x < tscale_edges[2]: 1
574 # ...
575 # tscale_edges[len(tscale_edges)-1] <= x: len(tscale_edges)
577 return int(num.searchsorted(tscale_edges, tscale))
580@squirrel_content
581class Station(Location):
582 '''
583 A seismic station.
584 '''
586 codes = CodesNSL.T()
588 tmin = Timestamp.T(optional=True)
589 tmax = Timestamp.T(optional=True)
591 description = Unicode.T(optional=True)
593 def __init__(self, **kwargs):
594 kwargs['codes'] = CodesNSL(kwargs['codes'])
595 Location.__init__(self, **kwargs)
597 @property
598 def time_span(self):
599 return (self.tmin, self.tmax)
601 def get_pyrocko_station(self):
602 from pyrocko import model
603 return model.Station(*self._get_pyrocko_station_args())
605 def _get_pyrocko_station_args(self):
606 return (
607 self.codes.network,
608 self.codes.station,
609 self.codes.location,
610 self.lat,
611 self.lon,
612 self.elevation,
613 self.depth,
614 self.north_shift,
615 self.east_shift)
618class Sensor(Location):
619 '''
620 Representation of a channel group.
621 '''
623 codes = CodesNSLCE.T()
625 tmin = Timestamp.T(optional=True)
626 tmax = Timestamp.T(optional=True)
628 deltat = Float.T(optional=True)
630 @property
631 def time_span(self):
632 return (self.tmin, self.tmax)
634 def __init__(self, **kwargs):
635 kwargs['codes'] = CodesNSLCE(kwargs['codes'])
636 Location.__init__(self, **kwargs)
638 def _get_sensor_args(self):
639 def getattr_rep(k):
640 if k == 'codes':
641 return self.codes.replace(
642 channel=self.codes.channel[:-1] + '?')
643 else:
644 return getattr(self, k)
646 return tuple(getattr_rep(k) for k in Sensor.T.propnames)
648 @classmethod
649 def from_channels(cls, channels):
650 groups = defaultdict(list)
651 for channel in channels:
652 groups[channel._get_sensor_args()].append(channel)
654 return [
655 cls(**dict((k, v) for (k, v) in zip(cls.T.propnames, args)))
656 for args, _ in groups.items()]
658 def _get_pyrocko_station_args(self):
659 return (
660 self.codes.network,
661 self.codes.station,
662 self.codes.location,
663 self.lat,
664 self.lon,
665 self.elevation,
666 self.depth,
667 self.north_shift,
668 self.east_shift)
671@squirrel_content
672class Channel(Sensor):
673 '''
674 A channel of a seismic station.
675 '''
677 dip = Float.T(optional=True)
678 azimuth = Float.T(optional=True)
680 @classmethod
681 def from_channels(cls, channels):
682 raise NotImplementedError()
684 def get_pyrocko_channel(self):
685 from pyrocko import model
686 return model.Channel(*self._get_pyrocko_channel_args())
688 def _get_pyrocko_channel_args(self):
689 return (
690 self.codes.channel,
691 self.azimuth,
692 self.dip)
695observational_quantities = [
696 'acceleration', 'velocity', 'displacement', 'pressure', 'rotation',
697 'temperature']
700technical_quantities = [
701 'voltage', 'counts']
704class QuantityType(StringChoice):
705 '''
706 Choice of observational or technical quantity.
708 SI units are used for all quantities, where applicable.
709 '''
710 choices = observational_quantities + technical_quantities
713class ResponseStage(Object):
714 '''
715 Representation of a response stage.
717 Components of a seismic recording system are represented as a sequence of
718 response stages, e.g. sensor, pre-amplifier, digitizer, digital
719 downsampling.
720 '''
721 input_quantity = QuantityType.T(optional=True)
722 input_sample_rate = Float.T(optional=True)
723 output_quantity = QuantityType.T(optional=True)
724 output_sample_rate = Float.T(optional=True)
725 elements = List.T(FrequencyResponse.T())
726 log = List.T(Tuple.T(3, String.T()))
728 @property
729 def stage_type(self):
730 if self.input_quantity in observational_quantities \
731 and self.output_quantity in observational_quantities:
732 return 'conversion'
734 if self.input_quantity in observational_quantities \
735 and self.output_quantity == 'voltage':
736 return 'sensor'
738 elif self.input_quantity == 'voltage' \
739 and self.output_quantity == 'voltage':
740 return 'electronics'
742 elif self.input_quantity == 'voltage' \
743 and self.output_quantity == 'counts':
744 return 'digitizer'
746 elif self.input_quantity == 'counts' \
747 and self.output_quantity == 'counts' \
748 and self.input_sample_rate != self.output_sample_rate:
749 return 'decimation'
751 elif self.input_quantity in observational_quantities \
752 and self.output_quantity == 'counts':
753 return 'combination'
755 else:
756 return 'unknown'
758 @property
759 def summary(self):
760 irate = self.input_sample_rate
761 orate = self.output_sample_rate
762 factor = None
763 if irate and orate:
764 factor = irate / orate
765 return 'ResponseStage, ' + (
766 '%s%s => %s%s%s' % (
767 self.input_quantity or '?',
768 ' @ %g Hz' % irate if irate else '',
769 self.output_quantity or '?',
770 ' @ %g Hz' % orate if orate else '',
771 ' :%g' % factor if factor else '')
772 )
774 def get_effective(self):
775 return MultiplyResponse(responses=list(self.elements))
778D = 'displacement'
779V = 'velocity'
780A = 'acceleration'
782g_converters = {
783 (V, D): IntegrationResponse(1),
784 (A, D): IntegrationResponse(2),
785 (D, V): DifferentiationResponse(1),
786 (A, V): IntegrationResponse(1),
787 (D, A): DifferentiationResponse(2),
788 (V, A): DifferentiationResponse(1)}
791def response_converters(input_quantity, output_quantity):
792 if input_quantity is None or input_quantity == output_quantity:
793 return []
795 if output_quantity is None:
796 raise ConversionError('Unspecified target quantity.')
798 try:
799 return [g_converters[input_quantity, output_quantity]]
801 except KeyError:
802 raise ConversionError('No rule to convert from "%s" to "%s".' % (
803 input_quantity, output_quantity))
806@squirrel_content
807class Response(Object):
808 '''
809 The instrument response of a seismic station channel.
810 '''
812 codes = CodesNSLCE.T()
813 tmin = Timestamp.T(optional=True)
814 tmax = Timestamp.T(optional=True)
816 stages = List.T(ResponseStage.T())
817 checkpoints = List.T(FrequencyResponseCheckpoint.T())
819 deltat = Float.T(optional=True)
820 log = List.T(Tuple.T(3, String.T()))
822 def __init__(self, **kwargs):
823 kwargs['codes'] = CodesNSLCE(kwargs['codes'])
824 Object.__init__(self, **kwargs)
826 @property
827 def time_span(self):
828 return (self.tmin, self.tmax)
830 @property
831 def nstages(self):
832 return len(self.stages)
834 @property
835 def input_quantity(self):
836 return self.stages[0].input_quantity if self.stages else None
838 @property
839 def output_quantity(self):
840 return self.stages[-1].input_quantity if self.stages else None
842 @property
843 def output_sample_rate(self):
844 return self.stages[-1].output_sample_rate if self.stages else None
846 @property
847 def stages_summary(self):
848 def grouped(xs):
849 xs = list(xs)
850 g = []
851 for i in range(len(xs)):
852 g.append(xs[i])
853 if i+1 < len(xs) and xs[i+1] != xs[i]:
854 yield g
855 g = []
857 if g:
858 yield g
860 return '+'.join(
861 '%s%s' % (g[0], '(%i)' % len(g) if len(g) > 1 else '')
862 for g in grouped(stage.stage_type for stage in self.stages))
864 @property
865 def summary(self):
866 orate = self.output_sample_rate
867 return '%s %-16s %s' % (
868 self.__class__.__name__, self.str_codes, self.str_time_span) \
869 + ', ' + ', '.join((
870 '%s => %s' % (
871 self.input_quantity or '?', self.output_quantity or '?')
872 + (' @ %g Hz' % orate if orate else ''),
873 self.stages_summary,
874 ))
876 def get_effective(self, input_quantity=None):
877 try:
878 elements = response_converters(input_quantity, self.input_quantity)
879 except ConversionError as e:
880 raise ConversionError(str(e) + ' (%s)' % self.summary)
882 elements.extend(
883 stage.get_effective() for stage in self.stages)
885 return MultiplyResponse(responses=simplify_responses(elements))
888@squirrel_content
889class Event(Object):
890 '''
891 A seismic event.
892 '''
894 name = String.T(optional=True)
895 time = Timestamp.T()
896 duration = Float.T(optional=True)
898 lat = Float.T()
899 lon = Float.T()
900 depth = Float.T(optional=True)
902 magnitude = Float.T(optional=True)
904 def get_pyrocko_event(self):
905 from pyrocko import model
906 model.Event(
907 name=self.name,
908 time=self.time,
909 lat=self.lat,
910 lon=self.lon,
911 depth=self.depth,
912 magnitude=self.magnitude,
913 duration=self.duration)
915 @property
916 def time_span(self):
917 return (self.time, self.time)
920def ehash(s):
921 return hashlib.sha1(s.encode('utf8')).hexdigest()
924def random_name(n=8):
925 return urlsafe_b64encode(urandom(n)).rstrip(b'=').decode('ascii')
928@squirrel_content
929class WaveformPromise(Object):
930 '''
931 Information about a waveform potentially downloadable from a remote site.
933 In the Squirrel framework, waveform promises are used to permit download of
934 selected waveforms from a remote site. They are typically generated by
935 calls to
936 :py:meth:`~pyrocko.squirrel.base.Squirrel.update_waveform_promises`.
937 Waveform promises are inserted and indexed in the database similar to
938 normal waveforms. When processing a waveform query, e.g. from
939 :py:meth:`~pyrocko.squirrel.base.Squirrel.get_waveform`, and no local
940 waveform is available for the queried time span, a matching promise can be
941 resolved, i.e. an attempt is made to download the waveform from the remote
942 site. The promise is removed after the download attempt (except when a
943 network error occurs). This prevents Squirrel from making unnecessary
944 queries for waveforms missing at the remote site.
945 '''
947 codes = CodesNSLCE.T()
948 tmin = Timestamp.T()
949 tmax = Timestamp.T()
951 deltat = Float.T(optional=True)
953 source_hash = String.T()
955 def __init__(self, **kwargs):
956 kwargs['codes'] = CodesNSLCE(kwargs['codes'])
957 Object.__init__(self, **kwargs)
959 @property
960 def time_span(self):
961 return (self.tmin, self.tmax)
964class InvalidWaveform(Exception):
965 pass
968class WaveformOrder(Object):
969 '''
970 Waveform request information.
971 '''
973 source_id = String.T()
974 codes = CodesNSLCE.T()
975 deltat = Float.T()
976 tmin = Timestamp.T()
977 tmax = Timestamp.T()
978 gaps = List.T(Tuple.T(2, Timestamp.T()))
980 @property
981 def client(self):
982 return self.source_id.split(':')[1]
984 def describe(self, site='?'):
985 return '%s:%s %s [%s - %s]' % (
986 self.client, site, str(self.codes),
987 util.time_to_str(self.tmin), util.time_to_str(self.tmax))
989 def validate(self, tr):
990 if tr.ydata.size == 0:
991 raise InvalidWaveform(
992 'waveform with zero data samples')
994 if tr.deltat != self.deltat:
995 raise InvalidWaveform(
996 'incorrect sampling interval - waveform: %g s, '
997 'meta-data: %g s' % (
998 tr.deltat, self.deltat))
1000 if not num.all(num.isfinite(tr.ydata)):
1001 raise InvalidWaveform('waveform has NaN values')
1004def order_summary(orders):
1005 codes_list = sorted(set(order.codes for order in orders))
1006 if len(codes_list) > 3:
1007 return '%i order%s: %s - %s' % (
1008 len(orders),
1009 util.plural_s(orders),
1010 str(codes_list[0]),
1011 str(codes_list[1]))
1013 else:
1014 return '%i order%s: %s' % (
1015 len(orders),
1016 util.plural_s(orders),
1017 ', '.join(str(codes) for codes in codes_list))
1020class Nut(Object):
1021 '''
1022 Index entry referencing an elementary piece of content.
1024 So-called *nuts* are used in Pyrocko's Squirrel framework to hold common
1025 meta-information about individual pieces of waveforms, stations, channels,
1026 etc. together with the information where it was found or generated.
1027 '''
1029 file_path = String.T(optional=True)
1030 file_format = String.T(optional=True)
1031 file_mtime = Timestamp.T(optional=True)
1032 file_size = Int.T(optional=True)
1034 file_segment = Int.T(optional=True)
1035 file_element = Int.T(optional=True)
1037 kind_id = Int.T()
1038 codes = Codes.T()
1040 tmin_seconds = Int.T(default=0)
1041 tmin_offset = Int.T(default=0, optional=True)
1042 tmax_seconds = Int.T(default=0)
1043 tmax_offset = Int.T(default=0, optional=True)
1045 deltat = Float.T(default=0.0)
1047 content = Any.T(optional=True)
1048 raw_content = Dict.T(String.T(), Any.T())
1050 content_in_db = False
1052 def __init__(
1053 self,
1054 file_path=None,
1055 file_format=None,
1056 file_mtime=None,
1057 file_size=None,
1058 file_segment=None,
1059 file_element=None,
1060 kind_id=0,
1061 codes=CodesX(''),
1062 tmin_seconds=None,
1063 tmin_offset=0,
1064 tmax_seconds=None,
1065 tmax_offset=0,
1066 deltat=None,
1067 content=None,
1068 raw_content=None,
1069 tmin=None,
1070 tmax=None,
1071 values_nocheck=None):
1073 if values_nocheck is not None:
1074 (self.file_path, self.file_format, self.file_mtime,
1075 self.file_size,
1076 self.file_segment, self.file_element,
1077 self.kind_id, codes_safe_str,
1078 self.tmin_seconds, self.tmin_offset,
1079 self.tmax_seconds, self.tmax_offset,
1080 self.deltat) = values_nocheck
1082 self.codes = to_codes_simple(self.kind_id, codes_safe_str)
1083 self.deltat = self.deltat or None
1084 self.raw_content = {}
1085 self.content = None
1086 else:
1087 if tmin is not None:
1088 tmin_seconds, tmin_offset = tsplit(tmin)
1090 if tmax is not None:
1091 tmax_seconds, tmax_offset = tsplit(tmax)
1093 self.kind_id = int(kind_id)
1094 self.codes = codes
1095 self.tmin_seconds = int_or_g_tmin(tmin_seconds)
1096 self.tmin_offset = int(tmin_offset)
1097 self.tmax_seconds = int_or_g_tmax(tmax_seconds)
1098 self.tmax_offset = int(tmax_offset)
1099 self.deltat = float_or_none(deltat)
1100 self.file_path = str_or_none(file_path)
1101 self.file_segment = int_or_none(file_segment)
1102 self.file_element = int_or_none(file_element)
1103 self.file_format = str_or_none(file_format)
1104 self.file_mtime = float_or_none(file_mtime)
1105 self.file_size = int_or_none(file_size)
1106 self.content = content
1107 if raw_content is None:
1108 self.raw_content = {}
1109 else:
1110 self.raw_content = raw_content
1112 Object.__init__(self, init_props=False)
1114 def __eq__(self, other):
1115 return (isinstance(other, Nut) and
1116 self.equality_values == other.equality_values)
1118 def hash(self):
1119 return ehash(','.join(str(x) for x in self.key))
1121 def __ne__(self, other):
1122 return not (self == other)
1124 def get_io_backend(self):
1125 from . import io
1126 return io.get_backend(self.file_format)
1128 def file_modified(self):
1129 return self.get_io_backend().get_stats(self.file_path) \
1130 != (self.file_mtime, self.file_size)
1132 @property
1133 def dkey(self):
1134 return (self.kind_id, self.tmin_seconds, self.tmin_offset, self.codes)
1136 @property
1137 def key(self):
1138 return (
1139 self.file_path,
1140 self.file_segment,
1141 self.file_element,
1142 self.file_mtime)
1144 @property
1145 def equality_values(self):
1146 return (
1147 self.file_segment, self.file_element,
1148 self.kind_id, self.codes,
1149 self.tmin_seconds, self.tmin_offset,
1150 self.tmax_seconds, self.tmax_offset, self.deltat)
1152 def diff(self, other):
1153 names = [
1154 'file_segment', 'file_element', 'kind_id', 'codes',
1155 'tmin_seconds', 'tmin_offset', 'tmax_seconds', 'tmax_offset',
1156 'deltat']
1158 d = []
1159 for name, a, b in zip(
1160 names, self.equality_values, other.equality_values):
1162 if a != b:
1163 d.append((name, a, b))
1165 return d
1167 @property
1168 def tmin(self):
1169 return tjoin(self.tmin_seconds, self.tmin_offset)
1171 @tmin.setter
1172 def tmin(self, t):
1173 self.tmin_seconds, self.tmin_offset = tsplit(t)
1175 @property
1176 def tmax(self):
1177 return tjoin(self.tmax_seconds, self.tmax_offset)
1179 @tmax.setter
1180 def tmax(self, t):
1181 self.tmax_seconds, self.tmax_offset = tsplit(t)
1183 @property
1184 def kscale(self):
1185 if self.tmin_seconds is None or self.tmax_seconds is None:
1186 return 0
1187 return tscale_to_kscale(self.tmax_seconds - self.tmin_seconds)
1189 @property
1190 def waveform_kwargs(self):
1191 network, station, location, channel, extra = self.codes
1193 return dict(
1194 network=network,
1195 station=station,
1196 location=location,
1197 channel=channel,
1198 extra=extra,
1199 tmin=self.tmin,
1200 tmax=self.tmax,
1201 deltat=self.deltat)
1203 @property
1204 def waveform_promise_kwargs(self):
1205 return dict(
1206 codes=self.codes,
1207 tmin=self.tmin,
1208 tmax=self.tmax,
1209 deltat=self.deltat)
1211 @property
1212 def station_kwargs(self):
1213 network, station, location = self.codes
1214 return dict(
1215 codes=self.codes,
1216 tmin=tmin_or_none(self.tmin),
1217 tmax=tmax_or_none(self.tmax))
1219 @property
1220 def channel_kwargs(self):
1221 network, station, location, channel, extra = self.codes
1222 return dict(
1223 codes=self.codes,
1224 tmin=tmin_or_none(self.tmin),
1225 tmax=tmax_or_none(self.tmax),
1226 deltat=self.deltat)
1228 @property
1229 def response_kwargs(self):
1230 return dict(
1231 codes=self.codes,
1232 tmin=tmin_or_none(self.tmin),
1233 tmax=tmax_or_none(self.tmax),
1234 deltat=self.deltat)
1236 @property
1237 def event_kwargs(self):
1238 return dict(
1239 name=self.codes,
1240 time=self.tmin,
1241 duration=(self.tmax - self.tmin) or None)
1243 @property
1244 def trace_kwargs(self):
1245 network, station, location, channel, extra = self.codes
1247 return dict(
1248 network=network,
1249 station=station,
1250 location=location,
1251 channel=channel,
1252 extra=extra,
1253 tmin=self.tmin,
1254 tmax=self.tmax-self.deltat,
1255 deltat=self.deltat)
1257 @property
1258 def dummy_trace(self):
1259 return DummyTrace(self)
1261 @property
1262 def summary(self):
1263 if self.tmin == self.tmax:
1264 ts = util.time_to_str(self.tmin)
1265 else:
1266 ts = '%s - %s' % (
1267 util.time_to_str(self.tmin),
1268 util.time_to_str(self.tmax))
1270 return ' '.join((
1271 ('%s,' % to_kind(self.kind_id)).ljust(9),
1272 ('%s,' % str(self.codes)).ljust(18),
1273 ts))
1276def make_waveform_nut(**kwargs):
1277 return Nut(kind_id=WAVEFORM, **kwargs)
1280def make_waveform_promise_nut(**kwargs):
1281 return Nut(kind_id=WAVEFORM_PROMISE, **kwargs)
1284def make_station_nut(**kwargs):
1285 return Nut(kind_id=STATION, **kwargs)
1288def make_channel_nut(**kwargs):
1289 return Nut(kind_id=CHANNEL, **kwargs)
1292def make_response_nut(**kwargs):
1293 return Nut(kind_id=RESPONSE, **kwargs)
1296def make_event_nut(**kwargs):
1297 return Nut(kind_id=EVENT, **kwargs)
1300def group_channels(nuts):
1301 by_ansl = {}
1302 for nut in nuts:
1303 if nut.kind_id != CHANNEL:
1304 continue
1306 ansl = nut.codes[:4]
1308 if ansl not in by_ansl:
1309 by_ansl[ansl] = {}
1311 group = by_ansl[ansl]
1313 k = nut.codes[4][:-1], nut.deltat, nut.tmin, nut.tmax
1315 if k not in group:
1316 group[k] = set()
1318 group.add(nut.codes[4])
1320 return by_ansl
1323class DummyTrace(object):
1325 def __init__(self, nut):
1326 self.nut = nut
1327 self.codes = nut.codes
1328 self.meta = {}
1330 @property
1331 def tmin(self):
1332 return self.nut.tmin
1334 @property
1335 def tmax(self):
1336 return self.nut.tmax
1338 @property
1339 def deltat(self):
1340 return self.nut.deltat
1342 @property
1343 def nslc_id(self):
1344 return self.codes.nslc
1346 @property
1347 def network(self):
1348 return self.codes.network
1350 @property
1351 def station(self):
1352 return self.codes.station
1354 @property
1355 def location(self):
1356 return self.codes.location
1358 @property
1359 def channel(self):
1360 return self.codes.channel
1362 @property
1363 def extra(self):
1364 return self.codes.extra
1366 def overlaps(self, tmin, tmax):
1367 return not (tmax < self.nut.tmin or self.nut.tmax < tmin)
1370def duration_to_str(t):
1371 if t > 24*3600:
1372 return '%gd' % (t / (24.*3600.))
1373 elif t > 3600:
1374 return '%gh' % (t / 3600.)
1375 elif t > 60:
1376 return '%gm' % (t / 60.)
1377 else:
1378 return '%gs' % t
1381class Coverage(Object):
1382 '''
1383 Information about times covered by a waveform or other time series data.
1384 '''
1385 kind_id = Int.T(
1386 help='Content type.')
1387 pattern = Codes.T(
1388 help='The codes pattern in the request, which caused this entry to '
1389 'match.')
1390 codes = Codes.T(
1391 help='NSLCE or NSL codes identifier of the time series.')
1392 deltat = Float.T(
1393 help='Sampling interval [s]',
1394 optional=True)
1395 tmin = Timestamp.T(
1396 help='Global start time of time series.',
1397 optional=True)
1398 tmax = Timestamp.T(
1399 help='Global end time of time series.',
1400 optional=True)
1401 changes = List.T(
1402 Tuple.T(2, Any.T()),
1403 help='List of change points, with entries of the form '
1404 '``(time, count)``, where a ``count`` of zero indicates start of '
1405 'a gap, a value of 1 start of normal data coverage and a higher '
1406 'value duplicate or redundant data coverage.')
1408 @classmethod
1409 def from_values(cls, args):
1410 pattern, codes, deltat, tmin, tmax, changes, kind_id = args
1411 return cls(
1412 kind_id=kind_id,
1413 pattern=pattern,
1414 codes=codes,
1415 deltat=deltat,
1416 tmin=tmin,
1417 tmax=tmax,
1418 changes=changes)
1420 @property
1421 def summary(self):
1422 ts = '%s - %s,' % (
1423 util.time_to_str(self.tmin),
1424 util.time_to_str(self.tmax))
1426 srate = self.sample_rate
1428 return ' '.join((
1429 ('%s,' % to_kind(self.kind_id)).ljust(9),
1430 ('%s,' % str(self.codes)).ljust(18),
1431 ts,
1432 '%10.3g,' % srate if srate else '',
1433 '%4i' % len(self.changes),
1434 '%s' % duration_to_str(self.total)))
1436 @property
1437 def sample_rate(self):
1438 if self.deltat is None:
1439 return None
1440 elif self.deltat == 0.0:
1441 return 0.0
1442 else:
1443 return 1.0 / self.deltat
1445 @property
1446 def labels(self):
1447 srate = self.sample_rate
1448 return (
1449 ('%s' % str(self.codes)),
1450 '%.3g' % srate if srate else '')
1452 @property
1453 def total(self):
1454 total_t = None
1455 for tmin, tmax, _ in self.iter_spans():
1456 total_t = (total_t or 0.0) + (tmax - tmin)
1458 return total_t
1460 def iter_spans(self):
1461 last = None
1462 for (t, count) in self.changes:
1463 if last is not None:
1464 last_t, last_count = last
1465 if last_count > 0:
1466 yield last_t, t, last_count
1468 last = (t, count)
1471class LocationPool(object):
1473 def __init__(self, squirrel, tmin, tmax):
1475 locations = {}
1476 for station in squirrel.get_stations(tmin=tmin, tmax=tmax):
1477 c = station.codes
1478 if c not in locations:
1479 locations[c] = station
1480 else:
1481 if locations[c] is not None \
1482 and not locations[c].same_location(station):
1484 locations[c] = None
1486 for channel in squirrel.get_channels(tmin=tmin, tmax=tmax):
1487 c = channel.codes
1488 if c not in locations:
1489 locations[c] = channel
1490 else:
1491 if locations[c] is not None \
1492 and not locations[c].same_location(channel):
1494 locations[c] = None
1496 self._locations = locations
1498 def get(self, codes):
1499 try:
1500 return self._locations[codes]
1501 except KeyError:
1502 pass
1504 try:
1505 return self._locations[codes.codes_nsl]
1506 except KeyError:
1507 pass
1509 try:
1510 return self._locations[codes.codes_nsl_star]
1511 except KeyError:
1512 return None
1515__all__ = [
1516 'to_codes',
1517 'to_codes_guess',
1518 'to_codes_simple',
1519 'to_kind',
1520 'to_kinds',
1521 'to_kind_id',
1522 'to_kind_ids',
1523 'CodesError',
1524 'Codes',
1525 'CodesNSLCE',
1526 'CodesNSL',
1527 'CodesX',
1528 'Station',
1529 'Channel',
1530 'Sensor',
1531 'Response',
1532 'Nut',
1533 'Coverage',
1534 'WaveformPromise',
1535]