1# http://pyrocko.org - GPLv3
2#
3# The Pyrocko Developers, 21st Century
4# ---|P------/S----------~Lg----------
6'''
7Data model and content types handled by the Squirrel framework.
9Squirrel uses flat content types to represent waveform, station, channel,
10response, event, and a few other objects. A common subset of the information in
11these objects is indexed in the database, currently: kind, codes, time interval
12and sampling rate. The :py:class:`Nut` objects encapsulate this information
13together with information about where and how to get the associated bulk data.
15Further content types are defined here to handle waveform orders, waveform
16promises, data coverage and sensor information.
17'''
19from __future__ import absolute_import, print_function
21import re
22import fnmatch
23import hashlib
24import numpy as num
25from os import urandom
26from base64 import urlsafe_b64encode
27from collections import defaultdict, namedtuple
29from pyrocko import util
30from pyrocko.guts import Object, SObject, String, Timestamp, Float, Int, \
31 Unicode, Tuple, List, StringChoice, Any, Dict
32from pyrocko.model import squirrel_content, Location
33from pyrocko.response import FrequencyResponse, MultiplyResponse, \
34 IntegrationResponse, DifferentiationResponse, simplify_responses, \
35 FrequencyResponseCheckpoint
37from .error import ConversionError, SquirrelError
40guts_prefix = 'squirrel'
43g_codes_pool = {}
46class CodesError(SquirrelError):
47 pass
50class Codes(SObject):
51 pass
54def normalize_nslce(*args, **kwargs):
55 if args and kwargs:
56 raise ValueError('Either *args or **kwargs accepted, not both.')
58 if len(args) == 1:
59 if isinstance(args[0], str):
60 args = tuple(args[0].split('.'))
61 elif isinstance(args[0], tuple):
62 args = args[0]
63 else:
64 raise ValueError('Invalid argument type: %s' % type(args[0]))
66 nargs = len(args)
67 if nargs == 5:
68 t = args
70 elif nargs == 4:
71 t = args + ('',)
73 elif nargs == 0:
74 d = dict(
75 network='',
76 station='',
77 location='',
78 channel='',
79 extra='')
81 d.update(kwargs)
82 t = tuple(kwargs.get(k, '') for k in (
83 'network', 'station', 'location', 'channel', 'extra'))
85 else:
86 raise CodesError(
87 'Does not match NSLC or NSLCE codes pattern: %s' % '.'.join(args))
89 if '.'.join(t).count('.') != 4:
90 raise CodesError(
91 'Codes may not contain a ".": "%s", "%s", "%s", "%s", "%s"' % t)
93 return t
96CodesNSLCEBase = namedtuple(
97 'CodesNSLCEBase', [
98 'network', 'station', 'location', 'channel', 'extra'])
101class CodesNSLCE(CodesNSLCEBase, Codes):
102 '''
103 Codes denominating a seismic channel (NSLC or NSLCE).
105 FDSN/SEED style NET.STA.LOC.CHA is accepted or NET.STA.LOC.CHA.EXTRA, where
106 the EXTRA part in the latter form can be used to identify a custom
107 processing applied to a channel.
108 '''
110 __slots__ = ()
111 __hash__ = CodesNSLCEBase.__hash__
113 as_dict = CodesNSLCEBase._asdict
115 def __new__(cls, *args, safe_str=None, **kwargs):
116 nargs = len(args)
117 if nargs == 1 and isinstance(args[0], CodesNSLCE):
118 return args[0]
119 elif nargs == 1 and isinstance(args[0], CodesNSL):
120 t = (args[0].nsl) + ('*', '*')
121 elif nargs == 1 and isinstance(args[0], CodesX):
122 t = ('*', '*', '*', '*', '*')
123 elif safe_str is not None:
124 t = safe_str.split('.')
125 else:
126 t = normalize_nslce(*args, **kwargs)
128 x = CodesNSLCEBase.__new__(cls, *t)
129 return g_codes_pool.setdefault(x, x)
131 def __init__(self, *args, **kwargs):
132 Codes.__init__(self)
134 def __str__(self):
135 if self.extra == '':
136 return '.'.join(self[:-1])
137 else:
138 return '.'.join(self)
140 def __eq__(self, other):
141 if not isinstance(other, CodesNSLCE):
142 other = CodesNSLCE(other)
144 return CodesNSLCEBase.__eq__(self, other)
146 @property
147 def safe_str(self):
148 return '.'.join(self)
150 @property
151 def nslce(self):
152 return self[:4]
154 @property
155 def nslc(self):
156 return self[:4]
158 @property
159 def nsl(self):
160 return self[:3]
162 @property
163 def ns(self):
164 return self[:2]
166 def as_tuple(self):
167 return tuple(self)
169 def replace(self, **kwargs):
170 x = CodesNSLCEBase._replace(self, **kwargs)
171 return g_codes_pool.setdefault(x, x)
174def normalize_nsl(*args, **kwargs):
175 if args and kwargs:
176 raise ValueError('Either *args or **kwargs accepted, not both.')
178 if len(args) == 1:
179 if isinstance(args[0], str):
180 args = tuple(args[0].split('.'))
181 elif isinstance(args[0], tuple):
182 args = args[0]
183 else:
184 raise ValueError('Invalid argument type: %s' % type(args[0]))
186 nargs = len(args)
187 if nargs == 3:
188 t = args
190 elif nargs == 0:
191 d = dict(
192 network='',
193 station='',
194 location='')
196 d.update(kwargs)
197 t = tuple(kwargs.get(k, '') for k in (
198 'network', 'station', 'location'))
200 else:
201 raise CodesError(
202 'Does not match NSL codes pattern: %s' % '.'.join(args))
204 if '.'.join(t).count('.') != 2:
205 raise CodesError(
206 'Codes may not contain a ".": "%s", "%s", "%s"' % t)
208 return t
211CodesNSLBase = namedtuple(
212 'CodesNSLBase', [
213 'network', 'station', 'location'])
216class CodesNSL(CodesNSLBase, Codes):
217 '''
218 Codes denominating a seismic station (NSL).
220 NET.STA.LOC is accepted, slightly different from SEED/StationXML, where
221 LOC is part of the channel. By setting location='*' is possible to get
222 compatible behaviour in most cases.
223 '''
225 __slots__ = ()
226 __hash__ = CodesNSLBase.__hash__
228 as_dict = CodesNSLBase._asdict
230 def __new__(cls, *args, safe_str=None, **kwargs):
231 nargs = len(args)
232 if nargs == 1 and isinstance(args[0], CodesNSL):
233 return args[0]
234 elif nargs == 1 and isinstance(args[0], CodesNSLCE):
235 t = args[0].nsl
236 elif nargs == 1 and isinstance(args[0], CodesX):
237 t = ('*', '*', '*')
238 elif safe_str is not None:
239 t = safe_str.split('.')
240 else:
241 t = normalize_nsl(*args, **kwargs)
243 x = CodesNSLBase.__new__(cls, *t)
244 return g_codes_pool.setdefault(x, x)
246 def __init__(self, *args, **kwargs):
247 Codes.__init__(self)
249 def __str__(self):
250 return '.'.join(self)
252 def __eq__(self, other):
253 if not isinstance(other, CodesNSL):
254 other = CodesNSL(other)
256 return CodesNSLBase.__eq__(self, other)
258 @property
259 def safe_str(self):
260 return '.'.join(self)
262 @property
263 def ns(self):
264 return self[:2]
266 @property
267 def nsl(self):
268 return self[:3]
270 def as_tuple(self):
271 return tuple(self)
273 def replace(self, **kwargs):
274 x = CodesNSLBase._replace(self, **kwargs)
275 return g_codes_pool.setdefault(x, x)
278CodesXBase = namedtuple(
279 'CodesXBase', [
280 'name'])
283class CodesX(CodesXBase, Codes):
284 '''
285 General purpose codes for anything other than channels or stations.
286 '''
288 __slots__ = ()
289 __hash__ = CodesXBase.__hash__
290 __eq__ = CodesXBase.__eq__
292 as_dict = CodesXBase._asdict
294 def __new__(cls, name='', safe_str=None):
295 if isinstance(name, CodesX):
296 return name
297 elif isinstance(name, (CodesNSLCE, CodesNSL)):
298 name = '*'
299 elif safe_str is not None:
300 name = safe_str
301 else:
302 if '.' in name:
303 raise CodesError('Code may not contain a ".": %s' % name)
305 x = CodesXBase.__new__(cls, name)
306 return g_codes_pool.setdefault(x, x)
308 def __init__(self, *args, **kwargs):
309 Codes.__init__(self)
311 def __str__(self):
312 return '.'.join(self)
314 @property
315 def safe_str(self):
316 return '.'.join(self)
318 @property
319 def ns(self):
320 return self[:2]
322 def as_tuple(self):
323 return tuple(self)
325 def replace(self, **kwargs):
326 x = CodesXBase._replace(self, **kwargs)
327 return g_codes_pool.setdefault(x, x)
330g_codes_patterns = {}
333def match_codes(pattern, codes):
334 spattern = pattern.safe_str
335 scodes = codes.safe_str
336 if spattern not in g_codes_patterns:
337 rpattern = re.compile(fnmatch.translate(spattern), re.I)
338 g_codes_patterns[spattern] = rpattern
340 rpattern = g_codes_patterns[spattern]
341 return bool(rpattern.match(scodes))
344g_content_kinds = [
345 'undefined',
346 'waveform',
347 'station',
348 'channel',
349 'response',
350 'event',
351 'waveform_promise']
354g_codes_classes = [
355 CodesX,
356 CodesNSLCE,
357 CodesNSL,
358 CodesNSLCE,
359 CodesNSLCE,
360 CodesX,
361 CodesNSLCE]
363g_codes_classes_ndot = {
364 0: CodesX,
365 2: CodesNSL,
366 3: CodesNSLCE,
367 4: CodesNSLCE}
370def to_codes_simple(kind_id, codes_safe_str):
371 return g_codes_classes[kind_id](safe_str=codes_safe_str)
374def to_codes(kind_id, obj):
375 return g_codes_classes[kind_id](obj)
378def to_codes_guess(s):
379 try:
380 return g_codes_classes_ndot[s.count('.')](s)
381 except KeyError:
382 raise CodesError('Cannot guess codes type: %s' % s)
385# derived list class to enable detection of already preprocessed codes patterns
386class codes_patterns_list(list):
387 pass
390def codes_patterns_for_kind(kind_id, codes):
391 if isinstance(codes, codes_patterns_list):
392 return codes
394 if isinstance(codes, list):
395 lcodes = codes_patterns_list()
396 for sc in codes:
397 lcodes.extend(codes_patterns_for_kind(kind_id, sc))
399 return lcodes
401 codes = to_codes(kind_id, codes)
403 lcodes = codes_patterns_list()
404 lcodes.append(codes)
405 if kind_id == STATION:
406 lcodes.append(codes.replace(location='[*]'))
408 return lcodes
411g_content_kind_ids = (
412 UNDEFINED, WAVEFORM, STATION, CHANNEL, RESPONSE, EVENT,
413 WAVEFORM_PROMISE) = range(len(g_content_kinds))
416g_tmin, g_tmax = util.get_working_system_time_range()[:2]
419try:
420 g_tmin_queries = max(g_tmin, util.str_to_time_fillup('1900-01-01'))
421except Exception:
422 g_tmin_queries = g_tmin
425def to_kind(kind_id):
426 return g_content_kinds[kind_id]
429def to_kinds(kind_ids):
430 return [g_content_kinds[kind_id] for kind_id in kind_ids]
433def to_kind_id(kind):
434 return g_content_kinds.index(kind)
437def to_kind_ids(kinds):
438 return [g_content_kinds.index(kind) for kind in kinds]
441g_kind_mask_all = 0xff
444def to_kind_mask(kinds):
445 if kinds:
446 return sum(1 << kind_id for kind_id in to_kind_ids(kinds))
447 else:
448 return g_kind_mask_all
451def str_or_none(x):
452 if x is None:
453 return None
454 else:
455 return str(x)
458def float_or_none(x):
459 if x is None:
460 return None
461 else:
462 return float(x)
465def int_or_none(x):
466 if x is None:
467 return None
468 else:
469 return int(x)
472def int_or_g_tmin(x):
473 if x is None:
474 return g_tmin
475 else:
476 return int(x)
479def int_or_g_tmax(x):
480 if x is None:
481 return g_tmax
482 else:
483 return int(x)
486def tmin_or_none(x):
487 if x == g_tmin:
488 return None
489 else:
490 return x
493def tmax_or_none(x):
494 if x == g_tmax:
495 return None
496 else:
497 return x
500def time_or_none_to_str(x):
501 if x is None:
502 return '...'.ljust(17)
503 else:
504 return util.time_to_str(x)
507def codes_to_str_abbreviated(codes, indent=' '):
508 codes = [str(x) for x in codes]
510 if len(codes) > 20:
511 scodes = '\n' + util.ewrap(codes[:10], indent=indent) \
512 + '\n%s[%i more]\n' % (indent, len(codes) - 20) \
513 + util.ewrap(codes[-10:], indent=' ')
514 else:
515 scodes = '\n' + util.ewrap(codes, indent=indent) \
516 if codes else '<none>'
518 return scodes
521g_offset_time_unit_inv = 1000000000
522g_offset_time_unit = 1.0 / g_offset_time_unit_inv
525def tsplit(t):
526 if t is None:
527 return None, 0.0
529 t = util.to_time_float(t)
530 if type(t) is float:
531 t = round(t, 5)
532 else:
533 t = round(t, 9)
535 seconds = num.floor(t)
536 offset = t - seconds
537 return int(seconds), int(round(offset * g_offset_time_unit_inv))
540def tjoin(seconds, offset):
541 if seconds is None:
542 return None
544 return util.to_time_float(seconds) \
545 + util.to_time_float(offset*g_offset_time_unit)
548tscale_min = 1
549tscale_max = 365 * 24 * 3600 # last edge is one above
550tscale_logbase = 20
552tscale_edges = [tscale_min]
553while True:
554 tscale_edges.append(tscale_edges[-1]*tscale_logbase)
555 if tscale_edges[-1] >= tscale_max:
556 break
559tscale_edges = num.array(tscale_edges)
562def tscale_to_kscale(tscale):
564 # 0 <= x < tscale_edges[1]: 0
565 # tscale_edges[1] <= x < tscale_edges[2]: 1
566 # ...
567 # tscale_edges[len(tscale_edges)-1] <= x: len(tscale_edges)
569 return int(num.searchsorted(tscale_edges, tscale))
572@squirrel_content
573class Station(Location):
574 '''
575 A seismic station.
576 '''
578 codes = CodesNSL.T()
580 tmin = Timestamp.T(optional=True)
581 tmax = Timestamp.T(optional=True)
583 description = Unicode.T(optional=True)
585 def __init__(self, **kwargs):
586 kwargs['codes'] = CodesNSL(kwargs['codes'])
587 Location.__init__(self, **kwargs)
589 @property
590 def time_span(self):
591 return (self.tmin, self.tmax)
593 def get_pyrocko_station(self):
594 from pyrocko import model
595 return model.Station(*self._get_pyrocko_station_args())
597 def _get_pyrocko_station_args(self):
598 return (
599 self.codes.network,
600 self.codes.station,
601 self.codes.location,
602 self.lat,
603 self.lon,
604 self.elevation,
605 self.depth,
606 self.north_shift,
607 self.east_shift)
610class Sensor(Location):
611 '''
612 Representation of a channel group.
613 '''
615 codes = CodesNSLCE.T()
617 tmin = Timestamp.T(optional=True)
618 tmax = Timestamp.T(optional=True)
620 deltat = Float.T(optional=True)
622 @property
623 def time_span(self):
624 return (self.tmin, self.tmax)
626 def __init__(self, **kwargs):
627 kwargs['codes'] = CodesNSLCE(kwargs['codes'])
628 Location.__init__(self, **kwargs)
630 def _get_sensor_args(self):
631 def getattr_rep(k):
632 if k == 'codes':
633 return self.codes.replace(
634 channel=self.codes.channel[:-1] + '?')
635 else:
636 return getattr(self, k)
638 return tuple(getattr_rep(k) for k in Sensor.T.propnames)
640 @classmethod
641 def from_channels(cls, channels):
642 groups = defaultdict(list)
643 for channel in channels:
644 groups[channel._get_sensor_args()].append(channel)
646 return [
647 cls(**dict((k, v) for (k, v) in zip(cls.T.propnames, args)))
648 for args, _ in groups.items()]
650 def _get_pyrocko_station_args(self):
651 return (
652 self.codes.network,
653 self.codes.station,
654 self.codes.location,
655 self.lat,
656 self.lon,
657 self.elevation,
658 self.depth,
659 self.north_shift,
660 self.east_shift)
663@squirrel_content
664class Channel(Sensor):
665 '''
666 A channel of a seismic station.
667 '''
669 dip = Float.T(optional=True)
670 azimuth = Float.T(optional=True)
672 @classmethod
673 def from_channels(cls, channels):
674 raise NotImplementedError()
676 def get_pyrocko_channel(self):
677 from pyrocko import model
678 return model.Channel(*self._get_pyrocko_channel_args())
680 def _get_pyrocko_channel_args(self):
681 return (
682 self.codes.channel,
683 self.azimuth,
684 self.dip)
687observational_quantities = [
688 'acceleration', 'velocity', 'displacement', 'pressure', 'rotation',
689 'temperature']
692technical_quantities = [
693 'voltage', 'counts']
696class QuantityType(StringChoice):
697 '''
698 Choice of observational or technical quantity.
700 SI units are used for all quantities, where applicable.
701 '''
702 choices = observational_quantities + technical_quantities
705class ResponseStage(Object):
706 '''
707 Representation of a response stage.
709 Components of a seismic recording system are represented as a sequence of
710 response stages, e.g. sensor, pre-amplifier, digitizer, digital
711 downsampling.
712 '''
713 input_quantity = QuantityType.T(optional=True)
714 input_sample_rate = Float.T(optional=True)
715 output_quantity = QuantityType.T(optional=True)
716 output_sample_rate = Float.T(optional=True)
717 elements = List.T(FrequencyResponse.T())
718 log = List.T(Tuple.T(3, String.T()))
720 @property
721 def stage_type(self):
722 if self.input_quantity in observational_quantities \
723 and self.output_quantity in observational_quantities:
724 return 'conversion'
726 if self.input_quantity in observational_quantities \
727 and self.output_quantity == 'voltage':
728 return 'sensor'
730 elif self.input_quantity == 'voltage' \
731 and self.output_quantity == 'voltage':
732 return 'electronics'
734 elif self.input_quantity == 'voltage' \
735 and self.output_quantity == 'counts':
736 return 'digitizer'
738 elif self.input_quantity == 'counts' \
739 and self.output_quantity == 'counts' \
740 and self.input_sample_rate != self.output_sample_rate:
741 return 'decimation'
743 elif self.input_quantity in observational_quantities \
744 and self.output_quantity == 'counts':
745 return 'combination'
747 else:
748 return 'unknown'
750 @property
751 def summary(self):
752 irate = self.input_sample_rate
753 orate = self.output_sample_rate
754 factor = None
755 if irate and orate:
756 factor = irate / orate
757 return 'ResponseStage, ' + (
758 '%s%s => %s%s%s' % (
759 self.input_quantity or '?',
760 ' @ %g Hz' % irate if irate else '',
761 self.output_quantity or '?',
762 ' @ %g Hz' % orate if orate else '',
763 ' :%g' % factor if factor else '')
764 )
766 def get_effective(self):
767 return MultiplyResponse(responses=list(self.elements))
770D = 'displacement'
771V = 'velocity'
772A = 'acceleration'
774g_converters = {
775 (V, D): IntegrationResponse(1),
776 (A, D): IntegrationResponse(2),
777 (D, V): DifferentiationResponse(1),
778 (A, V): IntegrationResponse(1),
779 (D, A): DifferentiationResponse(2),
780 (V, A): DifferentiationResponse(1)}
783def response_converters(input_quantity, output_quantity):
784 if input_quantity is None or input_quantity == output_quantity:
785 return []
787 if output_quantity is None:
788 raise ConversionError('Unspecified target quantity.')
790 try:
791 return [g_converters[input_quantity, output_quantity]]
793 except KeyError:
794 raise ConversionError('No rule to convert from "%s" to "%s".' % (
795 input_quantity, output_quantity))
798@squirrel_content
799class Response(Object):
800 '''
801 The instrument response of a seismic station channel.
802 '''
804 codes = CodesNSLCE.T()
805 tmin = Timestamp.T(optional=True)
806 tmax = Timestamp.T(optional=True)
808 stages = List.T(ResponseStage.T())
809 checkpoints = List.T(FrequencyResponseCheckpoint.T())
811 deltat = Float.T(optional=True)
812 log = List.T(Tuple.T(3, String.T()))
814 def __init__(self, **kwargs):
815 kwargs['codes'] = CodesNSLCE(kwargs['codes'])
816 Object.__init__(self, **kwargs)
818 @property
819 def time_span(self):
820 return (self.tmin, self.tmax)
822 @property
823 def nstages(self):
824 return len(self.stages)
826 @property
827 def input_quantity(self):
828 return self.stages[0].input_quantity if self.stages else None
830 @property
831 def output_quantity(self):
832 return self.stages[-1].input_quantity if self.stages else None
834 @property
835 def output_sample_rate(self):
836 return self.stages[-1].output_sample_rate if self.stages else None
838 @property
839 def stages_summary(self):
840 def grouped(xs):
841 xs = list(xs)
842 g = []
843 for i in range(len(xs)):
844 g.append(xs[i])
845 if i+1 < len(xs) and xs[i+1] != xs[i]:
846 yield g
847 g = []
849 if g:
850 yield g
852 return '+'.join(
853 '%s%s' % (g[0], '(%i)' % len(g) if len(g) > 1 else '')
854 for g in grouped(stage.stage_type for stage in self.stages))
856 @property
857 def summary(self):
858 orate = self.output_sample_rate
859 return '%s %-16s %s' % (
860 self.__class__.__name__, self.str_codes, self.str_time_span) \
861 + ', ' + ', '.join((
862 '%s => %s' % (
863 self.input_quantity or '?', self.output_quantity or '?')
864 + (' @ %g Hz' % orate if orate else ''),
865 self.stages_summary,
866 ))
868 def get_effective(self, input_quantity=None):
869 elements = response_converters(input_quantity, self.input_quantity)
871 elements.extend(
872 stage.get_effective() for stage in self.stages)
874 return MultiplyResponse(responses=simplify_responses(elements))
877@squirrel_content
878class Event(Object):
879 '''
880 A seismic event.
881 '''
883 name = String.T(optional=True)
884 time = Timestamp.T()
885 duration = Float.T(optional=True)
887 lat = Float.T()
888 lon = Float.T()
889 depth = Float.T(optional=True)
891 magnitude = Float.T(optional=True)
893 def get_pyrocko_event(self):
894 from pyrocko import model
895 model.Event(
896 name=self.name,
897 time=self.time,
898 lat=self.lat,
899 lon=self.lon,
900 depth=self.depth,
901 magnitude=self.magnitude,
902 duration=self.duration)
904 @property
905 def time_span(self):
906 return (self.time, self.time)
909def ehash(s):
910 return hashlib.sha1(s.encode('utf8')).hexdigest()
913def random_name(n=8):
914 return urlsafe_b64encode(urandom(n)).rstrip(b'=').decode('ascii')
917@squirrel_content
918class WaveformPromise(Object):
919 '''
920 Information about a waveform potentially downloadable from a remote site.
922 In the Squirrel framework, waveform promises are used to permit download of
923 selected waveforms from a remote site. They are typically generated by
924 calls to
925 :py:meth:`~pyrocko.squirrel.base.Squirrel.update_waveform_promises`.
926 Waveform promises are inserted and indexed in the database similar to
927 normal waveforms. When processing a waveform query, e.g. from
928 :py:meth:`~pyrocko.squirrel.base.Squirrel.get_waveform`, and no local
929 waveform is available for the queried time span, a matching promise can be
930 resolved, i.e. an attempt is made to download the waveform from the remote
931 site. The promise is removed after the download attempt (except when a
932 network error occurs). This prevents Squirrel from making unnecessary
933 queries for waveforms missing at the remote site.
934 '''
936 codes = CodesNSLCE.T()
937 tmin = Timestamp.T()
938 tmax = Timestamp.T()
940 deltat = Float.T(optional=True)
942 source_hash = String.T()
944 def __init__(self, **kwargs):
945 kwargs['codes'] = CodesNSLCE(kwargs['codes'])
946 Object.__init__(self, **kwargs)
948 @property
949 def time_span(self):
950 return (self.tmin, self.tmax)
953class InvalidWaveform(Exception):
954 pass
957class WaveformOrder(Object):
958 '''
959 Waveform request information.
960 '''
962 source_id = String.T()
963 codes = CodesNSLCE.T()
964 deltat = Float.T()
965 tmin = Timestamp.T()
966 tmax = Timestamp.T()
967 gaps = List.T(Tuple.T(2, Timestamp.T()))
969 @property
970 def client(self):
971 return self.source_id.split(':')[1]
973 def describe(self, site='?'):
974 return '%s:%s %s [%s - %s]' % (
975 self.client, site, str(self.codes),
976 util.time_to_str(self.tmin), util.time_to_str(self.tmax))
978 def validate(self, tr):
979 if tr.ydata.size == 0:
980 raise InvalidWaveform(
981 'waveform with zero data samples')
983 if tr.deltat != self.deltat:
984 raise InvalidWaveform(
985 'incorrect sampling interval - waveform: %g s, '
986 'meta-data: %g s' % (
987 tr.deltat, self.deltat))
989 if not num.all(num.isfinite(tr.ydata)):
990 raise InvalidWaveform('waveform has NaN values')
993def order_summary(orders):
994 codes_list = sorted(set(order.codes for order in orders))
995 if len(codes_list) > 3:
996 return '%i order%s: %s - %s' % (
997 len(orders),
998 util.plural_s(orders),
999 str(codes_list[0]),
1000 str(codes_list[1]))
1002 else:
1003 return '%i order%s: %s' % (
1004 len(orders),
1005 util.plural_s(orders),
1006 ', '.join(str(codes) for codes in codes_list))
1009class Nut(Object):
1010 '''
1011 Index entry referencing an elementary piece of content.
1013 So-called *nuts* are used in Pyrocko's Squirrel framework to hold common
1014 meta-information about individual pieces of waveforms, stations, channels,
1015 etc. together with the information where it was found or generated.
1016 '''
1018 file_path = String.T(optional=True)
1019 file_format = String.T(optional=True)
1020 file_mtime = Timestamp.T(optional=True)
1021 file_size = Int.T(optional=True)
1023 file_segment = Int.T(optional=True)
1024 file_element = Int.T(optional=True)
1026 kind_id = Int.T()
1027 codes = Codes.T()
1029 tmin_seconds = Int.T(default=0)
1030 tmin_offset = Int.T(default=0, optional=True)
1031 tmax_seconds = Int.T(default=0)
1032 tmax_offset = Int.T(default=0, optional=True)
1034 deltat = Float.T(default=0.0)
1036 content = Any.T(optional=True)
1037 raw_content = Dict.T(String.T(), Any.T())
1039 content_in_db = False
1041 def __init__(
1042 self,
1043 file_path=None,
1044 file_format=None,
1045 file_mtime=None,
1046 file_size=None,
1047 file_segment=None,
1048 file_element=None,
1049 kind_id=0,
1050 codes=CodesX(''),
1051 tmin_seconds=None,
1052 tmin_offset=0,
1053 tmax_seconds=None,
1054 tmax_offset=0,
1055 deltat=None,
1056 content=None,
1057 raw_content=None,
1058 tmin=None,
1059 tmax=None,
1060 values_nocheck=None):
1062 if values_nocheck is not None:
1063 (self.file_path, self.file_format, self.file_mtime,
1064 self.file_size,
1065 self.file_segment, self.file_element,
1066 self.kind_id, codes_safe_str,
1067 self.tmin_seconds, self.tmin_offset,
1068 self.tmax_seconds, self.tmax_offset,
1069 self.deltat) = values_nocheck
1071 self.codes = to_codes_simple(self.kind_id, codes_safe_str)
1072 self.raw_content = {}
1073 self.content = None
1074 else:
1075 if tmin is not None:
1076 tmin_seconds, tmin_offset = tsplit(tmin)
1078 if tmax is not None:
1079 tmax_seconds, tmax_offset = tsplit(tmax)
1081 self.kind_id = int(kind_id)
1082 self.codes = codes
1083 self.tmin_seconds = int_or_g_tmin(tmin_seconds)
1084 self.tmin_offset = int(tmin_offset)
1085 self.tmax_seconds = int_or_g_tmax(tmax_seconds)
1086 self.tmax_offset = int(tmax_offset)
1087 self.deltat = float_or_none(deltat)
1088 self.file_path = str_or_none(file_path)
1089 self.file_segment = int_or_none(file_segment)
1090 self.file_element = int_or_none(file_element)
1091 self.file_format = str_or_none(file_format)
1092 self.file_mtime = float_or_none(file_mtime)
1093 self.file_size = int_or_none(file_size)
1094 self.content = content
1095 if raw_content is None:
1096 self.raw_content = {}
1097 else:
1098 self.raw_content = raw_content
1100 Object.__init__(self, init_props=False)
1102 def __eq__(self, other):
1103 return (isinstance(other, Nut) and
1104 self.equality_values == other.equality_values)
1106 def hash(self):
1107 return ehash(','.join(str(x) for x in self.key))
1109 def __ne__(self, other):
1110 return not (self == other)
1112 def get_io_backend(self):
1113 from . import io
1114 return io.get_backend(self.file_format)
1116 def file_modified(self):
1117 return self.get_io_backend().get_stats(self.file_path) \
1118 != (self.file_mtime, self.file_size)
1120 @property
1121 def dkey(self):
1122 return (self.kind_id, self.tmin_seconds, self.tmin_offset, self.codes)
1124 @property
1125 def key(self):
1126 return (
1127 self.file_path,
1128 self.file_segment,
1129 self.file_element,
1130 self.file_mtime)
1132 @property
1133 def equality_values(self):
1134 return (
1135 self.file_segment, self.file_element,
1136 self.kind_id, self.codes,
1137 self.tmin_seconds, self.tmin_offset,
1138 self.tmax_seconds, self.tmax_offset, self.deltat)
1140 @property
1141 def tmin(self):
1142 return tjoin(self.tmin_seconds, self.tmin_offset)
1144 @tmin.setter
1145 def tmin(self, t):
1146 self.tmin_seconds, self.tmin_offset = tsplit(t)
1148 @property
1149 def tmax(self):
1150 return tjoin(self.tmax_seconds, self.tmax_offset)
1152 @tmax.setter
1153 def tmax(self, t):
1154 self.tmax_seconds, self.tmax_offset = tsplit(t)
1156 @property
1157 def kscale(self):
1158 if self.tmin_seconds is None or self.tmax_seconds is None:
1159 return 0
1160 return tscale_to_kscale(self.tmax_seconds - self.tmin_seconds)
1162 @property
1163 def waveform_kwargs(self):
1164 network, station, location, channel, extra = self.codes
1166 return dict(
1167 network=network,
1168 station=station,
1169 location=location,
1170 channel=channel,
1171 extra=extra,
1172 tmin=self.tmin,
1173 tmax=self.tmax,
1174 deltat=self.deltat)
1176 @property
1177 def waveform_promise_kwargs(self):
1178 return dict(
1179 codes=self.codes,
1180 tmin=self.tmin,
1181 tmax=self.tmax,
1182 deltat=self.deltat)
1184 @property
1185 def station_kwargs(self):
1186 network, station, location = self.codes
1187 return dict(
1188 codes=self.codes,
1189 tmin=tmin_or_none(self.tmin),
1190 tmax=tmax_or_none(self.tmax))
1192 @property
1193 def channel_kwargs(self):
1194 network, station, location, channel, extra = self.codes
1195 return dict(
1196 codes=self.codes,
1197 tmin=tmin_or_none(self.tmin),
1198 tmax=tmax_or_none(self.tmax),
1199 deltat=self.deltat)
1201 @property
1202 def response_kwargs(self):
1203 return dict(
1204 codes=self.codes,
1205 tmin=tmin_or_none(self.tmin),
1206 tmax=tmax_or_none(self.tmax),
1207 deltat=self.deltat)
1209 @property
1210 def event_kwargs(self):
1211 return dict(
1212 name=self.codes,
1213 time=self.tmin,
1214 duration=(self.tmax - self.tmin) or None)
1216 @property
1217 def trace_kwargs(self):
1218 network, station, location, channel, extra = self.codes
1220 return dict(
1221 network=network,
1222 station=station,
1223 location=location,
1224 channel=channel,
1225 extra=extra,
1226 tmin=self.tmin,
1227 tmax=self.tmax-self.deltat,
1228 deltat=self.deltat)
1230 @property
1231 def dummy_trace(self):
1232 return DummyTrace(self)
1234 @property
1235 def summary(self):
1236 if self.tmin == self.tmax:
1237 ts = util.time_to_str(self.tmin)
1238 else:
1239 ts = '%s - %s' % (
1240 util.time_to_str(self.tmin),
1241 util.time_to_str(self.tmax))
1243 return ' '.join((
1244 ('%s,' % to_kind(self.kind_id)).ljust(9),
1245 ('%s,' % str(self.codes)).ljust(18),
1246 ts))
1249def make_waveform_nut(**kwargs):
1250 return Nut(kind_id=WAVEFORM, **kwargs)
1253def make_waveform_promise_nut(**kwargs):
1254 return Nut(kind_id=WAVEFORM_PROMISE, **kwargs)
1257def make_station_nut(**kwargs):
1258 return Nut(kind_id=STATION, **kwargs)
1261def make_channel_nut(**kwargs):
1262 return Nut(kind_id=CHANNEL, **kwargs)
1265def make_response_nut(**kwargs):
1266 return Nut(kind_id=RESPONSE, **kwargs)
1269def make_event_nut(**kwargs):
1270 return Nut(kind_id=EVENT, **kwargs)
1273def group_channels(nuts):
1274 by_ansl = {}
1275 for nut in nuts:
1276 if nut.kind_id != CHANNEL:
1277 continue
1279 ansl = nut.codes[:4]
1281 if ansl not in by_ansl:
1282 by_ansl[ansl] = {}
1284 group = by_ansl[ansl]
1286 k = nut.codes[4][:-1], nut.deltat, nut.tmin, nut.tmax
1288 if k not in group:
1289 group[k] = set()
1291 group.add(nut.codes[4])
1293 return by_ansl
1296class DummyTrace(object):
1298 def __init__(self, nut):
1299 self.nut = nut
1300 self.codes = nut.codes
1301 self.meta = {}
1303 @property
1304 def tmin(self):
1305 return self.nut.tmin
1307 @property
1308 def tmax(self):
1309 return self.nut.tmax
1311 @property
1312 def deltat(self):
1313 return self.nut.deltat
1315 @property
1316 def nslc_id(self):
1317 return self.codes.nslc
1319 @property
1320 def network(self):
1321 return self.codes.network
1323 @property
1324 def station(self):
1325 return self.codes.station
1327 @property
1328 def location(self):
1329 return self.codes.location
1331 @property
1332 def channel(self):
1333 return self.codes.channel
1335 @property
1336 def extra(self):
1337 return self.codes.extra
1339 def overlaps(self, tmin, tmax):
1340 return not (tmax < self.nut.tmin or self.nut.tmax < tmin)
1343def duration_to_str(t):
1344 if t > 24*3600:
1345 return '%gd' % (t / (24.*3600.))
1346 elif t > 3600:
1347 return '%gh' % (t / 3600.)
1348 elif t > 60:
1349 return '%gm' % (t / 60.)
1350 else:
1351 return '%gs' % t
1354class Coverage(Object):
1355 '''
1356 Information about times covered by a waveform or other time series data.
1357 '''
1358 kind_id = Int.T(
1359 help='Content type.')
1360 pattern = Codes.T(
1361 help='The codes pattern in the request, which caused this entry to '
1362 'match.')
1363 codes = Codes.T(
1364 help='NSLCE or NSL codes identifier of the time series.')
1365 deltat = Float.T(
1366 help='Sampling interval [s]',
1367 optional=True)
1368 tmin = Timestamp.T(
1369 help='Global start time of time series.',
1370 optional=True)
1371 tmax = Timestamp.T(
1372 help='Global end time of time series.',
1373 optional=True)
1374 changes = List.T(
1375 Tuple.T(2, Any.T()),
1376 help='List of change points, with entries of the form '
1377 '``(time, count)``, where a ``count`` of zero indicates start of '
1378 'a gap, a value of 1 start of normal data coverage and a higher '
1379 'value duplicate or redundant data coverage.')
1381 @classmethod
1382 def from_values(cls, args):
1383 pattern, codes, deltat, tmin, tmax, changes, kind_id = args
1384 return cls(
1385 kind_id=kind_id,
1386 pattern=pattern,
1387 codes=codes,
1388 deltat=deltat,
1389 tmin=tmin,
1390 tmax=tmax,
1391 changes=changes)
1393 @property
1394 def summary(self):
1395 ts = '%s - %s,' % (
1396 util.time_to_str(self.tmin),
1397 util.time_to_str(self.tmax))
1399 srate = self.sample_rate
1401 return ' '.join((
1402 ('%s,' % to_kind(self.kind_id)).ljust(9),
1403 ('%s,' % str(self.codes)).ljust(18),
1404 ts,
1405 '%10.3g,' % srate if srate else '',
1406 '%4i' % len(self.changes),
1407 '%s' % duration_to_str(self.total)))
1409 @property
1410 def sample_rate(self):
1411 if self.deltat is None:
1412 return None
1413 elif self.deltat == 0.0:
1414 return 0.0
1415 else:
1416 return 1.0 / self.deltat
1418 @property
1419 def labels(self):
1420 srate = self.sample_rate
1421 return (
1422 ('%s' % str(self.codes)),
1423 '%.3g' % srate if srate else '')
1425 @property
1426 def total(self):
1427 total_t = None
1428 for tmin, tmax, _ in self.iter_spans():
1429 total_t = (total_t or 0.0) + (tmax - tmin)
1431 return total_t
1433 def iter_spans(self):
1434 last = None
1435 for (t, count) in self.changes:
1436 if last is not None:
1437 last_t, last_count = last
1438 if last_count > 0:
1439 yield last_t, t, last_count
1441 last = (t, count)
1444__all__ = [
1445 'to_codes',
1446 'to_codes_guess',
1447 'to_codes_simple',
1448 'to_kind',
1449 'to_kinds',
1450 'to_kind_id',
1451 'to_kind_ids',
1452 'CodesError',
1453 'Codes',
1454 'CodesNSLCE',
1455 'CodesNSL',
1456 'CodesX',
1457 'Station',
1458 'Channel',
1459 'Sensor',
1460 'Response',
1461 'Nut',
1462 'Coverage',
1463 'WaveformPromise',
1464]