Coverage for /usr/local/lib/python3.11/dist-packages/pyrocko/guts.py: 97%
1364 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-10-23 12:04 +0000
1# https://pyrocko.org - GPLv3
2#
3# The Pyrocko Developers, 21st Century
4# ---|P------/S----------~Lg----------
6'''
7Lightweight declarative YAML and XML data binding for Python.
8'''
10import datetime
11import calendar
12import re
13import sys
14import types
15import copy
16import os.path as op
17from collections import defaultdict
18from base64 import b64decode, b64encode
20from io import BytesIO
22try:
23 import numpy as num
24except ImportError:
25 num = None
27import yaml
28try:
29 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
30except ImportError:
31 from yaml import SafeLoader, SafeDumper
33from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \
34 get_time_float
37ALLOW_INCLUDE = False
40class GutsSafeDumper(SafeDumper):
41 pass
44class GutsSafeLoader(SafeLoader):
45 pass
48g_iprop = 0
50g_deferred = {}
51g_deferred_content = {}
53g_tagname_to_class = {}
54g_xmltagname_to_class = {}
55g_guessable_xmlns = {}
57guts_types = [
58 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float',
59 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern',
60 'UnicodePattern', 'StringChoice', 'IntChoice', 'List', 'Dict', 'Tuple',
61 'StringUnion', 'Choice', 'Any']
63us_to_cc_regex = re.compile(r'([a-z])_([a-z])')
66class literal(str):
67 pass
70class folded(str):
71 pass
74class singlequoted(str):
75 pass
78class doublequoted(str):
79 pass
82def make_str_presenter(style):
83 def presenter(dumper, data):
84 return dumper.represent_scalar(
85 'tag:yaml.org,2002:str', str(data), style=style)
87 return presenter
90str_style_map = {
91 None: lambda x: x,
92 '|': literal,
93 '>': folded,
94 "'": singlequoted,
95 '"': doublequoted}
97for (style, cls) in str_style_map.items():
98 if style:
99 GutsSafeDumper.add_representer(cls, make_str_presenter(style))
102class uliteral(str):
103 pass
106class ufolded(str):
107 pass
110class usinglequoted(str):
111 pass
114class udoublequoted(str):
115 pass
118def make_unicode_presenter(style):
119 def presenter(dumper, data):
120 return dumper.represent_scalar(
121 'tag:yaml.org,2002:str', str(data), style=style)
123 return presenter
126unicode_style_map = {
127 None: lambda x: x,
128 '|': literal,
129 '>': folded,
130 "'": singlequoted,
131 '"': doublequoted}
133for (style, cls) in unicode_style_map.items():
134 if style:
135 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style))
138class blist(list):
139 pass
142class flist(list):
143 pass
146list_style_map = {
147 None: list,
148 'block': blist,
149 'flow': flist}
152def make_list_presenter(flow_style):
153 def presenter(dumper, data):
154 return dumper.represent_sequence(
155 'tag:yaml.org,2002:seq', data, flow_style=flow_style)
157 return presenter
160GutsSafeDumper.add_representer(blist, make_list_presenter(False))
161GutsSafeDumper.add_representer(flist, make_list_presenter(True))
163if num:
164 def numpy_float_presenter(dumper, data):
165 return dumper.represent_float(float(data))
167 def numpy_int_presenter(dumper, data):
168 return dumper.represent_int(int(data))
170 for dtype in (num.float64, num.float32):
171 GutsSafeDumper.add_representer(dtype, numpy_float_presenter)
173 for dtype in (num.int32, num.int64):
174 GutsSafeDumper.add_representer(dtype, numpy_int_presenter)
177def us_to_cc(s):
178 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s)
181cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)')
182cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])')
185def cc_to_us(s):
186 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub(
187 '\\1_\\2\\3', s)).lower()
190re_frac = re.compile(r'\.[1-9]FRAC')
191frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789'])
194def encode_utf8(s):
195 return s.encode('utf-8')
198def no_encode(s):
199 return s
202def make_xmltagname_from_name(name):
203 return us_to_cc(name)
206def make_name_from_xmltagname(xmltagname):
207 return cc_to_us(xmltagname)
210def make_content_name(name):
211 if name.endswith('_list'):
212 return name[:-5]
213 elif name.endswith('s'):
214 return name[:-1]
215 else:
216 return name
219def classnames(cls):
220 if isinstance(cls, tuple):
221 return '(%s)' % ', '.join(x.__name__ for x in cls)
222 else:
223 return cls.__name__
226def expand_stream_args(mode):
227 def wrap(f):
228 '''
229 Decorator to enhance functions taking stream objects.
231 Wraps a function f(..., stream, ...) so that it can also be called as
232 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...).
233 '''
235 def g(*args, **kwargs):
236 stream = kwargs.pop('stream', None)
237 filename = kwargs.get('filename', None)
238 if mode != 'r':
239 filename = kwargs.pop('filename', None)
240 string = kwargs.pop('string', None)
242 assert sum(x is not None for x in (stream, filename, string)) <= 1
244 if stream is not None:
245 kwargs['stream'] = stream
246 return f(*args, **kwargs)
248 elif filename is not None:
249 stream = open(filename, mode+'b')
250 kwargs['stream'] = stream
251 retval = f(*args, **kwargs)
252 if isinstance(retval, types.GeneratorType):
253 def wrap_generator(gen):
254 try:
255 for x in gen:
256 yield x
258 except GeneratorExit:
259 pass
261 stream.close()
263 return wrap_generator(retval)
265 else:
266 stream.close()
267 return retval
269 elif string is not None:
270 assert mode == 'r', \
271 'Keyword argument string=... cannot be used in dumper ' \
272 'function.'
274 kwargs['stream'] = BytesIO(string.encode('utf-8'))
275 return f(*args, **kwargs)
277 else:
278 assert mode == 'w', \
279 'Use keyword argument stream=... or filename=... in ' \
280 'loader function.'
282 sout = BytesIO()
283 f(stream=sout, *args, **kwargs)
284 return sout.getvalue().decode('utf-8')
286 g.__doc__ = f.__doc__
287 return g
289 return wrap
292class Defer(object):
293 def __init__(self, classname, *args, **kwargs):
294 global g_iprop
295 if kwargs.get('position', None) is None:
296 kwargs['position'] = g_iprop
298 g_iprop += 1
300 self.classname = classname
301 self.args = args
302 self.kwargs = kwargs
305class TBase(object):
306 '''
307 Base class for Guts type definitions.
309 :param default:
310 Default value or :py:class:`DefaultMaker` object (see
311 :py:meth:`Object.D`) to be used to generate a default.
313 :param optional:
314 If ``True``, the attribute is optional and may be set to ``None``.
315 :type optional:
316 bool
318 :param xmlstyle:
319 Controls how the attribute is serialized in XML. :Choices:
320 ``'element'``, ``'attribute'``, ``'content'``. Only scalar and
321 :py:class:`SObject` values may be serialized as ``'attribute'``. Only
322 one attribute in a class may be serialized as ``'content'``.
323 :type xmlstyle:
324 str
326 :param xmltagname:
327 XML tag name to be used. By default, the attribute name converted to
328 camel-case is used.
329 :type xmltagname:
330 str
332 :param xmlns:
333 XML namespace to be used for this attribute.
334 :type xmlns:
335 str
337 :param help:
338 Description of the attribute used in documentation and online help.
339 :type help:
340 rst formatted :py:class:`str`
342 :param position:
343 Position index to be used when the attribute should be output in
344 non-default order.
345 :type position:
346 int
347 '''
349 strict = False
350 multivalued = None
351 force_regularize = False
352 propnames = []
353 _dummy_cls = None
354 _cls = None
355 _sphinx_doc_skip = False
357 @classmethod
358 def init_propertystuff(cls):
359 cls.properties = []
360 cls.xmltagname_to_name = {}
361 cls.xmltagname_to_name_multivalued = {}
362 cls.xmltagname_to_class = {}
363 cls.content_property = None
365 def __init__(
366 self,
367 default=None,
368 optional=False,
369 xmlstyle='element',
370 xmltagname=None,
371 xmlns=None,
372 help=None,
373 position=None):
375 global g_iprop
376 if position is not None:
377 self.position = position
378 else:
379 self.position = g_iprop
381 g_iprop += 1
382 self._default = default
384 self.optional = optional
385 self.name = None
386 self._xmltagname = xmltagname
387 self._xmlns = xmlns
388 self.parent = None
389 self.xmlstyle = xmlstyle
390 self.help = help
391 self._sphinx_doc_skip = True
393 def default(self):
394 return make_default(self._default)
396 def is_default(self, val):
397 if self._default is None:
398 return val is None
399 else:
400 return self._default == val
402 def has_default(self):
403 return self._default is not None
405 def xname(self):
406 if self.name is not None:
407 return self.name
408 elif self.parent is not None:
409 return 'element of %s' % self.parent.xname()
410 else:
411 return '?'
413 def set_xmlns(self, xmlns):
414 if self._xmlns is None and not self.xmlns:
415 self._xmlns = xmlns
417 if self.multivalued:
418 self.content_t.set_xmlns(xmlns)
420 def get_xmlns(self):
421 return self._xmlns or self.xmlns
423 def get_xmltagname(self):
424 if self._xmltagname is not None:
425 return self.get_xmlns() + ' ' + self._xmltagname
426 elif self.name:
427 return self.get_xmlns() + ' ' \
428 + make_xmltagname_from_name(self.name)
429 elif self.xmltagname:
430 return self.get_xmlns() + ' ' + self.xmltagname
431 else:
432 assert False
434 @classmethod
435 def get_property(cls, name):
436 for prop in cls.properties:
437 if prop.name == name:
438 return prop
440 raise ValueError()
442 @classmethod
443 def remove_property(cls, name):
445 prop = cls.get_property(name)
447 if not prop.multivalued:
448 del cls.xmltagname_to_class[prop.effective_xmltagname]
449 del cls.xmltagname_to_name[prop.effective_xmltagname]
450 else:
451 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname]
452 del cls.xmltagname_to_name_multivalued[
453 prop.content_t.effective_xmltagname]
455 if cls.content_property is prop:
456 cls.content_property = None
458 cls.properties.remove(prop)
459 cls.propnames.remove(name)
461 return prop
463 @classmethod
464 def add_property(cls, name, prop):
466 prop.instance = prop
467 prop.name = name
468 prop.set_xmlns(cls.xmlns)
470 if isinstance(prop, Choice.T):
471 for tc in prop.choices:
472 tc.effective_xmltagname = tc.get_xmltagname()
473 cls.xmltagname_to_class[tc.effective_xmltagname] = tc._cls
474 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name
475 elif not prop.multivalued:
476 prop.effective_xmltagname = prop.get_xmltagname()
477 cls.xmltagname_to_class[prop.effective_xmltagname] = prop._cls
478 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name
479 else:
480 prop.content_t.name = make_content_name(prop.name)
481 prop.content_t.effective_xmltagname = \
482 prop.content_t.get_xmltagname()
483 cls.xmltagname_to_class[
484 prop.content_t.effective_xmltagname] = prop.content_t._cls
485 cls.xmltagname_to_name_multivalued[
486 prop.content_t.effective_xmltagname] = prop.name
488 cls.properties.append(prop)
490 cls.properties.sort(key=lambda x: x.position)
492 cls.propnames = [p.name for p in cls.properties]
494 if prop.xmlstyle == 'content':
495 cls.content_property = prop
497 @classmethod
498 def ivals(cls, val):
499 for prop in cls.properties:
500 yield getattr(val, prop.name)
502 @classmethod
503 def ipropvals(cls, val):
504 for prop in cls.properties:
505 yield prop, getattr(val, prop.name)
507 @classmethod
508 def inamevals(cls, val):
509 for prop in cls.properties:
510 yield prop.name, getattr(val, prop.name)
512 @classmethod
513 def ipropvals_to_save(cls, val, xmlmode=False):
514 for prop in cls.properties:
515 v = getattr(val, prop.name)
516 if v is not None and (
517 not (prop.optional or (prop.multivalued and not v))
518 or (not prop.is_default(v))):
520 if xmlmode:
521 yield prop, prop.to_save_xml(v)
522 else:
523 yield prop, prop.to_save(v)
525 @classmethod
526 def inamevals_to_save(cls, val, xmlmode=False):
527 for prop, v in cls.ipropvals_to_save(val, xmlmode):
528 yield prop.name, v
530 @classmethod
531 def translate_from_xml(cls, list_of_pairs, strict):
532 d = {}
533 for k, v in list_of_pairs:
534 if k in cls.xmltagname_to_name_multivalued:
535 k2 = cls.xmltagname_to_name_multivalued[k]
536 if k2 not in d:
537 d[k2] = []
539 d[k2].append(v)
540 elif k in cls.xmltagname_to_name:
541 k2 = cls.xmltagname_to_name[k]
542 if k2 in d:
543 raise ArgumentError(
544 'Unexpectedly found more than one child element "%s" '
545 'within "%s".' % (k, cls.tagname))
547 d[k2] = v
548 elif k is None:
549 if cls.content_property:
550 k2 = cls.content_property.name
551 d[k2] = v
552 else:
553 if strict:
554 raise ArgumentError(
555 'Unexpected child element "%s" found within "%s".' % (
556 k, cls.tagname))
558 return d
560 def validate(self, val, regularize=False, depth=-1):
561 if self.optional and val is None:
562 return val
564 is_derived = isinstance(val, self._cls)
565 is_exact = type(val) == self._cls
567 not_ok = not self.strict and not is_derived or \
568 self.strict and not is_exact
570 if not_ok or self.force_regularize:
571 if regularize:
572 try:
573 val = self.regularize_extra(val)
574 except ValueError:
575 raise ValidationError(
576 '%s: could not convert "%s" to type %s' % (
577 self.xname(), val, classnames(self._cls)))
578 else:
579 raise ValidationError(
580 '%s: "%s" (type: %s) is not of type %s' % (
581 self.xname(), val, type(val), classnames(self._cls)))
583 validator = self
584 if isinstance(self._cls, tuple):
585 clss = self._cls
586 else:
587 clss = (self._cls,)
589 for cls in clss:
590 try:
591 if type(val) != cls and isinstance(val, cls):
592 validator = val.T.instance
594 except AttributeError:
595 pass
597 validator.validate_extra(val)
599 if depth != 0:
600 val = validator.validate_children(val, regularize, depth)
602 return val
604 def regularize_extra(self, val):
605 return self._cls(val)
607 def validate_extra(self, val):
608 pass
610 def validate_children(self, val, regularize, depth):
611 for prop, propval in self.ipropvals(val):
612 newpropval = prop.validate(propval, regularize, depth-1)
613 if regularize and (newpropval is not propval):
614 setattr(val, prop.name, newpropval)
616 return val
618 def to_save(self, val):
619 return val
621 def to_save_xml(self, val):
622 return self.to_save(val)
624 def extend_xmlelements(self, elems, v):
625 if self.multivalued:
626 for x in v:
627 elems.append((self.content_t.effective_xmltagname, x))
628 else:
629 elems.append((self.effective_xmltagname, v))
631 def deferred(self):
632 return []
634 def classname_for_help(self, strip_module=''):
636 if self._dummy_cls is not self._cls:
637 if self._dummy_cls.__module__ == strip_module:
638 sadd = ' (:py:class:`%s`)' % (
639 self._dummy_cls.__name__)
640 else:
641 sadd = ' (:py:class:`%s.%s`)' % (
642 self._dummy_cls.__module__, self._dummy_cls.__name__)
643 else:
644 sadd = ''
646 if self._dummy_cls in guts_plain_dummy_types:
647 return '``%s``' % self._cls.__name__
649 elif self._dummy_cls.dummy_for_description:
650 return '%s%s' % (self._dummy_cls.dummy_for_description, sadd)
652 else:
653 def sclass(cls):
654 mod = cls.__module__
655 clsn = cls.__name__
656 if mod == '__builtin__' or mod == 'builtins':
657 return '``%s``' % clsn
659 elif mod == strip_module:
660 return ':py:class:`%s`' % clsn
662 else:
663 return ':py:class:`%s.%s`' % (mod, clsn)
665 if isinstance(self._cls, tuple):
666 return '(%s)%s' % (
667 ' | '.join(sclass(cls) for cls in self._cls), sadd)
668 else:
669 return '%s%s' % (sclass(self._cls), sadd)
671 @classmethod
672 def props_help_string(cls):
673 baseprops = []
674 for base in cls._dummy_cls.__bases__:
675 if hasattr(base, 'T'):
676 baseprops.extend(base.T.properties)
678 hlp = []
679 hlp.append('')
680 for prop in cls.properties:
681 if prop in baseprops:
682 continue
684 descr = [
685 prop.classname_for_help(
686 strip_module=cls._dummy_cls.__module__)]
688 if prop.optional:
689 descr.append('*optional*')
691 if isinstance(prop._default, DefaultMaker):
692 descr.append('*default:* ``%s``' % repr(prop._default))
693 else:
694 d = prop.default()
695 if d is not None:
696 descr.append('*default:* ``%s``' % repr(d))
698 hlp.append(' .. py:gattribute:: %s' % prop.name)
699 hlp.append('')
700 hlp.append(' %s' % ', '.join(descr))
701 hlp.append(' ')
702 if prop.help is not None:
703 hlp.append(' %s' % prop.help)
704 hlp.append('')
706 return '\n'.join(hlp)
708 @classmethod
709 def class_help_string(cls):
710 return cls._dummy_cls.__doc_template__
712 @classmethod
713 def class_signature(cls):
714 r = []
715 for prop in cls.properties:
716 d = prop.default()
717 if d is not None:
718 arg = repr(d)
720 elif prop.optional:
721 arg = 'None'
723 else:
724 arg = '...'
726 r.append('%s=%s' % (prop.name, arg))
728 return '(%s)' % ', '.join(r)
730 @classmethod
731 def help(cls):
732 return cls.props_help_string()
735class ObjectMetaClass(type):
736 def __new__(meta, classname, bases, class_dict):
737 classname = class_dict.get('class_name', classname)
738 cls = type.__new__(meta, classname, bases, class_dict)
739 if classname != 'Object':
740 t_class_attr_name = '_%s__T' % classname
741 if not hasattr(cls, t_class_attr_name):
742 if hasattr(cls, 'T'):
743 class T(cls.T):
744 _sphinx_doc_skip = True
746 T.__doc__ = cls.T.__doc__
747 else:
748 class T(TBase):
749 _sphinx_doc_skip = True
751 T.__doc__ = TBase.__doc__
753 setattr(cls, t_class_attr_name, T)
755 T = getattr(cls, t_class_attr_name)
756 T.__name__ = 'T'
757 T.__qualname__ = T.__qualname__.replace('__T', 'T')
759 if cls.dummy_for is not None:
760 T._cls = cls.dummy_for
761 else:
762 T._cls = cls
764 T._dummy_cls = cls
766 if hasattr(cls, 'xmltagname'):
767 T.xmltagname = cls.xmltagname
768 else:
769 T.xmltagname = classname
771 mod = sys.modules[cls.__module__]
773 if hasattr(cls, 'xmlns'):
774 T.xmlns = cls.xmlns
775 elif hasattr(mod, 'guts_xmlns'):
776 T.xmlns = mod.guts_xmlns
777 else:
778 T.xmlns = ''
780 if T.xmlns and hasattr(cls, 'guessable_xmlns'):
781 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns
783 if hasattr(mod, 'guts_prefix'):
784 if mod.guts_prefix:
785 T.tagname = mod.guts_prefix + '.' + classname
786 else:
787 T.tagname = classname
788 else:
789 if cls.__module__ != '__main__':
790 T.tagname = cls.__module__ + '.' + classname
791 else:
792 T.tagname = classname
794 T.classname = classname
796 T.init_propertystuff()
798 for k in dir(cls):
799 prop = getattr(cls, k)
801 if k.endswith('__'):
802 k = k[:-2]
804 if isinstance(prop, TBase):
805 if prop.deferred():
806 for defer in prop.deferred():
807 g_deferred_content.setdefault(
808 defer.classname[:-2], []).append((prop, defer))
809 g_deferred.setdefault(
810 defer.classname[:-2], []).append((T, k, prop))
812 else:
813 T.add_property(k, prop)
815 elif isinstance(prop, Defer):
816 g_deferred.setdefault(prop.classname[:-2], []).append(
817 (T, k, prop))
819 if classname in g_deferred_content:
820 for prop, defer in g_deferred_content[classname]:
821 prop.process_deferred(
822 defer, T(*defer.args, **defer.kwargs))
824 del g_deferred_content[classname]
826 if classname in g_deferred:
827 for (T_, k_, prop_) in g_deferred.get(classname, []):
828 if isinstance(prop_, Defer):
829 prop_ = T(*prop_.args, **prop_.kwargs)
831 if not prop_.deferred():
832 T_.add_property(k_, prop_)
834 del g_deferred[classname]
836 g_tagname_to_class[T.tagname] = cls
837 if hasattr(cls, 'xmltagname'):
838 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls
840 cls.T = T
841 T.instance = T()
843 cls.__doc_template__ = cls.__doc__
844 cls.__doc__ = T.class_help_string()
846 if cls.__doc__ is None:
847 cls.__doc__ = 'Undocumented.'
849 cls.__doc__ += '\n' + T.props_help_string()
851 return cls
854class ValidationError(ValueError):
855 '''
856 Raised when an object is invalid according to its definition.
857 '''
858 pass
861class ArgumentError(ValueError):
862 '''
863 Raised when invalid arguments would be used in an object's initialization.
864 '''
865 pass
868def make_default(x):
869 if isinstance(x, DefaultMaker):
870 return x.make()
871 elif isinstance(x, Object):
872 return clone(x)
873 else:
874 return x
877class DefaultMaker(object):
878 '''
879 Base class for default value factories.
880 '''
881 def make(self):
882 '''
883 Create a new object.
884 '''
885 raise NotImplementedError
888class ObjectDefaultMaker(DefaultMaker):
889 '''
890 Default value factory for :py:class:`Object` derived classes.
891 '''
892 def __init__(self, cls, args, kwargs):
893 DefaultMaker.__init__(self)
894 self._cls = cls
895 self.args = args
896 self.kwargs = kwargs
897 self.instance = None
899 def make(self):
900 return self._cls(
901 *[make_default(x) for x in self.args],
902 **dict((k, make_default(v)) for (k, v) in self.kwargs.items()))
904 def __eq__(self, other):
905 if self.instance is None:
906 self.instance = self.make()
908 return self.instance == other
910 def __repr__(self):
911 sargs = []
912 for arg in self.args:
913 sargs.append(repr(arg))
915 for k, v in self.kwargs.items():
916 sargs.append(
917 '%s=%s' % (
918 k,
919 ' '.join(line.strip() for line in repr(v).splitlines())))
921 return '%s(%s)' % (self._cls.__name__, ', '.join(sargs))
924class TimestampDefaultMaker(DefaultMaker):
925 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'):
926 DefaultMaker.__init__(self)
927 self._stime = s
928 self._format = format
930 def make(self):
931 return str_to_time(self._stime, self._format)
933 def __repr__(self):
934 return 'str_to_time(%s)' % repr(self._stime)
937def with_metaclass(meta, *bases):
938 # inlined py2/py3 compat solution from python-future
939 class metaclass(meta):
940 __call__ = type.__call__
941 __init__ = type.__init__
943 def __new__(cls, name, this_bases, d):
944 if this_bases is None:
945 return type.__new__(cls, name, (), d)
946 return meta(name, bases, d)
948 return metaclass('temp', None, {})
951class Object(with_metaclass(ObjectMetaClass, object)):
952 '''
953 Base class for Guts objects.
955 :cvar dummy_for:
956 (class variable) If set, this indicates that the containing class is a
957 dummy for another type. This can be used to hold native Python objects
958 and non-Guts based objects as children of a Guts object.
959 :cvar dummy_for_description:
960 (class variable) Overrides the name shown in the "dummy for ..."
961 documentation strings.
962 '''
964 dummy_for = None
965 dummy_for_description = None
967 def __init__(self, **kwargs):
968 if not kwargs.get('init_props', True):
969 return
971 for prop in self.T.properties:
972 k = prop.name
973 if k in kwargs:
974 setattr(self, k, kwargs.pop(k))
975 else:
976 if not prop.optional and not prop.has_default():
977 raise ArgumentError('Missing argument to %s: %s' % (
978 self.T.tagname, prop.name))
979 else:
980 setattr(self, k, prop.default())
982 if kwargs:
983 raise ArgumentError('Invalid argument to %s: %s' % (
984 self.T.tagname, ', '.join(list(kwargs.keys()))))
986 @classmethod
987 def D(cls, *args, **kwargs):
988 '''
989 Get a default value factory for this class, configured with
990 specified arguments.
992 :returns:
993 Factory for default values.
994 :rtype:
995 :py:class:`ObjectDefaultMaker` object
996 '''
997 return ObjectDefaultMaker(cls, args, kwargs)
999 def validate(self, regularize=False, depth=-1):
1000 '''
1001 Validate this object.
1003 Raises :py:class:`ValidationError` when the object is invalid.
1005 :param depth:
1006 Maximum depth to descend into child objects.
1007 :type depth:
1008 int
1009 '''
1010 self.T.instance.validate(self, regularize, depth)
1012 def regularize(self, depth=-1):
1013 '''
1014 Regularize this object.
1016 Regularization tries to convert child objects of invalid types to the
1017 expected types.
1019 Raises :py:class:`ValidationError` when the object is invalid and
1020 cannot be regularized.
1022 :param depth:
1023 Maximum depth to descend into child objects.
1024 :type depth:
1025 int
1026 '''
1027 self.validate(regularize=True, depth=depth)
1029 def dump(self, stream=None, filename=None, header=False):
1030 '''
1031 Serialize to YAML.
1033 If neither ``stream`` nor ``filename`` is set, a string containing the
1034 serialized data is returned.
1036 :param stream:
1037 Output to stream.
1039 :param filename:
1040 Output to file of given name.
1041 :type filename:
1042 str
1044 :param header:
1045 File header to prepend to the output.
1046 :type header:
1047 str
1048 '''
1049 return dump(self, stream=stream, filename=filename, header=header)
1051 def dump_xml(
1052 self, stream=None, filename=None, header=False, ns_ignore=False):
1053 '''
1054 Serialize to XML.
1056 If neither ``stream`` nor ``filename`` is set, a string containing the
1057 serialized data is returned.
1059 :param stream:
1060 Output to stream.
1062 :param filename:
1063 Output to file of given name.
1064 :type filename:
1065 str
1067 :param header:
1068 File header to prepend to the output.
1069 :type header:
1070 str
1072 :param ns_ignore:
1073 Whether to ignore the XML namespace.
1074 :type ns_ignore:
1075 bool
1076 '''
1077 return dump_xml(
1078 self, stream=stream, filename=filename, header=header,
1079 ns_ignore=ns_ignore)
1081 @classmethod
1082 def load(cls, stream=None, filename=None, string=None):
1083 '''
1084 Deserialize from YAML.
1086 :param stream:
1087 Read input from stream.
1089 :param filename:
1090 Read input from file of given name.
1091 :type filename:
1092 str
1094 :param string:
1095 Read input from string.
1096 :type string:
1097 str
1098 '''
1099 return load(stream=stream, filename=filename, string=string)
1101 @classmethod
1102 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None,
1103 ns_ignore=False):
1104 '''
1105 Deserialize from XML.
1107 :param stream:
1108 Read input from stream.
1110 :param filename:
1111 Read input from file of given name.
1112 :type filename:
1113 str
1115 :param string:
1116 Read input from string.
1117 :type string:
1118 str
1119 '''
1121 if ns_hints is None:
1122 ns_hints = [cls.T.instance.get_xmlns()]
1124 return load_xml(
1125 stream=stream,
1126 filename=filename,
1127 string=string,
1128 ns_hints=ns_hints,
1129 ns_ignore=ns_ignore)
1131 def __str__(self):
1132 return self.dump()
1135def to_dict(obj):
1136 '''
1137 Get dict of guts object attributes.
1139 :param obj: :py:class`Object` object
1140 '''
1142 return dict(obj.T.inamevals(obj))
1145class SObject(Object):
1146 '''
1147 Base class for simple str-serializable Guts objects.
1149 Derived classes must support (de)serialization as in ``X(str(x))``.
1150 '''
1152 class __T(TBase):
1153 def regularize_extra(self, val):
1154 if isinstance(val, str):
1155 return self._cls(val)
1157 return val
1159 def to_save(self, val):
1160 return str(val)
1162 def to_save_xml(self, val):
1163 return str(val)
1166class Any(Object):
1167 '''
1168 Placeholder for any object.
1169 '''
1171 class __T(TBase):
1172 def validate(self, val, regularize=False, depth=-1):
1173 if isinstance(val, Object):
1174 val.validate(regularize, depth)
1176 return val
1179class Int(Object):
1180 '''
1181 Placeholder for :py:class:`int`.
1182 '''
1183 dummy_for = int
1185 class __T(TBase):
1186 strict = True
1188 def to_save_xml(self, value):
1189 return repr(value)
1192class Float(Object):
1193 '''
1194 Placeholder for :py:class:`float`.
1195 '''
1196 dummy_for = float
1198 class __T(TBase):
1199 strict = True
1201 def to_save_xml(self, value):
1202 return repr(value)
1205class Complex(Object):
1206 '''
1207 Placeholder for :py:class:`complex`.
1208 '''
1209 dummy_for = complex
1211 class __T(TBase):
1212 strict = True
1214 def regularize_extra(self, val):
1216 if isinstance(val, list) or isinstance(val, tuple):
1217 assert len(val) == 2
1218 val = complex(*val)
1220 elif not isinstance(val, complex):
1221 val = complex(val)
1223 return val
1225 def to_save(self, value):
1226 return repr(value)
1228 def to_save_xml(self, value):
1229 return repr(value)
1232class Bool(Object):
1233 '''
1234 Placeholder for :py:class:`bool`.
1235 '''
1236 dummy_for = bool
1238 class __T(TBase):
1239 strict = True
1241 def regularize_extra(self, val):
1242 if isinstance(val, str):
1243 if val.lower().strip() in ('0', 'false'):
1244 return False
1246 return bool(val)
1248 def to_save_xml(self, value):
1249 return repr(bool(value)).lower()
1252class String(Object):
1253 '''
1254 Placeholder for :py:class:`str`.
1255 '''
1256 dummy_for = str
1258 class __T(TBase):
1259 def __init__(self, *args, yamlstyle=None, **kwargs):
1260 TBase.__init__(self, *args, **kwargs)
1261 self.style_cls = str_style_map[yamlstyle]
1263 def to_save(self, val):
1264 return self.style_cls(val)
1267class Bytes(Object):
1268 '''
1269 Placeholder for :py:class:`bytes`.
1270 '''
1271 dummy_for = bytes
1273 class __T(TBase):
1275 def regularize_extra(self, val):
1276 if isinstance(val, str):
1277 val = b64decode(val)
1279 return val
1281 def to_save(self, val):
1282 return literal(b64encode(val).decode('utf-8'))
1285class Unicode(Object):
1286 '''
1287 Placeholder for :py:class:`str`.
1288 '''
1289 dummy_for = str
1291 class __T(TBase):
1292 def __init__(self, *args, yamlstyle=None, **kwargs):
1293 TBase.__init__(self, *args, **kwargs)
1294 self.style_cls = unicode_style_map[yamlstyle]
1296 def to_save(self, val):
1297 return self.style_cls(val)
1300guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool)
1303class Dict(Object):
1304 '''
1305 Placeholder for :py:class:`dict`.
1306 '''
1307 dummy_for = dict
1309 class __T(TBase):
1310 multivalued = dict
1312 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs):
1313 TBase.__init__(self, *args, **kwargs)
1314 assert isinstance(key_t, TBase)
1315 assert isinstance(content_t, TBase)
1316 self.key_t = key_t
1317 self.content_t = content_t
1318 self.content_t.parent = self
1320 def default(self):
1321 if self._default is not None:
1322 return dict(
1323 (make_default(k), make_default(v))
1324 for (k, v) in self._default.items())
1326 if self.optional:
1327 return None
1328 else:
1329 return {}
1331 def has_default(self):
1332 return True
1334 def validate(self, val, regularize, depth):
1335 return TBase.validate(self, val, regularize, depth+1)
1337 def validate_children(self, val, regularize, depth):
1338 for key, ele in list(val.items()):
1339 newkey = self.key_t.validate(key, regularize, depth-1)
1340 newele = self.content_t.validate(ele, regularize, depth-1)
1341 if regularize:
1342 if newkey is not key or newele is not ele:
1343 del val[key]
1344 val[newkey] = newele
1346 return val
1348 def to_save(self, val):
1349 return dict((self.key_t.to_save(k), self.content_t.to_save(v))
1350 for (k, v) in val.items())
1352 def to_save_xml(self, val):
1353 raise NotImplementedError
1355 def classname_for_help(self, strip_module=''):
1356 return '``dict`` of %s objects' % \
1357 self.content_t.classname_for_help(strip_module=strip_module)
1360class List(Object):
1361 '''
1362 Placeholder for :py:class:`list`.
1363 '''
1364 dummy_for = list
1366 class __T(TBase):
1367 multivalued = list
1369 def __init__(self, content_t=Any.T(), *args, yamlstyle=None, **kwargs):
1370 TBase.__init__(self, *args, **kwargs)
1371 assert isinstance(content_t, TBase) or isinstance(content_t, Defer)
1372 self.content_t = content_t
1373 self.content_t.parent = self
1374 self.style_cls = list_style_map[yamlstyle]
1376 def default(self):
1377 if self._default is not None:
1378 return [make_default(x) for x in self._default]
1379 if self.optional:
1380 return None
1381 else:
1382 return []
1384 def has_default(self):
1385 return True
1387 def validate(self, val, regularize, depth):
1388 return TBase.validate(self, val, regularize, depth+1)
1390 def validate_children(self, val, regularize, depth):
1391 for i, ele in enumerate(val):
1392 newele = self.content_t.validate(ele, regularize, depth-1)
1393 if regularize and newele is not ele:
1394 val[i] = newele
1396 return val
1398 def to_save(self, val):
1399 return self.style_cls(self.content_t.to_save(v) for v in val)
1401 def to_save_xml(self, val):
1402 return [self.content_t.to_save_xml(v) for v in val]
1404 def deferred(self):
1405 if isinstance(self.content_t, Defer):
1406 return [self.content_t]
1408 return []
1410 def process_deferred(self, defer, t_inst):
1411 if defer is self.content_t:
1412 self.content_t = t_inst
1414 def classname_for_help(self, strip_module=''):
1415 return '``list`` of %s objects' % \
1416 self.content_t.classname_for_help(strip_module=strip_module)
1419def make_typed_list_class(t):
1420 class TL(List):
1421 class __T(List.T):
1422 def __init__(self, *args, **kwargs):
1423 List.T.__init__(self, content_t=t.T(), *args, **kwargs)
1425 return TL
1428class Tuple(Object):
1429 '''
1430 Placeholder for :py:class:`tuple`.
1431 '''
1432 dummy_for = tuple
1434 class __T(TBase):
1435 multivalued = tuple
1437 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs):
1438 TBase.__init__(self, *args, **kwargs)
1439 assert isinstance(content_t, TBase)
1440 self.content_t = content_t
1441 self.content_t.parent = self
1442 self.n = n
1444 def default(self):
1445 if self._default is not None:
1446 return tuple(
1447 make_default(x) for x in self._default)
1449 elif self.optional:
1450 return None
1451 else:
1452 if self.n is not None:
1453 return tuple(
1454 self.content_t.default() for x in range(self.n))
1455 else:
1456 return tuple()
1458 def has_default(self):
1459 return True
1461 def validate(self, val, regularize, depth):
1462 return TBase.validate(self, val, regularize, depth+1)
1464 def validate_extra(self, val):
1465 if self.n is not None and len(val) != self.n:
1466 raise ValidationError(
1467 '%s should have length %i' % (self.xname(), self.n))
1469 def validate_children(self, val, regularize, depth):
1470 if not regularize:
1471 for ele in val:
1472 self.content_t.validate(ele, regularize, depth-1)
1474 return val
1475 else:
1476 newval = []
1477 isnew = False
1478 for ele in val:
1479 newele = self.content_t.validate(ele, regularize, depth-1)
1480 newval.append(newele)
1481 if newele is not ele:
1482 isnew = True
1484 if isnew:
1485 return tuple(newval)
1486 else:
1487 return val
1489 def to_save(self, val):
1490 return tuple(self.content_t.to_save(v) for v in val)
1492 def to_save_xml(self, val):
1493 return [self.content_t.to_save_xml(v) for v in val]
1495 def classname_for_help(self, strip_module=''):
1496 if self.n is not None:
1497 return '``tuple`` of %i %s objects' % (
1498 self.n, self.content_t.classname_for_help(
1499 strip_module=strip_module))
1500 else:
1501 return '``tuple`` of %s objects' % (
1502 self.content_t.classname_for_help(
1503 strip_module=strip_module))
1506duration_unit_factors = dict(
1507 s=1.0,
1508 m=60.0,
1509 h=3600.0,
1510 d=24*3600.0,
1511 y=365*24*3600.0)
1514def parse_duration(s):
1515 unit = s[-1]
1516 if unit in duration_unit_factors:
1517 return float(s[:-1]) * duration_unit_factors[unit]
1518 else:
1519 return float(s)
1522def str_duration(d):
1523 for k in 'ydhms':
1524 if abs(d) >= duration_unit_factors[k]:
1525 return '%g' % (d / duration_unit_factors[k]) + k
1527 return '%g' % d
1530class Duration(Object):
1531 '''
1532 Placeholder for :py:class:`float` time duration [s] with human-readable
1533 (de)serialization.
1535 Examples:
1537 - ``'1s'`` -> 1 second
1538 - ``'1m'`` -> 1 minute
1539 - ``'1h'`` -> 1 hour
1540 - ``'1d'`` -> 1 day
1541 - ``'1y'`` -> about 1 year = 365*24*3600 seconds
1542 '''
1543 dummy_for = float
1545 class __T(TBase):
1546 def regularize_extra(self, val):
1547 if isinstance(val, str):
1548 return parse_duration(val)
1550 return val
1552 def to_save(self, val):
1553 return str_duration(val)
1555 def to_save_xml(self, val):
1556 return str_duration(val)
1559re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$')
1562class Timestamp(Object):
1563 '''
1564 Placeholder for a UTC timestamp.
1565 '''
1566 dummy_for = (hpfloat, float)
1567 dummy_for_description = 'pyrocko.util.get_time_float'
1569 class __T(TBase):
1571 def regularize_extra(self, val):
1573 time_float = get_time_float()
1575 if isinstance(val, datetime.datetime):
1576 tt = val.utctimetuple()
1577 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1579 elif isinstance(val, datetime.date):
1580 tt = val.timetuple()
1581 val = time_float(calendar.timegm(tt))
1583 elif isinstance(val, str):
1584 val = val.strip()
1585 tz_offset = 0
1587 m = re_tz.search(val)
1588 if m:
1589 sh = m.group(2)
1590 sm = m.group(4)
1591 tz_offset = (int(sh)*3600 if sh else 0) \
1592 + (int(sm)*60 if sm else 0)
1594 val = re_tz.sub('', val)
1596 if len(val) > 10 and val[10] == 'T':
1597 val = val.replace('T', ' ', 1)
1599 try:
1600 val = str_to_time(val) - tz_offset
1601 except TimeStrError:
1602 raise ValidationError(
1603 '%s: cannot parse time/date: %s' % (self.xname(), val))
1605 elif isinstance(val, (int, float)):
1606 val = time_float(val)
1608 else:
1609 raise ValidationError(
1610 '%s: cannot convert "%s" to type %s' % (
1611 self.xname(), val, time_float))
1613 return val
1615 def to_save(self, val):
1616 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\
1617 .rstrip('0').rstrip('.')
1619 def to_save_xml(self, val):
1620 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\
1621 .rstrip('0').rstrip('.') + 'Z'
1623 @classmethod
1624 def D(self, s):
1625 return TimestampDefaultMaker(s)
1628class DateTimestamp(Object):
1629 '''
1630 Placeholder for a UTC timestamp which (de)serializes as a date string.
1631 '''
1632 dummy_for = (hpfloat, float)
1633 dummy_for_description = 'pyrocko.util.get_time_float'
1635 class __T(TBase):
1637 def regularize_extra(self, val):
1639 time_float = get_time_float()
1641 if isinstance(val, datetime.datetime):
1642 tt = val.utctimetuple()
1643 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1645 elif isinstance(val, datetime.date):
1646 tt = val.timetuple()
1647 val = time_float(calendar.timegm(tt))
1649 elif isinstance(val, str):
1650 val = str_to_time(val, format='%Y-%m-%d')
1652 elif isinstance(val, int):
1653 val = time_float(val)
1655 return val
1657 def to_save(self, val):
1658 return time_to_str(val, format='%Y-%m-%d')
1660 def to_save_xml(self, val):
1661 return time_to_str(val, format='%Y-%m-%d')
1663 @classmethod
1664 def D(self, s):
1665 return TimestampDefaultMaker(s, format='%Y-%m-%d')
1668class StringPattern(String):
1670 '''
1671 Any :py:class:`str` matching pattern ``%(pattern)s``.
1672 '''
1674 dummy_for = str
1675 pattern = '.*'
1677 class __T(String.T):
1678 def __init__(self, pattern=None, *args, **kwargs):
1679 String.T.__init__(self, *args, **kwargs)
1681 if pattern is not None:
1682 self.pattern = pattern
1683 else:
1684 self.pattern = self._dummy_cls.pattern
1686 def validate_extra(self, val):
1687 pat = self.pattern
1688 if not re.search(pat, val):
1689 raise ValidationError('%s: "%s" does not match pattern %s' % (
1690 self.xname(), val, repr(pat)))
1692 @classmethod
1693 def class_help_string(cls):
1694 dcls = cls._dummy_cls
1695 doc = dcls.__doc_template__ or StringPattern.__doc_template__
1696 return doc % {'pattern': repr(dcls.pattern)}
1699class UnicodePattern(Unicode):
1701 '''
1702 Any :py:class:`str` matching pattern ``%(pattern)s``.
1703 '''
1705 dummy_for = str
1706 pattern = '.*'
1708 class __T(TBase):
1709 def __init__(self, pattern=None, *args, **kwargs):
1710 TBase.__init__(self, *args, **kwargs)
1712 if pattern is not None:
1713 self.pattern = pattern
1714 else:
1715 self.pattern = self._dummy_cls.pattern
1717 def validate_extra(self, val):
1718 pat = self.pattern
1719 if not re.search(pat, val, flags=re.UNICODE):
1720 raise ValidationError('%s: "%s" does not match pattern %s' % (
1721 self.xname(), val, repr(pat)))
1723 @classmethod
1724 def class_help_string(cls):
1725 dcls = cls._dummy_cls
1726 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__
1727 return doc % {'pattern': repr(dcls.pattern)}
1730class StringChoice(String):
1732 '''
1733 Any :py:class:`str` out of ``%(choices)s``.
1735 :cvar choices:
1736 Allowed choices (:py:class:`list` of :py:class:`str`).
1737 :cvar ignore_case:
1738 Whether to behave case-insensitive (:py:class:`bool`, default:
1739 ``False``).
1740 '''
1742 dummy_for = str
1743 choices = []
1744 ignore_case = False
1746 class __T(String.T):
1747 def __init__(self, choices=None, ignore_case=None, *args, **kwargs):
1748 String.T.__init__(self, *args, **kwargs)
1750 if choices is not None:
1751 self.choices = choices
1752 else:
1753 self.choices = self._dummy_cls.choices
1755 if ignore_case is not None:
1756 self.ignore_case = ignore_case
1757 else:
1758 self.ignore_case = self._dummy_cls.ignore_case
1760 if self.ignore_case:
1761 self.choices = [x.upper() for x in self.choices]
1763 def validate_extra(self, val):
1764 if self.ignore_case:
1765 val = val.upper()
1767 if val not in self.choices:
1768 raise ValidationError(
1769 '%s: "%s" is not a valid choice out of %s' % (
1770 self.xname(), val, repr(self.choices)))
1772 @classmethod
1773 def class_help_string(cls):
1774 dcls = cls._dummy_cls
1775 doc = dcls.__doc_template__ or StringChoice.__doc_template__
1776 return doc % {'choices': repr(dcls.choices)}
1779class IntChoice(Int):
1781 '''
1782 Any :py:class:`int` out of ``%(choices)s``.
1783 '''
1785 dummy_for = int
1786 choices = []
1788 class __T(Int.T):
1789 def __init__(self, choices=None, *args, **kwargs):
1790 Int.T.__init__(self, *args, **kwargs)
1792 if choices is not None:
1793 self.choices = choices
1794 else:
1795 self.choices = self._dummy_cls.choices
1797 def validate_extra(self, val):
1798 if val not in self.choices:
1799 raise ValidationError(
1800 '%s: %i is not a valid choice out of %s' % (
1801 self.xname(), val, repr(self.choices)))
1803 @classmethod
1804 def class_help_string(cls):
1805 dcls = cls._dummy_cls
1806 doc = dcls.__doc_template__ or IntChoice.__doc_template__
1807 return doc % {'choices': repr(dcls.choices)}
1810# this will not always work...
1811class StringUnion(Object):
1812 '''
1813 Any :py:class:`str` matching any of a set of constraints.
1815 :cvar members:
1816 List of constraints, e.g. :py:class:`StringChoice`,
1817 :py:class:`StringPattern`, ... (:py:class:`list` of :py:class:`TBase`
1818 derived objects).
1820 '''
1822 members = []
1824 dummy_for = str
1826 class __T(TBase):
1827 def __init__(self, members=None, *args, **kwargs):
1828 TBase.__init__(self, *args, **kwargs)
1829 if members is not None:
1830 self.members = members
1831 else:
1832 self.members = self._dummy_cls.members
1834 def validate(self, val, regularize=False, depth=-1):
1835 assert self.members
1836 e2 = None
1837 for member in self.members:
1838 try:
1839 return member.validate(val, regularize, depth=depth)
1840 except ValidationError as e:
1841 e2 = e
1843 raise e2
1846class Choice(Object):
1847 '''
1848 Any out of a set of different types.
1850 :cvar choices:
1851 Allowed types (:py:class:`list` of :py:class:`TBase` derived objects).
1853 '''
1854 choices = []
1856 class __T(TBase):
1857 def __init__(self, choices=None, *args, **kwargs):
1858 TBase.__init__(self, *args, **kwargs)
1859 if choices is not None:
1860 self.choices = choices
1861 else:
1862 self.choices = self._dummy_cls.choices
1864 self.cls_to_xmltagname = dict(
1865 (t._cls, t.get_xmltagname()) for t in self.choices)
1867 def validate(self, val, regularize=False, depth=-1):
1868 if self.optional and val is None:
1869 return val
1871 t = None
1872 for tc in self.choices:
1873 is_derived = isinstance(val, tc._cls)
1874 is_exact = type(val) == tc._cls
1875 if not (not tc.strict and not is_derived or
1876 tc.strict and not is_exact):
1878 t = tc
1879 break
1881 if t is None:
1882 if regularize:
1883 ok = False
1884 for tc in self.choices:
1885 try:
1886 val = tc.regularize_extra(val)
1887 ok = True
1888 t = tc
1889 break
1890 except (ValidationError, ValueError):
1891 pass
1893 if not ok:
1894 raise ValidationError(
1895 '%s: could not convert "%s" to any type out of '
1896 '(%s)' % (self.xname(), val, ','.join(
1897 classnames(x._cls) for x in self.choices)))
1898 else:
1899 raise ValidationError(
1900 '%s: "%s" (type: %s) is not of any type out of '
1901 '(%s)' % (self.xname(), val, type(val), ','.join(
1902 classnames(x._cls) for x in self.choices)))
1904 validator = t
1906 if isinstance(t._cls, tuple):
1907 clss = t._cls
1908 else:
1909 clss = (t._cls,)
1911 for cls in clss:
1912 try:
1913 if type(val) != cls and isinstance(val, cls):
1914 validator = val.T.instance
1916 except AttributeError:
1917 pass
1919 validator.validate_extra(val)
1921 if depth != 0:
1922 val = validator.validate_children(val, regularize, depth)
1924 return val
1926 def extend_xmlelements(self, elems, v):
1927 elems.append((
1928 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v))
1931def _dump(
1932 object, stream,
1933 header=False,
1934 Dumper=GutsSafeDumper,
1935 _dump_function=yaml.dump):
1937 if not getattr(stream, 'encoding', None):
1938 enc = encode_utf8
1939 else:
1940 enc = no_encode
1942 if header:
1943 stream.write(enc(u'%YAML 1.1\n'))
1944 if isinstance(header, str):
1945 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
1946 stream.write(enc(banner))
1948 _dump_function(
1949 object,
1950 stream=stream,
1951 encoding='utf-8',
1952 explicit_start=True,
1953 Dumper=Dumper)
1956def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper):
1957 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all)
1960def _load(stream,
1961 Loader=GutsSafeLoader, allow_include=None, filename=None,
1962 included_files=None):
1964 class _Loader(Loader):
1965 _filename = filename
1966 _allow_include = allow_include
1967 _included_files = included_files or []
1969 return yaml.load(stream=stream, Loader=_Loader)
1972def _load_all(stream,
1973 Loader=GutsSafeLoader, allow_include=None, filename=None):
1975 class _Loader(Loader):
1976 _filename = filename
1977 _allow_include = allow_include
1979 return list(yaml.load_all(stream=stream, Loader=_Loader))
1982def _iload_all(stream,
1983 Loader=GutsSafeLoader, allow_include=None, filename=None):
1985 class _Loader(Loader):
1986 _filename = filename
1987 _allow_include = allow_include
1989 return yaml.load_all(stream=stream, Loader=_Loader)
1992def multi_representer(dumper, data):
1993 node = dumper.represent_mapping(
1994 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False)
1996 return node
1999# hack for compatibility with early GF Store versions
2000re_compatibility = re.compile(
2001 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.'
2002 r'(dummy|poel|qseis|qssp))\.'
2003)
2006def multi_constructor(loader, tag_suffix, node):
2007 tagname = str(tag_suffix)
2009 tagname = re_compatibility.sub('pf.', tagname)
2011 cls = g_tagname_to_class[tagname]
2012 kwargs = dict(iter(loader.construct_pairs(node, deep=True)))
2013 o = cls(**kwargs)
2014 o.validate(regularize=True, depth=1)
2015 return o
2018def include_constructor(loader, node):
2019 allow_include = loader._allow_include \
2020 if loader._allow_include is not None \
2021 else ALLOW_INCLUDE
2023 if not allow_include:
2024 raise EnvironmentError(
2025 'Not allowed to include YAML. Load with allow_include=True')
2027 if isinstance(node, yaml.nodes.ScalarNode):
2028 inc_file = loader.construct_scalar(node)
2029 else:
2030 raise TypeError('Unsupported YAML node %s' % repr(node))
2032 if loader._filename is not None and not op.isabs(inc_file):
2033 inc_file = op.join(op.dirname(loader._filename), inc_file)
2035 if not op.isfile(inc_file):
2036 raise FileNotFoundError(inc_file)
2038 included_files = list(loader._included_files)
2039 if loader._filename is not None:
2040 included_files.append(op.abspath(loader._filename))
2042 for included_file in loader._included_files:
2043 if op.samefile(inc_file, included_file):
2044 raise ImportError(
2045 'Circular import of file "%s". Include path: %s' % (
2046 op.abspath(inc_file),
2047 ' -> '.join('"%s"' % s for s in included_files)))
2049 with open(inc_file, 'rb') as f:
2050 return _load(
2051 f,
2052 Loader=loader.__class__, filename=inc_file,
2053 allow_include=True,
2054 included_files=included_files)
2057def dict_noflow_representer(dumper, data):
2058 return dumper.represent_mapping(
2059 'tag:yaml.org,2002:map', data, flow_style=False)
2062yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper)
2063yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader)
2064yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader)
2065yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper)
2068def str_representer(dumper, data):
2069 return dumper.represent_scalar(
2070 'tag:yaml.org,2002:str', str(data))
2073yaml.add_representer(str, str_representer, Dumper=GutsSafeDumper)
2076class Constructor(object):
2077 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None,
2078 ns_ignore=False):
2080 self.stack = []
2081 self.queue = []
2082 self.namespaces = defaultdict(list)
2083 self.add_namespace_maps = add_namespace_maps
2084 self.strict = strict
2085 self.ns_hints = ns_hints
2086 self.ns_ignore = ns_ignore
2088 def start_element(self, ns_name, attrs):
2089 if self.ns_ignore:
2090 ns_name = ns_name.split(' ')[-1]
2092 if -1 == ns_name.find(' '):
2093 if self.ns_hints is None and ns_name in g_guessable_xmlns:
2094 self.ns_hints = g_guessable_xmlns[ns_name]
2096 if self.ns_hints:
2097 ns_names = [
2098 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints]
2100 elif self.ns_hints is None:
2101 ns_names = [' ' + ns_name]
2103 else:
2104 ns_names = [ns_name]
2106 for ns_name in ns_names:
2107 if self.stack and self.stack[-1][1] is not None:
2108 cls = self.stack[-1][1].T.xmltagname_to_class.get(
2109 ns_name, None)
2111 if isinstance(cls, tuple):
2112 cls = None
2113 else:
2114 if cls is not None and (
2115 not issubclass(cls, Object)
2116 or issubclass(cls, SObject)):
2117 cls = None
2118 else:
2119 cls = g_xmltagname_to_class.get(ns_name, None)
2121 if cls:
2122 break
2124 self.stack.append((ns_name, cls, attrs, [], []))
2126 def end_element(self, _):
2127 ns_name, cls, attrs, content2, content1 = self.stack.pop()
2129 ns = ns_name.split(' ', 1)[0]
2131 if cls is not None:
2132 content2.extend(
2133 (ns + ' ' + k if -1 == k.find(' ') else k, v)
2134 for (k, v) in attrs.items())
2135 content2.append((None, ''.join(content1)))
2136 o = cls(**cls.T.translate_from_xml(content2, self.strict))
2137 o.validate(regularize=True, depth=1)
2138 if self.add_namespace_maps:
2139 o.namespace_map = self.get_current_namespace_map()
2141 if self.stack and not all(x[1] is None for x in self.stack):
2142 self.stack[-1][-2].append((ns_name, o))
2143 else:
2144 self.queue.append(o)
2145 else:
2146 content = [''.join(content1)]
2147 if self.stack:
2148 for c in content:
2149 self.stack[-1][-2].append((ns_name, c))
2151 def characters(self, char_content):
2152 if self.stack:
2153 self.stack[-1][-1].append(char_content)
2155 def start_namespace(self, ns, uri):
2156 self.namespaces[ns].append(uri)
2158 def end_namespace(self, ns):
2159 self.namespaces[ns].pop()
2161 def get_current_namespace_map(self):
2162 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v)
2164 def get_queued_elements(self):
2165 queue = self.queue
2166 self.queue = []
2167 return queue
2170def _iload_all_xml(
2171 stream,
2172 bufsize=100000,
2173 add_namespace_maps=False,
2174 strict=False,
2175 ns_hints=None,
2176 ns_ignore=False):
2178 from xml.parsers.expat import ParserCreate
2180 parser = ParserCreate('UTF-8', namespace_separator=' ')
2182 handler = Constructor(
2183 add_namespace_maps=add_namespace_maps,
2184 strict=strict,
2185 ns_hints=ns_hints,
2186 ns_ignore=ns_ignore)
2188 parser.StartElementHandler = handler.start_element
2189 parser.EndElementHandler = handler.end_element
2190 parser.CharacterDataHandler = handler.characters
2191 parser.StartNamespaceDeclHandler = handler.start_namespace
2192 parser.EndNamespaceDeclHandler = handler.end_namespace
2194 while True:
2195 data = stream.read(bufsize)
2196 parser.Parse(data, bool(not data))
2197 for element in handler.get_queued_elements():
2198 yield element
2200 if not data:
2201 break
2204def _load_all_xml(*args, **kwargs):
2205 return list(_iload_all_xml(*args, **kwargs))
2208def _load_xml(*args, **kwargs):
2209 g = _iload_all_xml(*args, **kwargs)
2210 return next(g)
2213def _dump_all_xml(objects, stream, root_element_name='root', header=True):
2215 if not getattr(stream, 'encoding', None):
2216 enc = encode_utf8
2217 else:
2218 enc = no_encode
2220 _dump_xml_header(stream, header)
2222 beg = u'<%s>\n' % root_element_name
2223 end = u'</%s>\n' % root_element_name
2225 stream.write(enc(beg))
2227 for ob in objects:
2228 _dump_xml(ob, stream=stream)
2230 stream.write(enc(end))
2233def _dump_xml_header(stream, banner=None):
2235 if not getattr(stream, 'encoding', None):
2236 enc = encode_utf8
2237 else:
2238 enc = no_encode
2240 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n'))
2241 if isinstance(banner, str):
2242 stream.write(enc(u'<!-- %s -->\n' % banner))
2245def _dump_xml(
2246 obj, stream, depth=0, ns_name=None, header=False, ns_map=[],
2247 ns_ignore=False):
2249 from xml.sax.saxutils import escape, quoteattr
2251 if not getattr(stream, 'encoding', None):
2252 enc = encode_utf8
2253 else:
2254 enc = no_encode
2256 if depth == 0 and header:
2257 _dump_xml_header(stream, header)
2259 indent = ' '*depth*2
2260 if ns_name is None:
2261 ns_name = obj.T.instance.get_xmltagname()
2263 if -1 != ns_name.find(' '):
2264 ns, name = ns_name.split(' ')
2265 else:
2266 ns, name = '', ns_name
2268 if isinstance(obj, Object):
2269 obj.validate(depth=1)
2270 attrs = []
2271 elems = []
2273 added_ns = False
2274 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns):
2275 attrs.append(('xmlns', ns))
2276 ns_map.append(ns)
2277 added_ns = True
2279 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True):
2280 if prop.xmlstyle == 'attribute':
2281 assert not prop.multivalued
2282 assert not isinstance(v, Object)
2283 attrs.append((prop.effective_xmltagname, v))
2285 elif prop.xmlstyle == 'content':
2286 assert not prop.multivalued
2287 assert not isinstance(v, Object)
2288 elems.append((None, v))
2290 else:
2291 prop.extend_xmlelements(elems, v)
2293 attr_str = ''
2294 if attrs:
2295 attr_str = ' ' + ' '.join(
2296 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v)))
2297 for (k, v) in attrs)
2299 if not elems:
2300 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str)))
2301 else:
2302 oneline = len(elems) == 1 and elems[0][0] is None
2303 stream.write(enc(u'%s<%s%s>%s' % (
2304 indent,
2305 name,
2306 attr_str,
2307 '' if oneline else '\n')))
2309 for (k, v) in elems:
2310 if k is None:
2311 stream.write(enc(escape(str(v), {'\0': '�'})))
2312 else:
2313 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore)
2315 stream.write(enc(u'%s</%s>\n' % (
2316 '' if oneline else indent, name)))
2318 if added_ns:
2319 ns_map.pop()
2321 else:
2322 stream.write(enc(u'%s<%s>%s</%s>\n' % (
2323 indent,
2324 name,
2325 escape(str(obj), {'\0': '�'}),
2326 name)))
2329def walk(x, typ=None, path=()):
2330 if typ is None or isinstance(x, typ):
2331 yield path, x
2333 if isinstance(x, Object):
2334 for (prop, val) in x.T.ipropvals(x):
2335 if prop.multivalued:
2336 if val is not None:
2337 for iele, ele in enumerate(val):
2338 for y in walk(ele, typ,
2339 path=path + ((prop.name, iele),)):
2340 yield y
2341 else:
2342 for y in walk(val, typ, path=path+(prop.name,)):
2343 yield y
2346def clone(x, pool=None):
2347 '''
2348 Clone guts object tree.
2350 Traverses guts object tree and recursively clones all guts attributes,
2351 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects
2352 deriving from :py:class:`Object` are instantiated using their respective
2353 init function. Multiply referenced objects in the source tree are multiply
2354 referenced also in the destination tree.
2356 This function can be used to clone guts objects ignoring any contained
2357 run-time state, i.e. any of their attributes not defined as a guts
2358 property.
2359 '''
2361 if pool is None:
2362 pool = {}
2364 if id(x) in pool:
2365 x_copy = pool[id(x)][1]
2367 else:
2368 if isinstance(x, SObject):
2369 x_copy = x.__class__(str(x))
2370 elif isinstance(x, Object):
2371 d = {}
2372 for (prop, y) in x.T.ipropvals(x):
2373 if y is not None:
2374 if not prop.multivalued:
2375 y_copy = clone(y, pool)
2376 elif prop.multivalued is dict:
2377 y_copy = dict(
2378 (clone(zk, pool), clone(zv, pool))
2379 for (zk, zv) in y.items())
2380 else:
2381 y_copy = type(y)(clone(z, pool) for z in y)
2382 else:
2383 y_copy = y
2385 d[prop.name] = y_copy
2387 x_copy = x.__class__(**d)
2389 else:
2390 x_copy = copy.deepcopy(x)
2392 pool[id(x)] = (x, x_copy)
2393 return x_copy
2396class YPathError(Exception):
2397 '''
2398 This exception is raised for invalid ypath specifications.
2399 '''
2400 pass
2403def _parse_yname(yname):
2404 ident = r'[a-zA-Z][a-zA-Z0-9_]*'
2405 rint = r'-?[0-9]+'
2406 m = re.match(
2407 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$'
2408 % (ident, rint, rint, rint), yname)
2410 if not m:
2411 raise YPathError('Syntax error in component: "%s"' % yname)
2413 d = dict(
2414 name=m.group(1))
2416 if m.group(2):
2417 if m.group(5):
2418 istart = iend = None
2419 if m.group(4):
2420 istart = int(m.group(4))
2421 if m.group(6):
2422 iend = int(m.group(6))
2424 d['slice'] = (istart, iend)
2425 else:
2426 d['index'] = int(m.group(7))
2428 return d
2431def _decend(obj, ynames):
2432 if ynames:
2433 for sobj in iter_elements(obj, ynames):
2434 yield sobj
2435 else:
2436 yield obj
2439def iter_elements(obj, ypath):
2440 '''
2441 Generator yielding elements matching a given ypath specification.
2443 :param obj: guts :py:class:`Object` instance
2444 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2445 To access list objects use slice notatation (e.g.
2446 'root.child[:].child[1:3].child[1]').
2448 Raises :py:exc:`YPathError` on failure.
2449 '''
2451 try:
2452 if isinstance(ypath, str):
2453 ynames = ypath.split('.')
2454 else:
2455 ynames = ypath
2457 yname = ynames[0]
2458 ynames = ynames[1:]
2459 d = _parse_yname(yname)
2460 if d['name'] not in obj.T.propnames:
2461 raise AttributeError(d['name'])
2463 obj = getattr(obj, d['name'])
2465 if 'index' in d:
2466 sobj = obj[d['index']]
2467 for ssobj in _decend(sobj, ynames):
2468 yield ssobj
2470 elif 'slice' in d:
2471 for i in range(*slice(*d['slice']).indices(len(obj))):
2472 sobj = obj[i]
2473 for ssobj in _decend(sobj, ynames):
2474 yield ssobj
2475 else:
2476 for sobj in _decend(obj, ynames):
2477 yield sobj
2479 except (AttributeError, IndexError) as e:
2480 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2483def get_elements(obj, ypath):
2484 '''
2485 Get all elements matching a given ypath specification.
2487 :param obj: guts :py:class:`Object` instance
2488 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2489 To access list objects use slice notatation (e.g.
2490 'root.child[:].child[1:3].child[1]').
2492 Raises :py:exc:`YPathError` on failure.
2493 '''
2494 return list(iter_elements(obj, ypath))
2497def set_elements(obj, ypath, value, validate=False, regularize=False):
2498 '''
2499 Set elements matching a given ypath specification.
2501 :param obj: guts :py:class:`Object` instance
2502 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2503 To access list objects use slice notatation (e.g.
2504 'root.child[:].child[1:3].child[1]').
2505 :param value: All matching elements will be set to `value`.
2506 :param validate: Whether to validate affected subtrees.
2507 :param regularize: Whether to regularize affected subtrees.
2509 Raises :py:exc:`YPathError` on failure.
2510 '''
2512 ynames = ypath.split('.')
2513 try:
2514 d = _parse_yname(ynames[-1])
2515 if ynames[:-1]:
2516 it = iter_elements(obj, ynames[:-1])
2517 else:
2518 it = [obj]
2520 for sobj in it:
2521 if d['name'] not in sobj.T.propnames:
2522 raise AttributeError(d['name'])
2524 if 'index' in d:
2525 ssobj = getattr(sobj, d['name'])
2526 ssobj[d['index']] = value
2527 elif 'slice' in d:
2528 ssobj = getattr(sobj, d['name'])
2529 for i in range(*slice(*d['slice']).indices(len(ssobj))):
2530 ssobj[i] = value
2531 else:
2532 setattr(sobj, d['name'], value)
2533 if regularize:
2534 sobj.regularize()
2535 if validate:
2536 sobj.validate()
2538 except (AttributeError, IndexError, YPathError) as e:
2539 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2542def zip_walk(x, typ=None, path=(), stack=()):
2543 if typ is None or isinstance(x, typ):
2544 yield path, stack + (x,)
2546 if isinstance(x, Object):
2547 for (prop, val) in x.T.ipropvals(x):
2548 if prop.multivalued:
2549 if val is not None:
2550 for iele, ele in enumerate(val):
2551 for y in zip_walk(
2552 ele, typ,
2553 path=path + ((prop.name, iele),),
2554 stack=stack + (x,)):
2556 yield y
2557 else:
2558 for y in zip_walk(val, typ,
2559 path=path+(prop.name,),
2560 stack=stack + (x,)):
2561 yield y
2564def path_element(x):
2565 if isinstance(x, tuple):
2566 if len(x) == 2:
2567 return '%s[%i]' % x
2568 elif len(x) == 3:
2569 return '%s[%i:%i]' % x
2571 else:
2572 return x
2575def path_to_str(path):
2576 return '.'.join(path_element(x) for x in path)
2579@expand_stream_args('w')
2580def dump(obj, stream, **kwargs):
2581 '''
2582 Serialize to YAML.
2584 If neither ``stream`` nor ``filename`` is set, a string containing the
2585 serialized data is returned.
2587 :param obj:
2588 Object to be serialized.
2589 :type obj:
2590 :py:class:`Object`
2592 :param stream:
2593 Output to stream.
2595 :param filename:
2596 Output to file of given name.
2597 :type filename:
2598 str
2600 :param header:
2601 File header to prepend to the output.
2602 :type header:
2603 str
2604 '''
2605 return _dump(obj, stream, **kwargs)
2608@expand_stream_args('r')
2609def load(stream, **kwargs):
2610 return _load(stream, **kwargs)
2613def load_string(s, **kwargs):
2614 return load(string=s, **kwargs)
2617@expand_stream_args('w')
2618def dump_all(obj, stream, **kwargs):
2619 return _dump_all(obj, stream, **kwargs)
2622@expand_stream_args('r')
2623def load_all(stream, **kwargs):
2624 return _load_all(stream, **kwargs)
2627@expand_stream_args('r')
2628def iload_all(stream, **kwargs):
2629 return _iload_all(stream, **kwargs)
2632@expand_stream_args('w')
2633def dump_xml(obj, stream, **kwargs):
2634 '''
2635 Serialize to XML.
2637 If neither ``stream`` nor ``filename`` is set, a string containing the
2638 serialized data is returned.
2640 :param obj:
2641 Object to be serialized.
2642 :type obj:
2643 :py:class:`Object`
2645 :param stream:
2646 Output to stream.
2648 :param filename:
2649 Output to file of given name.
2650 :type filename:
2651 str
2653 :param header:
2654 File header to prepend to the output.
2655 :type header:
2656 str
2658 :param ns_ignore:
2659 Whether to ignore the XML namespace.
2660 :type ns_ignore:
2661 bool
2662 '''
2663 return _dump_xml(obj, stream, **kwargs)
2666@expand_stream_args('r')
2667def load_xml(stream, **kwargs):
2668 kwargs.pop('filename', None)
2669 return _load_xml(stream, **kwargs)
2672def load_xml_string(s, **kwargs):
2673 return load_xml(string=s, **kwargs)
2676@expand_stream_args('w')
2677def dump_all_xml(obj, stream, **kwargs):
2678 return _dump_all_xml(obj, stream, **kwargs)
2681@expand_stream_args('r')
2682def load_all_xml(stream, **kwargs):
2683 kwargs.pop('filename', None)
2684 return _load_all_xml(stream, **kwargs)
2687@expand_stream_args('r')
2688def iload_all_xml(stream, **kwargs):
2689 kwargs.pop('filename', None)
2690 return _iload_all_xml(stream, **kwargs)
2693__all__ = guts_types + [
2694 'guts_types', 'TBase', 'ValidationError',
2695 'ArgumentError', 'Defer',
2696 'DefaultMaker', 'ObjectDefaultMaker',
2697 'clone',
2698 'dump', 'load',
2699 'dump_all', 'load_all', 'iload_all',
2700 'dump_xml', 'load_xml',
2701 'dump_all_xml', 'load_all_xml', 'iload_all_xml',
2702 'load_string',
2703 'load_xml_string',
2704 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str'
2705]