1# https://pyrocko.org - GPLv3
2#
3# The Pyrocko Developers, 21st Century
4# ---|P------/S----------~Lg----------
5'''
6Lightweight declarative YAML and XML data binding for Python.
7'''
9import datetime
10import calendar
11import re
12import sys
13import types
14import copy
15import os.path as op
16from collections import defaultdict
17from base64 import b64decode, b64encode
19from io import BytesIO
21try:
22 import numpy as num
23except ImportError:
24 num = None
26import yaml
27try:
28 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
29except ImportError:
30 from yaml import SafeLoader, SafeDumper
32from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \
33 get_time_float
36ALLOW_INCLUDE = False
39class GutsSafeDumper(SafeDumper):
40 pass
43class GutsSafeLoader(SafeLoader):
44 pass
47g_iprop = 0
49g_deferred = {}
50g_deferred_content = {}
52g_tagname_to_class = {}
53g_xmltagname_to_class = {}
54g_guessable_xmlns = {}
56guts_types = [
57 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float',
58 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern',
59 'UnicodePattern', 'StringChoice', 'IntChoice', 'List', 'Dict', 'Tuple',
60 'Union', 'Choice', 'Any']
62us_to_cc_regex = re.compile(r'([a-z])_([a-z])')
65class literal(str):
66 pass
69class folded(str):
70 pass
73class singlequoted(str):
74 pass
77class doublequoted(str):
78 pass
81def make_str_presenter(style):
82 def presenter(dumper, data):
83 return dumper.represent_scalar(
84 'tag:yaml.org,2002:str', str(data), style=style)
86 return presenter
89str_style_map = {
90 None: lambda x: x,
91 '|': literal,
92 '>': folded,
93 "'": singlequoted,
94 '"': doublequoted}
96for (style, cls) in str_style_map.items():
97 if style:
98 GutsSafeDumper.add_representer(cls, make_str_presenter(style))
101class uliteral(str):
102 pass
105class ufolded(str):
106 pass
109class usinglequoted(str):
110 pass
113class udoublequoted(str):
114 pass
117def make_unicode_presenter(style):
118 def presenter(dumper, data):
119 return dumper.represent_scalar(
120 'tag:yaml.org,2002:str', str(data), style=style)
122 return presenter
125unicode_style_map = {
126 None: lambda x: x,
127 '|': literal,
128 '>': folded,
129 "'": singlequoted,
130 '"': doublequoted}
132for (style, cls) in unicode_style_map.items():
133 if style:
134 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style))
137class blist(list):
138 pass
141class flist(list):
142 pass
145list_style_map = {
146 None: list,
147 'block': blist,
148 'flow': flist}
151def make_list_presenter(flow_style):
152 def presenter(dumper, data):
153 return dumper.represent_sequence(
154 'tag:yaml.org,2002:seq', data, flow_style=flow_style)
156 return presenter
159GutsSafeDumper.add_representer(blist, make_list_presenter(False))
160GutsSafeDumper.add_representer(flist, make_list_presenter(True))
162if num:
163 def numpy_float_presenter(dumper, data):
164 return dumper.represent_float(float(data))
166 def numpy_int_presenter(dumper, data):
167 return dumper.represent_int(int(data))
169 for dtype in (num.float64, num.float32):
170 GutsSafeDumper.add_representer(dtype, numpy_float_presenter)
172 for dtype in (num.int32, num.int64):
173 GutsSafeDumper.add_representer(dtype, numpy_int_presenter)
176def us_to_cc(s):
177 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s)
180cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)')
181cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])')
184def cc_to_us(s):
185 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub(
186 '\\1_\\2\\3', s)).lower()
189re_frac = re.compile(r'\.[1-9]FRAC')
190frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789'])
193def encode_utf8(s):
194 return s.encode('utf-8')
197def no_encode(s):
198 return s
201def make_xmltagname_from_name(name):
202 return us_to_cc(name)
205def make_name_from_xmltagname(xmltagname):
206 return cc_to_us(xmltagname)
209def make_content_name(name):
210 if name.endswith('_list'):
211 return name[:-5]
212 elif name.endswith('s'):
213 return name[:-1]
214 else:
215 return name
218def classnames(cls):
219 if isinstance(cls, tuple):
220 return '(%s)' % ', '.join(x.__name__ for x in cls)
221 else:
222 return cls.__name__
225def expand_stream_args(mode):
226 def wrap(f):
227 '''
228 Decorator to enhance functions taking stream objects.
230 Wraps a function f(..., stream, ...) so that it can also be called as
231 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...).
232 '''
234 def g(*args, **kwargs):
235 stream = kwargs.pop('stream', None)
236 filename = kwargs.get('filename', None)
237 if mode != 'r':
238 filename = kwargs.pop('filename', None)
239 string = kwargs.pop('string', None)
241 assert sum(x is not None for x in (stream, filename, string)) <= 1
243 if stream is not None:
244 kwargs['stream'] = stream
245 return f(*args, **kwargs)
247 elif filename is not None:
248 stream = open(filename, mode+'b')
249 kwargs['stream'] = stream
250 retval = f(*args, **kwargs)
251 if isinstance(retval, types.GeneratorType):
252 def wrap_generator(gen):
253 try:
254 for x in gen:
255 yield x
257 except GeneratorExit:
258 pass
260 stream.close()
262 return wrap_generator(retval)
264 else:
265 stream.close()
266 return retval
268 elif string is not None:
269 assert mode == 'r', \
270 'Keyword argument string=... cannot be used in dumper ' \
271 'function.'
273 kwargs['stream'] = BytesIO(string.encode('utf-8'))
274 return f(*args, **kwargs)
276 else:
277 assert mode == 'w', \
278 'Use keyword argument stream=... or filename=... in ' \
279 'loader function.'
281 sout = BytesIO()
282 f(stream=sout, *args, **kwargs)
283 return sout.getvalue().decode('utf-8')
285 return g
287 return wrap
290class Defer(object):
291 def __init__(self, classname, *args, **kwargs):
292 global g_iprop
293 if kwargs.get('position', None) is None:
294 kwargs['position'] = g_iprop
296 g_iprop += 1
298 self.classname = classname
299 self.args = args
300 self.kwargs = kwargs
303class TBase(object):
305 strict = False
306 multivalued = None
307 force_regularize = False
308 propnames = []
310 @classmethod
311 def init_propertystuff(cls):
312 cls.properties = []
313 cls.xmltagname_to_name = {}
314 cls.xmltagname_to_name_multivalued = {}
315 cls.xmltagname_to_class = {}
316 cls.content_property = None
318 def __init__(
319 self,
320 default=None,
321 optional=False,
322 xmlstyle='element',
323 xmltagname=None,
324 xmlns=None,
325 help=None,
326 position=None):
328 global g_iprop
329 if position is not None:
330 self.position = position
331 else:
332 self.position = g_iprop
334 g_iprop += 1
335 self._default = default
337 self.optional = optional
338 self.name = None
339 self._xmltagname = xmltagname
340 self._xmlns = xmlns
341 self.parent = None
342 self.xmlstyle = xmlstyle
343 self.help = help
345 def default(self):
346 return make_default(self._default)
348 def is_default(self, val):
349 if self._default is None:
350 return val is None
351 else:
352 return self._default == val
354 def has_default(self):
355 return self._default is not None
357 def xname(self):
358 if self.name is not None:
359 return self.name
360 elif self.parent is not None:
361 return 'element of %s' % self.parent.xname()
362 else:
363 return '?'
365 def set_xmlns(self, xmlns):
366 if self._xmlns is None and not self.xmlns:
367 self._xmlns = xmlns
369 if self.multivalued:
370 self.content_t.set_xmlns(xmlns)
372 def get_xmlns(self):
373 return self._xmlns or self.xmlns
375 def get_xmltagname(self):
376 if self._xmltagname is not None:
377 return self.get_xmlns() + ' ' + self._xmltagname
378 elif self.name:
379 return self.get_xmlns() + ' ' \
380 + make_xmltagname_from_name(self.name)
381 elif self.xmltagname:
382 return self.get_xmlns() + ' ' + self.xmltagname
383 else:
384 assert False
386 @classmethod
387 def get_property(cls, name):
388 for prop in cls.properties:
389 if prop.name == name:
390 return prop
392 raise ValueError()
394 @classmethod
395 def remove_property(cls, name):
397 prop = cls.get_property(name)
399 if not prop.multivalued:
400 del cls.xmltagname_to_class[prop.effective_xmltagname]
401 del cls.xmltagname_to_name[prop.effective_xmltagname]
402 else:
403 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname]
404 del cls.xmltagname_to_name_multivalued[
405 prop.content_t.effective_xmltagname]
407 if cls.content_property is prop:
408 cls.content_property = None
410 cls.properties.remove(prop)
411 cls.propnames.remove(name)
413 return prop
415 @classmethod
416 def add_property(cls, name, prop):
418 prop.instance = prop
419 prop.name = name
420 prop.set_xmlns(cls.xmlns)
422 if isinstance(prop, Choice.T):
423 for tc in prop.choices:
424 tc.effective_xmltagname = tc.get_xmltagname()
425 cls.xmltagname_to_class[tc.effective_xmltagname] = tc.cls
426 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name
427 elif not prop.multivalued:
428 prop.effective_xmltagname = prop.get_xmltagname()
429 cls.xmltagname_to_class[prop.effective_xmltagname] = prop.cls
430 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name
431 else:
432 prop.content_t.name = make_content_name(prop.name)
433 prop.content_t.effective_xmltagname = \
434 prop.content_t.get_xmltagname()
435 cls.xmltagname_to_class[
436 prop.content_t.effective_xmltagname] = prop.content_t.cls
437 cls.xmltagname_to_name_multivalued[
438 prop.content_t.effective_xmltagname] = prop.name
440 cls.properties.append(prop)
442 cls.properties.sort(key=lambda x: x.position)
444 cls.propnames = [p.name for p in cls.properties]
446 if prop.xmlstyle == 'content':
447 cls.content_property = prop
449 @classmethod
450 def ivals(cls, val):
451 for prop in cls.properties:
452 yield getattr(val, prop.name)
454 @classmethod
455 def ipropvals(cls, val):
456 for prop in cls.properties:
457 yield prop, getattr(val, prop.name)
459 @classmethod
460 def inamevals(cls, val):
461 for prop in cls.properties:
462 yield prop.name, getattr(val, prop.name)
464 @classmethod
465 def ipropvals_to_save(cls, val, xmlmode=False):
466 for prop in cls.properties:
467 v = getattr(val, prop.name)
468 if v is not None and (
469 not (prop.optional or (prop.multivalued and not v))
470 or (not prop.is_default(v))):
472 if xmlmode:
473 yield prop, prop.to_save_xml(v)
474 else:
475 yield prop, prop.to_save(v)
477 @classmethod
478 def inamevals_to_save(cls, val, xmlmode=False):
479 for prop, v in cls.ipropvals_to_save(val, xmlmode):
480 yield prop.name, v
482 @classmethod
483 def translate_from_xml(cls, list_of_pairs, strict):
484 d = {}
485 for k, v in list_of_pairs:
486 if k in cls.xmltagname_to_name_multivalued:
487 k2 = cls.xmltagname_to_name_multivalued[k]
488 if k2 not in d:
489 d[k2] = []
491 d[k2].append(v)
492 elif k in cls.xmltagname_to_name:
493 k2 = cls.xmltagname_to_name[k]
494 if k2 in d:
495 raise ArgumentError(
496 'Unexpectedly found more than one child element "%s" '
497 'within "%s".' % (k, cls.tagname))
499 d[k2] = v
500 elif k is None:
501 if cls.content_property:
502 k2 = cls.content_property.name
503 d[k2] = v
504 else:
505 if strict:
506 raise ArgumentError(
507 'Unexpected child element "%s" found within "%s".' % (
508 k, cls.tagname))
510 return d
512 def validate(self, val, regularize=False, depth=-1):
513 if self.optional and val is None:
514 return val
516 is_derived = isinstance(val, self.cls)
517 is_exact = type(val) == self.cls
519 not_ok = not self.strict and not is_derived or \
520 self.strict and not is_exact
522 if not_ok or self.force_regularize:
523 if regularize:
524 try:
525 val = self.regularize_extra(val)
526 except ValueError:
527 raise ValidationError(
528 '%s: could not convert "%s" to type %s' % (
529 self.xname(), val, classnames(self.cls)))
530 else:
531 raise ValidationError(
532 '%s: "%s" (type: %s) is not of type %s' % (
533 self.xname(), val, type(val), classnames(self.cls)))
535 validator = self
536 if isinstance(self.cls, tuple):
537 clss = self.cls
538 else:
539 clss = (self.cls,)
541 for cls in clss:
542 try:
543 if type(val) != cls and isinstance(val, cls):
544 validator = val.T.instance
546 except AttributeError:
547 pass
549 validator.validate_extra(val)
551 if depth != 0:
552 val = validator.validate_children(val, regularize, depth)
554 return val
556 def regularize_extra(self, val):
557 return self.cls(val)
559 def validate_extra(self, val):
560 pass
562 def validate_children(self, val, regularize, depth):
563 for prop, propval in self.ipropvals(val):
564 newpropval = prop.validate(propval, regularize, depth-1)
565 if regularize and (newpropval is not propval):
566 setattr(val, prop.name, newpropval)
568 return val
570 def to_save(self, val):
571 return val
573 def to_save_xml(self, val):
574 return self.to_save(val)
576 def extend_xmlelements(self, elems, v):
577 if self.multivalued:
578 for x in v:
579 elems.append((self.content_t.effective_xmltagname, x))
580 else:
581 elems.append((self.effective_xmltagname, v))
583 def deferred(self):
584 return []
586 def classname_for_help(self, strip_module=''):
588 if self.dummy_cls is not self.cls:
589 if self.dummy_cls.__module__ == strip_module:
590 sadd = ' (:py:class:`%s`)' % (
591 self.dummy_cls.__name__)
592 else:
593 sadd = ' (:py:class:`%s.%s`)' % (
594 self.dummy_cls.__module__, self.dummy_cls.__name__)
595 else:
596 sadd = ''
598 if self.dummy_cls in guts_plain_dummy_types:
599 return '``%s``' % self.cls.__name__
601 elif self.dummy_cls.dummy_for_description:
602 return '%s%s' % (self.dummy_cls.dummy_for_description, sadd)
604 else:
605 def sclass(cls):
606 mod = cls.__module__
607 clsn = cls.__name__
608 if mod == '__builtin__' or mod == 'builtins':
609 return '``%s``' % clsn
611 elif mod == strip_module:
612 return ':py:class:`%s`' % clsn
614 else:
615 return ':py:class:`%s.%s`' % (mod, clsn)
617 if isinstance(self.cls, tuple):
618 return '(%s)%s' % (
619 ' | '.join(sclass(cls) for cls in self.cls), sadd)
620 else:
621 return '%s%s' % (sclass(self.cls), sadd)
623 @classmethod
624 def props_help_string(cls):
625 baseprops = []
626 for base in cls.dummy_cls.__bases__:
627 if hasattr(base, 'T'):
628 baseprops.extend(base.T.properties)
630 hlp = []
631 hlp.append('')
632 for prop in cls.properties:
633 if prop in baseprops:
634 continue
636 descr = [
637 prop.classname_for_help(strip_module=cls.dummy_cls.__module__)]
639 if prop.optional:
640 descr.append('*optional*')
642 if isinstance(prop._default, DefaultMaker):
643 descr.append('*default:* ``%s``' % repr(prop._default))
644 else:
645 d = prop.default()
646 if d is not None:
647 descr.append('*default:* ``%s``' % repr(d))
649 hlp.append(' .. py:gattribute:: %s' % prop.name)
650 hlp.append('')
651 hlp.append(' %s' % ', '.join(descr))
652 hlp.append(' ')
653 if prop.help is not None:
654 hlp.append(' %s' % prop.help)
655 hlp.append('')
657 return '\n'.join(hlp)
659 @classmethod
660 def class_help_string(cls):
661 return cls.dummy_cls.__doc_template__
663 @classmethod
664 def class_signature(cls):
665 r = []
666 for prop in cls.properties:
667 d = prop.default()
668 if d is not None:
669 arg = repr(d)
671 elif prop.optional:
672 arg = 'None'
674 else:
675 arg = '...'
677 r.append('%s=%s' % (prop.name, arg))
679 return '(%s)' % ', '.join(r)
681 @classmethod
682 def help(cls):
683 return cls.props_help_string()
686class ObjectMetaClass(type):
687 def __new__(meta, classname, bases, class_dict):
688 classname = class_dict.get('class_name', classname)
689 cls = type.__new__(meta, classname, bases, class_dict)
690 if classname != 'Object':
691 t_class_attr_name = '_%s__T' % classname
692 if not hasattr(cls, t_class_attr_name):
693 if hasattr(cls, 'T'):
694 class T(cls.T):
695 pass
696 else:
697 class T(TBase):
698 pass
700 setattr(cls, t_class_attr_name, T)
702 T = getattr(cls, t_class_attr_name)
704 if cls.dummy_for is not None:
705 T.cls = cls.dummy_for
706 else:
707 T.cls = cls
709 T.dummy_cls = cls
711 if hasattr(cls, 'xmltagname'):
712 T.xmltagname = cls.xmltagname
713 else:
714 T.xmltagname = classname
716 mod = sys.modules[cls.__module__]
718 if hasattr(cls, 'xmlns'):
719 T.xmlns = cls.xmlns
720 elif hasattr(mod, 'guts_xmlns'):
721 T.xmlns = mod.guts_xmlns
722 else:
723 T.xmlns = ''
725 if T.xmlns and hasattr(cls, 'guessable_xmlns'):
726 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns
728 if hasattr(mod, 'guts_prefix'):
729 if mod.guts_prefix:
730 T.tagname = mod.guts_prefix + '.' + classname
731 else:
732 T.tagname = classname
733 else:
734 if cls.__module__ != '__main__':
735 T.tagname = cls.__module__ + '.' + classname
736 else:
737 T.tagname = classname
739 T.classname = classname
741 T.init_propertystuff()
743 for k in dir(cls):
744 prop = getattr(cls, k)
746 if k.endswith('__'):
747 k = k[:-2]
749 if isinstance(prop, TBase):
750 if prop.deferred():
751 for defer in prop.deferred():
752 g_deferred_content.setdefault(
753 defer.classname[:-2], []).append((prop, defer))
754 g_deferred.setdefault(
755 defer.classname[:-2], []).append((T, k, prop))
757 else:
758 T.add_property(k, prop)
760 elif isinstance(prop, Defer):
761 g_deferred.setdefault(prop.classname[:-2], []).append(
762 (T, k, prop))
764 if classname in g_deferred_content:
765 for prop, defer in g_deferred_content[classname]:
766 prop.process_deferred(
767 defer, T(*defer.args, **defer.kwargs))
769 del g_deferred_content[classname]
771 if classname in g_deferred:
772 for (T_, k_, prop_) in g_deferred.get(classname, []):
773 if isinstance(prop_, Defer):
774 prop_ = T(*prop_.args, **prop_.kwargs)
776 if not prop_.deferred():
777 T_.add_property(k_, prop_)
779 del g_deferred[classname]
781 g_tagname_to_class[T.tagname] = cls
782 if hasattr(cls, 'xmltagname'):
783 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls
785 cls.T = T
786 T.instance = T()
788 cls.__doc_template__ = cls.__doc__
789 cls.__doc__ = T.class_help_string()
791 if cls.__doc__ is None:
792 cls.__doc__ = 'Undocumented.'
794 cls.__doc__ += '\n' + T.props_help_string()
796 return cls
799class ValidationError(Exception):
800 pass
803class ArgumentError(Exception):
804 pass
807def make_default(x):
808 if isinstance(x, DefaultMaker):
809 return x.make()
810 elif isinstance(x, Object):
811 return clone(x)
812 else:
813 return x
816class DefaultMaker(object):
817 def make(self):
818 raise NotImplementedError('Schould be implemented in subclass.')
821class ObjectDefaultMaker(DefaultMaker):
822 def __init__(self, cls, args, kwargs):
823 DefaultMaker.__init__(self)
824 self.cls = cls
825 self.args = args
826 self.kwargs = kwargs
827 self.instance = None
829 def make(self):
830 return self.cls(
831 *[make_default(x) for x in self.args],
832 **dict((k, make_default(v)) for (k, v) in self.kwargs.items()))
834 def __eq__(self, other):
835 if self.instance is None:
836 self.instance = self.make()
838 return self.instance == other
840 def __repr__(self):
841 sargs = []
842 for arg in self.args:
843 sargs.append(repr(arg))
845 for k, v in self.kwargs.items():
846 sargs.append('%s=%s' % (k, repr(v)))
848 return '%s(%s)' % (self.cls.__name__, ', '.join(sargs))
851class TimestampDefaultMaker(DefaultMaker):
852 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'):
853 DefaultMaker.__init__(self)
854 self._stime = s
855 self._format = format
857 def make(self):
858 return str_to_time(self._stime, self._format)
860 def __repr__(self):
861 return 'str_to_time(%s)' % repr(self._stime)
864def with_metaclass(meta, *bases):
865 # inlined py2/py3 compat solution from python-future
866 class metaclass(meta):
867 __call__ = type.__call__
868 __init__ = type.__init__
870 def __new__(cls, name, this_bases, d):
871 if this_bases is None:
872 return type.__new__(cls, name, (), d)
873 return meta(name, bases, d)
875 return metaclass('temp', None, {})
878class Object(with_metaclass(ObjectMetaClass, object)):
879 dummy_for = None
880 dummy_for_description = None
882 def __init__(self, **kwargs):
883 if not kwargs.get('init_props', True):
884 return
886 for prop in self.T.properties:
887 k = prop.name
888 if k in kwargs:
889 setattr(self, k, kwargs.pop(k))
890 else:
891 if not prop.optional and not prop.has_default():
892 raise ArgumentError('Missing argument to %s: %s' % (
893 self.T.tagname, prop.name))
894 else:
895 setattr(self, k, prop.default())
897 if kwargs:
898 raise ArgumentError('Invalid argument to %s: %s' % (
899 self.T.tagname, ', '.join(list(kwargs.keys()))))
901 @classmethod
902 def D(cls, *args, **kwargs):
903 return ObjectDefaultMaker(cls, args, kwargs)
905 def validate(self, regularize=False, depth=-1):
906 self.T.instance.validate(self, regularize, depth)
908 def regularize(self, depth=-1):
909 self.validate(regularize=True, depth=depth)
911 def dump(self, stream=None, filename=None, header=False):
912 return dump(self, stream=stream, filename=filename, header=header)
914 def dump_xml(
915 self, stream=None, filename=None, header=False, ns_ignore=False):
916 return dump_xml(
917 self, stream=stream, filename=filename, header=header,
918 ns_ignore=ns_ignore)
920 @classmethod
921 def load(cls, stream=None, filename=None, string=None):
922 return load(stream=stream, filename=filename, string=string)
924 @classmethod
925 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None,
926 ns_ignore=False):
928 if ns_hints is None:
929 ns_hints = [cls.T.instance.get_xmlns()]
931 return load_xml(
932 stream=stream,
933 filename=filename,
934 string=string,
935 ns_hints=ns_hints,
936 ns_ignore=ns_ignore)
938 def __str__(self):
939 return self.dump()
942def to_dict(obj):
943 '''
944 Get dict of guts object attributes.
946 :param obj: :py:class`Object` object
947 '''
949 return dict(obj.T.inamevals(obj))
952class SObject(Object):
954 class __T(TBase):
955 def regularize_extra(self, val):
956 if isinstance(val, str):
957 return self.cls(val)
959 return val
961 def to_save(self, val):
962 return str(val)
964 def to_save_xml(self, val):
965 return str(val)
968class Any(Object):
970 class __T(TBase):
971 def validate(self, val, regularize=False, depth=-1):
972 if isinstance(val, Object):
973 val.validate(regularize, depth)
975 return val
978class Int(Object):
979 dummy_for = int
981 class __T(TBase):
982 strict = True
984 def to_save_xml(self, value):
985 return repr(value)
988class Float(Object):
989 dummy_for = float
991 class __T(TBase):
992 strict = True
994 def to_save_xml(self, value):
995 return repr(value)
998class Complex(Object):
999 dummy_for = complex
1001 class __T(TBase):
1002 strict = True
1004 def regularize_extra(self, val):
1006 if isinstance(val, list) or isinstance(val, tuple):
1007 assert len(val) == 2
1008 val = complex(*val)
1010 elif not isinstance(val, complex):
1011 val = complex(val)
1013 return val
1015 def to_save(self, value):
1016 return repr(value)
1018 def to_save_xml(self, value):
1019 return repr(value)
1022class Bool(Object):
1023 dummy_for = bool
1025 class __T(TBase):
1026 strict = True
1028 def regularize_extra(self, val):
1029 if isinstance(val, str):
1030 if val.lower().strip() in ('0', 'false'):
1031 return False
1033 return bool(val)
1035 def to_save_xml(self, value):
1036 return repr(bool(value)).lower()
1039class String(Object):
1040 dummy_for = str
1042 class __T(TBase):
1043 def __init__(self, *args, **kwargs):
1044 yamlstyle = kwargs.pop('yamlstyle', None)
1045 TBase.__init__(self, *args, **kwargs)
1046 self.style_cls = str_style_map[yamlstyle]
1048 def to_save(self, val):
1049 return self.style_cls(val)
1052class Bytes(Object):
1053 dummy_for = bytes
1055 class __T(TBase):
1057 def regularize_extra(self, val):
1058 if isinstance(val, str):
1059 val = b64decode(val)
1061 return val
1063 def to_save(self, val):
1064 return literal(b64encode(val).decode('utf-8'))
1067class Unicode(Object):
1068 dummy_for = str
1070 class __T(TBase):
1071 def __init__(self, *args, **kwargs):
1072 yamlstyle = kwargs.pop('yamlstyle', None)
1073 TBase.__init__(self, *args, **kwargs)
1074 self.style_cls = unicode_style_map[yamlstyle]
1076 def to_save(self, val):
1077 return self.style_cls(val)
1080guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool)
1083class Dict(Object):
1084 dummy_for = dict
1086 class __T(TBase):
1087 multivalued = dict
1089 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs):
1090 TBase.__init__(self, *args, **kwargs)
1091 assert isinstance(key_t, TBase)
1092 assert isinstance(content_t, TBase)
1093 self.key_t = key_t
1094 self.content_t = content_t
1095 self.content_t.parent = self
1097 def default(self):
1098 if self._default is not None:
1099 return dict(
1100 (make_default(k), make_default(v))
1101 for (k, v) in self._default.items())
1103 if self.optional:
1104 return None
1105 else:
1106 return {}
1108 def has_default(self):
1109 return True
1111 def validate(self, val, regularize, depth):
1112 return TBase.validate(self, val, regularize, depth+1)
1114 def validate_children(self, val, regularize, depth):
1115 for key, ele in list(val.items()):
1116 newkey = self.key_t.validate(key, regularize, depth-1)
1117 newele = self.content_t.validate(ele, regularize, depth-1)
1118 if regularize:
1119 if newkey is not key or newele is not ele:
1120 del val[key]
1121 val[newkey] = newele
1123 return val
1125 def to_save(self, val):
1126 return dict((self.key_t.to_save(k), self.content_t.to_save(v))
1127 for (k, v) in val.items())
1129 def to_save_xml(self, val):
1130 raise NotImplementedError()
1132 def classname_for_help(self, strip_module=''):
1133 return '``dict`` of %s objects' % \
1134 self.content_t.classname_for_help(strip_module=strip_module)
1137class List(Object):
1138 dummy_for = list
1140 class __T(TBase):
1141 multivalued = list
1143 def __init__(self, content_t=Any.T(), *args, **kwargs):
1144 yamlstyle = kwargs.pop('yamlstyle', None)
1145 TBase.__init__(self, *args, **kwargs)
1146 assert isinstance(content_t, TBase) or isinstance(content_t, Defer)
1147 self.content_t = content_t
1148 self.content_t.parent = self
1149 self.style_cls = list_style_map[yamlstyle]
1151 def default(self):
1152 if self._default is not None:
1153 return [make_default(x) for x in self._default]
1154 if self.optional:
1155 return None
1156 else:
1157 return []
1159 def has_default(self):
1160 return True
1162 def validate(self, val, regularize, depth):
1163 return TBase.validate(self, val, regularize, depth+1)
1165 def validate_children(self, val, regularize, depth):
1166 for i, ele in enumerate(val):
1167 newele = self.content_t.validate(ele, regularize, depth-1)
1168 if regularize and newele is not ele:
1169 val[i] = newele
1171 return val
1173 def to_save(self, val):
1174 return self.style_cls(self.content_t.to_save(v) for v in val)
1176 def to_save_xml(self, val):
1177 return [self.content_t.to_save_xml(v) for v in val]
1179 def deferred(self):
1180 if isinstance(self.content_t, Defer):
1181 return [self.content_t]
1183 return []
1185 def process_deferred(self, defer, t_inst):
1186 if defer is self.content_t:
1187 self.content_t = t_inst
1189 def classname_for_help(self, strip_module=''):
1190 return '``list`` of %s objects' % \
1191 self.content_t.classname_for_help(strip_module=strip_module)
1194def make_typed_list_class(t):
1195 class TL(List):
1196 class __T(List.T):
1197 def __init__(self, *args, **kwargs):
1198 List.T.__init__(self, content_t=t.T(), *args, **kwargs)
1200 return TL
1203class Tuple(Object):
1204 dummy_for = tuple
1206 class __T(TBase):
1207 multivalued = tuple
1209 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs):
1210 TBase.__init__(self, *args, **kwargs)
1211 assert isinstance(content_t, TBase)
1212 self.content_t = content_t
1213 self.content_t.parent = self
1214 self.n = n
1216 def default(self):
1217 if self._default is not None:
1218 return tuple(
1219 make_default(x) for x in self._default)
1221 elif self.optional:
1222 return None
1223 else:
1224 if self.n is not None:
1225 return tuple(
1226 self.content_t.default() for x in range(self.n))
1227 else:
1228 return tuple()
1230 def has_default(self):
1231 return True
1233 def validate(self, val, regularize, depth):
1234 return TBase.validate(self, val, regularize, depth+1)
1236 def validate_extra(self, val):
1237 if self.n is not None and len(val) != self.n:
1238 raise ValidationError(
1239 '%s should have length %i' % (self.xname(), self.n))
1241 def validate_children(self, val, regularize, depth):
1242 if not regularize:
1243 for ele in val:
1244 self.content_t.validate(ele, regularize, depth-1)
1246 return val
1247 else:
1248 newval = []
1249 isnew = False
1250 for ele in val:
1251 newele = self.content_t.validate(ele, regularize, depth-1)
1252 newval.append(newele)
1253 if newele is not ele:
1254 isnew = True
1256 if isnew:
1257 return tuple(newval)
1258 else:
1259 return val
1261 def to_save(self, val):
1262 return tuple(self.content_t.to_save(v) for v in val)
1264 def to_save_xml(self, val):
1265 return [self.content_t.to_save_xml(v) for v in val]
1267 def classname_for_help(self, strip_module=''):
1268 if self.n is not None:
1269 return '``tuple`` of %i %s objects' % (
1270 self.n, self.content_t.classname_for_help(
1271 strip_module=strip_module))
1272 else:
1273 return '``tuple`` of %s objects' % (
1274 self.content_t.classname_for_help(
1275 strip_module=strip_module))
1278duration_unit_factors = dict(
1279 s=1.0,
1280 m=60.0,
1281 h=3600.0,
1282 d=24*3600.0,
1283 y=365*24*3600.0)
1286def parse_duration(s):
1287 unit = s[-1]
1288 if unit in duration_unit_factors:
1289 return float(s[:-1]) * duration_unit_factors[unit]
1290 else:
1291 return float(s)
1294def str_duration(d):
1295 for k in 'ydhms':
1296 if abs(d) >= duration_unit_factors[k]:
1297 return '%g' % (d / duration_unit_factors[k]) + k
1299 return '%g' % d
1302class Duration(Object):
1303 dummy_for = float
1305 class __T(TBase):
1306 def regularize_extra(self, val):
1307 if isinstance(val, str):
1308 return parse_duration(val)
1310 return val
1312 def to_save(self, val):
1313 return str_duration(val)
1315 def to_save_xml(self, val):
1316 return str_duration(val)
1319re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$')
1322class Timestamp(Object):
1323 dummy_for = (hpfloat, float)
1324 dummy_for_description = 'time_float'
1326 class __T(TBase):
1328 def regularize_extra(self, val):
1330 time_float = get_time_float()
1332 if isinstance(val, datetime.datetime):
1333 tt = val.utctimetuple()
1334 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1336 elif isinstance(val, datetime.date):
1337 tt = val.timetuple()
1338 val = time_float(calendar.timegm(tt))
1340 elif isinstance(val, str):
1341 val = val.strip()
1342 tz_offset = 0
1344 m = re_tz.search(val)
1345 if m:
1346 sh = m.group(2)
1347 sm = m.group(4)
1348 tz_offset = (int(sh)*3600 if sh else 0) \
1349 + (int(sm)*60 if sm else 0)
1351 val = re_tz.sub('', val)
1353 if len(val) > 10 and val[10] == 'T':
1354 val = val.replace('T', ' ', 1)
1356 try:
1357 val = str_to_time(val) - tz_offset
1358 except TimeStrError:
1359 raise ValidationError(
1360 '%s: cannot parse time/date: %s' % (self.xname(), val))
1362 elif isinstance(val, (int, float)):
1363 val = time_float(val)
1365 else:
1366 raise ValidationError(
1367 '%s: cannot convert "%s" to type %s' % (
1368 self.xname(), val, time_float))
1370 return val
1372 def to_save(self, val):
1373 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\
1374 .rstrip('0').rstrip('.')
1376 def to_save_xml(self, val):
1377 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\
1378 .rstrip('0').rstrip('.') + 'Z'
1380 @classmethod
1381 def D(self, s):
1382 return TimestampDefaultMaker(s)
1385class DateTimestamp(Object):
1386 dummy_for = (hpfloat, float)
1387 dummy_for_description = 'time_float'
1389 class __T(TBase):
1391 def regularize_extra(self, val):
1393 time_float = get_time_float()
1395 if isinstance(val, datetime.datetime):
1396 tt = val.utctimetuple()
1397 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1399 elif isinstance(val, datetime.date):
1400 tt = val.timetuple()
1401 val = time_float(calendar.timegm(tt))
1403 elif isinstance(val, str):
1404 val = str_to_time(val, format='%Y-%m-%d')
1406 elif isinstance(val, int):
1407 val = time_float(val)
1409 return val
1411 def to_save(self, val):
1412 return time_to_str(val, format='%Y-%m-%d')
1414 def to_save_xml(self, val):
1415 return time_to_str(val, format='%Y-%m-%d')
1417 @classmethod
1418 def D(self, s):
1419 return TimestampDefaultMaker(s, format='%Y-%m-%d')
1422class StringPattern(String):
1424 '''
1425 Any ``str`` matching pattern ``%(pattern)s``.
1426 '''
1428 dummy_for = str
1429 pattern = '.*'
1431 class __T(String.T):
1432 def __init__(self, pattern=None, *args, **kwargs):
1433 String.T.__init__(self, *args, **kwargs)
1435 if pattern is not None:
1436 self.pattern = pattern
1437 else:
1438 self.pattern = self.dummy_cls.pattern
1440 def validate_extra(self, val):
1441 pat = self.pattern
1442 if not re.search(pat, val):
1443 raise ValidationError('%s: "%s" does not match pattern %s' % (
1444 self.xname(), val, repr(pat)))
1446 @classmethod
1447 def class_help_string(cls):
1448 dcls = cls.dummy_cls
1449 doc = dcls.__doc_template__ or StringPattern.__doc_template__
1450 return doc % {'pattern': repr(dcls.pattern)}
1453class UnicodePattern(Unicode):
1455 '''
1456 Any ``str`` matching pattern ``%(pattern)s``.
1457 '''
1459 dummy_for = str
1460 pattern = '.*'
1462 class __T(TBase):
1463 def __init__(self, pattern=None, *args, **kwargs):
1464 TBase.__init__(self, *args, **kwargs)
1466 if pattern is not None:
1467 self.pattern = pattern
1468 else:
1469 self.pattern = self.dummy_cls.pattern
1471 def validate_extra(self, val):
1472 pat = self.pattern
1473 if not re.search(pat, val, flags=re.UNICODE):
1474 raise ValidationError('%s: "%s" does not match pattern %s' % (
1475 self.xname(), val, repr(pat)))
1477 @classmethod
1478 def class_help_string(cls):
1479 dcls = cls.dummy_cls
1480 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__
1481 return doc % {'pattern': repr(dcls.pattern)}
1484class StringChoice(String):
1486 '''
1487 Any ``str`` out of ``%(choices)s``.
1488 '''
1490 dummy_for = str
1491 choices = []
1492 ignore_case = False
1494 class __T(String.T):
1495 def __init__(self, choices=None, ignore_case=None, *args, **kwargs):
1496 String.T.__init__(self, *args, **kwargs)
1498 if choices is not None:
1499 self.choices = choices
1500 else:
1501 self.choices = self.dummy_cls.choices
1503 if ignore_case is not None:
1504 self.ignore_case = ignore_case
1505 else:
1506 self.ignore_case = self.dummy_cls.ignore_case
1508 if self.ignore_case:
1509 self.choices = [x.upper() for x in self.choices]
1511 def validate_extra(self, val):
1512 if self.ignore_case:
1513 val = val.upper()
1515 if val not in self.choices:
1516 raise ValidationError(
1517 '%s: "%s" is not a valid choice out of %s' % (
1518 self.xname(), val, repr(self.choices)))
1520 @classmethod
1521 def class_help_string(cls):
1522 dcls = cls.dummy_cls
1523 doc = dcls.__doc_template__ or StringChoice.__doc_template__
1524 return doc % {'choices': repr(dcls.choices)}
1527class IntChoice(Int):
1529 '''
1530 Any ``int`` out of ``%(choices)s``.
1531 '''
1533 dummy_for = int
1534 choices = []
1536 class __T(Int.T):
1537 def __init__(self, choices=None, *args, **kwargs):
1538 Int.T.__init__(self, *args, **kwargs)
1540 if choices is not None:
1541 self.choices = choices
1542 else:
1543 self.choices = self.dummy_cls.choices
1545 def validate_extra(self, val):
1546 if val not in self.choices:
1547 raise ValidationError(
1548 '%s: %i is not a valid choice out of %s' % (
1549 self.xname(), val, repr(self.choices)))
1551 @classmethod
1552 def class_help_string(cls):
1553 dcls = cls.dummy_cls
1554 doc = dcls.__doc_template__ or IntChoice.__doc_template__
1555 return doc % {'choices': repr(dcls.choices)}
1558# this will not always work...
1559class Union(Object):
1560 members = []
1561 dummy_for = str
1563 class __T(TBase):
1564 def __init__(self, members=None, *args, **kwargs):
1565 TBase.__init__(self, *args, **kwargs)
1566 if members is not None:
1567 self.members = members
1568 else:
1569 self.members = self.dummy_cls.members
1571 def validate(self, val, regularize=False, depth=-1):
1572 assert self.members
1573 e2 = None
1574 for member in self.members:
1575 try:
1576 return member.validate(val, regularize, depth=depth)
1577 except ValidationError as e:
1578 e2 = e
1580 raise e2
1583class Choice(Object):
1584 choices = []
1586 class __T(TBase):
1587 def __init__(self, choices=None, *args, **kwargs):
1588 TBase.__init__(self, *args, **kwargs)
1589 if choices is not None:
1590 self.choices = choices
1591 else:
1592 self.choices = self.dummy_cls.choices
1594 self.cls_to_xmltagname = dict(
1595 (t.cls, t.get_xmltagname()) for t in self.choices)
1597 def validate(self, val, regularize=False, depth=-1):
1598 if self.optional and val is None:
1599 return val
1601 t = None
1602 for tc in self.choices:
1603 is_derived = isinstance(val, tc.cls)
1604 is_exact = type(val) == tc.cls
1605 if not (not tc.strict and not is_derived or
1606 tc.strict and not is_exact):
1608 t = tc
1609 break
1611 if t is None:
1612 if regularize:
1613 ok = False
1614 for tc in self.choices:
1615 try:
1616 val = tc.regularize_extra(val)
1617 ok = True
1618 t = tc
1619 break
1620 except (ValidationError, ValueError):
1621 pass
1623 if not ok:
1624 raise ValidationError(
1625 '%s: could not convert "%s" to any type out of '
1626 '(%s)' % (self.xname(), val, ','.join(
1627 classnames(x.cls) for x in self.choices)))
1628 else:
1629 raise ValidationError(
1630 '%s: "%s" (type: %s) is not of any type out of '
1631 '(%s)' % (self.xname(), val, type(val), ','.join(
1632 classnames(x.cls) for x in self.choices)))
1634 validator = t
1636 if isinstance(t.cls, tuple):
1637 clss = t.cls
1638 else:
1639 clss = (t.cls,)
1641 for cls in clss:
1642 try:
1643 if type(val) != cls and isinstance(val, cls):
1644 validator = val.T.instance
1646 except AttributeError:
1647 pass
1649 validator.validate_extra(val)
1651 if depth != 0:
1652 val = validator.validate_children(val, regularize, depth)
1654 return val
1656 def extend_xmlelements(self, elems, v):
1657 elems.append((
1658 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v))
1661def _dump(
1662 object, stream,
1663 header=False,
1664 Dumper=GutsSafeDumper,
1665 _dump_function=yaml.dump):
1667 if not getattr(stream, 'encoding', None):
1668 enc = encode_utf8
1669 else:
1670 enc = no_encode
1672 if header:
1673 stream.write(enc(u'%YAML 1.1\n'))
1674 if isinstance(header, str):
1675 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
1676 stream.write(enc(banner))
1678 _dump_function(
1679 object,
1680 stream=stream,
1681 encoding='utf-8',
1682 explicit_start=True,
1683 Dumper=Dumper)
1686def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper):
1687 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all)
1690def _load(stream,
1691 Loader=GutsSafeLoader, allow_include=None, filename=None,
1692 included_files=None):
1694 class _Loader(Loader):
1695 _filename = filename
1696 _allow_include = allow_include
1697 _included_files = included_files or []
1699 return yaml.load(stream=stream, Loader=_Loader)
1702def _load_all(stream,
1703 Loader=GutsSafeLoader, allow_include=None, filename=None):
1705 class _Loader(Loader):
1706 _filename = filename
1707 _allow_include = allow_include
1709 return list(yaml.load_all(stream=stream, Loader=_Loader))
1712def _iload_all(stream,
1713 Loader=GutsSafeLoader, allow_include=None, filename=None):
1715 class _Loader(Loader):
1716 _filename = filename
1717 _allow_include = allow_include
1719 return yaml.load_all(stream=stream, Loader=_Loader)
1722def multi_representer(dumper, data):
1723 node = dumper.represent_mapping(
1724 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False)
1726 return node
1729# hack for compatibility with early GF Store versions
1730re_compatibility = re.compile(
1731 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.'
1732 r'(dummy|poel|qseis|qssp))\.'
1733)
1736def multi_constructor(loader, tag_suffix, node):
1737 tagname = str(tag_suffix)
1739 tagname = re_compatibility.sub('pf.', tagname)
1741 cls = g_tagname_to_class[tagname]
1742 kwargs = dict(iter(loader.construct_pairs(node, deep=True)))
1743 o = cls(**kwargs)
1744 o.validate(regularize=True, depth=1)
1745 return o
1748def include_constructor(loader, node):
1749 allow_include = loader._allow_include \
1750 if loader._allow_include is not None \
1751 else ALLOW_INCLUDE
1753 if not allow_include:
1754 raise EnvironmentError(
1755 'Not allowed to include YAML. Load with allow_include=True')
1757 if isinstance(node, yaml.nodes.ScalarNode):
1758 inc_file = loader.construct_scalar(node)
1759 else:
1760 raise TypeError('Unsupported YAML node %s' % repr(node))
1762 if loader._filename is not None and not op.isabs(inc_file):
1763 inc_file = op.join(op.dirname(loader._filename), inc_file)
1765 if not op.isfile(inc_file):
1766 raise FileNotFoundError(inc_file)
1768 included_files = list(loader._included_files)
1769 if loader._filename is not None:
1770 included_files.append(op.abspath(loader._filename))
1772 for included_file in loader._included_files:
1773 if op.samefile(inc_file, included_file):
1774 raise ImportError(
1775 'Circular import of file "%s". Include path: %s' % (
1776 op.abspath(inc_file),
1777 ' -> '.join('"%s"' % s for s in included_files)))
1779 with open(inc_file, 'rb') as f:
1780 return _load(
1781 f,
1782 Loader=loader.__class__, filename=inc_file,
1783 allow_include=True,
1784 included_files=included_files)
1787def dict_noflow_representer(dumper, data):
1788 return dumper.represent_mapping(
1789 'tag:yaml.org,2002:map', data, flow_style=False)
1792yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper)
1793yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader)
1794yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader)
1795yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper)
1798def str_representer(dumper, data):
1799 return dumper.represent_scalar(
1800 'tag:yaml.org,2002:str', str(data))
1803yaml.add_representer(str, str_representer, Dumper=GutsSafeDumper)
1806class Constructor(object):
1807 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None,
1808 ns_ignore=False):
1810 self.stack = []
1811 self.queue = []
1812 self.namespaces = defaultdict(list)
1813 self.add_namespace_maps = add_namespace_maps
1814 self.strict = strict
1815 self.ns_hints = ns_hints
1816 self.ns_ignore = ns_ignore
1818 def start_element(self, ns_name, attrs):
1819 if self.ns_ignore:
1820 ns_name = ns_name.split(' ')[-1]
1822 if -1 == ns_name.find(' '):
1823 if self.ns_hints is None and ns_name in g_guessable_xmlns:
1824 self.ns_hints = g_guessable_xmlns[ns_name]
1826 if self.ns_hints:
1827 ns_names = [
1828 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints]
1830 elif self.ns_hints is None:
1831 ns_names = [' ' + ns_name]
1833 else:
1834 ns_names = [ns_name]
1836 for ns_name in ns_names:
1837 if self.stack and self.stack[-1][1] is not None:
1838 cls = self.stack[-1][1].T.xmltagname_to_class.get(
1839 ns_name, None)
1841 if isinstance(cls, tuple):
1842 cls = None
1843 else:
1844 if cls is not None and (
1845 not issubclass(cls, Object)
1846 or issubclass(cls, SObject)):
1847 cls = None
1848 else:
1849 cls = g_xmltagname_to_class.get(ns_name, None)
1851 if cls:
1852 break
1854 self.stack.append((ns_name, cls, attrs, [], []))
1856 def end_element(self, _):
1857 ns_name, cls, attrs, content2, content1 = self.stack.pop()
1859 ns = ns_name.split(' ', 1)[0]
1861 if cls is not None:
1862 content2.extend(
1863 (ns + ' ' + k if -1 == k.find(' ') else k, v)
1864 for (k, v) in attrs.items())
1865 content2.append((None, ''.join(content1)))
1866 o = cls(**cls.T.translate_from_xml(content2, self.strict))
1867 o.validate(regularize=True, depth=1)
1868 if self.add_namespace_maps:
1869 o.namespace_map = self.get_current_namespace_map()
1871 if self.stack and not all(x[1] is None for x in self.stack):
1872 self.stack[-1][-2].append((ns_name, o))
1873 else:
1874 self.queue.append(o)
1875 else:
1876 content = [''.join(content1)]
1877 if self.stack:
1878 for c in content:
1879 self.stack[-1][-2].append((ns_name, c))
1881 def characters(self, char_content):
1882 if self.stack:
1883 self.stack[-1][-1].append(char_content)
1885 def start_namespace(self, ns, uri):
1886 self.namespaces[ns].append(uri)
1888 def end_namespace(self, ns):
1889 self.namespaces[ns].pop()
1891 def get_current_namespace_map(self):
1892 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v)
1894 def get_queued_elements(self):
1895 queue = self.queue
1896 self.queue = []
1897 return queue
1900def _iload_all_xml(
1901 stream,
1902 bufsize=100000,
1903 add_namespace_maps=False,
1904 strict=False,
1905 ns_hints=None,
1906 ns_ignore=False):
1908 from xml.parsers.expat import ParserCreate
1910 parser = ParserCreate('UTF-8', namespace_separator=' ')
1912 handler = Constructor(
1913 add_namespace_maps=add_namespace_maps,
1914 strict=strict,
1915 ns_hints=ns_hints,
1916 ns_ignore=ns_ignore)
1918 parser.StartElementHandler = handler.start_element
1919 parser.EndElementHandler = handler.end_element
1920 parser.CharacterDataHandler = handler.characters
1921 parser.StartNamespaceDeclHandler = handler.start_namespace
1922 parser.EndNamespaceDeclHandler = handler.end_namespace
1924 while True:
1925 data = stream.read(bufsize)
1926 parser.Parse(data, bool(not data))
1927 for element in handler.get_queued_elements():
1928 yield element
1930 if not data:
1931 break
1934def _load_all_xml(*args, **kwargs):
1935 return list(_iload_all_xml(*args, **kwargs))
1938def _load_xml(*args, **kwargs):
1939 g = _iload_all_xml(*args, **kwargs)
1940 return next(g)
1943def _dump_all_xml(objects, stream, root_element_name='root', header=True):
1945 if not getattr(stream, 'encoding', None):
1946 enc = encode_utf8
1947 else:
1948 enc = no_encode
1950 _dump_xml_header(stream, header)
1952 beg = u'<%s>\n' % root_element_name
1953 end = u'</%s>\n' % root_element_name
1955 stream.write(enc(beg))
1957 for ob in objects:
1958 _dump_xml(ob, stream=stream)
1960 stream.write(enc(end))
1963def _dump_xml_header(stream, banner=None):
1965 if not getattr(stream, 'encoding', None):
1966 enc = encode_utf8
1967 else:
1968 enc = no_encode
1970 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n'))
1971 if isinstance(banner, str):
1972 stream.write(enc(u'<!-- %s -->\n' % banner))
1975def _dump_xml(
1976 obj, stream, depth=0, ns_name=None, header=False, ns_map=[],
1977 ns_ignore=False):
1979 from xml.sax.saxutils import escape, quoteattr
1981 if not getattr(stream, 'encoding', None):
1982 enc = encode_utf8
1983 else:
1984 enc = no_encode
1986 if depth == 0 and header:
1987 _dump_xml_header(stream, header)
1989 indent = ' '*depth*2
1990 if ns_name is None:
1991 ns_name = obj.T.instance.get_xmltagname()
1993 if -1 != ns_name.find(' '):
1994 ns, name = ns_name.split(' ')
1995 else:
1996 ns, name = '', ns_name
1998 if isinstance(obj, Object):
1999 obj.validate(depth=1)
2000 attrs = []
2001 elems = []
2003 added_ns = False
2004 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns):
2005 attrs.append(('xmlns', ns))
2006 ns_map.append(ns)
2007 added_ns = True
2009 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True):
2010 if prop.xmlstyle == 'attribute':
2011 assert not prop.multivalued
2012 assert not isinstance(v, Object)
2013 attrs.append((prop.effective_xmltagname, v))
2015 elif prop.xmlstyle == 'content':
2016 assert not prop.multivalued
2017 assert not isinstance(v, Object)
2018 elems.append((None, v))
2020 else:
2021 prop.extend_xmlelements(elems, v)
2023 attr_str = ''
2024 if attrs:
2025 attr_str = ' ' + ' '.join(
2026 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v)))
2027 for (k, v) in attrs)
2029 if not elems:
2030 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str)))
2031 else:
2032 oneline = len(elems) == 1 and elems[0][0] is None
2033 stream.write(enc(u'%s<%s%s>%s' % (
2034 indent,
2035 name,
2036 attr_str,
2037 '' if oneline else '\n')))
2039 for (k, v) in elems:
2040 if k is None:
2041 stream.write(enc(escape(str(v), {'\0': '�'})))
2042 else:
2043 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore)
2045 stream.write(enc(u'%s</%s>\n' % (
2046 '' if oneline else indent, name)))
2048 if added_ns:
2049 ns_map.pop()
2051 else:
2052 stream.write(enc(u'%s<%s>%s</%s>\n' % (
2053 indent,
2054 name,
2055 escape(str(obj), {'\0': '�'}),
2056 name)))
2059def walk(x, typ=None, path=()):
2060 if typ is None or isinstance(x, typ):
2061 yield path, x
2063 if isinstance(x, Object):
2064 for (prop, val) in x.T.ipropvals(x):
2065 if prop.multivalued:
2066 if val is not None:
2067 for iele, ele in enumerate(val):
2068 for y in walk(ele, typ,
2069 path=path + ((prop.name, iele),)):
2070 yield y
2071 else:
2072 for y in walk(val, typ, path=path+(prop.name,)):
2073 yield y
2076def clone(x, pool=None):
2077 '''
2078 Clone guts object tree.
2080 Traverses guts object tree and recursively clones all guts attributes,
2081 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects
2082 deriving from :py:class:`Object` are instantiated using their respective
2083 init function. Multiply referenced objects in the source tree are multiply
2084 referenced also in the destination tree.
2086 This function can be used to clone guts objects ignoring any contained
2087 run-time state, i.e. any of their attributes not defined as a guts
2088 property.
2089 '''
2091 if pool is None:
2092 pool = {}
2094 if id(x) in pool:
2095 x_copy = pool[id(x)]
2097 else:
2098 if isinstance(x, SObject):
2099 x_copy = x.__class__(str(x))
2100 elif isinstance(x, Object):
2101 d = {}
2102 for (prop, y) in x.T.ipropvals(x):
2103 if y is not None:
2104 if not prop.multivalued:
2105 y_copy = clone(y, pool)
2106 elif prop.multivalued is dict:
2107 y_copy = dict(
2108 (clone(zk, pool), clone(zv, pool))
2109 for (zk, zv) in y.items())
2110 else:
2111 y_copy = type(y)(clone(z, pool) for z in y)
2112 else:
2113 y_copy = y
2115 d[prop.name] = y_copy
2117 x_copy = x.__class__(**d)
2119 else:
2120 x_copy = copy.deepcopy(x)
2122 pool[id(x)] = x_copy
2123 return x_copy
2126class YPathError(Exception):
2127 '''
2128 This exception is raised for invalid ypath specifications.
2129 '''
2130 pass
2133def _parse_yname(yname):
2134 ident = r'[a-zA-Z][a-zA-Z0-9_]*'
2135 rint = r'-?[0-9]+'
2136 m = re.match(
2137 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$'
2138 % (ident, rint, rint, rint), yname)
2140 if not m:
2141 raise YPathError('Syntax error in component: "%s"' % yname)
2143 d = dict(
2144 name=m.group(1))
2146 if m.group(2):
2147 if m.group(5):
2148 istart = iend = None
2149 if m.group(4):
2150 istart = int(m.group(4))
2151 if m.group(6):
2152 iend = int(m.group(6))
2154 d['slice'] = (istart, iend)
2155 else:
2156 d['index'] = int(m.group(7))
2158 return d
2161def _decend(obj, ynames):
2162 if ynames:
2163 for sobj in iter_elements(obj, ynames):
2164 yield sobj
2165 else:
2166 yield obj
2169def iter_elements(obj, ypath):
2170 '''
2171 Generator yielding elements matching a given ypath specification.
2173 :param obj: guts :py:class:`Object` instance
2174 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2175 To access list objects use slice notatation (e.g.
2176 'root.child[:].child[1:3].child[1]').
2178 Raises :py:exc:`YPathError` on failure.
2179 '''
2181 try:
2182 if isinstance(ypath, str):
2183 ynames = ypath.split('.')
2184 else:
2185 ynames = ypath
2187 yname = ynames[0]
2188 ynames = ynames[1:]
2189 d = _parse_yname(yname)
2190 if d['name'] not in obj.T.propnames:
2191 raise AttributeError(d['name'])
2193 obj = getattr(obj, d['name'])
2195 if 'index' in d:
2196 sobj = obj[d['index']]
2197 for ssobj in _decend(sobj, ynames):
2198 yield ssobj
2200 elif 'slice' in d:
2201 for i in range(*slice(*d['slice']).indices(len(obj))):
2202 sobj = obj[i]
2203 for ssobj in _decend(sobj, ynames):
2204 yield ssobj
2205 else:
2206 for sobj in _decend(obj, ynames):
2207 yield sobj
2209 except (AttributeError, IndexError) as e:
2210 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2213def get_elements(obj, ypath):
2214 '''
2215 Get all elements matching a given ypath specification.
2217 :param obj: guts :py:class:`Object` instance
2218 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2219 To access list objects use slice notatation (e.g.
2220 'root.child[:].child[1:3].child[1]').
2222 Raises :py:exc:`YPathError` on failure.
2223 '''
2224 return list(iter_elements(obj, ypath))
2227def set_elements(obj, ypath, value, validate=False, regularize=False):
2228 '''
2229 Set elements matching a given ypath specification.
2231 :param obj: guts :py:class:`Object` instance
2232 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2233 To access list objects use slice notatation (e.g.
2234 'root.child[:].child[1:3].child[1]').
2235 :param value: All matching elements will be set to `value`.
2236 :param validate: Whether to validate affected subtrees.
2237 :param regularize: Whether to regularize affected subtrees.
2239 Raises :py:exc:`YPathError` on failure.
2240 '''
2242 ynames = ypath.split('.')
2243 try:
2244 d = _parse_yname(ynames[-1])
2245 if ynames[:-1]:
2246 it = iter_elements(obj, ynames[:-1])
2247 else:
2248 it = [obj]
2250 for sobj in it:
2251 if d['name'] not in sobj.T.propnames:
2252 raise AttributeError(d['name'])
2254 if 'index' in d:
2255 ssobj = getattr(sobj, d['name'])
2256 ssobj[d['index']] = value
2257 elif 'slice' in d:
2258 ssobj = getattr(sobj, d['name'])
2259 for i in range(*slice(*d['slice']).indices(len(ssobj))):
2260 ssobj[i] = value
2261 else:
2262 setattr(sobj, d['name'], value)
2263 if regularize:
2264 sobj.regularize()
2265 if validate:
2266 sobj.validate()
2268 except (AttributeError, IndexError, YPathError) as e:
2269 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2272def zip_walk(x, typ=None, path=(), stack=()):
2273 if typ is None or isinstance(x, typ):
2274 yield path, stack + (x,)
2276 if isinstance(x, Object):
2277 for (prop, val) in x.T.ipropvals(x):
2278 if prop.multivalued:
2279 if val is not None:
2280 for iele, ele in enumerate(val):
2281 for y in zip_walk(
2282 ele, typ,
2283 path=path + ((prop.name, iele),),
2284 stack=stack + (x,)):
2286 yield y
2287 else:
2288 for y in zip_walk(val, typ,
2289 path=path+(prop.name,),
2290 stack=stack + (x,)):
2291 yield y
2294def path_element(x):
2295 if isinstance(x, tuple):
2296 if len(x) == 2:
2297 return '%s[%i]' % x
2298 elif len(x) == 3:
2299 return '%s[%i:%i]' % x
2301 else:
2302 return x
2305def path_to_str(path):
2306 return '.'.join(path_element(x) for x in path)
2309@expand_stream_args('w')
2310def dump(*args, **kwargs):
2311 return _dump(*args, **kwargs)
2314@expand_stream_args('r')
2315def load(*args, **kwargs):
2316 return _load(*args, **kwargs)
2319def load_string(s, *args, **kwargs):
2320 return load(string=s, *args, **kwargs)
2323@expand_stream_args('w')
2324def dump_all(*args, **kwargs):
2325 return _dump_all(*args, **kwargs)
2328@expand_stream_args('r')
2329def load_all(*args, **kwargs):
2330 return _load_all(*args, **kwargs)
2333@expand_stream_args('r')
2334def iload_all(*args, **kwargs):
2335 return _iload_all(*args, **kwargs)
2338@expand_stream_args('w')
2339def dump_xml(*args, **kwargs):
2340 return _dump_xml(*args, **kwargs)
2343@expand_stream_args('r')
2344def load_xml(*args, **kwargs):
2345 kwargs.pop('filename', None)
2346 return _load_xml(*args, **kwargs)
2349def load_xml_string(s, *args, **kwargs):
2350 return load_xml(string=s, *args, **kwargs)
2353@expand_stream_args('w')
2354def dump_all_xml(*args, **kwargs):
2355 return _dump_all_xml(*args, **kwargs)
2358@expand_stream_args('r')
2359def load_all_xml(*args, **kwargs):
2360 kwargs.pop('filename', None)
2361 return _load_all_xml(*args, **kwargs)
2364@expand_stream_args('r')
2365def iload_all_xml(*args, **kwargs):
2366 kwargs.pop('filename', None)
2367 return _iload_all_xml(*args, **kwargs)
2370__all__ = guts_types + [
2371 'guts_types', 'TBase', 'ValidationError',
2372 'ArgumentError', 'Defer',
2373 'dump', 'load',
2374 'dump_all', 'load_all', 'iload_all',
2375 'dump_xml', 'load_xml',
2376 'dump_all_xml', 'load_all_xml', 'iload_all_xml',
2377 'load_string',
2378 'load_xml_string',
2379 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str'
2380]