1# http://pyrocko.org - GPLv3
2#
3# The Pyrocko Developers, 21st Century
4# ---|P------/S----------~Lg----------
5'''
6Lightweight declarative YAML and XML data binding for Python.
7'''
9import datetime
10import calendar
11import re
12import sys
13import types
14import copy
15import os.path as op
16from collections import defaultdict
18from io import BytesIO
20try:
21 import numpy as num
22except ImportError:
23 num = None
25import yaml
26try:
27 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
28except ImportError:
29 from yaml import SafeLoader, SafeDumper
31from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \
32 get_time_float
35ALLOW_INCLUDE = False
38class GutsSafeDumper(SafeDumper):
39 pass
42class GutsSafeLoader(SafeLoader):
43 pass
46g_iprop = 0
48g_deferred = {}
49g_deferred_content = {}
51g_tagname_to_class = {}
52g_xmltagname_to_class = {}
53g_guessable_xmlns = {}
55guts_types = [
56 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float',
57 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern',
58 'UnicodePattern', 'StringChoice', 'IntChoice', 'List', 'Dict', 'Tuple',
59 'Union', 'Choice', 'Any']
61us_to_cc_regex = re.compile(r'([a-z])_([a-z])')
64class literal(str):
65 pass
68class folded(str):
69 pass
72class singlequoted(str):
73 pass
76class doublequoted(str):
77 pass
80def make_str_presenter(style):
81 def presenter(dumper, data):
82 return dumper.represent_scalar(
83 'tag:yaml.org,2002:str', str(data), style=style)
85 return presenter
88str_style_map = {
89 None: lambda x: x,
90 '|': literal,
91 '>': folded,
92 "'": singlequoted,
93 '"': doublequoted}
95for (style, cls) in str_style_map.items():
96 if style:
97 GutsSafeDumper.add_representer(cls, make_str_presenter(style))
100class uliteral(str):
101 pass
104class ufolded(str):
105 pass
108class usinglequoted(str):
109 pass
112class udoublequoted(str):
113 pass
116def make_unicode_presenter(style):
117 def presenter(dumper, data):
118 return dumper.represent_scalar(
119 'tag:yaml.org,2002:str', str(data), style=style)
121 return presenter
124unicode_style_map = {
125 None: lambda x: x,
126 '|': literal,
127 '>': folded,
128 "'": singlequoted,
129 '"': doublequoted}
131for (style, cls) in unicode_style_map.items():
132 if style:
133 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style))
136class blist(list):
137 pass
140class flist(list):
141 pass
144list_style_map = {
145 None: list,
146 'block': blist,
147 'flow': flist}
150def make_list_presenter(flow_style):
151 def presenter(dumper, data):
152 return dumper.represent_sequence(
153 'tag:yaml.org,2002:seq', data, flow_style=flow_style)
155 return presenter
158GutsSafeDumper.add_representer(blist, make_list_presenter(False))
159GutsSafeDumper.add_representer(flist, make_list_presenter(True))
161if num:
162 def numpy_float_presenter(dumper, data):
163 return dumper.represent_float(float(data))
165 def numpy_int_presenter(dumper, data):
166 return dumper.represent_int(int(data))
168 for dtype in (num.float64, num.float32):
169 GutsSafeDumper.add_representer(dtype, numpy_float_presenter)
171 for dtype in (num.int32, num.int64):
172 GutsSafeDumper.add_representer(dtype, numpy_int_presenter)
175def us_to_cc(s):
176 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s)
179cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)')
180cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])')
183def cc_to_us(s):
184 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub(
185 '\\1_\\2\\3', s)).lower()
188re_frac = re.compile(r'\.[1-9]FRAC')
189frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789'])
192def encode_utf8(s):
193 return s.encode('utf-8')
196def no_encode(s):
197 return s
200def make_xmltagname_from_name(name):
201 return us_to_cc(name)
204def make_name_from_xmltagname(xmltagname):
205 return cc_to_us(xmltagname)
208def make_content_name(name):
209 if name.endswith('_list'):
210 return name[:-5]
211 elif name.endswith('s'):
212 return name[:-1]
213 else:
214 return name
217def classnames(cls):
218 if isinstance(cls, tuple):
219 return '(%s)' % ', '.join(x.__name__ for x in cls)
220 else:
221 return cls.__name__
224def expand_stream_args(mode):
225 def wrap(f):
226 '''
227 Decorator to enhance functions taking stream objects.
229 Wraps a function f(..., stream, ...) so that it can also be called as
230 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...).
231 '''
233 def g(*args, **kwargs):
234 stream = kwargs.pop('stream', None)
235 filename = kwargs.get('filename', None)
236 if mode != 'r':
237 filename = kwargs.pop('filename', None)
238 string = kwargs.pop('string', None)
240 assert sum(x is not None for x in (stream, filename, string)) <= 1
242 if stream is not None:
243 kwargs['stream'] = stream
244 return f(*args, **kwargs)
246 elif filename is not None:
247 stream = open(filename, mode+'b')
248 kwargs['stream'] = stream
249 retval = f(*args, **kwargs)
250 if isinstance(retval, types.GeneratorType):
251 def wrap_generator(gen):
252 try:
253 for x in gen:
254 yield x
256 except GeneratorExit:
257 pass
259 stream.close()
261 return wrap_generator(retval)
263 else:
264 stream.close()
265 return retval
267 elif string is not None:
268 assert mode == 'r', \
269 'Keyword argument string=... cannot be used in dumper ' \
270 'function.'
272 kwargs['stream'] = BytesIO(string.encode('utf-8'))
273 return f(*args, **kwargs)
275 else:
276 assert mode == 'w', \
277 'Use keyword argument stream=... or filename=... in ' \
278 'loader function.'
280 sout = BytesIO()
281 f(stream=sout, *args, **kwargs)
282 return sout.getvalue().decode('utf-8')
284 return g
286 return wrap
289class Defer(object):
290 def __init__(self, classname, *args, **kwargs):
291 global g_iprop
292 if kwargs.get('position', None) is None:
293 kwargs['position'] = g_iprop
295 g_iprop += 1
297 self.classname = classname
298 self.args = args
299 self.kwargs = kwargs
302class TBase(object):
304 strict = False
305 multivalued = None
306 force_regularize = False
307 propnames = []
309 @classmethod
310 def init_propertystuff(cls):
311 cls.properties = []
312 cls.xmltagname_to_name = {}
313 cls.xmltagname_to_name_multivalued = {}
314 cls.xmltagname_to_class = {}
315 cls.content_property = None
317 def __init__(
318 self,
319 default=None,
320 optional=False,
321 xmlstyle='element',
322 xmltagname=None,
323 xmlns=None,
324 help=None,
325 position=None):
327 global g_iprop
328 if position is not None:
329 self.position = position
330 else:
331 self.position = g_iprop
333 g_iprop += 1
334 self._default = default
336 self.optional = optional
337 self.name = None
338 self._xmltagname = xmltagname
339 self._xmlns = xmlns
340 self.parent = None
341 self.xmlstyle = xmlstyle
342 self.help = help
344 def default(self):
345 return make_default(self._default)
347 def is_default(self, val):
348 if self._default is None:
349 return val is None
350 else:
351 return self._default == val
353 def has_default(self):
354 return self._default is not None
356 def xname(self):
357 if self.name is not None:
358 return self.name
359 elif self.parent is not None:
360 return 'element of %s' % self.parent.xname()
361 else:
362 return '?'
364 def set_xmlns(self, xmlns):
365 if self._xmlns is None and not self.xmlns:
366 self._xmlns = xmlns
368 if self.multivalued:
369 self.content_t.set_xmlns(xmlns)
371 def get_xmlns(self):
372 return self._xmlns or self.xmlns
374 def get_xmltagname(self):
375 if self._xmltagname is not None:
376 return self.get_xmlns() + ' ' + self._xmltagname
377 elif self.name:
378 return self.get_xmlns() + ' ' \
379 + make_xmltagname_from_name(self.name)
380 elif self.xmltagname:
381 return self.get_xmlns() + ' ' + self.xmltagname
382 else:
383 assert False
385 @classmethod
386 def get_property(cls, name):
387 for prop in cls.properties:
388 if prop.name == name:
389 return prop
391 raise ValueError()
393 @classmethod
394 def remove_property(cls, name):
396 prop = cls.get_property(name)
398 if not prop.multivalued:
399 del cls.xmltagname_to_class[prop.effective_xmltagname]
400 del cls.xmltagname_to_name[prop.effective_xmltagname]
401 else:
402 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname]
403 del cls.xmltagname_to_name_multivalued[
404 prop.content_t.effective_xmltagname]
406 if cls.content_property is prop:
407 cls.content_property = None
409 cls.properties.remove(prop)
410 cls.propnames.remove(name)
412 return prop
414 @classmethod
415 def add_property(cls, name, prop):
417 prop.instance = prop
418 prop.name = name
419 prop.set_xmlns(cls.xmlns)
421 if isinstance(prop, Choice.T):
422 for tc in prop.choices:
423 tc.effective_xmltagname = tc.get_xmltagname()
424 cls.xmltagname_to_class[tc.effective_xmltagname] = tc.cls
425 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name
426 elif not prop.multivalued:
427 prop.effective_xmltagname = prop.get_xmltagname()
428 cls.xmltagname_to_class[prop.effective_xmltagname] = prop.cls
429 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name
430 else:
431 prop.content_t.name = make_content_name(prop.name)
432 prop.content_t.effective_xmltagname = \
433 prop.content_t.get_xmltagname()
434 cls.xmltagname_to_class[
435 prop.content_t.effective_xmltagname] = prop.content_t.cls
436 cls.xmltagname_to_name_multivalued[
437 prop.content_t.effective_xmltagname] = prop.name
439 cls.properties.append(prop)
441 cls.properties.sort(key=lambda x: x.position)
443 cls.propnames = [p.name for p in cls.properties]
445 if prop.xmlstyle == 'content':
446 cls.content_property = prop
448 @classmethod
449 def ivals(cls, val):
450 for prop in cls.properties:
451 yield getattr(val, prop.name)
453 @classmethod
454 def ipropvals(cls, val):
455 for prop in cls.properties:
456 yield prop, getattr(val, prop.name)
458 @classmethod
459 def inamevals(cls, val):
460 for prop in cls.properties:
461 yield prop.name, getattr(val, prop.name)
463 @classmethod
464 def ipropvals_to_save(cls, val, xmlmode=False):
465 for prop in cls.properties:
466 v = getattr(val, prop.name)
467 if v is not None and (
468 not (prop.optional or (prop.multivalued and not v))
469 or (not prop.is_default(v))):
471 if xmlmode:
472 yield prop, prop.to_save_xml(v)
473 else:
474 yield prop, prop.to_save(v)
476 @classmethod
477 def inamevals_to_save(cls, val, xmlmode=False):
478 for prop, v in cls.ipropvals_to_save(val, xmlmode):
479 yield prop.name, v
481 @classmethod
482 def translate_from_xml(cls, list_of_pairs, strict):
483 d = {}
484 for k, v in list_of_pairs:
485 if k in cls.xmltagname_to_name_multivalued:
486 k2 = cls.xmltagname_to_name_multivalued[k]
487 if k2 not in d:
488 d[k2] = []
490 d[k2].append(v)
491 elif k in cls.xmltagname_to_name:
492 k2 = cls.xmltagname_to_name[k]
493 if k2 in d:
494 raise ArgumentError(
495 'Unexpectedly found more than one child element "%s" '
496 'within "%s".' % (k, cls.tagname))
498 d[k2] = v
499 elif k is None:
500 if cls.content_property:
501 k2 = cls.content_property.name
502 d[k2] = v
503 else:
504 if strict:
505 raise ArgumentError(
506 'Unexpected child element "%s" found within "%s".' % (
507 k, cls.tagname))
509 return d
511 def validate(self, val, regularize=False, depth=-1):
512 if self.optional and val is None:
513 return val
515 is_derived = isinstance(val, self.cls)
516 is_exact = type(val) == self.cls
518 not_ok = not self.strict and not is_derived or \
519 self.strict and not is_exact
521 if not_ok or self.force_regularize:
522 if regularize:
523 try:
524 val = self.regularize_extra(val)
525 except ValueError:
526 raise ValidationError(
527 '%s: could not convert "%s" to type %s' % (
528 self.xname(), val, classnames(self.cls)))
529 else:
530 raise ValidationError(
531 '%s: "%s" (type: %s) is not of type %s' % (
532 self.xname(), val, type(val), classnames(self.cls)))
534 validator = self
535 if isinstance(self.cls, tuple):
536 clss = self.cls
537 else:
538 clss = (self.cls,)
540 for cls in clss:
541 try:
542 if type(val) != cls and isinstance(val, cls):
543 validator = val.T.instance
545 except AttributeError:
546 pass
548 validator.validate_extra(val)
550 if depth != 0:
551 val = validator.validate_children(val, regularize, depth)
553 return val
555 def regularize_extra(self, val):
556 return self.cls(val)
558 def validate_extra(self, val):
559 pass
561 def validate_children(self, val, regularize, depth):
562 for prop, propval in self.ipropvals(val):
563 newpropval = prop.validate(propval, regularize, depth-1)
564 if regularize and (newpropval is not propval):
565 setattr(val, prop.name, newpropval)
567 return val
569 def to_save(self, val):
570 return val
572 def to_save_xml(self, val):
573 return self.to_save(val)
575 def extend_xmlelements(self, elems, v):
576 if self.multivalued:
577 for x in v:
578 elems.append((self.content_t.effective_xmltagname, x))
579 else:
580 elems.append((self.effective_xmltagname, v))
582 def deferred(self):
583 return []
585 def classname_for_help(self, strip_module=''):
587 if self.dummy_cls is not self.cls:
588 if self.dummy_cls.__module__ == strip_module:
589 sadd = ' (:py:class:`%s`)' % (
590 self.dummy_cls.__name__)
591 else:
592 sadd = ' (:py:class:`%s.%s`)' % (
593 self.dummy_cls.__module__, self.dummy_cls.__name__)
594 else:
595 sadd = ''
597 if self.dummy_cls in guts_plain_dummy_types:
598 return '``%s``' % self.cls.__name__
600 elif self.dummy_cls.dummy_for_description:
601 return '%s%s' % (self.dummy_cls.dummy_for_description, sadd)
603 else:
604 def sclass(cls):
605 mod = cls.__module__
606 clsn = cls.__name__
607 if mod == '__builtin__' or mod == 'builtins':
608 return '``%s``' % clsn
610 elif mod == strip_module:
611 return ':py:class:`%s`' % clsn
613 else:
614 return ':py:class:`%s.%s`' % (mod, clsn)
616 if isinstance(self.cls, tuple):
617 return '(%s)%s' % (
618 ' | '.join(sclass(cls) for cls in self.cls), sadd)
619 else:
620 return '%s%s' % (sclass(self.cls), sadd)
622 @classmethod
623 def props_help_string(cls):
624 baseprops = []
625 for base in cls.dummy_cls.__bases__:
626 if hasattr(base, 'T'):
627 baseprops.extend(base.T.properties)
629 hlp = []
630 hlp.append('')
631 for prop in cls.properties:
632 if prop in baseprops:
633 continue
635 descr = [
636 prop.classname_for_help(strip_module=cls.dummy_cls.__module__)]
638 if prop.optional:
639 descr.append('*optional*')
641 if isinstance(prop._default, DefaultMaker):
642 descr.append('*default:* ``%s``' % repr(prop._default))
643 else:
644 d = prop.default()
645 if d is not None:
646 descr.append('*default:* ``%s``' % repr(d))
648 hlp.append(' .. py:gattribute:: %s' % prop.name)
649 hlp.append('')
650 hlp.append(' %s' % ', '.join(descr))
651 hlp.append(' ')
652 if prop.help is not None:
653 hlp.append(' %s' % prop.help)
654 hlp.append('')
656 return '\n'.join(hlp)
658 @classmethod
659 def class_help_string(cls):
660 return cls.dummy_cls.__doc_template__
662 @classmethod
663 def class_signature(cls):
664 r = []
665 for prop in cls.properties:
666 d = prop.default()
667 if d is not None:
668 arg = repr(d)
670 elif prop.optional:
671 arg = 'None'
673 else:
674 arg = '...'
676 r.append('%s=%s' % (prop.name, arg))
678 return '(%s)' % ', '.join(r)
680 @classmethod
681 def help(cls):
682 return cls.props_help_string()
685class ObjectMetaClass(type):
686 def __new__(meta, classname, bases, class_dict):
687 cls = type.__new__(meta, classname, bases, class_dict)
688 if classname != 'Object':
689 t_class_attr_name = '_%s__T' % classname
690 if not hasattr(cls, t_class_attr_name):
691 if hasattr(cls, 'T'):
692 class T(cls.T):
693 pass
694 else:
695 class T(TBase):
696 pass
698 setattr(cls, t_class_attr_name, T)
700 T = getattr(cls, t_class_attr_name)
702 if cls.dummy_for is not None:
703 T.cls = cls.dummy_for
704 else:
705 T.cls = cls
707 T.dummy_cls = cls
709 if hasattr(cls, 'xmltagname'):
710 T.xmltagname = cls.xmltagname
711 else:
712 T.xmltagname = classname
714 mod = sys.modules[cls.__module__]
716 if hasattr(cls, 'xmlns'):
717 T.xmlns = cls.xmlns
718 elif hasattr(mod, 'guts_xmlns'):
719 T.xmlns = mod.guts_xmlns
720 else:
721 T.xmlns = ''
723 if T.xmlns and hasattr(cls, 'guessable_xmlns'):
724 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns
726 if hasattr(mod, 'guts_prefix'):
727 if mod.guts_prefix:
728 T.tagname = mod.guts_prefix + '.' + classname
729 else:
730 T.tagname = classname
731 else:
732 if cls.__module__ != '__main__':
733 T.tagname = cls.__module__ + '.' + classname
734 else:
735 T.tagname = classname
737 T.classname = classname
739 T.init_propertystuff()
741 for k in dir(cls):
742 prop = getattr(cls, k)
744 if k.endswith('__'):
745 k = k[:-2]
747 if isinstance(prop, TBase):
748 if prop.deferred():
749 for defer in prop.deferred():
750 g_deferred_content.setdefault(
751 defer.classname[:-2], []).append((prop, defer))
752 g_deferred.setdefault(
753 defer.classname[:-2], []).append((T, k, prop))
755 else:
756 T.add_property(k, prop)
758 elif isinstance(prop, Defer):
759 g_deferred.setdefault(prop.classname[:-2], []).append(
760 (T, k, prop))
762 if classname in g_deferred_content:
763 for prop, defer in g_deferred_content[classname]:
764 prop.process_deferred(
765 defer, T(*defer.args, **defer.kwargs))
767 del g_deferred_content[classname]
769 if classname in g_deferred:
770 for (T_, k_, prop_) in g_deferred.get(classname, []):
771 if isinstance(prop_, Defer):
772 prop_ = T(*prop_.args, **prop_.kwargs)
774 if not prop_.deferred():
775 T_.add_property(k_, prop_)
777 del g_deferred[classname]
779 g_tagname_to_class[T.tagname] = cls
780 if hasattr(cls, 'xmltagname'):
781 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls
783 cls.T = T
784 T.instance = T()
786 cls.__doc_template__ = cls.__doc__
787 cls.__doc__ = T.class_help_string()
789 if cls.__doc__ is None:
790 cls.__doc__ = 'Undocumented.'
792 cls.__doc__ += '\n' + T.props_help_string()
794 return cls
797class ValidationError(Exception):
798 pass
801class ArgumentError(Exception):
802 pass
805def make_default(x):
806 if isinstance(x, DefaultMaker):
807 return x.make()
808 elif isinstance(x, Object):
809 return clone(x)
810 else:
811 return x
814class DefaultMaker(object):
815 def make(self):
816 raise NotImplementedError('Schould be implemented in subclass.')
819class ObjectDefaultMaker(DefaultMaker):
820 def __init__(self, cls, args, kwargs):
821 DefaultMaker.__init__(self)
822 self.cls = cls
823 self.args = args
824 self.kwargs = kwargs
825 self.instance = None
827 def make(self):
828 return self.cls(
829 *[make_default(x) for x in self.args],
830 **dict((k, make_default(v)) for (k, v) in self.kwargs.items()))
832 def __eq__(self, other):
833 if self.instance is None:
834 self.instance = self.make()
836 return self.instance == other
838 def __repr__(self):
839 sargs = []
840 for arg in self.args:
841 sargs.append(repr(arg))
843 for k, v in self.kwargs.items():
844 sargs.append('%s=%s' % (k, repr(v)))
846 return '%s(%s)' % (self.cls.__name__, ', '.join(sargs))
849class TimestampDefaultMaker(DefaultMaker):
850 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'):
851 DefaultMaker.__init__(self)
852 self._stime = s
853 self._format = format
855 def make(self):
856 return str_to_time(self._stime, self._format)
858 def __repr__(self):
859 return "str_to_time(%s)" % repr(self._stime)
862def with_metaclass(meta, *bases):
863 # inlined py2/py3 compat solution from python-future
864 class metaclass(meta):
865 __call__ = type.__call__
866 __init__ = type.__init__
868 def __new__(cls, name, this_bases, d):
869 if this_bases is None:
870 return type.__new__(cls, name, (), d)
871 return meta(name, bases, d)
873 return metaclass('temp', None, {})
876class Object(with_metaclass(ObjectMetaClass, object)):
877 dummy_for = None
878 dummy_for_description = None
880 def __init__(self, **kwargs):
881 if not kwargs.get('init_props', True):
882 return
884 for prop in self.T.properties:
885 k = prop.name
886 if k in kwargs:
887 setattr(self, k, kwargs.pop(k))
888 else:
889 if not prop.optional and not prop.has_default():
890 raise ArgumentError('Missing argument to %s: %s' % (
891 self.T.tagname, prop.name))
892 else:
893 setattr(self, k, prop.default())
895 if kwargs:
896 raise ArgumentError('Invalid argument to %s: %s' % (
897 self.T.tagname, ', '.join(list(kwargs.keys()))))
899 @classmethod
900 def D(cls, *args, **kwargs):
901 return ObjectDefaultMaker(cls, args, kwargs)
903 def validate(self, regularize=False, depth=-1):
904 self.T.instance.validate(self, regularize, depth)
906 def regularize(self, depth=-1):
907 self.validate(regularize=True, depth=depth)
909 def dump(self, stream=None, filename=None, header=False):
910 return dump(self, stream=stream, filename=filename, header=header)
912 def dump_xml(
913 self, stream=None, filename=None, header=False, ns_ignore=False):
914 return dump_xml(
915 self, stream=stream, filename=filename, header=header,
916 ns_ignore=ns_ignore)
918 @classmethod
919 def load(cls, stream=None, filename=None, string=None):
920 return load(stream=stream, filename=filename, string=string)
922 @classmethod
923 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None,
924 ns_ignore=False):
926 if ns_hints is None:
927 ns_hints = [cls.T.instance.get_xmlns()]
929 return load_xml(
930 stream=stream,
931 filename=filename,
932 string=string,
933 ns_hints=ns_hints,
934 ns_ignore=ns_ignore)
936 def __str__(self):
937 return self.dump()
940def to_dict(obj):
941 '''
942 Get dict of guts object attributes.
944 :param obj: :py:class`Object` object
945 '''
947 return dict(obj.T.inamevals(obj))
950class SObject(Object):
952 class __T(TBase):
953 def regularize_extra(self, val):
954 if isinstance(val, str):
955 return self.cls(val)
957 return val
959 def to_save(self, val):
960 return str(val)
962 def to_save_xml(self, val):
963 return str(val)
966class Any(Object):
968 class __T(TBase):
969 def validate(self, val, regularize=False, depth=-1):
970 if isinstance(val, Object):
971 val.validate(regularize, depth)
973 return val
976class Int(Object):
977 dummy_for = int
979 class __T(TBase):
980 strict = True
982 def to_save_xml(self, value):
983 return repr(value)
986class Float(Object):
987 dummy_for = float
989 class __T(TBase):
990 strict = True
992 def to_save_xml(self, value):
993 return repr(value)
996class Complex(Object):
997 dummy_for = complex
999 class __T(TBase):
1000 strict = True
1002 def regularize_extra(self, val):
1004 if isinstance(val, list) or isinstance(val, tuple):
1005 assert len(val) == 2
1006 val = complex(*val)
1008 elif not isinstance(val, complex):
1009 val = complex(val)
1011 return val
1013 def to_save(self, value):
1014 return repr(value)
1016 def to_save_xml(self, value):
1017 return repr(value)
1020class Bool(Object):
1021 dummy_for = bool
1023 class __T(TBase):
1024 strict = True
1026 def regularize_extra(self, val):
1027 if isinstance(val, str):
1028 if val.lower().strip() in ('0', 'false'):
1029 return False
1031 return bool(val)
1033 def to_save_xml(self, value):
1034 return repr(bool(value)).lower()
1037class String(Object):
1038 dummy_for = str
1040 class __T(TBase):
1041 def __init__(self, *args, **kwargs):
1042 yamlstyle = kwargs.pop('yamlstyle', None)
1043 TBase.__init__(self, *args, **kwargs)
1044 self.style_cls = str_style_map[yamlstyle]
1046 def to_save(self, val):
1047 return self.style_cls(val)
1050class Unicode(Object):
1051 dummy_for = str
1053 class __T(TBase):
1054 def __init__(self, *args, **kwargs):
1055 yamlstyle = kwargs.pop('yamlstyle', None)
1056 TBase.__init__(self, *args, **kwargs)
1057 self.style_cls = unicode_style_map[yamlstyle]
1059 def to_save(self, val):
1060 return self.style_cls(val)
1063guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool)
1066class Dict(Object):
1067 dummy_for = dict
1069 class __T(TBase):
1070 multivalued = dict
1072 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs):
1073 TBase.__init__(self, *args, **kwargs)
1074 assert isinstance(key_t, TBase)
1075 assert isinstance(content_t, TBase)
1076 self.key_t = key_t
1077 self.content_t = content_t
1078 self.content_t.parent = self
1080 def default(self):
1081 if self._default is not None:
1082 return dict(
1083 (make_default(k), make_default(v))
1084 for (k, v) in self._default.items())
1086 if self.optional:
1087 return None
1088 else:
1089 return {}
1091 def has_default(self):
1092 return True
1094 def validate(self, val, regularize, depth):
1095 return TBase.validate(self, val, regularize, depth+1)
1097 def validate_children(self, val, regularize, depth):
1098 for key, ele in list(val.items()):
1099 newkey = self.key_t.validate(key, regularize, depth-1)
1100 newele = self.content_t.validate(ele, regularize, depth-1)
1101 if regularize:
1102 if newkey is not key or newele is not ele:
1103 del val[key]
1104 val[newkey] = newele
1106 return val
1108 def to_save(self, val):
1109 return dict((self.key_t.to_save(k), self.content_t.to_save(v))
1110 for (k, v) in val.items())
1112 def to_save_xml(self, val):
1113 raise NotImplementedError()
1115 def classname_for_help(self, strip_module=''):
1116 return '``dict`` of %s objects' % \
1117 self.content_t.classname_for_help(strip_module=strip_module)
1120class List(Object):
1121 dummy_for = list
1123 class __T(TBase):
1124 multivalued = list
1126 def __init__(self, content_t=Any.T(), *args, **kwargs):
1127 yamlstyle = kwargs.pop('yamlstyle', None)
1128 TBase.__init__(self, *args, **kwargs)
1129 assert isinstance(content_t, TBase) or isinstance(content_t, Defer)
1130 self.content_t = content_t
1131 self.content_t.parent = self
1132 self.style_cls = list_style_map[yamlstyle]
1134 def default(self):
1135 if self._default is not None:
1136 return [make_default(x) for x in self._default]
1137 if self.optional:
1138 return None
1139 else:
1140 return []
1142 def has_default(self):
1143 return True
1145 def validate(self, val, regularize, depth):
1146 return TBase.validate(self, val, regularize, depth+1)
1148 def validate_children(self, val, regularize, depth):
1149 for i, ele in enumerate(val):
1150 newele = self.content_t.validate(ele, regularize, depth-1)
1151 if regularize and newele is not ele:
1152 val[i] = newele
1154 return val
1156 def to_save(self, val):
1157 return self.style_cls(self.content_t.to_save(v) for v in val)
1159 def to_save_xml(self, val):
1160 return [self.content_t.to_save_xml(v) for v in val]
1162 def deferred(self):
1163 if isinstance(self.content_t, Defer):
1164 return [self.content_t]
1166 return []
1168 def process_deferred(self, defer, t_inst):
1169 if defer is self.content_t:
1170 self.content_t = t_inst
1172 def classname_for_help(self, strip_module=''):
1173 return '``list`` of %s objects' % \
1174 self.content_t.classname_for_help(strip_module=strip_module)
1177def make_typed_list_class(t):
1178 class TL(List):
1179 class __T(List.T):
1180 def __init__(self, *args, **kwargs):
1181 List.T.__init__(self, content_t=t.T(), *args, **kwargs)
1183 return TL
1186class Tuple(Object):
1187 dummy_for = tuple
1189 class __T(TBase):
1190 multivalued = tuple
1192 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs):
1193 TBase.__init__(self, *args, **kwargs)
1194 assert isinstance(content_t, TBase)
1195 self.content_t = content_t
1196 self.content_t.parent = self
1197 self.n = n
1199 def default(self):
1200 if self._default is not None:
1201 return tuple(
1202 make_default(x) for x in self._default)
1204 elif self.optional:
1205 return None
1206 else:
1207 if self.n is not None:
1208 return tuple(
1209 self.content_t.default() for x in range(self.n))
1210 else:
1211 return tuple()
1213 def has_default(self):
1214 return True
1216 def validate(self, val, regularize, depth):
1217 return TBase.validate(self, val, regularize, depth+1)
1219 def validate_extra(self, val):
1220 if self.n is not None and len(val) != self.n:
1221 raise ValidationError(
1222 '%s should have length %i' % (self.xname(), self.n))
1224 def validate_children(self, val, regularize, depth):
1225 if not regularize:
1226 for ele in val:
1227 self.content_t.validate(ele, regularize, depth-1)
1229 return val
1230 else:
1231 newval = []
1232 isnew = False
1233 for ele in val:
1234 newele = self.content_t.validate(ele, regularize, depth-1)
1235 newval.append(newele)
1236 if newele is not ele:
1237 isnew = True
1239 if isnew:
1240 return tuple(newval)
1241 else:
1242 return val
1244 def to_save(self, val):
1245 return tuple(self.content_t.to_save(v) for v in val)
1247 def to_save_xml(self, val):
1248 return [self.content_t.to_save_xml(v) for v in val]
1250 def classname_for_help(self, strip_module=''):
1251 if self.n is not None:
1252 return '``tuple`` of %i %s objects' % (
1253 self.n, self.content_t.classname_for_help(
1254 strip_module=strip_module))
1255 else:
1256 return '``tuple`` of %s objects' % (
1257 self.content_t.classname_for_help(
1258 strip_module=strip_module))
1261unit_factors = dict(
1262 s=1.0,
1263 m=60.0,
1264 h=3600.0,
1265 d=24*3600.0,
1266 y=365*24*3600.0)
1269class Duration(Object):
1270 dummy_for = float
1272 class __T(TBase):
1274 def regularize_extra(self, val):
1275 if isinstance(val, str):
1276 unit = val[-1]
1277 if unit in unit_factors:
1278 return float(val[:-1]) * unit_factors[unit]
1279 else:
1280 return float(val)
1282 return val
1285re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$')
1288class Timestamp(Object):
1289 dummy_for = (hpfloat, float)
1290 dummy_for_description = 'time_float'
1292 class __T(TBase):
1294 def regularize_extra(self, val):
1296 time_float = get_time_float()
1298 if isinstance(val, datetime.datetime):
1299 tt = val.utctimetuple()
1300 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1302 elif isinstance(val, datetime.date):
1303 tt = val.timetuple()
1304 val = time_float(calendar.timegm(tt))
1306 elif isinstance(val, str):
1307 val = val.strip()
1308 tz_offset = 0
1310 m = re_tz.search(val)
1311 if m:
1312 sh = m.group(2)
1313 sm = m.group(4)
1314 tz_offset = (int(sh)*3600 if sh else 0) \
1315 + (int(sm)*60 if sm else 0)
1317 val = re_tz.sub('', val)
1319 if len(val) > 10 and val[10] == 'T':
1320 val = val.replace('T', ' ', 1)
1322 try:
1323 val = str_to_time(val) - tz_offset
1324 except TimeStrError:
1325 raise ValidationError(
1326 '%s: cannot parse time/date: %s' % (self.xname(), val))
1328 elif isinstance(val, (int, float)):
1329 val = time_float(val)
1331 else:
1332 raise ValidationError(
1333 '%s: cannot convert "%s" to type %s' % (
1334 self.xname(), val, time_float))
1336 return val
1338 def to_save(self, val):
1339 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\
1340 .rstrip('0').rstrip('.')
1342 def to_save_xml(self, val):
1343 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\
1344 .rstrip('0').rstrip('.') + 'Z'
1346 @classmethod
1347 def D(self, s):
1348 return TimestampDefaultMaker(s)
1351class DateTimestamp(Object):
1352 dummy_for = (hpfloat, float)
1353 dummy_for_description = 'time_float'
1355 class __T(TBase):
1357 def regularize_extra(self, val):
1359 time_float = get_time_float()
1361 if isinstance(val, datetime.datetime):
1362 tt = val.utctimetuple()
1363 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1365 elif isinstance(val, datetime.date):
1366 tt = val.timetuple()
1367 val = time_float(calendar.timegm(tt))
1369 elif isinstance(val, str):
1370 val = str_to_time(val, format='%Y-%m-%d')
1372 elif isinstance(val, int):
1373 val = time_float(val)
1375 return val
1377 def to_save(self, val):
1378 return time_to_str(val, format='%Y-%m-%d')
1380 def to_save_xml(self, val):
1381 return time_to_str(val, format='%Y-%m-%d')
1383 @classmethod
1384 def D(self, s):
1385 return TimestampDefaultMaker(s, format='%Y-%m-%d')
1388class StringPattern(String):
1390 '''
1391 Any ``str`` matching pattern ``%(pattern)s``.
1392 '''
1394 dummy_for = str
1395 pattern = '.*'
1397 class __T(String.T):
1398 def __init__(self, pattern=None, *args, **kwargs):
1399 String.T.__init__(self, *args, **kwargs)
1401 if pattern is not None:
1402 self.pattern = pattern
1403 else:
1404 self.pattern = self.dummy_cls.pattern
1406 def validate_extra(self, val):
1407 pat = self.pattern
1408 if not re.search(pat, val):
1409 raise ValidationError('%s: "%s" does not match pattern %s' % (
1410 self.xname(), val, repr(pat)))
1412 @classmethod
1413 def class_help_string(cls):
1414 dcls = cls.dummy_cls
1415 doc = dcls.__doc_template__ or StringPattern.__doc_template__
1416 return doc % {'pattern': repr(dcls.pattern)}
1419class UnicodePattern(Unicode):
1421 '''
1422 Any ``str`` matching pattern ``%(pattern)s``.
1423 '''
1425 dummy_for = str
1426 pattern = '.*'
1428 class __T(TBase):
1429 def __init__(self, pattern=None, *args, **kwargs):
1430 TBase.__init__(self, *args, **kwargs)
1432 if pattern is not None:
1433 self.pattern = pattern
1434 else:
1435 self.pattern = self.dummy_cls.pattern
1437 def validate_extra(self, val):
1438 pat = self.pattern
1439 if not re.search(pat, val, flags=re.UNICODE):
1440 raise ValidationError('%s: "%s" does not match pattern %s' % (
1441 self.xname(), val, repr(pat)))
1443 @classmethod
1444 def class_help_string(cls):
1445 dcls = cls.dummy_cls
1446 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__
1447 return doc % {'pattern': repr(dcls.pattern)}
1450class StringChoice(String):
1452 '''
1453 Any ``str`` out of ``%(choices)s``.
1454 '''
1456 dummy_for = str
1457 choices = []
1458 ignore_case = False
1460 class __T(String.T):
1461 def __init__(self, choices=None, ignore_case=None, *args, **kwargs):
1462 String.T.__init__(self, *args, **kwargs)
1464 if choices is not None:
1465 self.choices = choices
1466 else:
1467 self.choices = self.dummy_cls.choices
1469 if ignore_case is not None:
1470 self.ignore_case = ignore_case
1471 else:
1472 self.ignore_case = self.dummy_cls.ignore_case
1474 if self.ignore_case:
1475 self.choices = [x.upper() for x in self.choices]
1477 def validate_extra(self, val):
1478 if self.ignore_case:
1479 val = val.upper()
1481 if val not in self.choices:
1482 raise ValidationError(
1483 '%s: "%s" is not a valid choice out of %s' % (
1484 self.xname(), val, repr(self.choices)))
1486 @classmethod
1487 def class_help_string(cls):
1488 dcls = cls.dummy_cls
1489 doc = dcls.__doc_template__ or StringChoice.__doc_template__
1490 return doc % {'choices': repr(dcls.choices)}
1493class IntChoice(Int):
1495 '''
1496 Any ``int`` out of ``%(choices)s``.
1497 '''
1499 dummy_for = int
1500 choices = []
1502 class __T(Int.T):
1503 def __init__(self, choices=None, *args, **kwargs):
1504 Int.T.__init__(self, *args, **kwargs)
1506 if choices is not None:
1507 self.choices = choices
1508 else:
1509 self.choices = self.dummy_cls.choices
1511 def validate_extra(self, val):
1512 if val not in self.choices:
1513 raise ValidationError(
1514 '%s: %i is not a valid choice out of %s' % (
1515 self.xname(), val, repr(self.choices)))
1517 @classmethod
1518 def class_help_string(cls):
1519 dcls = cls.dummy_cls
1520 doc = dcls.__doc_template__ or IntChoice.__doc_template__
1521 return doc % {'choices': repr(dcls.choices)}
1524# this will not always work...
1525class Union(Object):
1526 members = []
1527 dummy_for = str
1529 class __T(TBase):
1530 def __init__(self, members=None, *args, **kwargs):
1531 TBase.__init__(self, *args, **kwargs)
1532 if members is not None:
1533 self.members = members
1534 else:
1535 self.members = self.dummy_cls.members
1537 def validate(self, val, regularize=False, depth=-1):
1538 assert self.members
1539 e2 = None
1540 for member in self.members:
1541 try:
1542 return member.validate(val, regularize, depth=depth)
1543 except ValidationError as e:
1544 e2 = e
1546 raise e2
1549class Choice(Object):
1550 choices = []
1552 class __T(TBase):
1553 def __init__(self, choices=None, *args, **kwargs):
1554 TBase.__init__(self, *args, **kwargs)
1555 if choices is not None:
1556 self.choices = choices
1557 else:
1558 self.choices = self.dummy_cls.choices
1560 self.cls_to_xmltagname = dict(
1561 (t.cls, t.get_xmltagname()) for t in self.choices)
1563 def validate(self, val, regularize=False, depth=-1):
1564 if self.optional and val is None:
1565 return val
1567 t = None
1568 for tc in self.choices:
1569 is_derived = isinstance(val, tc.cls)
1570 is_exact = type(val) == tc.cls
1571 if not (not tc.strict and not is_derived or
1572 tc.strict and not is_exact):
1574 t = tc
1575 break
1577 if t is None:
1578 if regularize:
1579 ok = False
1580 for tc in self.choices:
1581 try:
1582 val = tc.regularize_extra(val)
1583 ok = True
1584 t = tc
1585 break
1586 except (ValidationError, ValueError):
1587 pass
1589 if not ok:
1590 raise ValidationError(
1591 '%s: could not convert "%s" to any type out of '
1592 '(%s)' % (self.xname(), val, ','.join(
1593 classnames(x.cls) for x in self.choices)))
1594 else:
1595 raise ValidationError(
1596 '%s: "%s" (type: %s) is not of any type out of '
1597 '(%s)' % (self.xname(), val, type(val), ','.join(
1598 classnames(x.cls) for x in self.choices)))
1600 validator = t
1602 if isinstance(t.cls, tuple):
1603 clss = t.cls
1604 else:
1605 clss = (t.cls,)
1607 for cls in clss:
1608 try:
1609 if type(val) != cls and isinstance(val, cls):
1610 validator = val.T.instance
1612 except AttributeError:
1613 pass
1615 validator.validate_extra(val)
1617 if depth != 0:
1618 val = validator.validate_children(val, regularize, depth)
1620 return val
1622 def extend_xmlelements(self, elems, v):
1623 elems.append((
1624 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v))
1627def _dump(
1628 object, stream,
1629 header=False,
1630 Dumper=GutsSafeDumper,
1631 _dump_function=yaml.dump):
1633 if not getattr(stream, 'encoding', None):
1634 enc = encode_utf8
1635 else:
1636 enc = no_encode
1638 if header:
1639 stream.write(enc(u'%YAML 1.1\n'))
1640 if isinstance(header, str):
1641 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
1642 stream.write(enc(banner))
1644 _dump_function(
1645 object,
1646 stream=stream,
1647 encoding='utf-8',
1648 explicit_start=True,
1649 Dumper=Dumper)
1652def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper):
1653 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all)
1656def _load(stream,
1657 Loader=GutsSafeLoader, allow_include=None, filename=None,
1658 included_files=None):
1660 class _Loader(Loader):
1661 _filename = filename
1662 _allow_include = allow_include
1663 _included_files = included_files or []
1665 return yaml.load(stream=stream, Loader=_Loader)
1668def _load_all(stream,
1669 Loader=GutsSafeLoader, allow_include=None, filename=None):
1671 class _Loader(Loader):
1672 _filename = filename
1673 _allow_include = allow_include
1675 return list(yaml.load_all(stream=stream, Loader=_Loader))
1678def _iload_all(stream,
1679 Loader=GutsSafeLoader, allow_include=None, filename=None):
1681 class _Loader(Loader):
1682 _filename = filename
1683 _allow_include = allow_include
1685 return yaml.load_all(stream=stream, Loader=_Loader)
1688def multi_representer(dumper, data):
1689 node = dumper.represent_mapping(
1690 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False)
1692 return node
1695# hack for compatibility with early GF Store versions
1696re_compatibility = re.compile(
1697 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.'
1698 r'(dummy|poel|qseis|qssp))\.'
1699)
1702def multi_constructor(loader, tag_suffix, node):
1703 tagname = str(tag_suffix)
1705 tagname = re_compatibility.sub('pf.', tagname)
1707 cls = g_tagname_to_class[tagname]
1708 kwargs = dict(iter(loader.construct_pairs(node, deep=True)))
1709 o = cls(**kwargs)
1710 o.validate(regularize=True, depth=1)
1711 return o
1714def include_constructor(loader, node):
1715 allow_include = loader._allow_include \
1716 if loader._allow_include is not None \
1717 else ALLOW_INCLUDE
1719 if not allow_include:
1720 raise EnvironmentError(
1721 'Not allowed to include YAML. Load with allow_include=True')
1723 if isinstance(node, yaml.nodes.ScalarNode):
1724 inc_file = loader.construct_scalar(node)
1725 else:
1726 raise TypeError('Unsupported YAML node %s' % repr(node))
1728 if loader._filename is not None and not op.isabs(inc_file):
1729 inc_file = op.join(op.dirname(loader._filename), inc_file)
1731 if not op.isfile(inc_file):
1732 raise FileNotFoundError(inc_file)
1734 included_files = list(loader._included_files)
1735 if loader._filename is not None:
1736 included_files.append(op.abspath(loader._filename))
1738 for included_file in loader._included_files:
1739 if op.samefile(inc_file, included_file):
1740 raise ImportError(
1741 'Circular import of file "%s". Include path: %s' % (
1742 op.abspath(inc_file),
1743 ' -> '.join('"%s"' % s for s in included_files)))
1745 with open(inc_file, 'rb') as f:
1746 return _load(
1747 f,
1748 Loader=loader.__class__, filename=inc_file,
1749 allow_include=True,
1750 included_files=included_files)
1753def dict_noflow_representer(dumper, data):
1754 return dumper.represent_mapping(
1755 'tag:yaml.org,2002:map', data, flow_style=False)
1758yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper)
1759yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader)
1760yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader)
1761yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper)
1764def str_representer(dumper, data):
1765 return dumper.represent_scalar(
1766 'tag:yaml.org,2002:str', str(data))
1769yaml.add_representer(str, str_representer, Dumper=GutsSafeDumper)
1772class Constructor(object):
1773 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None,
1774 ns_ignore=False):
1776 self.stack = []
1777 self.queue = []
1778 self.namespaces = defaultdict(list)
1779 self.add_namespace_maps = add_namespace_maps
1780 self.strict = strict
1781 self.ns_hints = ns_hints
1782 self.ns_ignore = ns_ignore
1784 def start_element(self, ns_name, attrs):
1785 if self.ns_ignore:
1786 ns_name = ns_name.split(' ')[-1]
1788 if -1 == ns_name.find(' '):
1789 if self.ns_hints is None and ns_name in g_guessable_xmlns:
1790 self.ns_hints = g_guessable_xmlns[ns_name]
1792 if self.ns_hints:
1793 ns_names = [
1794 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints]
1796 elif self.ns_hints is None:
1797 ns_names = [' ' + ns_name]
1799 else:
1800 ns_names = [ns_name]
1802 for ns_name in ns_names:
1803 if self.stack and self.stack[-1][1] is not None:
1804 cls = self.stack[-1][1].T.xmltagname_to_class.get(
1805 ns_name, None)
1807 if isinstance(cls, tuple):
1808 cls = None
1809 else:
1810 if cls is not None and (
1811 not issubclass(cls, Object)
1812 or issubclass(cls, SObject)):
1813 cls = None
1814 else:
1815 cls = g_xmltagname_to_class.get(ns_name, None)
1817 if cls:
1818 break
1820 self.stack.append((ns_name, cls, attrs, [], []))
1822 def end_element(self, _):
1823 ns_name, cls, attrs, content2, content1 = self.stack.pop()
1825 ns = ns_name.split(' ', 1)[0]
1827 if cls is not None:
1828 content2.extend(
1829 (ns + ' ' + k if -1 == k.find(' ') else k, v)
1830 for (k, v) in attrs.items())
1831 content2.append((None, ''.join(content1)))
1832 o = cls(**cls.T.translate_from_xml(content2, self.strict))
1833 o.validate(regularize=True, depth=1)
1834 if self.add_namespace_maps:
1835 o.namespace_map = self.get_current_namespace_map()
1837 if self.stack and not all(x[1] is None for x in self.stack):
1838 self.stack[-1][-2].append((ns_name, o))
1839 else:
1840 self.queue.append(o)
1841 else:
1842 content = [''.join(content1)]
1843 if self.stack:
1844 for c in content:
1845 self.stack[-1][-2].append((ns_name, c))
1847 def characters(self, char_content):
1848 if self.stack:
1849 self.stack[-1][-1].append(char_content)
1851 def start_namespace(self, ns, uri):
1852 self.namespaces[ns].append(uri)
1854 def end_namespace(self, ns):
1855 self.namespaces[ns].pop()
1857 def get_current_namespace_map(self):
1858 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v)
1860 def get_queued_elements(self):
1861 queue = self.queue
1862 self.queue = []
1863 return queue
1866def _iload_all_xml(
1867 stream,
1868 bufsize=100000,
1869 add_namespace_maps=False,
1870 strict=False,
1871 ns_hints=None,
1872 ns_ignore=False):
1874 from xml.parsers.expat import ParserCreate
1876 parser = ParserCreate('UTF-8', namespace_separator=' ')
1878 handler = Constructor(
1879 add_namespace_maps=add_namespace_maps,
1880 strict=strict,
1881 ns_hints=ns_hints,
1882 ns_ignore=ns_ignore)
1884 parser.StartElementHandler = handler.start_element
1885 parser.EndElementHandler = handler.end_element
1886 parser.CharacterDataHandler = handler.characters
1887 parser.StartNamespaceDeclHandler = handler.start_namespace
1888 parser.EndNamespaceDeclHandler = handler.end_namespace
1890 while True:
1891 data = stream.read(bufsize)
1892 parser.Parse(data, bool(not data))
1893 for element in handler.get_queued_elements():
1894 yield element
1896 if not data:
1897 break
1900def _load_all_xml(*args, **kwargs):
1901 return list(_iload_all_xml(*args, **kwargs))
1904def _load_xml(*args, **kwargs):
1905 g = _iload_all_xml(*args, **kwargs)
1906 return next(g)
1909def _dump_all_xml(objects, stream, root_element_name='root', header=True):
1911 if not getattr(stream, 'encoding', None):
1912 enc = encode_utf8
1913 else:
1914 enc = no_encode
1916 _dump_xml_header(stream, header)
1918 beg = u'<%s>\n' % root_element_name
1919 end = u'</%s>\n' % root_element_name
1921 stream.write(enc(beg))
1923 for ob in objects:
1924 _dump_xml(ob, stream=stream)
1926 stream.write(enc(end))
1929def _dump_xml_header(stream, banner=None):
1931 if not getattr(stream, 'encoding', None):
1932 enc = encode_utf8
1933 else:
1934 enc = no_encode
1936 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n'))
1937 if isinstance(banner, str):
1938 stream.write(enc(u'<!-- %s -->\n' % banner))
1941def _dump_xml(
1942 obj, stream, depth=0, ns_name=None, header=False, ns_map=[],
1943 ns_ignore=False):
1945 from xml.sax.saxutils import escape, quoteattr
1947 if not getattr(stream, 'encoding', None):
1948 enc = encode_utf8
1949 else:
1950 enc = no_encode
1952 if depth == 0 and header:
1953 _dump_xml_header(stream, header)
1955 indent = ' '*depth*2
1956 if ns_name is None:
1957 ns_name = obj.T.instance.get_xmltagname()
1959 if -1 != ns_name.find(' '):
1960 ns, name = ns_name.split(' ')
1961 else:
1962 ns, name = '', ns_name
1964 if isinstance(obj, Object):
1965 obj.validate(depth=1)
1966 attrs = []
1967 elems = []
1969 added_ns = False
1970 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns):
1971 attrs.append(('xmlns', ns))
1972 ns_map.append(ns)
1973 added_ns = True
1975 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True):
1976 if prop.xmlstyle == 'attribute':
1977 assert not prop.multivalued
1978 assert not isinstance(v, Object)
1979 attrs.append((prop.effective_xmltagname, v))
1981 elif prop.xmlstyle == 'content':
1982 assert not prop.multivalued
1983 assert not isinstance(v, Object)
1984 elems.append((None, v))
1986 else:
1987 prop.extend_xmlelements(elems, v)
1989 attr_str = ''
1990 if attrs:
1991 attr_str = ' ' + ' '.join(
1992 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v)))
1993 for (k, v) in attrs)
1995 if not elems:
1996 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str)))
1997 else:
1998 oneline = len(elems) == 1 and elems[0][0] is None
1999 stream.write(enc(u'%s<%s%s>%s' % (
2000 indent,
2001 name,
2002 attr_str,
2003 '' if oneline else '\n')))
2005 for (k, v) in elems:
2006 if k is None:
2007 stream.write(enc(escape(str(v), {'\0': '�'})))
2008 else:
2009 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore)
2011 stream.write(enc(u'%s</%s>\n' % (
2012 '' if oneline else indent, name)))
2014 if added_ns:
2015 ns_map.pop()
2017 else:
2018 stream.write(enc(u'%s<%s>%s</%s>\n' % (
2019 indent,
2020 name,
2021 escape(str(obj), {'\0': '�'}),
2022 name)))
2025def walk(x, typ=None, path=()):
2026 if typ is None or isinstance(x, typ):
2027 yield path, x
2029 if isinstance(x, Object):
2030 for (prop, val) in x.T.ipropvals(x):
2031 if prop.multivalued:
2032 if val is not None:
2033 for iele, ele in enumerate(val):
2034 for y in walk(ele, typ,
2035 path=path + ((prop.name, iele),)):
2036 yield y
2037 else:
2038 for y in walk(val, typ, path=path+(prop.name,)):
2039 yield y
2042def clone(x, pool=None):
2043 '''
2044 Clone guts object tree.
2046 Traverses guts object tree and recursively clones all guts attributes,
2047 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects
2048 deriving from :py:class:`Object` are instantiated using their respective
2049 init function. Multiply referenced objects in the source tree are multiply
2050 referenced also in the destination tree.
2052 This function can be used to clone guts objects ignoring any contained
2053 run-time state, i.e. any of their attributes not defined as a guts
2054 property.
2055 '''
2057 if pool is None:
2058 pool = {}
2060 if id(x) in pool:
2061 x_copy = pool[id(x)]
2063 else:
2064 if isinstance(x, SObject):
2065 x_copy = x.__class__(str(x))
2066 elif isinstance(x, Object):
2067 d = {}
2068 for (prop, y) in x.T.ipropvals(x):
2069 if y is not None:
2070 if not prop.multivalued:
2071 y_copy = clone(y, pool)
2072 elif prop.multivalued is dict:
2073 y_copy = dict(
2074 (clone(zk, pool), clone(zv, pool))
2075 for (zk, zv) in y.items())
2076 else:
2077 y_copy = type(y)(clone(z, pool) for z in y)
2078 else:
2079 y_copy = y
2081 d[prop.name] = y_copy
2083 x_copy = x.__class__(**d)
2085 else:
2086 x_copy = copy.deepcopy(x)
2088 pool[id(x)] = x_copy
2089 return x_copy
2092class YPathError(Exception):
2093 '''
2094 This exception is raised for invalid ypath specifications.
2095 '''
2096 pass
2099def _parse_yname(yname):
2100 ident = r'[a-zA-Z][a-zA-Z0-9_]*'
2101 rint = r'-?[0-9]+'
2102 m = re.match(
2103 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$'
2104 % (ident, rint, rint, rint), yname)
2106 if not m:
2107 raise YPathError('Syntax error in component: "%s"' % yname)
2109 d = dict(
2110 name=m.group(1))
2112 if m.group(2):
2113 if m.group(5):
2114 istart = iend = None
2115 if m.group(4):
2116 istart = int(m.group(4))
2117 if m.group(6):
2118 iend = int(m.group(6))
2120 d['slice'] = (istart, iend)
2121 else:
2122 d['index'] = int(m.group(7))
2124 return d
2127def _decend(obj, ynames):
2128 if ynames:
2129 for sobj in iter_elements(obj, ynames):
2130 yield sobj
2131 else:
2132 yield obj
2135def iter_elements(obj, ypath):
2136 '''
2137 Generator yielding elements matching a given ypath specification.
2139 :param obj: guts :py:class:`Object` instance
2140 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2141 To access list objects use slice notatation (e.g.
2142 'root.child[:].child[1:3].child[1]').
2144 Raises :py:exc:`YPathError` on failure.
2145 '''
2147 try:
2148 if isinstance(ypath, str):
2149 ynames = ypath.split('.')
2150 else:
2151 ynames = ypath
2153 yname = ynames[0]
2154 ynames = ynames[1:]
2155 d = _parse_yname(yname)
2156 if d['name'] not in obj.T.propnames:
2157 raise AttributeError(d['name'])
2159 obj = getattr(obj, d['name'])
2161 if 'index' in d:
2162 sobj = obj[d['index']]
2163 for ssobj in _decend(sobj, ynames):
2164 yield ssobj
2166 elif 'slice' in d:
2167 for i in range(*slice(*d['slice']).indices(len(obj))):
2168 sobj = obj[i]
2169 for ssobj in _decend(sobj, ynames):
2170 yield ssobj
2171 else:
2172 for sobj in _decend(obj, ynames):
2173 yield sobj
2175 except (AttributeError, IndexError) as e:
2176 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2179def get_elements(obj, ypath):
2180 '''
2181 Get all elements matching a given ypath specification.
2183 :param obj: guts :py:class:`Object` instance
2184 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2185 To access list objects use slice notatation (e.g.
2186 'root.child[:].child[1:3].child[1]').
2188 Raises :py:exc:`YPathError` on failure.
2189 '''
2190 return list(iter_elements(obj, ypath))
2193def set_elements(obj, ypath, value, validate=False, regularize=False):
2194 '''
2195 Set elements matching a given ypath specification.
2197 :param obj: guts :py:class:`Object` instance
2198 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2199 To access list objects use slice notatation (e.g.
2200 'root.child[:].child[1:3].child[1]').
2201 :param value: All matching elements will be set to `value`.
2202 :param validate: Whether to validate affected subtrees.
2203 :param regularize: Whether to regularize affected subtrees.
2205 Raises :py:exc:`YPathError` on failure.
2206 '''
2208 ynames = ypath.split('.')
2209 try:
2210 d = _parse_yname(ynames[-1])
2211 for sobj in iter_elements(obj, ynames[:-1]):
2212 if d['name'] not in sobj.T.propnames:
2213 raise AttributeError(d['name'])
2215 if 'index' in d:
2216 ssobj = getattr(sobj, d['name'])
2217 ssobj[d['index']] = value
2218 elif 'slice' in d:
2219 ssobj = getattr(sobj, d['name'])
2220 for i in range(*slice(*d['slice']).indices(len(ssobj))):
2221 ssobj[i] = value
2222 else:
2223 setattr(sobj, d['name'], value)
2224 if regularize:
2225 sobj.regularize()
2226 if validate:
2227 sobj.validate()
2229 except (AttributeError, IndexError, YPathError) as e:
2230 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2233def zip_walk(x, typ=None, path=(), stack=()):
2234 if typ is None or isinstance(x, typ):
2235 yield path, stack + (x,)
2237 if isinstance(x, Object):
2238 for (prop, val) in x.T.ipropvals(x):
2239 if prop.multivalued:
2240 if val is not None:
2241 for iele, ele in enumerate(val):
2242 for y in zip_walk(
2243 ele, typ,
2244 path=path + ((prop.name, iele),),
2245 stack=stack + (x,)):
2247 yield y
2248 else:
2249 for y in zip_walk(val, typ,
2250 path=path+(prop.name,),
2251 stack=stack + (x,)):
2252 yield y
2255def path_element(x):
2256 if isinstance(x, tuple):
2257 return '%s[%i]' % x
2258 else:
2259 return x
2262def path_to_str(path):
2263 return '.'.join(path_element(x) for x in path)
2266@expand_stream_args('w')
2267def dump(*args, **kwargs):
2268 return _dump(*args, **kwargs)
2271@expand_stream_args('r')
2272def load(*args, **kwargs):
2273 return _load(*args, **kwargs)
2276def load_string(s, *args, **kwargs):
2277 return load(string=s, *args, **kwargs)
2280@expand_stream_args('w')
2281def dump_all(*args, **kwargs):
2282 return _dump_all(*args, **kwargs)
2285@expand_stream_args('r')
2286def load_all(*args, **kwargs):
2287 return _load_all(*args, **kwargs)
2290@expand_stream_args('r')
2291def iload_all(*args, **kwargs):
2292 return _iload_all(*args, **kwargs)
2295@expand_stream_args('w')
2296def dump_xml(*args, **kwargs):
2297 return _dump_xml(*args, **kwargs)
2300@expand_stream_args('r')
2301def load_xml(*args, **kwargs):
2302 kwargs.pop('filename', None)
2303 return _load_xml(*args, **kwargs)
2306def load_xml_string(s, *args, **kwargs):
2307 return load_xml(string=s, *args, **kwargs)
2310@expand_stream_args('w')
2311def dump_all_xml(*args, **kwargs):
2312 return _dump_all_xml(*args, **kwargs)
2315@expand_stream_args('r')
2316def load_all_xml(*args, **kwargs):
2317 kwargs.pop('filename', None)
2318 return _load_all_xml(*args, **kwargs)
2321@expand_stream_args('r')
2322def iload_all_xml(*args, **kwargs):
2323 kwargs.pop('filename', None)
2324 return _iload_all_xml(*args, **kwargs)
2327__all__ = guts_types + [
2328 'guts_types', 'TBase', 'ValidationError',
2329 'ArgumentError', 'Defer',
2330 'dump', 'load',
2331 'dump_all', 'load_all', 'iload_all',
2332 'dump_xml', 'load_xml',
2333 'dump_all_xml', 'load_all_xml', 'iload_all_xml',
2334 'load_string',
2335 'load_xml_string',
2336 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str'
2337]