1# -*- coding: utf-8 -*-
2# http://pyrocko.org - GPLv3
3#
4# The Pyrocko Developers, 21st Century
5# ---|P------/S----------~Lg----------
6'''
7Lightweight declarative YAML and XML data binding for Python.
8'''
9from __future__ import absolute_import, print_function
11import datetime
12import calendar
13import re
14import sys
15import types
16import copy
17import os.path as op
18from collections import defaultdict
20from io import BytesIO
22try:
23 import numpy as num
24except ImportError:
25 num = None
27import yaml
28try:
29 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
30except ImportError:
31 from yaml import SafeLoader, SafeDumper
33from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \
34 get_time_float
36try:
37 newstr = unicode
38 range = xrange
39except NameError:
40 newstr = str
42try:
43 # needed for py2/py3 compatibility to allow
44 # from pyrocko.guts import FileNotFoundError
45 FileNotFoundError = FileNotFoundError
46except NameError:
47 class FileNotFoundError(EnvironmentError):
48 pass
51ALLOW_INCLUDE = False
54class GutsSafeDumper(SafeDumper):
55 pass
58class GutsSafeLoader(SafeLoader):
59 pass
62try:
63 unicode
64except NameError:
65 unicode = str
68g_iprop = 0
70g_deferred = {}
71g_deferred_content = {}
73g_tagname_to_class = {}
74g_xmltagname_to_class = {}
75g_guessable_xmlns = {}
77guts_types = [
78 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float',
79 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern',
80 'UnicodePattern', 'StringChoice', 'IntChoice', 'List', 'Dict', 'Tuple',
81 'Union', 'Choice', 'Any']
83us_to_cc_regex = re.compile(r'([a-z])_([a-z])')
86class literal(str):
87 pass
90class folded(str):
91 pass
94class singlequoted(str):
95 pass
98class doublequoted(str):
99 pass
102def make_str_presenter(style):
103 def presenter(dumper, data):
104 return dumper.represent_scalar(
105 'tag:yaml.org,2002:str', str(data), style=style)
107 return presenter
110str_style_map = {
111 None: lambda x: x,
112 '|': literal,
113 '>': folded,
114 "'": singlequoted,
115 '"': doublequoted}
117for (style, cls) in str_style_map.items():
118 if style:
119 GutsSafeDumper.add_representer(cls, make_str_presenter(style))
122class uliteral(unicode):
123 pass
126class ufolded(unicode):
127 pass
130class usinglequoted(unicode):
131 pass
134class udoublequoted(unicode):
135 pass
138def make_unicode_presenter(style):
139 def presenter(dumper, data):
140 return dumper.represent_scalar(
141 'tag:yaml.org,2002:str', unicode(data), style=style)
143 return presenter
146unicode_style_map = {
147 None: lambda x: x,
148 '|': literal,
149 '>': folded,
150 "'": singlequoted,
151 '"': doublequoted}
153for (style, cls) in unicode_style_map.items():
154 if style:
155 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style))
158class blist(list):
159 pass
162class flist(list):
163 pass
166list_style_map = {
167 None: list,
168 'block': blist,
169 'flow': flist}
172def make_list_presenter(flow_style):
173 def presenter(dumper, data):
174 return dumper.represent_sequence(
175 'tag:yaml.org,2002:seq', data, flow_style=flow_style)
177 return presenter
180GutsSafeDumper.add_representer(blist, make_list_presenter(False))
181GutsSafeDumper.add_representer(flist, make_list_presenter(True))
183if num:
184 def numpy_float_presenter(dumper, data):
185 return dumper.represent_float(float(data))
187 def numpy_int_presenter(dumper, data):
188 return dumper.represent_int(int(data))
190 for dtype in (num.float64, num.float32):
191 GutsSafeDumper.add_representer(dtype, numpy_float_presenter)
193 for dtype in (num.int32, num.int64):
194 GutsSafeDumper.add_representer(dtype, numpy_int_presenter)
197def us_to_cc(s):
198 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s)
201cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)')
202cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])')
205def cc_to_us(s):
206 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub(
207 '\\1_\\2\\3', s)).lower()
210re_frac = re.compile(r'\.[1-9]FRAC')
211frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789'])
214def encode_utf8(s):
215 return s.encode('utf-8')
218def no_encode(s):
219 return s
222def make_xmltagname_from_name(name):
223 return us_to_cc(name)
226def make_name_from_xmltagname(xmltagname):
227 return cc_to_us(xmltagname)
230def make_content_name(name):
231 if name.endswith('_list'):
232 return name[:-5]
233 elif name.endswith('s'):
234 return name[:-1]
235 else:
236 return name
239def classnames(cls):
240 if isinstance(cls, tuple):
241 return '(%s)' % ', '.join(x.__name__ for x in cls)
242 else:
243 return cls.__name__
246def expand_stream_args(mode):
247 def wrap(f):
248 '''
249 Decorator to enhance functions taking stream objects.
251 Wraps a function f(..., stream, ...) so that it can also be called as
252 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...).
253 '''
255 def g(*args, **kwargs):
256 stream = kwargs.pop('stream', None)
257 filename = kwargs.get('filename', None)
258 if mode != 'r':
259 filename = kwargs.pop('filename', None)
260 string = kwargs.pop('string', None)
262 assert sum(x is not None for x in (stream, filename, string)) <= 1
264 if stream is not None:
265 kwargs['stream'] = stream
266 return f(*args, **kwargs)
268 elif filename is not None:
269 stream = open(filename, mode+'b')
270 kwargs['stream'] = stream
271 retval = f(*args, **kwargs)
272 if isinstance(retval, types.GeneratorType):
273 def wrap_generator(gen):
274 try:
275 for x in gen:
276 yield x
278 except GeneratorExit:
279 pass
281 stream.close()
283 return wrap_generator(retval)
285 else:
286 stream.close()
287 return retval
289 elif string is not None:
290 assert mode == 'r', \
291 'Keyword argument string=... cannot be used in dumper ' \
292 'function.'
294 kwargs['stream'] = BytesIO(string.encode('utf-8'))
295 return f(*args, **kwargs)
297 else:
298 assert mode == 'w', \
299 'Use keyword argument stream=... or filename=... in ' \
300 'loader function.'
302 sout = BytesIO()
303 f(stream=sout, *args, **kwargs)
304 return sout.getvalue().decode('utf-8')
306 return g
308 return wrap
311class Defer(object):
312 def __init__(self, classname, *args, **kwargs):
313 global g_iprop
314 if kwargs.get('position', None) is None:
315 kwargs['position'] = g_iprop
317 g_iprop += 1
319 self.classname = classname
320 self.args = args
321 self.kwargs = kwargs
324class TBase(object):
326 strict = False
327 multivalued = None
328 force_regularize = False
329 propnames = []
331 @classmethod
332 def init_propertystuff(cls):
333 cls.properties = []
334 cls.xmltagname_to_name = {}
335 cls.xmltagname_to_name_multivalued = {}
336 cls.xmltagname_to_class = {}
337 cls.content_property = None
339 def __init__(
340 self,
341 default=None,
342 optional=False,
343 xmlstyle='element',
344 xmltagname=None,
345 xmlns=None,
346 help=None,
347 position=None):
349 global g_iprop
350 if position is not None:
351 self.position = position
352 else:
353 self.position = g_iprop
355 g_iprop += 1
356 self._default = default
358 self.optional = optional
359 self.name = None
360 self._xmltagname = xmltagname
361 self._xmlns = xmlns
362 self.parent = None
363 self.xmlstyle = xmlstyle
364 self.help = help
366 def default(self):
367 return make_default(self._default)
369 def is_default(self, val):
370 if self._default is None:
371 return val is None
372 else:
373 return self._default == val
375 def has_default(self):
376 return self._default is not None
378 def xname(self):
379 if self.name is not None:
380 return self.name
381 elif self.parent is not None:
382 return 'element of %s' % self.parent.xname()
383 else:
384 return '?'
386 def set_xmlns(self, xmlns):
387 if self._xmlns is None and not self.xmlns:
388 self._xmlns = xmlns
390 if self.multivalued:
391 self.content_t.set_xmlns(xmlns)
393 def get_xmlns(self):
394 return self._xmlns or self.xmlns
396 def get_xmltagname(self):
397 if self._xmltagname is not None:
398 return self.get_xmlns() + ' ' + self._xmltagname
399 elif self.name:
400 return self.get_xmlns() + ' ' \
401 + make_xmltagname_from_name(self.name)
402 elif self.xmltagname:
403 return self.get_xmlns() + ' ' + self.xmltagname
404 else:
405 assert False
407 @classmethod
408 def get_property(cls, name):
409 for prop in cls.properties:
410 if prop.name == name:
411 return prop
413 raise ValueError()
415 @classmethod
416 def remove_property(cls, name):
418 prop = cls.get_property(name)
420 if not prop.multivalued:
421 del cls.xmltagname_to_class[prop.effective_xmltagname]
422 del cls.xmltagname_to_name[prop.effective_xmltagname]
423 else:
424 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname]
425 del cls.xmltagname_to_name_multivalued[
426 prop.content_t.effective_xmltagname]
428 if cls.content_property is prop:
429 cls.content_property = None
431 cls.properties.remove(prop)
432 cls.propnames.remove(name)
434 return prop
436 @classmethod
437 def add_property(cls, name, prop):
439 prop.instance = prop
440 prop.name = name
441 prop.set_xmlns(cls.xmlns)
443 if isinstance(prop, Choice.T):
444 for tc in prop.choices:
445 tc.effective_xmltagname = tc.get_xmltagname()
446 cls.xmltagname_to_class[tc.effective_xmltagname] = tc.cls
447 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name
448 elif not prop.multivalued:
449 prop.effective_xmltagname = prop.get_xmltagname()
450 cls.xmltagname_to_class[prop.effective_xmltagname] = prop.cls
451 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name
452 else:
453 prop.content_t.name = make_content_name(prop.name)
454 prop.content_t.effective_xmltagname = \
455 prop.content_t.get_xmltagname()
456 cls.xmltagname_to_class[
457 prop.content_t.effective_xmltagname] = prop.content_t.cls
458 cls.xmltagname_to_name_multivalued[
459 prop.content_t.effective_xmltagname] = prop.name
461 cls.properties.append(prop)
463 cls.properties.sort(key=lambda x: x.position)
465 cls.propnames = [p.name for p in cls.properties]
467 if prop.xmlstyle == 'content':
468 cls.content_property = prop
470 @classmethod
471 def ivals(cls, val):
472 for prop in cls.properties:
473 yield getattr(val, prop.name)
475 @classmethod
476 def ipropvals(cls, val):
477 for prop in cls.properties:
478 yield prop, getattr(val, prop.name)
480 @classmethod
481 def inamevals(cls, val):
482 for prop in cls.properties:
483 yield prop.name, getattr(val, prop.name)
485 @classmethod
486 def ipropvals_to_save(cls, val, xmlmode=False):
487 for prop in cls.properties:
488 v = getattr(val, prop.name)
489 if v is not None and (
490 not (prop.optional or (prop.multivalued and not v))
491 or (not prop.is_default(v))):
493 if xmlmode:
494 yield prop, prop.to_save_xml(v)
495 else:
496 yield prop, prop.to_save(v)
498 @classmethod
499 def inamevals_to_save(cls, val, xmlmode=False):
500 for prop, v in cls.ipropvals_to_save(val, xmlmode):
501 yield prop.name, v
503 @classmethod
504 def translate_from_xml(cls, list_of_pairs, strict):
505 d = {}
506 for k, v in list_of_pairs:
507 if k in cls.xmltagname_to_name_multivalued:
508 k2 = cls.xmltagname_to_name_multivalued[k]
509 if k2 not in d:
510 d[k2] = []
512 d[k2].append(v)
513 elif k in cls.xmltagname_to_name:
514 k2 = cls.xmltagname_to_name[k]
515 if k2 in d:
516 raise ArgumentError(
517 'Unexpectedly found more than one child element "%s" '
518 'within "%s".' % (k, cls.tagname))
520 d[k2] = v
521 elif k is None:
522 if cls.content_property:
523 k2 = cls.content_property.name
524 d[k2] = v
525 else:
526 if strict:
527 raise ArgumentError(
528 'Unexpected child element "%s" found within "%s".' % (
529 k, cls.tagname))
531 return d
533 def validate(self, val, regularize=False, depth=-1):
534 if self.optional and val is None:
535 return val
537 is_derived = isinstance(val, self.cls)
538 is_exact = type(val) == self.cls
540 not_ok = not self.strict and not is_derived or \
541 self.strict and not is_exact
543 if not_ok or self.force_regularize:
544 if regularize:
545 try:
546 val = self.regularize_extra(val)
547 except ValueError:
548 raise ValidationError(
549 '%s: could not convert "%s" to type %s' % (
550 self.xname(), val, classnames(self.cls)))
551 else:
552 raise ValidationError(
553 '%s: "%s" (type: %s) is not of type %s' % (
554 self.xname(), val, type(val), classnames(self.cls)))
556 validator = self
557 if isinstance(self.cls, tuple):
558 clss = self.cls
559 else:
560 clss = (self.cls,)
562 for cls in clss:
563 try:
564 if type(val) != cls and isinstance(val, cls):
565 validator = val.T.instance
567 except AttributeError:
568 pass
570 validator.validate_extra(val)
572 if depth != 0:
573 val = validator.validate_children(val, regularize, depth)
575 return val
577 def regularize_extra(self, val):
578 return self.cls(val)
580 def validate_extra(self, val):
581 pass
583 def validate_children(self, val, regularize, depth):
584 for prop, propval in self.ipropvals(val):
585 newpropval = prop.validate(propval, regularize, depth-1)
586 if regularize and (newpropval is not propval):
587 setattr(val, prop.name, newpropval)
589 return val
591 def to_save(self, val):
592 return val
594 def to_save_xml(self, val):
595 return self.to_save(val)
597 def extend_xmlelements(self, elems, v):
598 if self.multivalued:
599 for x in v:
600 elems.append((self.content_t.effective_xmltagname, x))
601 else:
602 elems.append((self.effective_xmltagname, v))
604 def deferred(self):
605 return []
607 def classname_for_help(self, strip_module=''):
608 if self.dummy_cls in guts_plain_dummy_types:
609 return '``%s``' % self.cls.__name__
611 elif self.dummy_cls.dummy_for_description:
612 return self.dummy_cls.dummy_for_description
614 else:
615 if self.dummy_cls is not self.cls:
616 if self.dummy_cls.__module__ == strip_module:
617 sadd = ' (:py:class:`%s`)' % (
618 self.dummy_cls.__name__)
619 else:
620 sadd = ' (:py:class:`%s.%s`)' % (
621 self.dummy_cls.__module__, self.dummy_cls.__name__)
622 else:
623 sadd = ''
625 def sclass(cls):
626 mod = cls.__module__
627 clsn = cls.__name__
628 if mod == '__builtin__' or mod == 'builtins':
629 return '``%s``' % clsn
631 elif mod == strip_module:
632 return ':py:class:`%s`' % clsn
634 else:
635 return ':py:class:`%s.%s`' % (mod, clsn)
637 if isinstance(self.cls, tuple):
638 return '(%s)%s' % (
639 ' | '.join(sclass(cls) for cls in self.cls), sadd)
640 else:
641 return '%s%s' % (sclass(self.cls), sadd)
643 @classmethod
644 def props_help_string(cls):
645 baseprops = []
646 for base in cls.dummy_cls.__bases__:
647 if hasattr(base, 'T'):
648 baseprops.extend(base.T.properties)
650 hlp = []
651 hlp.append('')
652 for prop in cls.properties:
653 if prop in baseprops:
654 continue
656 descr = [
657 prop.classname_for_help(strip_module=cls.dummy_cls.__module__)]
659 if prop.optional:
660 descr.append('*optional*')
662 if isinstance(prop._default, DefaultMaker):
663 descr.append('*default:* ``%s``' % repr(prop._default))
664 else:
665 d = prop.default()
666 if d is not None:
667 descr.append('*default:* ``%s``' % repr(d))
669 hlp.append(' .. py:gattribute:: %s' % prop.name)
670 hlp.append('')
671 hlp.append(' %s' % ', '.join(descr))
672 hlp.append(' ')
673 if prop.help is not None:
674 hlp.append(' %s' % prop.help)
675 hlp.append('')
677 return '\n'.join(hlp)
679 @classmethod
680 def class_help_string(cls):
681 return cls.dummy_cls.__doc_template__
683 @classmethod
684 def class_signature(cls):
685 r = []
686 for prop in cls.properties:
687 d = prop.default()
688 if d is not None:
689 arg = repr(d)
691 elif prop.optional:
692 arg = 'None'
694 else:
695 arg = '...'
697 r.append('%s=%s' % (prop.name, arg))
699 return '(%s)' % ', '.join(r)
701 @classmethod
702 def help(cls):
703 return cls.props_help_string()
706class ObjectMetaClass(type):
707 def __new__(meta, classname, bases, class_dict):
708 cls = type.__new__(meta, classname, bases, class_dict)
709 if classname != 'Object':
710 t_class_attr_name = '_%s__T' % classname
711 if not hasattr(cls, t_class_attr_name):
712 if hasattr(cls, 'T'):
713 class T(cls.T):
714 pass
715 else:
716 class T(TBase):
717 pass
719 setattr(cls, t_class_attr_name, T)
721 T = getattr(cls, t_class_attr_name)
723 if cls.dummy_for is not None:
724 T.cls = cls.dummy_for
725 else:
726 T.cls = cls
728 T.dummy_cls = cls
730 if hasattr(cls, 'xmltagname'):
731 T.xmltagname = cls.xmltagname
732 else:
733 T.xmltagname = classname
735 mod = sys.modules[cls.__module__]
737 if hasattr(cls, 'xmlns'):
738 T.xmlns = cls.xmlns
739 elif hasattr(mod, 'guts_xmlns'):
740 T.xmlns = mod.guts_xmlns
741 else:
742 T.xmlns = ''
744 if T.xmlns and hasattr(cls, 'guessable_xmlns'):
745 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns
747 if hasattr(mod, 'guts_prefix'):
748 if mod.guts_prefix:
749 T.tagname = mod.guts_prefix + '.' + classname
750 else:
751 T.tagname = classname
752 else:
753 if cls.__module__ != '__main__':
754 T.tagname = cls.__module__ + '.' + classname
755 else:
756 T.tagname = classname
758 T.classname = classname
760 T.init_propertystuff()
762 for k in dir(cls):
763 prop = getattr(cls, k)
765 if k.endswith('__'):
766 k = k[:-2]
768 if isinstance(prop, TBase):
769 if prop.deferred():
770 for defer in prop.deferred():
771 g_deferred_content.setdefault(
772 defer.classname[:-2], []).append((prop, defer))
773 g_deferred.setdefault(
774 defer.classname[:-2], []).append((T, k, prop))
776 else:
777 T.add_property(k, prop)
779 elif isinstance(prop, Defer):
780 g_deferred.setdefault(prop.classname[:-2], []).append(
781 (T, k, prop))
783 if classname in g_deferred_content:
784 for prop, defer in g_deferred_content[classname]:
785 prop.process_deferred(
786 defer, T(*defer.args, **defer.kwargs))
788 del g_deferred_content[classname]
790 if classname in g_deferred:
791 for (T_, k_, prop_) in g_deferred.get(classname, []):
792 if isinstance(prop_, Defer):
793 prop_ = T(*prop_.args, **prop_.kwargs)
795 if not prop_.deferred():
796 T_.add_property(k_, prop_)
798 del g_deferred[classname]
800 g_tagname_to_class[T.tagname] = cls
801 if hasattr(cls, 'xmltagname'):
802 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls
804 cls.T = T
805 T.instance = T()
807 cls.__doc_template__ = cls.__doc__
808 cls.__doc__ = T.class_help_string()
810 if cls.__doc__ is None:
811 cls.__doc__ = 'Undocumented.'
813 cls.__doc__ += '\n' + T.props_help_string()
815 return cls
818class ValidationError(Exception):
819 pass
822class ArgumentError(Exception):
823 pass
826def make_default(x):
827 if isinstance(x, DefaultMaker):
828 return x.make()
829 elif isinstance(x, Object):
830 return clone(x)
831 else:
832 return x
835class DefaultMaker(object):
836 def make(self):
837 raise NotImplementedError('Schould be implemented in subclass.')
840class ObjectDefaultMaker(DefaultMaker):
841 def __init__(self, cls, args, kwargs):
842 DefaultMaker.__init__(self)
843 self.cls = cls
844 self.args = args
845 self.kwargs = kwargs
846 self.instance = None
848 def make(self):
849 return self.cls(
850 *[make_default(x) for x in self.args],
851 **dict((k, make_default(v)) for (k, v) in self.kwargs.items()))
853 def __eq__(self, other):
854 if self.instance is None:
855 self.instance = self.make()
857 return self.instance == other
859 def __repr__(self):
860 sargs = []
861 for arg in self.args:
862 sargs.append(repr(arg))
864 for k, v in self.kwargs.items():
865 sargs.append('%s=%s' % (k, repr(v)))
867 return '%s(%s)' % (self.cls.__name__, ', '.join(sargs))
870class TimestampDefaultMaker(DefaultMaker):
871 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'):
872 DefaultMaker.__init__(self)
873 self._stime = s
874 self._format = format
876 def make(self):
877 return str_to_time(self._stime, self._format)
879 def __repr__(self):
880 return "str_to_time(%s)" % repr(self._stime)
883def with_metaclass(meta, *bases):
884 # inlined py2/py3 compat solution from python-future
885 class metaclass(meta):
886 __call__ = type.__call__
887 __init__ = type.__init__
889 def __new__(cls, name, this_bases, d):
890 if this_bases is None:
891 return type.__new__(cls, name, (), d)
892 return meta(name, bases, d)
894 return metaclass('temp', None, {})
897class Object(with_metaclass(ObjectMetaClass, object)):
898 dummy_for = None
899 dummy_for_description = None
901 def __init__(self, **kwargs):
902 if not kwargs.get('init_props', True):
903 return
905 for prop in self.T.properties:
906 k = prop.name
907 if k in kwargs:
908 setattr(self, k, kwargs.pop(k))
909 else:
910 if not prop.optional and not prop.has_default():
911 raise ArgumentError('Missing argument to %s: %s' % (
912 self.T.tagname, prop.name))
913 else:
914 setattr(self, k, prop.default())
916 if kwargs:
917 raise ArgumentError('Invalid argument to %s: %s' % (
918 self.T.tagname, ', '.join(list(kwargs.keys()))))
920 @classmethod
921 def D(cls, *args, **kwargs):
922 return ObjectDefaultMaker(cls, args, kwargs)
924 def validate(self, regularize=False, depth=-1):
925 self.T.instance.validate(self, regularize, depth)
927 def regularize(self, depth=-1):
928 self.validate(regularize=True, depth=depth)
930 def dump(self, stream=None, filename=None, header=False):
931 return dump(self, stream=stream, filename=filename, header=header)
933 def dump_xml(
934 self, stream=None, filename=None, header=False, ns_ignore=False):
935 return dump_xml(
936 self, stream=stream, filename=filename, header=header,
937 ns_ignore=ns_ignore)
939 @classmethod
940 def load(cls, stream=None, filename=None, string=None):
941 return load(stream=stream, filename=filename, string=string)
943 @classmethod
944 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None,
945 ns_ignore=False):
947 if ns_hints is None:
948 ns_hints = [cls.T.instance.get_xmlns()]
950 return load_xml(
951 stream=stream,
952 filename=filename,
953 string=string,
954 ns_hints=ns_hints,
955 ns_ignore=ns_ignore)
957 def __str__(self):
958 return self.dump()
961def to_dict(obj):
962 '''
963 Get dict of guts object attributes.
965 :param obj: :py:class`Object` object
966 '''
968 return dict(obj.T.inamevals(obj))
971class SObject(Object):
973 class __T(TBase):
974 def regularize_extra(self, val):
975 if isinstance(val, (str, newstr)):
976 return self.cls(val)
978 return val
980 def to_save(self, val):
981 return str(val)
983 def to_save_xml(self, val):
984 return str(val)
987class Any(Object):
989 class __T(TBase):
990 def validate(self, val, regularize=False, depth=-1):
991 if isinstance(val, Object):
992 val.validate(regularize, depth)
994 return val
997class Int(Object):
998 dummy_for = int
1000 class __T(TBase):
1001 strict = True
1003 def to_save_xml(self, value):
1004 return repr(value)
1007class Float(Object):
1008 dummy_for = float
1010 class __T(TBase):
1011 strict = True
1013 def to_save_xml(self, value):
1014 return repr(value)
1017class Complex(Object):
1018 dummy_for = complex
1020 class __T(TBase):
1021 strict = True
1023 def regularize_extra(self, val):
1025 if isinstance(val, list) or isinstance(val, tuple):
1026 assert len(val) == 2
1027 val = complex(*val)
1029 elif not isinstance(val, complex):
1030 val = complex(val)
1032 return val
1034 def to_save(self, value):
1035 return repr(value)
1037 def to_save_xml(self, value):
1038 return repr(value)
1041class Bool(Object):
1042 dummy_for = bool
1044 class __T(TBase):
1045 strict = True
1047 def regularize_extra(self, val):
1048 if isinstance(val, (str, newstr)):
1049 if val.lower().strip() in ('0', 'false'):
1050 return False
1052 return bool(val)
1054 def to_save_xml(self, value):
1055 return repr(bool(value)).lower()
1058class String(Object):
1059 dummy_for = str
1061 class __T(TBase):
1062 def __init__(self, *args, **kwargs):
1063 yamlstyle = kwargs.pop('yamlstyle', None)
1064 TBase.__init__(self, *args, **kwargs)
1065 self.style_cls = str_style_map[yamlstyle]
1067 def to_save(self, val):
1068 return self.style_cls(val)
1071class Unicode(Object):
1072 dummy_for = newstr
1074 class __T(TBase):
1075 def __init__(self, *args, **kwargs):
1076 yamlstyle = kwargs.pop('yamlstyle', None)
1077 TBase.__init__(self, *args, **kwargs)
1078 self.style_cls = unicode_style_map[yamlstyle]
1080 def to_save(self, val):
1081 return self.style_cls(val)
1084guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool)
1087class Dict(Object):
1088 dummy_for = dict
1090 class __T(TBase):
1091 multivalued = dict
1093 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs):
1094 TBase.__init__(self, *args, **kwargs)
1095 assert isinstance(key_t, TBase)
1096 assert isinstance(content_t, TBase)
1097 self.key_t = key_t
1098 self.content_t = content_t
1099 self.content_t.parent = self
1101 def default(self):
1102 if self._default is not None:
1103 return dict(
1104 (make_default(k), make_default(v))
1105 for (k, v) in self._default.items())
1107 if self.optional:
1108 return None
1109 else:
1110 return {}
1112 def has_default(self):
1113 return True
1115 def validate(self, val, regularize, depth):
1116 return TBase.validate(self, val, regularize, depth+1)
1118 def validate_children(self, val, regularize, depth):
1119 for key, ele in list(val.items()):
1120 newkey = self.key_t.validate(key, regularize, depth-1)
1121 newele = self.content_t.validate(ele, regularize, depth-1)
1122 if regularize:
1123 if newkey is not key or newele is not ele:
1124 del val[key]
1125 val[newkey] = newele
1127 return val
1129 def to_save(self, val):
1130 return dict((self.key_t.to_save(k), self.content_t.to_save(v))
1131 for (k, v) in val.items())
1133 def to_save_xml(self, val):
1134 raise NotImplementedError()
1136 def classname_for_help(self, strip_module=''):
1137 return '``dict`` of %s objects' % \
1138 self.content_t.classname_for_help(strip_module=strip_module)
1141class List(Object):
1142 dummy_for = list
1144 class __T(TBase):
1145 multivalued = list
1147 def __init__(self, content_t=Any.T(), *args, **kwargs):
1148 yamlstyle = kwargs.pop('yamlstyle', None)
1149 TBase.__init__(self, *args, **kwargs)
1150 assert isinstance(content_t, TBase) or isinstance(content_t, Defer)
1151 self.content_t = content_t
1152 self.content_t.parent = self
1153 self.style_cls = list_style_map[yamlstyle]
1155 def default(self):
1156 if self._default is not None:
1157 return [make_default(x) for x in self._default]
1158 if self.optional:
1159 return None
1160 else:
1161 return []
1163 def has_default(self):
1164 return True
1166 def validate(self, val, regularize, depth):
1167 return TBase.validate(self, val, regularize, depth+1)
1169 def validate_children(self, val, regularize, depth):
1170 for i, ele in enumerate(val):
1171 newele = self.content_t.validate(ele, regularize, depth-1)
1172 if regularize and newele is not ele:
1173 val[i] = newele
1175 return val
1177 def to_save(self, val):
1178 return self.style_cls(self.content_t.to_save(v) for v in val)
1180 def to_save_xml(self, val):
1181 return [self.content_t.to_save_xml(v) for v in val]
1183 def deferred(self):
1184 if isinstance(self.content_t, Defer):
1185 return [self.content_t]
1187 return []
1189 def process_deferred(self, defer, t_inst):
1190 if defer is self.content_t:
1191 self.content_t = t_inst
1193 def classname_for_help(self, strip_module=''):
1194 return '``list`` of %s objects' % \
1195 self.content_t.classname_for_help(strip_module=strip_module)
1198def make_typed_list_class(t):
1199 class TL(List):
1200 class __T(List.T):
1201 def __init__(self, *args, **kwargs):
1202 List.T.__init__(self, content_t=t.T(), *args, **kwargs)
1204 return TL
1207class Tuple(Object):
1208 dummy_for = tuple
1210 class __T(TBase):
1211 multivalued = tuple
1213 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs):
1214 TBase.__init__(self, *args, **kwargs)
1215 assert isinstance(content_t, TBase)
1216 self.content_t = content_t
1217 self.content_t.parent = self
1218 self.n = n
1220 def default(self):
1221 if self._default is not None:
1222 return tuple(
1223 make_default(x) for x in self._default)
1225 elif self.optional:
1226 return None
1227 else:
1228 if self.n is not None:
1229 return tuple(
1230 self.content_t.default() for x in range(self.n))
1231 else:
1232 return tuple()
1234 def has_default(self):
1235 return True
1237 def validate(self, val, regularize, depth):
1238 return TBase.validate(self, val, regularize, depth+1)
1240 def validate_extra(self, val):
1241 if self.n is not None and len(val) != self.n:
1242 raise ValidationError(
1243 '%s should have length %i' % (self.xname(), self.n))
1245 def validate_children(self, val, regularize, depth):
1246 if not regularize:
1247 for ele in val:
1248 self.content_t.validate(ele, regularize, depth-1)
1250 return val
1251 else:
1252 newval = []
1253 isnew = False
1254 for ele in val:
1255 newele = self.content_t.validate(ele, regularize, depth-1)
1256 newval.append(newele)
1257 if newele is not ele:
1258 isnew = True
1260 if isnew:
1261 return tuple(newval)
1262 else:
1263 return val
1265 def to_save(self, val):
1266 return tuple(self.content_t.to_save(v) for v in val)
1268 def to_save_xml(self, val):
1269 return [self.content_t.to_save_xml(v) for v in val]
1271 def classname_for_help(self, strip_module=''):
1272 if self.n is not None:
1273 return '``tuple`` of %i %s objects' % (
1274 self.n, self.content_t.classname_for_help(
1275 strip_module=strip_module))
1276 else:
1277 return '``tuple`` of %s objects' % (
1278 self.content_t.classname_for_help(
1279 strip_module=strip_module))
1282unit_factors = dict(
1283 s=1.0,
1284 m=60.0,
1285 h=3600.0,
1286 d=24*3600.0,
1287 y=365*24*3600.0)
1290class Duration(Object):
1291 dummy_for = float
1293 class __T(TBase):
1295 def regularize_extra(self, val):
1296 if isinstance(val, (str, newstr)):
1297 unit = val[-1]
1298 if unit in unit_factors:
1299 return float(val[:-1]) * unit_factors[unit]
1300 else:
1301 return float(val)
1303 return val
1306re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$')
1309class Timestamp(Object):
1310 dummy_for = (hpfloat, float)
1311 dummy_for_description = 'time_float'
1313 class __T(TBase):
1315 def regularize_extra(self, val):
1317 time_float = get_time_float()
1319 if isinstance(val, datetime.datetime):
1320 tt = val.utctimetuple()
1321 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1323 elif isinstance(val, datetime.date):
1324 tt = val.timetuple()
1325 val = time_float(calendar.timegm(tt))
1327 elif isinstance(val, (str, newstr)):
1328 val = val.strip()
1329 tz_offset = 0
1331 m = re_tz.search(val)
1332 if m:
1333 sh = m.group(2)
1334 sm = m.group(4)
1335 tz_offset = (int(sh)*3600 if sh else 0) \
1336 + (int(sm)*60 if sm else 0)
1338 val = re_tz.sub('', val)
1340 if len(val) > 10 and val[10] == 'T':
1341 val = val.replace('T', ' ', 1)
1343 try:
1344 val = str_to_time(val) - tz_offset
1345 except TimeStrError:
1346 raise ValidationError(
1347 '%s: cannot parse time/date: %s' % (self.xname(), val))
1349 elif isinstance(val, (int, float)):
1350 val = time_float(val)
1352 else:
1353 raise ValidationError(
1354 '%s: cannot convert "%s" to type %s' % (
1355 self.xname(), val, time_float))
1357 return val
1359 def to_save(self, val):
1360 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\
1361 .rstrip('0').rstrip('.')
1363 def to_save_xml(self, val):
1364 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\
1365 .rstrip('0').rstrip('.') + 'Z'
1367 @classmethod
1368 def D(self, s):
1369 return TimestampDefaultMaker(s)
1372class DateTimestamp(Object):
1373 dummy_for = (hpfloat, float)
1374 dummy_for_description = 'time_float'
1376 class __T(TBase):
1378 def regularize_extra(self, val):
1380 time_float = get_time_float()
1382 if isinstance(val, datetime.datetime):
1383 tt = val.utctimetuple()
1384 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1386 elif isinstance(val, datetime.date):
1387 tt = val.timetuple()
1388 val = time_float(calendar.timegm(tt))
1390 elif isinstance(val, (str, newstr)):
1391 val = str_to_time(val, format='%Y-%m-%d')
1393 elif isinstance(val, int):
1394 val = time_float(val)
1396 return val
1398 def to_save(self, val):
1399 return time_to_str(val, format='%Y-%m-%d')
1401 def to_save_xml(self, val):
1402 return time_to_str(val, format='%Y-%m-%d')
1404 @classmethod
1405 def D(self, s):
1406 return TimestampDefaultMaker(s, format='%Y-%m-%d')
1409class StringPattern(String):
1411 '''
1412 Any ``str`` matching pattern ``%(pattern)s``.
1413 '''
1415 dummy_for = str
1416 pattern = '.*'
1418 class __T(String.T):
1419 def __init__(self, pattern=None, *args, **kwargs):
1420 String.T.__init__(self, *args, **kwargs)
1422 if pattern is not None:
1423 self.pattern = pattern
1424 else:
1425 self.pattern = self.dummy_cls.pattern
1427 def validate_extra(self, val):
1428 pat = self.pattern
1429 if not re.search(pat, val):
1430 raise ValidationError('%s: "%s" does not match pattern %s' % (
1431 self.xname(), val, repr(pat)))
1433 @classmethod
1434 def class_help_string(cls):
1435 dcls = cls.dummy_cls
1436 doc = dcls.__doc_template__ or StringPattern.__doc_template__
1437 return doc % {'pattern': repr(dcls.pattern)}
1440class UnicodePattern(Unicode):
1442 '''
1443 Any ``unicode`` matching pattern ``%(pattern)s``.
1444 '''
1446 dummy_for = newstr
1447 pattern = '.*'
1449 class __T(TBase):
1450 def __init__(self, pattern=None, *args, **kwargs):
1451 TBase.__init__(self, *args, **kwargs)
1453 if pattern is not None:
1454 self.pattern = pattern
1455 else:
1456 self.pattern = self.dummy_cls.pattern
1458 def validate_extra(self, val):
1459 pat = self.pattern
1460 if not re.search(pat, val, flags=re.UNICODE):
1461 raise ValidationError('%s: "%s" does not match pattern %s' % (
1462 self.xname(), val, repr(pat)))
1464 @classmethod
1465 def class_help_string(cls):
1466 dcls = cls.dummy_cls
1467 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__
1468 return doc % {'pattern': repr(dcls.pattern)}
1471class StringChoice(String):
1473 '''
1474 Any ``str`` out of ``%(choices)s``.
1475 '''
1477 dummy_for = str
1478 choices = []
1479 ignore_case = False
1481 class __T(String.T):
1482 def __init__(self, choices=None, ignore_case=None, *args, **kwargs):
1483 String.T.__init__(self, *args, **kwargs)
1485 if choices is not None:
1486 self.choices = choices
1487 else:
1488 self.choices = self.dummy_cls.choices
1490 if ignore_case is not None:
1491 self.ignore_case = ignore_case
1492 else:
1493 self.ignore_case = self.dummy_cls.ignore_case
1495 if self.ignore_case:
1496 self.choices = [x.upper() for x in self.choices]
1498 def validate_extra(self, val):
1499 if self.ignore_case:
1500 val = val.upper()
1502 if val not in self.choices:
1503 raise ValidationError(
1504 '%s: "%s" is not a valid choice out of %s' % (
1505 self.xname(), val, repr(self.choices)))
1507 @classmethod
1508 def class_help_string(cls):
1509 dcls = cls.dummy_cls
1510 doc = dcls.__doc_template__ or StringChoice.__doc_template__
1511 return doc % {'choices': repr(dcls.choices)}
1514class IntChoice(Int):
1516 '''
1517 Any ``int`` out of ``%(choices)s``.
1518 '''
1520 dummy_for = int
1521 choices = []
1523 class __T(Int.T):
1524 def __init__(self, choices=None, *args, **kwargs):
1525 Int.T.__init__(self, *args, **kwargs)
1527 if choices is not None:
1528 self.choices = choices
1529 else:
1530 self.choices = self.dummy_cls.choices
1532 def validate_extra(self, val):
1533 if val not in self.choices:
1534 raise ValidationError(
1535 '%s: %i is not a valid choice out of %s' % (
1536 self.xname(), val, repr(self.choices)))
1538 @classmethod
1539 def class_help_string(cls):
1540 dcls = cls.dummy_cls
1541 doc = dcls.__doc_template__ or IntChoice.__doc_template__
1542 return doc % {'choices': repr(dcls.choices)}
1545# this will not always work...
1546class Union(Object):
1547 members = []
1548 dummy_for = str
1550 class __T(TBase):
1551 def __init__(self, members=None, *args, **kwargs):
1552 TBase.__init__(self, *args, **kwargs)
1553 if members is not None:
1554 self.members = members
1555 else:
1556 self.members = self.dummy_cls.members
1558 def validate(self, val, regularize=False, depth=-1):
1559 assert self.members
1560 e2 = None
1561 for member in self.members:
1562 try:
1563 return member.validate(val, regularize, depth=depth)
1564 except ValidationError as e:
1565 e2 = e
1567 raise e2
1570class Choice(Object):
1571 choices = []
1573 class __T(TBase):
1574 def __init__(self, choices=None, *args, **kwargs):
1575 TBase.__init__(self, *args, **kwargs)
1576 if choices is not None:
1577 self.choices = choices
1578 else:
1579 self.choices = self.dummy_cls.choices
1581 self.cls_to_xmltagname = dict(
1582 (t.cls, t.get_xmltagname()) for t in self.choices)
1584 def validate(self, val, regularize=False, depth=-1):
1585 if self.optional and val is None:
1586 return val
1588 t = None
1589 for tc in self.choices:
1590 is_derived = isinstance(val, tc.cls)
1591 is_exact = type(val) == tc.cls
1592 if not (not tc.strict and not is_derived or
1593 tc.strict and not is_exact):
1595 t = tc
1596 break
1598 if t is None:
1599 if regularize:
1600 ok = False
1601 for tc in self.choices:
1602 try:
1603 val = tc.regularize_extra(val)
1604 ok = True
1605 t = tc
1606 break
1607 except (ValidationError, ValueError):
1608 pass
1610 if not ok:
1611 raise ValidationError(
1612 '%s: could not convert "%s" to any type out of '
1613 '(%s)' % (self.xname(), val, ','.join(
1614 classnames(x.cls) for x in self.choices)))
1615 else:
1616 raise ValidationError(
1617 '%s: "%s" (type: %s) is not of any type out of '
1618 '(%s)' % (self.xname(), val, type(val), ','.join(
1619 classnames(x.cls) for x in self.choices)))
1621 validator = t
1623 if isinstance(t.cls, tuple):
1624 clss = t.cls
1625 else:
1626 clss = (t.cls,)
1628 for cls in clss:
1629 try:
1630 if type(val) != cls and isinstance(val, cls):
1631 validator = val.T.instance
1633 except AttributeError:
1634 pass
1636 validator.validate_extra(val)
1638 if depth != 0:
1639 val = validator.validate_children(val, regularize, depth)
1641 return val
1643 def extend_xmlelements(self, elems, v):
1644 elems.append((
1645 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v))
1648def _dump(
1649 object, stream,
1650 header=False,
1651 Dumper=GutsSafeDumper,
1652 _dump_function=yaml.dump):
1654 if not getattr(stream, 'encoding', None):
1655 enc = encode_utf8
1656 else:
1657 enc = no_encode
1659 if header:
1660 stream.write(enc(u'%YAML 1.1\n'))
1661 if isinstance(header, (str, newstr)):
1662 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
1663 stream.write(enc(banner))
1665 _dump_function(
1666 object,
1667 stream=stream,
1668 encoding='utf-8',
1669 explicit_start=True,
1670 Dumper=Dumper)
1673def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper):
1674 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all)
1677def _load(stream,
1678 Loader=GutsSafeLoader, allow_include=None, filename=None,
1679 included_files=None):
1681 class _Loader(Loader):
1682 _filename = filename
1683 _allow_include = allow_include
1684 _included_files = included_files or []
1686 return yaml.load(stream=stream, Loader=_Loader)
1689def _load_all(stream,
1690 Loader=GutsSafeLoader, allow_include=None, filename=None):
1692 class _Loader(Loader):
1693 _filename = filename
1694 _allow_include = allow_include
1696 return list(yaml.load_all(stream=stream, Loader=_Loader))
1699def _iload_all(stream,
1700 Loader=GutsSafeLoader, allow_include=None, filename=None):
1702 class _Loader(Loader):
1703 _filename = filename
1704 _allow_include = allow_include
1706 return yaml.load_all(stream=stream, Loader=_Loader)
1709def multi_representer(dumper, data):
1710 node = dumper.represent_mapping(
1711 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False)
1713 return node
1716# hack for compatibility with early GF Store versions
1717re_compatibility = re.compile(
1718 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.'
1719 r'(dummy|poel|qseis|qssp))\.'
1720)
1723def multi_constructor(loader, tag_suffix, node):
1724 tagname = str(tag_suffix)
1726 tagname = re_compatibility.sub('pf.', tagname)
1728 cls = g_tagname_to_class[tagname]
1729 kwargs = dict(iter(loader.construct_pairs(node, deep=True)))
1730 o = cls(**kwargs)
1731 o.validate(regularize=True, depth=1)
1732 return o
1735def include_constructor(loader, node):
1736 allow_include = loader._allow_include \
1737 if loader._allow_include is not None \
1738 else ALLOW_INCLUDE
1740 if not allow_include:
1741 raise EnvironmentError(
1742 'Not allowed to include YAML. Load with allow_include=True')
1744 if isinstance(node, yaml.nodes.ScalarNode):
1745 inc_file = loader.construct_scalar(node)
1746 else:
1747 raise TypeError('Unsupported YAML node %s' % repr(node))
1749 if loader._filename is not None and not op.isabs(inc_file):
1750 inc_file = op.join(op.dirname(loader._filename), inc_file)
1752 if not op.isfile(inc_file):
1753 raise FileNotFoundError(inc_file)
1755 included_files = list(loader._included_files)
1756 if loader._filename is not None:
1757 included_files.append(op.abspath(loader._filename))
1759 for included_file in loader._included_files:
1760 if op.samefile(inc_file, included_file):
1761 raise ImportError(
1762 'Circular import of file "%s". Include path: %s' % (
1763 op.abspath(inc_file),
1764 ' -> '.join('"%s"' % s for s in included_files)))
1766 with open(inc_file) as f:
1767 return _load(
1768 f,
1769 Loader=loader.__class__, filename=inc_file,
1770 allow_include=True,
1771 included_files=included_files)
1774def dict_noflow_representer(dumper, data):
1775 return dumper.represent_mapping(
1776 'tag:yaml.org,2002:map', data, flow_style=False)
1779yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper)
1780yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader)
1781yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader)
1782yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper)
1785def newstr_representer(dumper, data):
1786 return dumper.represent_scalar(
1787 'tag:yaml.org,2002:str', unicode(data))
1790yaml.add_representer(newstr, newstr_representer, Dumper=GutsSafeDumper)
1793class Constructor(object):
1794 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None,
1795 ns_ignore=False):
1797 self.stack = []
1798 self.queue = []
1799 self.namespaces = defaultdict(list)
1800 self.add_namespace_maps = add_namespace_maps
1801 self.strict = strict
1802 self.ns_hints = ns_hints
1803 self.ns_ignore = ns_ignore
1805 def start_element(self, ns_name, attrs):
1806 if self.ns_ignore:
1807 ns_name = ns_name.split(' ')[-1]
1809 if -1 == ns_name.find(' '):
1810 if self.ns_hints is None and ns_name in g_guessable_xmlns:
1811 self.ns_hints = g_guessable_xmlns[ns_name]
1813 if self.ns_hints:
1814 ns_names = [
1815 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints]
1817 elif self.ns_hints is None:
1818 ns_names = [' ' + ns_name]
1820 else:
1821 ns_names = [ns_name]
1823 for ns_name in ns_names:
1824 if self.stack and self.stack[-1][1] is not None:
1825 cls = self.stack[-1][1].T.xmltagname_to_class.get(
1826 ns_name, None)
1828 if isinstance(cls, tuple):
1829 cls = None
1830 else:
1831 if cls is not None and (
1832 not issubclass(cls, Object)
1833 or issubclass(cls, SObject)):
1834 cls = None
1835 else:
1836 cls = g_xmltagname_to_class.get(ns_name, None)
1838 if cls:
1839 break
1841 self.stack.append((ns_name, cls, attrs, [], []))
1843 def end_element(self, _):
1844 ns_name, cls, attrs, content2, content1 = self.stack.pop()
1846 ns = ns_name.split(' ', 1)[0]
1848 if cls is not None:
1849 content2.extend(
1850 (ns + ' ' + k if -1 == k.find(' ') else k, v)
1851 for (k, v) in attrs.items())
1852 content2.append((None, ''.join(content1)))
1853 o = cls(**cls.T.translate_from_xml(content2, self.strict))
1854 o.validate(regularize=True, depth=1)
1855 if self.add_namespace_maps:
1856 o.namespace_map = self.get_current_namespace_map()
1858 if self.stack and not all(x[1] is None for x in self.stack):
1859 self.stack[-1][-2].append((ns_name, o))
1860 else:
1861 self.queue.append(o)
1862 else:
1863 content = [''.join(content1)]
1864 if self.stack:
1865 for c in content:
1866 self.stack[-1][-2].append((ns_name, c))
1868 def characters(self, char_content):
1869 if self.stack:
1870 self.stack[-1][-1].append(char_content)
1872 def start_namespace(self, ns, uri):
1873 self.namespaces[ns].append(uri)
1875 def end_namespace(self, ns):
1876 self.namespaces[ns].pop()
1878 def get_current_namespace_map(self):
1879 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v)
1881 def get_queued_elements(self):
1882 queue = self.queue
1883 self.queue = []
1884 return queue
1887def _iload_all_xml(
1888 stream,
1889 bufsize=100000,
1890 add_namespace_maps=False,
1891 strict=False,
1892 ns_hints=None,
1893 ns_ignore=False):
1895 from xml.parsers.expat import ParserCreate
1897 parser = ParserCreate('UTF-8', namespace_separator=' ')
1899 handler = Constructor(
1900 add_namespace_maps=add_namespace_maps,
1901 strict=strict,
1902 ns_hints=ns_hints,
1903 ns_ignore=ns_ignore)
1905 parser.StartElementHandler = handler.start_element
1906 parser.EndElementHandler = handler.end_element
1907 parser.CharacterDataHandler = handler.characters
1908 parser.StartNamespaceDeclHandler = handler.start_namespace
1909 parser.EndNamespaceDeclHandler = handler.end_namespace
1911 while True:
1912 data = stream.read(bufsize)
1913 parser.Parse(data, bool(not data))
1914 for element in handler.get_queued_elements():
1915 yield element
1917 if not data:
1918 break
1921def _load_all_xml(*args, **kwargs):
1922 return list(_iload_all_xml(*args, **kwargs))
1925def _load_xml(*args, **kwargs):
1926 g = _iload_all_xml(*args, **kwargs)
1927 return next(g)
1930def _dump_all_xml(objects, stream, root_element_name='root', header=True):
1932 if not getattr(stream, 'encoding', None):
1933 enc = encode_utf8
1934 else:
1935 enc = no_encode
1937 _dump_xml_header(stream, header)
1939 beg = u'<%s>\n' % root_element_name
1940 end = u'</%s>\n' % root_element_name
1942 stream.write(enc(beg))
1944 for ob in objects:
1945 _dump_xml(ob, stream=stream)
1947 stream.write(enc(end))
1950def _dump_xml_header(stream, banner=None):
1952 if not getattr(stream, 'encoding', None):
1953 enc = encode_utf8
1954 else:
1955 enc = no_encode
1957 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n'))
1958 if isinstance(banner, (str, newstr)):
1959 stream.write(enc(u'<!-- %s -->\n' % banner))
1962def _dump_xml(
1963 obj, stream, depth=0, ns_name=None, header=False, ns_map=[],
1964 ns_ignore=False):
1966 from xml.sax.saxutils import escape, quoteattr
1968 if not getattr(stream, 'encoding', None):
1969 enc = encode_utf8
1970 else:
1971 enc = no_encode
1973 if depth == 0 and header:
1974 _dump_xml_header(stream, header)
1976 indent = ' '*depth*2
1977 if ns_name is None:
1978 ns_name = obj.T.instance.get_xmltagname()
1980 if -1 != ns_name.find(' '):
1981 ns, name = ns_name.split(' ')
1982 else:
1983 ns, name = '', ns_name
1985 if isinstance(obj, Object):
1986 obj.validate(depth=1)
1987 attrs = []
1988 elems = []
1990 added_ns = False
1991 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns):
1992 attrs.append(('xmlns', ns))
1993 ns_map.append(ns)
1994 added_ns = True
1996 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True):
1997 if prop.xmlstyle == 'attribute':
1998 assert not prop.multivalued
1999 assert not isinstance(v, Object)
2000 attrs.append((prop.effective_xmltagname, v))
2002 elif prop.xmlstyle == 'content':
2003 assert not prop.multivalued
2004 assert not isinstance(v, Object)
2005 elems.append((None, v))
2007 else:
2008 prop.extend_xmlelements(elems, v)
2010 attr_str = ''
2011 if attrs:
2012 attr_str = ' ' + ' '.join(
2013 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v)))
2014 for (k, v) in attrs)
2016 if not elems:
2017 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str)))
2018 else:
2019 oneline = len(elems) == 1 and elems[0][0] is None
2020 stream.write(enc(u'%s<%s%s>%s' % (
2021 indent,
2022 name,
2023 attr_str,
2024 '' if oneline else '\n')))
2026 for (k, v) in elems:
2027 if k is None:
2028 stream.write(enc(escape(newstr(v), {'\0': '�'})))
2029 else:
2030 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore)
2032 stream.write(enc(u'%s</%s>\n' % (
2033 '' if oneline else indent, name)))
2035 if added_ns:
2036 ns_map.pop()
2038 else:
2039 stream.write(enc(u'%s<%s>%s</%s>\n' % (
2040 indent,
2041 name,
2042 escape(newstr(obj), {'\0': '�'}),
2043 name)))
2046def walk(x, typ=None, path=()):
2047 if typ is None or isinstance(x, typ):
2048 yield path, x
2050 if isinstance(x, Object):
2051 for (prop, val) in x.T.ipropvals(x):
2052 if prop.multivalued:
2053 if val is not None:
2054 for iele, ele in enumerate(val):
2055 for y in walk(ele, typ,
2056 path=path + ((prop.name, iele),)):
2057 yield y
2058 else:
2059 for y in walk(val, typ, path=path+(prop.name,)):
2060 yield y
2063def clone(x, pool=None):
2064 '''
2065 Clone guts object tree.
2067 Traverses guts object tree and recursively clones all guts attributes,
2068 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects
2069 deriving from :py:class:`Object` are instantiated using their respective
2070 init function. Multiply referenced objects in the source tree are multiply
2071 referenced also in the destination tree.
2073 This function can be used to clone guts objects ignoring any contained
2074 run-time state, i.e. any of their attributes not defined as a guts
2075 property.
2076 '''
2078 if pool is None:
2079 pool = {}
2081 if id(x) in pool:
2082 x_copy = pool[id(x)]
2084 else:
2085 if isinstance(x, Object):
2086 d = {}
2087 for (prop, y) in x.T.ipropvals(x):
2088 if y is not None:
2089 if not prop.multivalued:
2090 y_copy = clone(y, pool)
2091 elif prop.multivalued is dict:
2092 y_copy = dict(
2093 (clone(zk, pool), clone(zv, pool))
2094 for (zk, zv) in y.items())
2095 else:
2096 y_copy = type(y)(clone(z, pool) for z in y)
2097 else:
2098 y_copy = y
2100 d[prop.name] = y_copy
2102 x_copy = x.__class__(**d)
2104 else:
2105 x_copy = copy.deepcopy(x)
2107 pool[id(x)] = x_copy
2108 return x_copy
2111class YPathError(Exception):
2112 '''
2113 This exception is raised for invalid ypath specifications.
2114 '''
2115 pass
2118def _parse_yname(yname):
2119 ident = r'[a-zA-Z][a-zA-Z0-9_]*'
2120 rint = r'-?[0-9]+'
2121 m = re.match(
2122 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$'
2123 % (ident, rint, rint, rint), yname)
2125 if not m:
2126 raise YPathError('Syntax error in component: "%s"' % yname)
2128 d = dict(
2129 name=m.group(1))
2131 if m.group(2):
2132 if m.group(5):
2133 istart = iend = None
2134 if m.group(4):
2135 istart = int(m.group(4))
2136 if m.group(6):
2137 iend = int(m.group(6))
2139 d['slice'] = (istart, iend)
2140 else:
2141 d['index'] = int(m.group(7))
2143 return d
2146def _decend(obj, ynames):
2147 if ynames:
2148 for sobj in iter_elements(obj, ynames):
2149 yield sobj
2150 else:
2151 yield obj
2154def iter_elements(obj, ypath):
2155 '''
2156 Generator yielding elements matching a given ypath specification.
2158 :param obj: guts :py:class:`Object` instance
2159 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2160 To access list objects use slice notatation (e.g.
2161 'root.child[:].child[1:3].child[1]').
2163 Raises :py:exc:`YPathError` on failure.
2164 '''
2166 try:
2167 if isinstance(ypath, str):
2168 ynames = ypath.split('.')
2169 else:
2170 ynames = ypath
2172 yname = ynames[0]
2173 ynames = ynames[1:]
2174 d = _parse_yname(yname)
2175 if d['name'] not in obj.T.propnames:
2176 raise AttributeError(d['name'])
2178 obj = getattr(obj, d['name'])
2180 if 'index' in d:
2181 sobj = obj[d['index']]
2182 for ssobj in _decend(sobj, ynames):
2183 yield ssobj
2185 elif 'slice' in d:
2186 for i in range(*slice(*d['slice']).indices(len(obj))):
2187 sobj = obj[i]
2188 for ssobj in _decend(sobj, ynames):
2189 yield ssobj
2190 else:
2191 for sobj in _decend(obj, ynames):
2192 yield sobj
2194 except (AttributeError, IndexError) as e:
2195 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2198def get_elements(obj, ypath):
2199 '''
2200 Get all elements matching a given ypath specification.
2202 :param obj: guts :py:class:`Object` instance
2203 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2204 To access list objects use slice notatation (e.g.
2205 'root.child[:].child[1:3].child[1]').
2207 Raises :py:exc:`YPathError` on failure.
2208 '''
2209 return list(iter_elements(obj, ypath))
2212def set_elements(obj, ypath, value, validate=False, regularize=False):
2213 '''
2214 Set elements matching a given ypath specification.
2216 :param obj: guts :py:class:`Object` instance
2217 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2218 To access list objects use slice notatation (e.g.
2219 'root.child[:].child[1:3].child[1]').
2220 :param value: All matching elements will be set to `value`.
2221 :param validate: Whether to validate affected subtrees.
2222 :param regularize: Whether to regularize affected subtrees.
2224 Raises :py:exc:`YPathError` on failure.
2225 '''
2227 ynames = ypath.split('.')
2228 try:
2229 d = _parse_yname(ynames[-1])
2230 for sobj in iter_elements(obj, ynames[:-1]):
2231 if d['name'] not in sobj.T.propnames:
2232 raise AttributeError(d['name'])
2234 if 'index' in d:
2235 ssobj = getattr(sobj, d['name'])
2236 ssobj[d['index']] = value
2237 elif 'slice' in d:
2238 ssobj = getattr(sobj, d['name'])
2239 for i in range(*slice(*d['slice']).indices(len(ssobj))):
2240 ssobj[i] = value
2241 else:
2242 setattr(sobj, d['name'], value)
2243 if regularize:
2244 sobj.regularize()
2245 if validate:
2246 sobj.validate()
2248 except (AttributeError, IndexError, YPathError) as e:
2249 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2252def zip_walk(x, typ=None, path=(), stack=()):
2253 if typ is None or isinstance(x, typ):
2254 yield path, stack + (x,)
2256 if isinstance(x, Object):
2257 for (prop, val) in x.T.ipropvals(x):
2258 if prop.multivalued:
2259 if val is not None:
2260 for iele, ele in enumerate(val):
2261 for y in zip_walk(
2262 ele, typ,
2263 path=path + ((prop.name, iele),),
2264 stack=stack + (x,)):
2266 yield y
2267 else:
2268 for y in zip_walk(val, typ,
2269 path=path+(prop.name,),
2270 stack=stack + (x,)):
2271 yield y
2274def path_element(x):
2275 if isinstance(x, tuple):
2276 return '%s[%i]' % x
2277 else:
2278 return x
2281def path_to_str(path):
2282 return '.'.join(path_element(x) for x in path)
2285@expand_stream_args('w')
2286def dump(*args, **kwargs):
2287 return _dump(*args, **kwargs)
2290@expand_stream_args('r')
2291def load(*args, **kwargs):
2292 return _load(*args, **kwargs)
2295def load_string(s, *args, **kwargs):
2296 return load(string=s, *args, **kwargs)
2299@expand_stream_args('w')
2300def dump_all(*args, **kwargs):
2301 return _dump_all(*args, **kwargs)
2304@expand_stream_args('r')
2305def load_all(*args, **kwargs):
2306 return _load_all(*args, **kwargs)
2309@expand_stream_args('r')
2310def iload_all(*args, **kwargs):
2311 return _iload_all(*args, **kwargs)
2314@expand_stream_args('w')
2315def dump_xml(*args, **kwargs):
2316 return _dump_xml(*args, **kwargs)
2319@expand_stream_args('r')
2320def load_xml(*args, **kwargs):
2321 kwargs.pop('filename', None)
2322 return _load_xml(*args, **kwargs)
2325def load_xml_string(s, *args, **kwargs):
2326 return load_xml(string=s, *args, **kwargs)
2329@expand_stream_args('w')
2330def dump_all_xml(*args, **kwargs):
2331 return _dump_all_xml(*args, **kwargs)
2334@expand_stream_args('r')
2335def load_all_xml(*args, **kwargs):
2336 kwargs.pop('filename', None)
2337 return _load_all_xml(*args, **kwargs)
2340@expand_stream_args('r')
2341def iload_all_xml(*args, **kwargs):
2342 kwargs.pop('filename', None)
2343 return _iload_all_xml(*args, **kwargs)
2346__all__ = guts_types + [
2347 'guts_types', 'TBase', 'ValidationError',
2348 'ArgumentError', 'Defer',
2349 'dump', 'load',
2350 'dump_all', 'load_all', 'iload_all',
2351 'dump_xml', 'load_xml',
2352 'dump_all_xml', 'load_all_xml', 'iload_all_xml',
2353 'load_string',
2354 'load_xml_string',
2355 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str'
2356]