1# -*- coding: utf-8 -*-
2# http://pyrocko.org - GPLv3
3#
4# The Pyrocko Developers, 21st Century
5# ---|P------/S----------~Lg----------
6'''
7Lightweight declarative YAML and XML data binding for Python.
8'''
9from __future__ import absolute_import, print_function
11import datetime
12import calendar
13import re
14import sys
15import types
16import copy
17import os.path as op
18from collections import defaultdict
20from io import BytesIO
22try:
23 import numpy as num
24except ImportError:
25 num = None
27import yaml
28try:
29 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
30except ImportError:
31 from yaml import SafeLoader, SafeDumper
33from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \
34 get_time_float
36try:
37 newstr = unicode
38 range = xrange
39except NameError:
40 newstr = str
42try:
43 # needed for py2/py3 compatibility to allow
44 # from pyrocko.guts import FileNotFoundError
45 FileNotFoundError = FileNotFoundError
46except NameError:
47 class FileNotFoundError(EnvironmentError):
48 pass
51ALLOW_INCLUDE = False
54class GutsSafeDumper(SafeDumper):
55 pass
58class GutsSafeLoader(SafeLoader):
59 pass
62try:
63 unicode
64except NameError:
65 unicode = str
68g_iprop = 0
70g_deferred = {}
71g_deferred_content = {}
73g_tagname_to_class = {}
74g_xmltagname_to_class = {}
75g_guessable_xmlns = {}
77guts_types = [
78 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float',
79 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern',
80 'UnicodePattern', 'StringChoice', 'List', 'Dict', 'Tuple', 'Union',
81 'Choice', 'Any']
83us_to_cc_regex = re.compile(r'([a-z])_([a-z])')
86class literal(str):
87 pass
90class folded(str):
91 pass
94class singlequoted(str):
95 pass
98class doublequoted(str):
99 pass
102def make_str_presenter(style):
103 def presenter(dumper, data):
104 return dumper.represent_scalar(
105 'tag:yaml.org,2002:str', str(data), style=style)
107 return presenter
110str_style_map = {
111 None: lambda x: x,
112 '|': literal,
113 '>': folded,
114 "'": singlequoted,
115 '"': doublequoted}
117for (style, cls) in str_style_map.items():
118 if style:
119 GutsSafeDumper.add_representer(cls, make_str_presenter(style))
122class uliteral(unicode):
123 pass
126class ufolded(unicode):
127 pass
130class usinglequoted(unicode):
131 pass
134class udoublequoted(unicode):
135 pass
138def make_unicode_presenter(style):
139 def presenter(dumper, data):
140 return dumper.represent_scalar(
141 'tag:yaml.org,2002:str', unicode(data), style=style)
143 return presenter
146unicode_style_map = {
147 None: lambda x: x,
148 '|': literal,
149 '>': folded,
150 "'": singlequoted,
151 '"': doublequoted}
153for (style, cls) in unicode_style_map.items():
154 if style:
155 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style))
158class blist(list):
159 pass
162class flist(list):
163 pass
166list_style_map = {
167 None: list,
168 'block': blist,
169 'flow': flist}
172def make_list_presenter(flow_style):
173 def presenter(dumper, data):
174 return dumper.represent_sequence(
175 'tag:yaml.org,2002:seq', data, flow_style=flow_style)
177 return presenter
180GutsSafeDumper.add_representer(blist, make_list_presenter(False))
181GutsSafeDumper.add_representer(flist, make_list_presenter(True))
183if num:
184 def numpy_float_presenter(dumper, data):
185 return dumper.represent_float(float(data))
187 def numpy_int_presenter(dumper, data):
188 return dumper.represent_int(int(data))
190 for dtype in (num.float64, num.float32):
191 GutsSafeDumper.add_representer(dtype, numpy_float_presenter)
193 for dtype in (num.int32, num.int64):
194 GutsSafeDumper.add_representer(dtype, numpy_int_presenter)
197def us_to_cc(s):
198 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s)
201cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)')
202cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])')
205def cc_to_us(s):
206 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub(
207 '\\1_\\2\\3', s)).lower()
210re_frac = re.compile(r'\.[1-9]FRAC')
211frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789'])
214def encode_utf8(s):
215 return s.encode('utf-8')
218def no_encode(s):
219 return s
222def make_xmltagname_from_name(name):
223 return us_to_cc(name)
226def make_name_from_xmltagname(xmltagname):
227 return cc_to_us(xmltagname)
230def make_content_name(name):
231 if name.endswith('_list'):
232 return name[:-5]
233 elif name.endswith('s'):
234 return name[:-1]
235 else:
236 return name
239def classnames(cls):
240 if isinstance(cls, tuple):
241 return '(%s)' % ', '.join(x.__name__ for x in cls)
242 else:
243 return cls.__name__
246def expand_stream_args(mode):
247 def wrap(f):
248 '''
249 Decorator to enhance functions taking stream objects.
251 Wraps a function f(..., stream, ...) so that it can also be called as
252 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...).
253 '''
255 def g(*args, **kwargs):
256 stream = kwargs.pop('stream', None)
257 filename = kwargs.get('filename', None)
258 if mode != 'r':
259 filename = kwargs.pop('filename', None)
260 string = kwargs.pop('string', None)
262 assert sum(x is not None for x in (stream, filename, string)) <= 1
264 if stream is not None:
265 kwargs['stream'] = stream
266 return f(*args, **kwargs)
268 elif filename is not None:
269 stream = open(filename, mode+'b')
270 kwargs['stream'] = stream
271 retval = f(*args, **kwargs)
272 if isinstance(retval, types.GeneratorType):
273 def wrap_generator(gen):
274 try:
275 for x in gen:
276 yield x
278 except GeneratorExit:
279 pass
281 stream.close()
283 return wrap_generator(retval)
285 else:
286 stream.close()
287 return retval
289 elif string is not None:
290 assert mode == 'r', \
291 'Keyword argument string=... cannot be used in dumper ' \
292 'function.'
294 kwargs['stream'] = BytesIO(string.encode('utf-8'))
295 return f(*args, **kwargs)
297 else:
298 assert mode == 'w', \
299 'Use keyword argument stream=... or filename=... in ' \
300 'loader function.'
302 sout = BytesIO()
303 f(stream=sout, *args, **kwargs)
304 return sout.getvalue().decode('utf-8')
306 return g
308 return wrap
311class Defer(object):
312 def __init__(self, classname, *args, **kwargs):
313 global g_iprop
314 if kwargs.get('position', None) is None:
315 kwargs['position'] = g_iprop
317 g_iprop += 1
319 self.classname = classname
320 self.args = args
321 self.kwargs = kwargs
324class TBase(object):
326 strict = False
327 multivalued = None
328 force_regularize = False
329 propnames = []
331 @classmethod
332 def init_propertystuff(cls):
333 cls.properties = []
334 cls.xmltagname_to_name = {}
335 cls.xmltagname_to_name_multivalued = {}
336 cls.xmltagname_to_class = {}
337 cls.content_property = None
339 def __init__(
340 self,
341 default=None,
342 optional=False,
343 xmlstyle='element',
344 xmltagname=None,
345 xmlns=None,
346 help=None,
347 position=None):
349 global g_iprop
350 if position is not None:
351 self.position = position
352 else:
353 self.position = g_iprop
355 g_iprop += 1
356 self._default = default
358 self.optional = optional
359 self.name = None
360 self._xmltagname = xmltagname
361 self._xmlns = xmlns
362 self.parent = None
363 self.xmlstyle = xmlstyle
364 self.help = help
366 def default(self):
367 return make_default(self._default)
369 def is_default(self, val):
370 if self._default is None:
371 return val is None
372 else:
373 return self._default == val
375 def has_default(self):
376 return self._default is not None
378 def xname(self):
379 if self.name is not None:
380 return self.name
381 elif self.parent is not None:
382 return 'element of %s' % self.parent.xname()
383 else:
384 return '?'
386 def set_xmlns(self, xmlns):
387 if self._xmlns is None and not self.xmlns:
388 self._xmlns = xmlns
390 if self.multivalued:
391 self.content_t.set_xmlns(xmlns)
393 def get_xmlns(self):
394 return self._xmlns or self.xmlns
396 def get_xmltagname(self):
397 if self._xmltagname is not None:
398 return self.get_xmlns() + ' ' + self._xmltagname
399 elif self.name:
400 return self.get_xmlns() + ' ' \
401 + make_xmltagname_from_name(self.name)
402 elif self.xmltagname:
403 return self.get_xmlns() + ' ' + self.xmltagname
404 else:
405 assert False
407 @classmethod
408 def get_property(cls, name):
409 for prop in cls.properties:
410 if prop.name == name:
411 return prop
413 raise ValueError()
415 @classmethod
416 def remove_property(cls, name):
418 prop = cls.get_property(name)
420 if not prop.multivalued:
421 del cls.xmltagname_to_class[prop.effective_xmltagname]
422 del cls.xmltagname_to_name[prop.effective_xmltagname]
423 else:
424 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname]
425 del cls.xmltagname_to_name_multivalued[
426 prop.content_t.effective_xmltagname]
428 if cls.content_property is prop:
429 cls.content_property = None
431 cls.properties.remove(prop)
432 cls.propnames.remove(name)
434 return prop
436 @classmethod
437 def add_property(cls, name, prop):
439 prop.instance = prop
440 prop.name = name
441 prop.set_xmlns(cls.xmlns)
443 if isinstance(prop, Choice.T):
444 for tc in prop.choices:
445 tc.effective_xmltagname = tc.get_xmltagname()
446 cls.xmltagname_to_class[tc.effective_xmltagname] = tc.cls
447 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name
448 elif not prop.multivalued:
449 prop.effective_xmltagname = prop.get_xmltagname()
450 cls.xmltagname_to_class[prop.effective_xmltagname] = prop.cls
451 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name
452 else:
453 prop.content_t.name = make_content_name(prop.name)
454 prop.content_t.effective_xmltagname = \
455 prop.content_t.get_xmltagname()
456 cls.xmltagname_to_class[
457 prop.content_t.effective_xmltagname] = prop.content_t.cls
458 cls.xmltagname_to_name_multivalued[
459 prop.content_t.effective_xmltagname] = prop.name
461 cls.properties.append(prop)
463 cls.properties.sort(key=lambda x: x.position)
465 cls.propnames = [p.name for p in cls.properties]
467 if prop.xmlstyle == 'content':
468 cls.content_property = prop
470 @classmethod
471 def ivals(cls, val):
472 for prop in cls.properties:
473 yield getattr(val, prop.name)
475 @classmethod
476 def ipropvals(cls, val):
477 for prop in cls.properties:
478 yield prop, getattr(val, prop.name)
480 @classmethod
481 def inamevals(cls, val):
482 for prop in cls.properties:
483 yield prop.name, getattr(val, prop.name)
485 @classmethod
486 def ipropvals_to_save(cls, val, xmlmode=False):
487 for prop in cls.properties:
488 v = getattr(val, prop.name)
489 if v is not None and (
490 not (prop.optional or (prop.multivalued and not v))
491 or (not prop.is_default(v))):
493 if xmlmode:
494 yield prop, prop.to_save_xml(v)
495 else:
496 yield prop, prop.to_save(v)
498 @classmethod
499 def inamevals_to_save(cls, val, xmlmode=False):
500 for prop, v in cls.ipropvals_to_save(val, xmlmode):
501 yield prop.name, v
503 @classmethod
504 def translate_from_xml(cls, list_of_pairs, strict):
505 d = {}
506 for k, v in list_of_pairs:
507 if k in cls.xmltagname_to_name_multivalued:
508 k2 = cls.xmltagname_to_name_multivalued[k]
509 if k2 not in d:
510 d[k2] = []
512 d[k2].append(v)
513 elif k in cls.xmltagname_to_name:
514 k2 = cls.xmltagname_to_name[k]
515 if k2 in d:
516 raise ArgumentError(
517 'Unexpectedly found more than one child element "%s" '
518 'within "%s".' % (k, cls.tagname))
520 d[k2] = v
521 elif k is None:
522 if cls.content_property:
523 k2 = cls.content_property.name
524 d[k2] = v
525 else:
526 if strict:
527 raise ArgumentError(
528 'Unexpected child element "%s" found within "%s".' % (
529 k, cls.tagname))
531 return d
533 def validate(self, val, regularize=False, depth=-1):
534 if self.optional and val is None:
535 return val
537 is_derived = isinstance(val, self.cls)
538 is_exact = type(val) == self.cls
540 not_ok = not self.strict and not is_derived or \
541 self.strict and not is_exact
543 if not_ok or self.force_regularize:
544 if regularize:
545 try:
546 val = self.regularize_extra(val)
547 except ValueError:
548 raise ValidationError(
549 '%s: could not convert "%s" to type %s' % (
550 self.xname(), val, classnames(self.cls)))
551 else:
552 raise ValidationError(
553 '%s: "%s" (type: %s) is not of type %s' % (
554 self.xname(), val, type(val), classnames(self.cls)))
556 validator = self
557 if isinstance(self.cls, tuple):
558 clss = self.cls
559 else:
560 clss = (self.cls,)
562 for cls in clss:
563 try:
564 if type(val) != cls and isinstance(val, cls):
565 validator = val.T.instance
567 except AttributeError:
568 pass
570 validator.validate_extra(val)
572 if depth != 0:
573 val = validator.validate_children(val, regularize, depth)
575 return val
577 def regularize_extra(self, val):
578 return self.cls(val)
580 def validate_extra(self, val):
581 pass
583 def validate_children(self, val, regularize, depth):
584 for prop, propval in self.ipropvals(val):
585 newpropval = prop.validate(propval, regularize, depth-1)
586 if regularize and (newpropval is not propval):
587 setattr(val, prop.name, newpropval)
589 return val
591 def to_save(self, val):
592 return val
594 def to_save_xml(self, val):
595 return self.to_save(val)
597 def extend_xmlelements(self, elems, v):
598 if self.multivalued:
599 for x in v:
600 elems.append((self.content_t.effective_xmltagname, x))
601 else:
602 elems.append((self.effective_xmltagname, v))
604 def deferred(self):
605 return []
607 def classname_for_help(self, strip_module=''):
608 if self.dummy_cls in guts_plain_dummy_types:
609 return '``%s``' % self.cls.__name__
611 elif self.dummy_cls.dummy_for_description:
612 return self.dummy_cls.dummy_for_description
614 else:
615 if self.dummy_cls is not self.cls:
616 if self.dummy_cls.__module__ == strip_module:
617 sadd = ' (:py:class:`%s`)' % (
618 self.dummy_cls.__name__)
619 else:
620 sadd = ' (:py:class:`%s.%s`)' % (
621 self.dummy_cls.__module__, self.dummy_cls.__name__)
622 else:
623 sadd = ''
625 def sclass(cls):
626 mod = cls.__module__
627 clsn = cls.__name__
628 if mod == '__builtin__' or mod == 'builtins':
629 return '``%s``' % clsn
631 elif mod == strip_module:
632 return ':py:class:`%s`' % clsn
634 else:
635 return ':py:class:`%s.%s`' % (mod, clsn)
637 if isinstance(self.cls, tuple):
638 return '(%s)%s' % (
639 ' | '.join(sclass(cls) for cls in self.cls), sadd)
640 else:
641 return '%s%s' % (sclass(self.cls), sadd)
643 @classmethod
644 def props_help_string(cls):
645 baseprops = []
646 for base in cls.dummy_cls.__bases__:
647 if hasattr(base, 'T'):
648 baseprops.extend(base.T.properties)
650 hlp = []
651 hlp.append('')
652 for prop in cls.properties:
653 if prop in baseprops:
654 continue
656 descr = [
657 prop.classname_for_help(strip_module=cls.dummy_cls.__module__)]
659 if prop.optional:
660 descr.append('*optional*')
662 if isinstance(prop._default, DefaultMaker):
663 descr.append('*default:* ``%s``' % repr(prop._default))
664 else:
665 d = prop.default()
666 if d is not None:
667 descr.append('*default:* ``%s``' % repr(d))
669 hlp.append(' .. py:gattribute:: %s' % prop.name)
670 hlp.append('')
671 hlp.append(' %s' % ', '.join(descr))
672 hlp.append(' ')
673 if prop.help is not None:
674 hlp.append(' %s' % prop.help)
675 hlp.append('')
677 return '\n'.join(hlp)
679 @classmethod
680 def class_help_string(cls):
681 return cls.dummy_cls.__doc_template__
683 @classmethod
684 def class_signature(cls):
685 r = []
686 for prop in cls.properties:
687 d = prop.default()
688 if d is not None:
689 arg = repr(d)
691 elif prop.optional:
692 arg = 'None'
694 else:
695 arg = '...'
697 r.append('%s=%s' % (prop.name, arg))
699 return '(%s)' % ', '.join(r)
701 @classmethod
702 def help(cls):
703 return cls.props_help_string()
706class ObjectMetaClass(type):
707 def __new__(meta, classname, bases, class_dict):
708 cls = type.__new__(meta, classname, bases, class_dict)
709 if classname != 'Object':
710 t_class_attr_name = '_%s__T' % classname
711 if not hasattr(cls, t_class_attr_name):
712 if hasattr(cls, 'T'):
713 class T(cls.T):
714 pass
715 else:
716 class T(TBase):
717 pass
719 setattr(cls, t_class_attr_name, T)
721 T = getattr(cls, t_class_attr_name)
723 if cls.dummy_for is not None:
724 T.cls = cls.dummy_for
725 else:
726 T.cls = cls
728 T.dummy_cls = cls
730 if hasattr(cls, 'xmltagname'):
731 T.xmltagname = cls.xmltagname
732 else:
733 T.xmltagname = classname
735 mod = sys.modules[cls.__module__]
737 if hasattr(cls, 'xmlns'):
738 T.xmlns = cls.xmlns
739 elif hasattr(mod, 'guts_xmlns'):
740 T.xmlns = mod.guts_xmlns
741 else:
742 T.xmlns = ''
744 if T.xmlns and hasattr(cls, 'guessable_xmlns'):
745 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns
747 if hasattr(mod, 'guts_prefix'):
748 if mod.guts_prefix:
749 T.tagname = mod.guts_prefix + '.' + classname
750 else:
751 T.tagname = classname
752 else:
753 if cls.__module__ != '__main__':
754 T.tagname = cls.__module__ + '.' + classname
755 else:
756 T.tagname = classname
758 T.classname = classname
760 T.init_propertystuff()
762 for k in dir(cls):
763 prop = getattr(cls, k)
765 if k.endswith('__'):
766 k = k[:-2]
768 if isinstance(prop, TBase):
769 if prop.deferred():
770 for defer in prop.deferred():
771 g_deferred_content.setdefault(
772 defer.classname[:-2], []).append((prop, defer))
773 g_deferred.setdefault(
774 defer.classname[:-2], []).append((T, k, prop))
776 else:
777 T.add_property(k, prop)
779 elif isinstance(prop, Defer):
780 g_deferred.setdefault(prop.classname[:-2], []).append(
781 (T, k, prop))
783 if classname in g_deferred_content:
784 for prop, defer in g_deferred_content[classname]:
785 prop.process_deferred(
786 defer, T(*defer.args, **defer.kwargs))
788 del g_deferred_content[classname]
790 if classname in g_deferred:
791 for (T_, k_, prop_) in g_deferred.get(classname, []):
792 if isinstance(prop_, Defer):
793 prop_ = T(*prop_.args, **prop_.kwargs)
795 if not prop_.deferred():
796 T_.add_property(k_, prop_)
798 del g_deferred[classname]
800 g_tagname_to_class[T.tagname] = cls
801 if hasattr(cls, 'xmltagname'):
802 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls
804 cls.T = T
805 T.instance = T()
807 cls.__doc_template__ = cls.__doc__
808 cls.__doc__ = T.class_help_string()
810 if cls.__doc__ is None:
811 cls.__doc__ = 'Undocumented.'
813 cls.__doc__ += '\n' + T.props_help_string()
815 return cls
818class ValidationError(Exception):
819 pass
822class ArgumentError(Exception):
823 pass
826def make_default(x):
827 if isinstance(x, DefaultMaker):
828 return x.make()
829 elif isinstance(x, Object):
830 return clone(x)
831 else:
832 return x
835class DefaultMaker(object):
836 def make(self):
837 raise NotImplementedError('Schould be implemented in subclass.')
840class ObjectDefaultMaker(DefaultMaker):
841 def __init__(self, cls, args, kwargs):
842 DefaultMaker.__init__(self)
843 self.cls = cls
844 self.args = args
845 self.kwargs = kwargs
846 self.instance = None
848 def make(self):
849 return self.cls(
850 *[make_default(x) for x in self.args],
851 **dict((k, make_default(v)) for (k, v) in self.kwargs.items()))
853 def __eq__(self, other):
854 if self.instance is None:
855 self.instance = self.make()
857 return self.instance == other
859 def __repr__(self):
860 sargs = []
861 for arg in self.args:
862 sargs.append(repr(arg))
864 for k, v in self.kwargs.items():
865 sargs.append('%s=%s' % (k, repr(v)))
867 return '%s(%s)' % (self.cls.__name__, ', '.join(sargs))
870class TimestampDefaultMaker(DefaultMaker):
871 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'):
872 DefaultMaker.__init__(self)
873 self._stime = s
874 self._format = format
876 def make(self):
877 return str_to_time(self._stime, self._format)
879 def __repr__(self):
880 return "str_to_time(%s)" % repr(self._stime)
883def with_metaclass(meta, *bases):
884 # inlined py2/py3 compat solution from python-future
885 class metaclass(meta):
886 __call__ = type.__call__
887 __init__ = type.__init__
889 def __new__(cls, name, this_bases, d):
890 if this_bases is None:
891 return type.__new__(cls, name, (), d)
892 return meta(name, bases, d)
894 return metaclass('temp', None, {})
897class Object(with_metaclass(ObjectMetaClass, object)):
898 dummy_for = None
899 dummy_for_description = None
901 def __init__(self, **kwargs):
902 if not kwargs.get('init_props', True):
903 return
905 for prop in self.T.properties:
906 k = prop.name
907 if k in kwargs:
908 setattr(self, k, kwargs.pop(k))
909 else:
910 if not prop.optional and not prop.has_default():
911 raise ArgumentError('Missing argument to %s: %s' % (
912 self.T.tagname, prop.name))
913 else:
914 setattr(self, k, prop.default())
916 if kwargs:
917 raise ArgumentError('Invalid argument to %s: %s' % (
918 self.T.tagname, ', '.join(list(kwargs.keys()))))
920 @classmethod
921 def D(cls, *args, **kwargs):
922 return ObjectDefaultMaker(cls, args, kwargs)
924 def validate(self, regularize=False, depth=-1):
925 self.T.instance.validate(self, regularize, depth)
927 def regularize(self, depth=-1):
928 self.validate(regularize=True, depth=depth)
930 def dump(self, stream=None, filename=None, header=False):
931 return dump(self, stream=stream, filename=filename, header=header)
933 def dump_xml(
934 self, stream=None, filename=None, header=False, ns_ignore=False):
935 return dump_xml(
936 self, stream=stream, filename=filename, header=header,
937 ns_ignore=ns_ignore)
939 @classmethod
940 def load(cls, stream=None, filename=None, string=None):
941 return load(stream=stream, filename=filename, string=string)
943 @classmethod
944 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None,
945 ns_ignore=False):
947 if ns_hints is None:
948 ns_hints = [cls.T.instance.get_xmlns()]
950 return load_xml(
951 stream=stream,
952 filename=filename,
953 string=string,
954 ns_hints=ns_hints,
955 ns_ignore=ns_ignore)
957 def __str__(self):
958 return self.dump()
961def to_dict(obj):
962 '''
963 Get dict of guts object attributes.
965 :param obj: :py:class`Object` object
966 '''
968 return dict(obj.T.inamevals(obj))
971class SObject(Object):
973 class __T(TBase):
974 def regularize_extra(self, val):
975 if isinstance(val, (str, newstr)):
976 return self.cls(val)
978 return val
980 def to_save(self, val):
981 return str(val)
983 def to_save_xml(self, val):
984 return str(val)
987class Any(Object):
989 class __T(TBase):
990 def validate(self, val, regularize=False, depth=-1):
991 if isinstance(val, Object):
992 val.validate(regularize, depth)
994 return val
997class Int(Object):
998 dummy_for = int
1000 class __T(TBase):
1001 strict = True
1003 def to_save_xml(self, value):
1004 return repr(value)
1007class Float(Object):
1008 dummy_for = float
1010 class __T(TBase):
1011 strict = True
1013 def to_save_xml(self, value):
1014 return repr(value)
1017class Complex(Object):
1018 dummy_for = complex
1020 class __T(TBase):
1021 strict = True
1023 def regularize_extra(self, val):
1025 if isinstance(val, list) or isinstance(val, tuple):
1026 assert len(val) == 2
1027 val = complex(*val)
1029 elif not isinstance(val, complex):
1030 val = complex(val)
1032 return val
1034 def to_save(self, value):
1035 return repr(value)
1037 def to_save_xml(self, value):
1038 return repr(value)
1041class Bool(Object):
1042 dummy_for = bool
1044 class __T(TBase):
1045 strict = True
1047 def regularize_extra(self, val):
1048 if isinstance(val, (str, newstr)):
1049 if val.lower().strip() in ('0', 'false'):
1050 return False
1052 return bool(val)
1054 def to_save_xml(self, value):
1055 return repr(bool(value)).lower()
1058class String(Object):
1059 dummy_for = str
1061 class __T(TBase):
1062 def __init__(self, *args, **kwargs):
1063 yamlstyle = kwargs.pop('yamlstyle', None)
1064 TBase.__init__(self, *args, **kwargs)
1065 self.style_cls = str_style_map[yamlstyle]
1067 def to_save(self, val):
1068 return self.style_cls(val)
1071class Unicode(Object):
1072 dummy_for = newstr
1074 class __T(TBase):
1075 def __init__(self, *args, **kwargs):
1076 yamlstyle = kwargs.pop('yamlstyle', None)
1077 TBase.__init__(self, *args, **kwargs)
1078 self.style_cls = unicode_style_map[yamlstyle]
1080 def to_save(self, val):
1081 return self.style_cls(val)
1084guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool)
1087class Dict(Object):
1088 dummy_for = dict
1090 class __T(TBase):
1091 multivalued = dict
1093 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs):
1094 TBase.__init__(self, *args, **kwargs)
1095 assert isinstance(key_t, TBase)
1096 assert isinstance(content_t, TBase)
1097 self.key_t = key_t
1098 self.content_t = content_t
1099 self.content_t.parent = self
1101 def default(self):
1102 if self._default is not None:
1103 return dict(
1104 (make_default(k), make_default(v))
1105 for (k, v) in self._default.items())
1107 if self.optional:
1108 return None
1109 else:
1110 return {}
1112 def has_default(self):
1113 return True
1115 def validate(self, val, regularize, depth):
1116 return TBase.validate(self, val, regularize, depth+1)
1118 def validate_children(self, val, regularize, depth):
1119 for key, ele in list(val.items()):
1120 newkey = self.key_t.validate(key, regularize, depth-1)
1121 newele = self.content_t.validate(ele, regularize, depth-1)
1122 if regularize:
1123 if newkey is not key or newele is not ele:
1124 del val[key]
1125 val[newkey] = newele
1127 return val
1129 def to_save(self, val):
1130 return dict((self.key_t.to_save(k), self.content_t.to_save(v))
1131 for (k, v) in val.items())
1133 def to_save_xml(self, val):
1134 raise NotImplementedError()
1136 def classname_for_help(self, strip_module=''):
1137 return '``dict`` of %s objects' % \
1138 self.content_t.classname_for_help(strip_module=strip_module)
1141class List(Object):
1142 dummy_for = list
1144 class __T(TBase):
1145 multivalued = list
1147 def __init__(self, content_t=Any.T(), *args, **kwargs):
1148 yamlstyle = kwargs.pop('yamlstyle', None)
1149 TBase.__init__(self, *args, **kwargs)
1150 assert isinstance(content_t, TBase) or isinstance(content_t, Defer)
1151 self.content_t = content_t
1152 self.content_t.parent = self
1153 self.style_cls = list_style_map[yamlstyle]
1155 def default(self):
1156 if self._default is not None:
1157 return [make_default(x) for x in self._default]
1158 if self.optional:
1159 return None
1160 else:
1161 return []
1163 def has_default(self):
1164 return True
1166 def validate(self, val, regularize, depth):
1167 return TBase.validate(self, val, regularize, depth+1)
1169 def validate_children(self, val, regularize, depth):
1170 for i, ele in enumerate(val):
1171 newele = self.content_t.validate(ele, regularize, depth-1)
1172 if regularize and newele is not ele:
1173 val[i] = newele
1175 return val
1177 def to_save(self, val):
1178 return self.style_cls(self.content_t.to_save(v) for v in val)
1180 def to_save_xml(self, val):
1181 return [self.content_t.to_save_xml(v) for v in val]
1183 def deferred(self):
1184 if isinstance(self.content_t, Defer):
1185 return [self.content_t]
1187 return []
1189 def process_deferred(self, defer, t_inst):
1190 if defer is self.content_t:
1191 self.content_t = t_inst
1193 def classname_for_help(self, strip_module=''):
1194 return '``list`` of %s objects' % \
1195 self.content_t.classname_for_help(strip_module=strip_module)
1198def make_typed_list_class(t):
1199 class TL(List):
1200 class __T(List.T):
1201 def __init__(self, *args, **kwargs):
1202 List.T.__init__(self, content_t=t.T(), *args, **kwargs)
1204 return TL
1207class Tuple(Object):
1208 dummy_for = tuple
1210 class __T(TBase):
1211 multivalued = tuple
1213 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs):
1214 TBase.__init__(self, *args, **kwargs)
1215 assert isinstance(content_t, TBase)
1216 self.content_t = content_t
1217 self.content_t.parent = self
1218 self.n = n
1220 def default(self):
1221 if self._default is not None:
1222 return tuple(
1223 make_default(x) for x in self._default)
1225 elif self.optional:
1226 return None
1227 else:
1228 if self.n is not None:
1229 return tuple(
1230 self.content_t.default() for x in range(self.n))
1231 else:
1232 return tuple()
1234 def has_default(self):
1235 return True
1237 def validate(self, val, regularize, depth):
1238 return TBase.validate(self, val, regularize, depth+1)
1240 def validate_extra(self, val):
1241 if self.n is not None and len(val) != self.n:
1242 raise ValidationError(
1243 '%s should have length %i' % (self.xname(), self.n))
1245 def validate_children(self, val, regularize, depth):
1246 if not regularize:
1247 for ele in val:
1248 self.content_t.validate(ele, regularize, depth-1)
1250 return val
1251 else:
1252 newval = []
1253 isnew = False
1254 for ele in val:
1255 newele = self.content_t.validate(ele, regularize, depth-1)
1256 newval.append(newele)
1257 if newele is not ele:
1258 isnew = True
1260 if isnew:
1261 return tuple(newval)
1262 else:
1263 return val
1265 def to_save(self, val):
1266 return tuple(self.content_t.to_save(v) for v in val)
1268 def to_save_xml(self, val):
1269 return [self.content_t.to_save_xml(v) for v in val]
1271 def classname_for_help(self, strip_module=''):
1272 if self.n is not None:
1273 return '``tuple`` of %i %s objects' % (
1274 self.n, self.content_t.classname_for_help(
1275 strip_module=strip_module))
1276 else:
1277 return '``tuple`` of %s objects' % (
1278 self.content_t.classname_for_help(
1279 strip_module=strip_module))
1282re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$')
1285class Timestamp(Object):
1286 dummy_for = (hpfloat, float)
1287 dummy_for_description = 'time_float'
1289 class __T(TBase):
1291 def regularize_extra(self, val):
1293 time_float = get_time_float()
1295 if isinstance(val, datetime.datetime):
1296 tt = val.utctimetuple()
1297 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1299 elif isinstance(val, datetime.date):
1300 tt = val.timetuple()
1301 val = time_float(calendar.timegm(tt))
1303 elif isinstance(val, (str, newstr)):
1304 val = val.strip()
1305 tz_offset = 0
1307 m = re_tz.search(val)
1308 if m:
1309 sh = m.group(2)
1310 sm = m.group(4)
1311 tz_offset = (int(sh)*3600 if sh else 0) \
1312 + (int(sm)*60 if sm else 0)
1314 val = re_tz.sub('', val)
1316 if len(val) > 10 and val[10] == 'T':
1317 val = val.replace('T', ' ', 1)
1319 try:
1320 val = str_to_time(val) - tz_offset
1321 except TimeStrError:
1322 raise ValidationError(
1323 '%s: cannot parse time/date: %s' % (self.xname(), val))
1325 elif isinstance(val, (int, float)):
1326 val = time_float(val)
1328 else:
1329 raise ValidationError(
1330 '%s: cannot convert "%s" to type %s' % (
1331 self.xname(), val, time_float))
1333 return val
1335 def to_save(self, val):
1336 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\
1337 .rstrip('0').rstrip('.')
1339 def to_save_xml(self, val):
1340 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\
1341 .rstrip('0').rstrip('.') + 'Z'
1343 @classmethod
1344 def D(self, s):
1345 return TimestampDefaultMaker(s)
1348class DateTimestamp(Object):
1349 dummy_for = (hpfloat, float)
1350 dummy_for_description = 'time_float'
1352 class __T(TBase):
1354 def regularize_extra(self, val):
1356 time_float = get_time_float()
1358 if isinstance(val, datetime.datetime):
1359 tt = val.utctimetuple()
1360 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1362 elif isinstance(val, datetime.date):
1363 tt = val.timetuple()
1364 val = time_float(calendar.timegm(tt))
1366 elif isinstance(val, (str, newstr)):
1367 val = str_to_time(val, format='%Y-%m-%d')
1369 elif isinstance(val, int):
1370 val = time_float(val)
1372 return val
1374 def to_save(self, val):
1375 return time_to_str(val, format='%Y-%m-%d')
1377 def to_save_xml(self, val):
1378 return time_to_str(val, format='%Y-%m-%d')
1380 @classmethod
1381 def D(self, s):
1382 return TimestampDefaultMaker(s, format='%Y-%m-%d')
1385class StringPattern(String):
1387 '''
1388 Any ``str`` matching pattern ``%(pattern)s``.
1389 '''
1391 dummy_for = str
1392 pattern = '.*'
1394 class __T(String.T):
1395 def __init__(self, pattern=None, *args, **kwargs):
1396 String.T.__init__(self, *args, **kwargs)
1398 if pattern is not None:
1399 self.pattern = pattern
1400 else:
1401 self.pattern = self.dummy_cls.pattern
1403 def validate_extra(self, val):
1404 pat = self.pattern
1405 if not re.search(pat, val):
1406 raise ValidationError('%s: "%s" does not match pattern %s' % (
1407 self.xname(), val, repr(pat)))
1409 @classmethod
1410 def class_help_string(cls):
1411 dcls = cls.dummy_cls
1412 doc = dcls.__doc_template__ or StringPattern.__doc_template__
1413 return doc % {'pattern': repr(dcls.pattern)}
1416class UnicodePattern(Unicode):
1418 '''
1419 Any ``unicode`` matching pattern ``%(pattern)s``.
1420 '''
1422 dummy_for = newstr
1423 pattern = '.*'
1425 class __T(TBase):
1426 def __init__(self, pattern=None, *args, **kwargs):
1427 TBase.__init__(self, *args, **kwargs)
1429 if pattern is not None:
1430 self.pattern = pattern
1431 else:
1432 self.pattern = self.dummy_cls.pattern
1434 def validate_extra(self, val):
1435 pat = self.pattern
1436 if not re.search(pat, val, flags=re.UNICODE):
1437 raise ValidationError('%s: "%s" does not match pattern %s' % (
1438 self.xname(), val, repr(pat)))
1440 @classmethod
1441 def class_help_string(cls):
1442 dcls = cls.dummy_cls
1443 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__
1444 return doc % {'pattern': repr(dcls.pattern)}
1447class StringChoice(String):
1449 '''
1450 Any ``str`` out of ``%(choices)s``.
1451 '''
1453 dummy_for = str
1454 choices = []
1455 ignore_case = False
1457 class __T(String.T):
1458 def __init__(self, choices=None, ignore_case=None, *args, **kwargs):
1459 String.T.__init__(self, *args, **kwargs)
1461 if choices is not None:
1462 self.choices = choices
1463 else:
1464 self.choices = self.dummy_cls.choices
1466 if ignore_case is not None:
1467 self.ignore_case = ignore_case
1468 else:
1469 self.ignore_case = self.dummy_cls.ignore_case
1471 if self.ignore_case:
1472 self.choices = [x.upper() for x in self.choices]
1474 def validate_extra(self, val):
1475 if self.ignore_case:
1476 val = val.upper()
1478 if val not in self.choices:
1479 raise ValidationError(
1480 '%s: "%s" is not a valid choice out of %s' % (
1481 self.xname(), val, repr(self.choices)))
1483 @classmethod
1484 def class_help_string(cls):
1485 dcls = cls.dummy_cls
1486 doc = dcls.__doc_template__ or StringChoice.__doc_template__
1487 return doc % {'choices': repr(dcls.choices)}
1490# this will not always work...
1491class Union(Object):
1492 members = []
1493 dummy_for = str
1495 class __T(TBase):
1496 def __init__(self, members=None, *args, **kwargs):
1497 TBase.__init__(self, *args, **kwargs)
1498 if members is not None:
1499 self.members = members
1500 else:
1501 self.members = self.dummy_cls.members
1503 def validate(self, val, regularize=False, depth=-1):
1504 assert self.members
1505 e2 = None
1506 for member in self.members:
1507 try:
1508 return member.validate(val, regularize, depth=depth)
1509 except ValidationError as e:
1510 e2 = e
1512 raise e2
1515class Choice(Object):
1516 choices = []
1518 class __T(TBase):
1519 def __init__(self, choices=None, *args, **kwargs):
1520 TBase.__init__(self, *args, **kwargs)
1521 if choices is not None:
1522 self.choices = choices
1523 else:
1524 self.choices = self.dummy_cls.choices
1526 self.cls_to_xmltagname = dict(
1527 (t.cls, t.get_xmltagname()) for t in self.choices)
1529 def validate(self, val, regularize=False, depth=-1):
1530 if self.optional and val is None:
1531 return val
1533 t = None
1534 for tc in self.choices:
1535 is_derived = isinstance(val, tc.cls)
1536 is_exact = type(val) == tc.cls
1537 if not (not tc.strict and not is_derived or
1538 tc.strict and not is_exact):
1540 t = tc
1541 break
1543 if t is None:
1544 if regularize:
1545 ok = False
1546 for tc in self.choices:
1547 try:
1548 val = tc.regularize_extra(val)
1549 ok = True
1550 t = tc
1551 break
1552 except (ValidationError, ValueError):
1553 pass
1555 if not ok:
1556 raise ValidationError(
1557 '%s: could not convert "%s" to any type out of '
1558 '(%s)' % (self.xname(), val, ','.join(
1559 classnames(x.cls) for x in self.choices)))
1560 else:
1561 raise ValidationError(
1562 '%s: "%s" (type: %s) is not of any type out of '
1563 '(%s)' % (self.xname(), val, type(val), ','.join(
1564 classnames(x.cls) for x in self.choices)))
1566 validator = t
1568 if isinstance(t.cls, tuple):
1569 clss = t.cls
1570 else:
1571 clss = (t.cls,)
1573 for cls in clss:
1574 try:
1575 if type(val) != cls and isinstance(val, cls):
1576 validator = val.T.instance
1578 except AttributeError:
1579 pass
1581 validator.validate_extra(val)
1583 if depth != 0:
1584 val = validator.validate_children(val, regularize, depth)
1586 return val
1588 def extend_xmlelements(self, elems, v):
1589 elems.append((
1590 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v))
1593def _dump(
1594 object, stream,
1595 header=False,
1596 Dumper=GutsSafeDumper,
1597 _dump_function=yaml.dump):
1599 if not getattr(stream, 'encoding', None):
1600 enc = encode_utf8
1601 else:
1602 enc = no_encode
1604 if header:
1605 stream.write(enc(u'%YAML 1.1\n'))
1606 if isinstance(header, (str, newstr)):
1607 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
1608 stream.write(enc(banner))
1610 _dump_function(
1611 object,
1612 stream=stream,
1613 encoding='utf-8',
1614 explicit_start=True,
1615 Dumper=Dumper)
1618def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper):
1619 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all)
1622def _load(stream,
1623 Loader=GutsSafeLoader, allow_include=None, filename=None,
1624 included_files=None):
1626 class _Loader(Loader):
1627 _filename = filename
1628 _allow_include = allow_include
1629 _included_files = included_files or []
1631 return yaml.load(stream=stream, Loader=_Loader)
1634def _load_all(stream,
1635 Loader=GutsSafeLoader, allow_include=None, filename=None):
1637 class _Loader(Loader):
1638 _filename = filename
1639 _allow_include = allow_include
1641 return list(yaml.load_all(stream=stream, Loader=_Loader))
1644def _iload_all(stream,
1645 Loader=GutsSafeLoader, allow_include=None, filename=None):
1647 class _Loader(Loader):
1648 _filename = filename
1649 _allow_include = allow_include
1651 return yaml.load_all(stream=stream, Loader=_Loader)
1654def multi_representer(dumper, data):
1655 node = dumper.represent_mapping(
1656 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False)
1658 return node
1661# hack for compatibility with early GF Store versions
1662re_compatibility = re.compile(
1663 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.'
1664 r'(dummy|poel|qseis|qssp))\.'
1665)
1668def multi_constructor(loader, tag_suffix, node):
1669 tagname = str(tag_suffix)
1671 tagname = re_compatibility.sub('pf.', tagname)
1673 cls = g_tagname_to_class[tagname]
1674 kwargs = dict(iter(loader.construct_pairs(node, deep=True)))
1675 o = cls(**kwargs)
1676 o.validate(regularize=True, depth=1)
1677 return o
1680def include_constructor(loader, node):
1681 allow_include = loader._allow_include \
1682 if loader._allow_include is not None \
1683 else ALLOW_INCLUDE
1685 if not allow_include:
1686 raise EnvironmentError(
1687 'Not allowed to include YAML. Load with allow_include=True')
1689 if isinstance(node, yaml.nodes.ScalarNode):
1690 inc_file = loader.construct_scalar(node)
1691 else:
1692 raise TypeError('Unsupported YAML node %s' % repr(node))
1694 if loader._filename is not None and not op.isabs(inc_file):
1695 inc_file = op.join(op.dirname(loader._filename), inc_file)
1697 if not op.isfile(inc_file):
1698 raise FileNotFoundError(inc_file)
1700 included_files = list(loader._included_files)
1701 if loader._filename is not None:
1702 included_files.append(op.abspath(loader._filename))
1704 for included_file in loader._included_files:
1705 if op.samefile(inc_file, included_file):
1706 raise ImportError(
1707 'Circular import of file "%s". Include path: %s' % (
1708 op.abspath(inc_file),
1709 ' -> '.join('"%s"' % s for s in included_files)))
1711 with open(inc_file) as f:
1712 return _load(
1713 f,
1714 Loader=loader.__class__, filename=inc_file,
1715 allow_include=True,
1716 included_files=included_files)
1719def dict_noflow_representer(dumper, data):
1720 return dumper.represent_mapping(
1721 'tag:yaml.org,2002:map', data, flow_style=False)
1724yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper)
1725yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader)
1726yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader)
1727yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper)
1730def newstr_representer(dumper, data):
1731 return dumper.represent_scalar(
1732 'tag:yaml.org,2002:str', unicode(data))
1735yaml.add_representer(newstr, newstr_representer, Dumper=GutsSafeDumper)
1738class Constructor(object):
1739 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None,
1740 ns_ignore=False):
1742 self.stack = []
1743 self.queue = []
1744 self.namespaces = defaultdict(list)
1745 self.add_namespace_maps = add_namespace_maps
1746 self.strict = strict
1747 self.ns_hints = ns_hints
1748 self.ns_ignore = ns_ignore
1750 def start_element(self, ns_name, attrs):
1751 if self.ns_ignore:
1752 ns_name = ns_name.split(' ')[-1]
1754 if -1 == ns_name.find(' '):
1755 if self.ns_hints is None and ns_name in g_guessable_xmlns:
1756 self.ns_hints = g_guessable_xmlns[ns_name]
1758 if self.ns_hints:
1759 ns_names = [
1760 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints]
1762 elif self.ns_hints is None:
1763 ns_names = [' ' + ns_name]
1765 else:
1766 ns_names = [ns_name]
1768 for ns_name in ns_names:
1769 if self.stack and self.stack[-1][1] is not None:
1770 cls = self.stack[-1][1].T.xmltagname_to_class.get(
1771 ns_name, None)
1773 if isinstance(cls, tuple):
1774 cls = None
1775 else:
1776 if cls is not None and (
1777 not issubclass(cls, Object)
1778 or issubclass(cls, SObject)):
1779 cls = None
1780 else:
1781 cls = g_xmltagname_to_class.get(ns_name, None)
1783 if cls:
1784 break
1786 self.stack.append((ns_name, cls, attrs, [], []))
1788 def end_element(self, _):
1789 ns_name, cls, attrs, content2, content1 = self.stack.pop()
1791 ns = ns_name.split(' ', 1)[0]
1793 if cls is not None:
1794 content2.extend(
1795 (ns + ' ' + k if -1 == k.find(' ') else k, v)
1796 for (k, v) in attrs.items())
1797 content2.append((None, ''.join(content1)))
1798 o = cls(**cls.T.translate_from_xml(content2, self.strict))
1799 o.validate(regularize=True, depth=1)
1800 if self.add_namespace_maps:
1801 o.namespace_map = self.get_current_namespace_map()
1803 if self.stack and not all(x[1] is None for x in self.stack):
1804 self.stack[-1][-2].append((ns_name, o))
1805 else:
1806 self.queue.append(o)
1807 else:
1808 content = [''.join(content1)]
1809 if self.stack:
1810 for c in content:
1811 self.stack[-1][-2].append((ns_name, c))
1813 def characters(self, char_content):
1814 if self.stack:
1815 self.stack[-1][-1].append(char_content)
1817 def start_namespace(self, ns, uri):
1818 self.namespaces[ns].append(uri)
1820 def end_namespace(self, ns):
1821 self.namespaces[ns].pop()
1823 def get_current_namespace_map(self):
1824 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v)
1826 def get_queued_elements(self):
1827 queue = self.queue
1828 self.queue = []
1829 return queue
1832def _iload_all_xml(
1833 stream,
1834 bufsize=100000,
1835 add_namespace_maps=False,
1836 strict=False,
1837 ns_hints=None,
1838 ns_ignore=False):
1840 from xml.parsers.expat import ParserCreate
1842 parser = ParserCreate('UTF-8', namespace_separator=' ')
1844 handler = Constructor(
1845 add_namespace_maps=add_namespace_maps,
1846 strict=strict,
1847 ns_hints=ns_hints,
1848 ns_ignore=ns_ignore)
1850 parser.StartElementHandler = handler.start_element
1851 parser.EndElementHandler = handler.end_element
1852 parser.CharacterDataHandler = handler.characters
1853 parser.StartNamespaceDeclHandler = handler.start_namespace
1854 parser.EndNamespaceDeclHandler = handler.end_namespace
1856 while True:
1857 data = stream.read(bufsize)
1858 parser.Parse(data, bool(not data))
1859 for element in handler.get_queued_elements():
1860 yield element
1862 if not data:
1863 break
1866def _load_all_xml(*args, **kwargs):
1867 return list(_iload_all_xml(*args, **kwargs))
1870def _load_xml(*args, **kwargs):
1871 g = _iload_all_xml(*args, **kwargs)
1872 return next(g)
1875def _dump_all_xml(objects, stream, root_element_name='root', header=True):
1877 if not getattr(stream, 'encoding', None):
1878 enc = encode_utf8
1879 else:
1880 enc = no_encode
1882 _dump_xml_header(stream, header)
1884 beg = u'<%s>\n' % root_element_name
1885 end = u'</%s>\n' % root_element_name
1887 stream.write(enc(beg))
1889 for ob in objects:
1890 _dump_xml(ob, stream=stream)
1892 stream.write(enc(end))
1895def _dump_xml_header(stream, banner=None):
1897 if not getattr(stream, 'encoding', None):
1898 enc = encode_utf8
1899 else:
1900 enc = no_encode
1902 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n'))
1903 if isinstance(banner, (str, newstr)):
1904 stream.write(enc(u'<!-- %s -->\n' % banner))
1907def _dump_xml(
1908 obj, stream, depth=0, ns_name=None, header=False, ns_map=[],
1909 ns_ignore=False):
1911 from xml.sax.saxutils import escape, quoteattr
1913 if not getattr(stream, 'encoding', None):
1914 enc = encode_utf8
1915 else:
1916 enc = no_encode
1918 if depth == 0 and header:
1919 _dump_xml_header(stream, header)
1921 indent = ' '*depth*2
1922 if ns_name is None:
1923 ns_name = obj.T.instance.get_xmltagname()
1925 if -1 != ns_name.find(' '):
1926 ns, name = ns_name.split(' ')
1927 else:
1928 ns, name = '', ns_name
1930 if isinstance(obj, Object):
1931 obj.validate(depth=1)
1932 attrs = []
1933 elems = []
1935 added_ns = False
1936 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns):
1937 attrs.append(('xmlns', ns))
1938 ns_map.append(ns)
1939 added_ns = True
1941 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True):
1942 if prop.xmlstyle == 'attribute':
1943 assert not prop.multivalued
1944 assert not isinstance(v, Object)
1945 attrs.append((prop.effective_xmltagname, v))
1947 elif prop.xmlstyle == 'content':
1948 assert not prop.multivalued
1949 assert not isinstance(v, Object)
1950 elems.append((None, v))
1952 else:
1953 prop.extend_xmlelements(elems, v)
1955 attr_str = ''
1956 if attrs:
1957 attr_str = ' ' + ' '.join(
1958 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v)))
1959 for (k, v) in attrs)
1961 if not elems:
1962 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str)))
1963 else:
1964 oneline = len(elems) == 1 and elems[0][0] is None
1965 stream.write(enc(u'%s<%s%s>%s' % (
1966 indent,
1967 name,
1968 attr_str,
1969 '' if oneline else '\n')))
1971 for (k, v) in elems:
1972 if k is None:
1973 stream.write(enc(escape(newstr(v), {'\0': '�'})))
1974 else:
1975 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore)
1977 stream.write(enc(u'%s</%s>\n' % (
1978 '' if oneline else indent, name)))
1980 if added_ns:
1981 ns_map.pop()
1983 else:
1984 stream.write(enc(u'%s<%s>%s</%s>\n' % (
1985 indent,
1986 name,
1987 escape(newstr(obj), {'\0': '�'}),
1988 name)))
1991def walk(x, typ=None, path=()):
1992 if typ is None or isinstance(x, typ):
1993 yield path, x
1995 if isinstance(x, Object):
1996 for (prop, val) in x.T.ipropvals(x):
1997 if prop.multivalued:
1998 if val is not None:
1999 for iele, ele in enumerate(val):
2000 for y in walk(ele, typ,
2001 path=path + ((prop.name, iele),)):
2002 yield y
2003 else:
2004 for y in walk(val, typ, path=path+(prop.name,)):
2005 yield y
2008def clone(x, pool=None):
2009 '''
2010 Clone guts object tree.
2012 Traverses guts object tree and recursively clones all guts attributes,
2013 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects
2014 deriving from :py:class:`Object` are instantiated using their respective
2015 init function. Multiply referenced objects in the source tree are multiply
2016 referenced also in the destination tree.
2018 This function can be used to clone guts objects ignoring any contained
2019 run-time state, i.e. any of their attributes not defined as a guts
2020 property.
2021 '''
2023 if pool is None:
2024 pool = {}
2026 if id(x) in pool:
2027 x_copy = pool[id(x)]
2029 else:
2030 if isinstance(x, Object):
2031 d = {}
2032 for (prop, y) in x.T.ipropvals(x):
2033 if y is not None:
2034 if not prop.multivalued:
2035 y_copy = clone(y, pool)
2036 elif prop.multivalued is dict:
2037 y_copy = dict(
2038 (clone(zk, pool), clone(zv, pool))
2039 for (zk, zv) in y.items())
2040 else:
2041 y_copy = type(y)(clone(z, pool) for z in y)
2042 else:
2043 y_copy = y
2045 d[prop.name] = y_copy
2047 x_copy = x.__class__(**d)
2049 else:
2050 x_copy = copy.deepcopy(x)
2052 pool[id(x)] = x_copy
2053 return x_copy
2056class YPathError(Exception):
2057 '''
2058 This exception is raised for invalid ypath specifications.
2059 '''
2060 pass
2063def _parse_yname(yname):
2064 ident = r'[a-zA-Z][a-zA-Z0-9_]*'
2065 rint = r'-?[0-9]+'
2066 m = re.match(
2067 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$'
2068 % (ident, rint, rint, rint), yname)
2070 if not m:
2071 raise YPathError('Syntax error in component: "%s"' % yname)
2073 d = dict(
2074 name=m.group(1))
2076 if m.group(2):
2077 if m.group(5):
2078 istart = iend = None
2079 if m.group(4):
2080 istart = int(m.group(4))
2081 if m.group(6):
2082 iend = int(m.group(6))
2084 d['slice'] = (istart, iend)
2085 else:
2086 d['index'] = int(m.group(7))
2088 return d
2091def _decend(obj, ynames):
2092 if ynames:
2093 for sobj in iter_elements(obj, ynames):
2094 yield sobj
2095 else:
2096 yield obj
2099def iter_elements(obj, ypath):
2100 '''
2101 Generator yielding elements matching a given ypath specification.
2103 :param obj: guts :py:class:`Object` instance
2104 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2105 To access list objects use slice notatation (e.g.
2106 'root.child[:].child[1:3].child[1]').
2108 Raises :py:exc:`YPathError` on failure.
2109 '''
2111 try:
2112 if isinstance(ypath, str):
2113 ynames = ypath.split('.')
2114 else:
2115 ynames = ypath
2117 yname = ynames[0]
2118 ynames = ynames[1:]
2119 d = _parse_yname(yname)
2120 if d['name'] not in obj.T.propnames:
2121 raise AttributeError(d['name'])
2123 obj = getattr(obj, d['name'])
2125 if 'index' in d:
2126 sobj = obj[d['index']]
2127 for ssobj in _decend(sobj, ynames):
2128 yield ssobj
2130 elif 'slice' in d:
2131 for i in range(*slice(*d['slice']).indices(len(obj))):
2132 sobj = obj[i]
2133 for ssobj in _decend(sobj, ynames):
2134 yield ssobj
2135 else:
2136 for sobj in _decend(obj, ynames):
2137 yield sobj
2139 except (AttributeError, IndexError) as e:
2140 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2143def get_elements(obj, ypath):
2144 '''
2145 Get all elements matching a given ypath specification.
2147 :param obj: guts :py:class:`Object` instance
2148 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2149 To access list objects use slice notatation (e.g.
2150 'root.child[:].child[1:3].child[1]').
2152 Raises :py:exc:`YPathError` on failure.
2153 '''
2154 return list(iter_elements(obj, ypath))
2157def set_elements(obj, ypath, value, validate=False, regularize=False):
2158 '''
2159 Set elements matching a given ypath specification.
2161 :param obj: guts :py:class:`Object` instance
2162 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2163 To access list objects use slice notatation (e.g.
2164 'root.child[:].child[1:3].child[1]').
2165 :param value: All matching elements will be set to `value`.
2166 :param validate: Whether to validate affected subtrees.
2167 :param regularize: Whether to regularize affected subtrees.
2169 Raises :py:exc:`YPathError` on failure.
2170 '''
2172 ynames = ypath.split('.')
2173 try:
2174 d = _parse_yname(ynames[-1])
2175 for sobj in iter_elements(obj, ynames[:-1]):
2176 if d['name'] not in sobj.T.propnames:
2177 raise AttributeError(d['name'])
2179 if 'index' in d:
2180 ssobj = getattr(sobj, d['name'])
2181 ssobj[d['index']] = value
2182 elif 'slice' in d:
2183 ssobj = getattr(sobj, d['name'])
2184 for i in range(*slice(*d['slice']).indices(len(ssobj))):
2185 ssobj[i] = value
2186 else:
2187 setattr(sobj, d['name'], value)
2188 if regularize:
2189 sobj.regularize()
2190 if validate:
2191 sobj.validate()
2193 except (AttributeError, IndexError, YPathError) as e:
2194 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2197def zip_walk(x, typ=None, path=(), stack=()):
2198 if typ is None or isinstance(x, typ):
2199 yield path, stack + (x,)
2201 if isinstance(x, Object):
2202 for (prop, val) in x.T.ipropvals(x):
2203 if prop.multivalued:
2204 if val is not None:
2205 for iele, ele in enumerate(val):
2206 for y in zip_walk(
2207 ele, typ,
2208 path=path + ((prop.name, iele),),
2209 stack=stack + (x,)):
2211 yield y
2212 else:
2213 for y in zip_walk(val, typ,
2214 path=path+(prop.name,),
2215 stack=stack + (x,)):
2216 yield y
2219def path_element(x):
2220 if isinstance(x, tuple):
2221 return '%s[%i]' % x
2222 else:
2223 return x
2226def path_to_str(path):
2227 return '.'.join(path_element(x) for x in path)
2230@expand_stream_args('w')
2231def dump(*args, **kwargs):
2232 return _dump(*args, **kwargs)
2235@expand_stream_args('r')
2236def load(*args, **kwargs):
2237 return _load(*args, **kwargs)
2240def load_string(s, *args, **kwargs):
2241 return load(string=s, *args, **kwargs)
2244@expand_stream_args('w')
2245def dump_all(*args, **kwargs):
2246 return _dump_all(*args, **kwargs)
2249@expand_stream_args('r')
2250def load_all(*args, **kwargs):
2251 return _load_all(*args, **kwargs)
2254@expand_stream_args('r')
2255def iload_all(*args, **kwargs):
2256 return _iload_all(*args, **kwargs)
2259@expand_stream_args('w')
2260def dump_xml(*args, **kwargs):
2261 return _dump_xml(*args, **kwargs)
2264@expand_stream_args('r')
2265def load_xml(*args, **kwargs):
2266 kwargs.pop('filename', None)
2267 return _load_xml(*args, **kwargs)
2270def load_xml_string(s, *args, **kwargs):
2271 return load_xml(string=s, *args, **kwargs)
2274@expand_stream_args('w')
2275def dump_all_xml(*args, **kwargs):
2276 return _dump_all_xml(*args, **kwargs)
2279@expand_stream_args('r')
2280def load_all_xml(*args, **kwargs):
2281 kwargs.pop('filename', None)
2282 return _load_all_xml(*args, **kwargs)
2285@expand_stream_args('r')
2286def iload_all_xml(*args, **kwargs):
2287 kwargs.pop('filename', None)
2288 return _iload_all_xml(*args, **kwargs)
2291__all__ = guts_types + [
2292 'guts_types', 'TBase', 'ValidationError',
2293 'ArgumentError', 'Defer',
2294 'dump', 'load',
2295 'dump_all', 'load_all', 'iload_all',
2296 'dump_xml', 'load_xml',
2297 'dump_all_xml', 'load_all_xml', 'iload_all_xml',
2298 'load_string',
2299 'load_xml_string',
2300 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str'
2301]