1# -*- coding: utf-8 -*-
2# http://pyrocko.org - GPLv3
3#
4# The Pyrocko Developers, 21st Century
5# ---|P------/S----------~Lg----------
6'''
7Lightweight declarative YAML and XML data binding for Python.
8'''
9from __future__ import absolute_import, print_function
11import datetime
12import calendar
13import re
14import sys
15import types
16import copy
17import os.path as op
18from collections import defaultdict
20from io import BytesIO
22try:
23 import numpy as num
24except ImportError:
25 num = None
27import yaml
28try:
29 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
30except ImportError:
31 from yaml import SafeLoader, SafeDumper
33from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \
34 get_time_float
36try:
37 newstr = unicode
38 range = xrange
39except NameError:
40 newstr = str
42try:
43 # needed for py2/py3 compatibility to allow
44 # from pyrocko.guts import FileNotFoundError
45 FileNotFoundError = FileNotFoundError
46except NameError:
47 class FileNotFoundError(EnvironmentError):
48 pass
51ALLOW_INCLUDE = False
54class GutsSafeDumper(SafeDumper):
55 pass
58class GutsSafeLoader(SafeLoader):
59 pass
62try:
63 unicode
64except NameError:
65 unicode = str
68g_iprop = 0
70g_deferred = {}
71g_deferred_content = {}
73g_tagname_to_class = {}
74g_xmltagname_to_class = {}
75g_guessable_xmlns = {}
77guts_types = [
78 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float',
79 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern',
80 'UnicodePattern', 'StringChoice', 'IntChoice', 'List', 'Dict', 'Tuple',
81 'Union', 'Choice', 'Any']
83us_to_cc_regex = re.compile(r'([a-z])_([a-z])')
86class literal(str):
87 pass
90class folded(str):
91 pass
94class singlequoted(str):
95 pass
98class doublequoted(str):
99 pass
102def make_str_presenter(style):
103 def presenter(dumper, data):
104 return dumper.represent_scalar(
105 'tag:yaml.org,2002:str', str(data), style=style)
107 return presenter
110str_style_map = {
111 None: lambda x: x,
112 '|': literal,
113 '>': folded,
114 "'": singlequoted,
115 '"': doublequoted}
117for (style, cls) in str_style_map.items():
118 if style:
119 GutsSafeDumper.add_representer(cls, make_str_presenter(style))
122class uliteral(unicode):
123 pass
126class ufolded(unicode):
127 pass
130class usinglequoted(unicode):
131 pass
134class udoublequoted(unicode):
135 pass
138def make_unicode_presenter(style):
139 def presenter(dumper, data):
140 return dumper.represent_scalar(
141 'tag:yaml.org,2002:str', unicode(data), style=style)
143 return presenter
146unicode_style_map = {
147 None: lambda x: x,
148 '|': literal,
149 '>': folded,
150 "'": singlequoted,
151 '"': doublequoted}
153for (style, cls) in unicode_style_map.items():
154 if style:
155 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style))
158class blist(list):
159 pass
162class flist(list):
163 pass
166list_style_map = {
167 None: list,
168 'block': blist,
169 'flow': flist}
172def make_list_presenter(flow_style):
173 def presenter(dumper, data):
174 return dumper.represent_sequence(
175 'tag:yaml.org,2002:seq', data, flow_style=flow_style)
177 return presenter
180GutsSafeDumper.add_representer(blist, make_list_presenter(False))
181GutsSafeDumper.add_representer(flist, make_list_presenter(True))
183if num:
184 def numpy_float_presenter(dumper, data):
185 return dumper.represent_float(float(data))
187 def numpy_int_presenter(dumper, data):
188 return dumper.represent_int(int(data))
190 for dtype in (num.float64, num.float32):
191 GutsSafeDumper.add_representer(dtype, numpy_float_presenter)
193 for dtype in (num.int32, num.int64):
194 GutsSafeDumper.add_representer(dtype, numpy_int_presenter)
197def us_to_cc(s):
198 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s)
201cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)')
202cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])')
205def cc_to_us(s):
206 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub(
207 '\\1_\\2\\3', s)).lower()
210re_frac = re.compile(r'\.[1-9]FRAC')
211frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789'])
214def encode_utf8(s):
215 return s.encode('utf-8')
218def no_encode(s):
219 return s
222def make_xmltagname_from_name(name):
223 return us_to_cc(name)
226def make_name_from_xmltagname(xmltagname):
227 return cc_to_us(xmltagname)
230def make_content_name(name):
231 if name.endswith('_list'):
232 return name[:-5]
233 elif name.endswith('s'):
234 return name[:-1]
235 else:
236 return name
239def classnames(cls):
240 if isinstance(cls, tuple):
241 return '(%s)' % ', '.join(x.__name__ for x in cls)
242 else:
243 return cls.__name__
246def expand_stream_args(mode):
247 def wrap(f):
248 '''
249 Decorator to enhance functions taking stream objects.
251 Wraps a function f(..., stream, ...) so that it can also be called as
252 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...).
253 '''
255 def g(*args, **kwargs):
256 stream = kwargs.pop('stream', None)
257 filename = kwargs.get('filename', None)
258 if mode != 'r':
259 filename = kwargs.pop('filename', None)
260 string = kwargs.pop('string', None)
262 assert sum(x is not None for x in (stream, filename, string)) <= 1
264 if stream is not None:
265 kwargs['stream'] = stream
266 return f(*args, **kwargs)
268 elif filename is not None:
269 stream = open(filename, mode+'b')
270 kwargs['stream'] = stream
271 retval = f(*args, **kwargs)
272 if isinstance(retval, types.GeneratorType):
273 def wrap_generator(gen):
274 try:
275 for x in gen:
276 yield x
278 except GeneratorExit:
279 pass
281 stream.close()
283 return wrap_generator(retval)
285 else:
286 stream.close()
287 return retval
289 elif string is not None:
290 assert mode == 'r', \
291 'Keyword argument string=... cannot be used in dumper ' \
292 'function.'
294 kwargs['stream'] = BytesIO(string.encode('utf-8'))
295 return f(*args, **kwargs)
297 else:
298 assert mode == 'w', \
299 'Use keyword argument stream=... or filename=... in ' \
300 'loader function.'
302 sout = BytesIO()
303 f(stream=sout, *args, **kwargs)
304 return sout.getvalue().decode('utf-8')
306 return g
308 return wrap
311class Defer(object):
312 def __init__(self, classname, *args, **kwargs):
313 global g_iprop
314 if kwargs.get('position', None) is None:
315 kwargs['position'] = g_iprop
317 g_iprop += 1
319 self.classname = classname
320 self.args = args
321 self.kwargs = kwargs
324class TBase(object):
326 strict = False
327 multivalued = None
328 force_regularize = False
329 propnames = []
331 @classmethod
332 def init_propertystuff(cls):
333 cls.properties = []
334 cls.xmltagname_to_name = {}
335 cls.xmltagname_to_name_multivalued = {}
336 cls.xmltagname_to_class = {}
337 cls.content_property = None
339 def __init__(
340 self,
341 default=None,
342 optional=False,
343 xmlstyle='element',
344 xmltagname=None,
345 xmlns=None,
346 help=None,
347 position=None):
349 global g_iprop
350 if position is not None:
351 self.position = position
352 else:
353 self.position = g_iprop
355 g_iprop += 1
356 self._default = default
358 self.optional = optional
359 self.name = None
360 self._xmltagname = xmltagname
361 self._xmlns = xmlns
362 self.parent = None
363 self.xmlstyle = xmlstyle
364 self.help = help
366 def default(self):
367 return make_default(self._default)
369 def is_default(self, val):
370 if self._default is None:
371 return val is None
372 else:
373 return self._default == val
375 def has_default(self):
376 return self._default is not None
378 def xname(self):
379 if self.name is not None:
380 return self.name
381 elif self.parent is not None:
382 return 'element of %s' % self.parent.xname()
383 else:
384 return '?'
386 def set_xmlns(self, xmlns):
387 if self._xmlns is None and not self.xmlns:
388 self._xmlns = xmlns
390 if self.multivalued:
391 self.content_t.set_xmlns(xmlns)
393 def get_xmlns(self):
394 return self._xmlns or self.xmlns
396 def get_xmltagname(self):
397 if self._xmltagname is not None:
398 return self.get_xmlns() + ' ' + self._xmltagname
399 elif self.name:
400 return self.get_xmlns() + ' ' \
401 + make_xmltagname_from_name(self.name)
402 elif self.xmltagname:
403 return self.get_xmlns() + ' ' + self.xmltagname
404 else:
405 assert False
407 @classmethod
408 def get_property(cls, name):
409 for prop in cls.properties:
410 if prop.name == name:
411 return prop
413 raise ValueError()
415 @classmethod
416 def remove_property(cls, name):
418 prop = cls.get_property(name)
420 if not prop.multivalued:
421 del cls.xmltagname_to_class[prop.effective_xmltagname]
422 del cls.xmltagname_to_name[prop.effective_xmltagname]
423 else:
424 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname]
425 del cls.xmltagname_to_name_multivalued[
426 prop.content_t.effective_xmltagname]
428 if cls.content_property is prop:
429 cls.content_property = None
431 cls.properties.remove(prop)
432 cls.propnames.remove(name)
434 return prop
436 @classmethod
437 def add_property(cls, name, prop):
439 prop.instance = prop
440 prop.name = name
441 prop.set_xmlns(cls.xmlns)
443 if isinstance(prop, Choice.T):
444 for tc in prop.choices:
445 tc.effective_xmltagname = tc.get_xmltagname()
446 cls.xmltagname_to_class[tc.effective_xmltagname] = tc.cls
447 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name
448 elif not prop.multivalued:
449 prop.effective_xmltagname = prop.get_xmltagname()
450 cls.xmltagname_to_class[prop.effective_xmltagname] = prop.cls
451 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name
452 else:
453 prop.content_t.name = make_content_name(prop.name)
454 prop.content_t.effective_xmltagname = \
455 prop.content_t.get_xmltagname()
456 cls.xmltagname_to_class[
457 prop.content_t.effective_xmltagname] = prop.content_t.cls
458 cls.xmltagname_to_name_multivalued[
459 prop.content_t.effective_xmltagname] = prop.name
461 cls.properties.append(prop)
463 cls.properties.sort(key=lambda x: x.position)
465 cls.propnames = [p.name for p in cls.properties]
467 if prop.xmlstyle == 'content':
468 cls.content_property = prop
470 @classmethod
471 def ivals(cls, val):
472 for prop in cls.properties:
473 yield getattr(val, prop.name)
475 @classmethod
476 def ipropvals(cls, val):
477 for prop in cls.properties:
478 yield prop, getattr(val, prop.name)
480 @classmethod
481 def inamevals(cls, val):
482 for prop in cls.properties:
483 yield prop.name, getattr(val, prop.name)
485 @classmethod
486 def ipropvals_to_save(cls, val, xmlmode=False):
487 for prop in cls.properties:
488 v = getattr(val, prop.name)
489 if v is not None and (
490 not (prop.optional or (prop.multivalued and not v))
491 or (not prop.is_default(v))):
493 if xmlmode:
494 yield prop, prop.to_save_xml(v)
495 else:
496 yield prop, prop.to_save(v)
498 @classmethod
499 def inamevals_to_save(cls, val, xmlmode=False):
500 for prop, v in cls.ipropvals_to_save(val, xmlmode):
501 yield prop.name, v
503 @classmethod
504 def translate_from_xml(cls, list_of_pairs, strict):
505 d = {}
506 for k, v in list_of_pairs:
507 if k in cls.xmltagname_to_name_multivalued:
508 k2 = cls.xmltagname_to_name_multivalued[k]
509 if k2 not in d:
510 d[k2] = []
512 d[k2].append(v)
513 elif k in cls.xmltagname_to_name:
514 k2 = cls.xmltagname_to_name[k]
515 if k2 in d:
516 raise ArgumentError(
517 'Unexpectedly found more than one child element "%s" '
518 'within "%s".' % (k, cls.tagname))
520 d[k2] = v
521 elif k is None:
522 if cls.content_property:
523 k2 = cls.content_property.name
524 d[k2] = v
525 else:
526 if strict:
527 raise ArgumentError(
528 'Unexpected child element "%s" found within "%s".' % (
529 k, cls.tagname))
531 return d
533 def validate(self, val, regularize=False, depth=-1):
534 if self.optional and val is None:
535 return val
537 is_derived = isinstance(val, self.cls)
538 is_exact = type(val) == self.cls
540 not_ok = not self.strict and not is_derived or \
541 self.strict and not is_exact
543 if not_ok or self.force_regularize:
544 if regularize:
545 try:
546 val = self.regularize_extra(val)
547 except ValueError:
548 raise ValidationError(
549 '%s: could not convert "%s" to type %s' % (
550 self.xname(), val, classnames(self.cls)))
551 else:
552 raise ValidationError(
553 '%s: "%s" (type: %s) is not of type %s' % (
554 self.xname(), val, type(val), classnames(self.cls)))
556 validator = self
557 if isinstance(self.cls, tuple):
558 clss = self.cls
559 else:
560 clss = (self.cls,)
562 for cls in clss:
563 try:
564 if type(val) != cls and isinstance(val, cls):
565 validator = val.T.instance
567 except AttributeError:
568 pass
570 validator.validate_extra(val)
572 if depth != 0:
573 val = validator.validate_children(val, regularize, depth)
575 return val
577 def regularize_extra(self, val):
578 return self.cls(val)
580 def validate_extra(self, val):
581 pass
583 def validate_children(self, val, regularize, depth):
584 for prop, propval in self.ipropvals(val):
585 newpropval = prop.validate(propval, regularize, depth-1)
586 if regularize and (newpropval is not propval):
587 setattr(val, prop.name, newpropval)
589 return val
591 def to_save(self, val):
592 return val
594 def to_save_xml(self, val):
595 return self.to_save(val)
597 def extend_xmlelements(self, elems, v):
598 if self.multivalued:
599 for x in v:
600 elems.append((self.content_t.effective_xmltagname, x))
601 else:
602 elems.append((self.effective_xmltagname, v))
604 def deferred(self):
605 return []
607 def classname_for_help(self, strip_module=''):
609 if self.dummy_cls is not self.cls:
610 if self.dummy_cls.__module__ == strip_module:
611 sadd = ' (:py:class:`%s`)' % (
612 self.dummy_cls.__name__)
613 else:
614 sadd = ' (:py:class:`%s.%s`)' % (
615 self.dummy_cls.__module__, self.dummy_cls.__name__)
616 else:
617 sadd = ''
619 if self.dummy_cls in guts_plain_dummy_types:
620 return '``%s``' % self.cls.__name__
622 elif self.dummy_cls.dummy_for_description:
623 return '%s%s' % (self.dummy_cls.dummy_for_description, sadd)
625 else:
626 def sclass(cls):
627 mod = cls.__module__
628 clsn = cls.__name__
629 if mod == '__builtin__' or mod == 'builtins':
630 return '``%s``' % clsn
632 elif mod == strip_module:
633 return ':py:class:`%s`' % clsn
635 else:
636 return ':py:class:`%s.%s`' % (mod, clsn)
638 if isinstance(self.cls, tuple):
639 return '(%s)%s' % (
640 ' | '.join(sclass(cls) for cls in self.cls), sadd)
641 else:
642 return '%s%s' % (sclass(self.cls), sadd)
644 @classmethod
645 def props_help_string(cls):
646 baseprops = []
647 for base in cls.dummy_cls.__bases__:
648 if hasattr(base, 'T'):
649 baseprops.extend(base.T.properties)
651 hlp = []
652 hlp.append('')
653 for prop in cls.properties:
654 if prop in baseprops:
655 continue
657 descr = [
658 prop.classname_for_help(strip_module=cls.dummy_cls.__module__)]
660 if prop.optional:
661 descr.append('*optional*')
663 if isinstance(prop._default, DefaultMaker):
664 descr.append('*default:* ``%s``' % repr(prop._default))
665 else:
666 d = prop.default()
667 if d is not None:
668 descr.append('*default:* ``%s``' % repr(d))
670 hlp.append(' .. py:gattribute:: %s' % prop.name)
671 hlp.append('')
672 hlp.append(' %s' % ', '.join(descr))
673 hlp.append(' ')
674 if prop.help is not None:
675 hlp.append(' %s' % prop.help)
676 hlp.append('')
678 return '\n'.join(hlp)
680 @classmethod
681 def class_help_string(cls):
682 return cls.dummy_cls.__doc_template__
684 @classmethod
685 def class_signature(cls):
686 r = []
687 for prop in cls.properties:
688 d = prop.default()
689 if d is not None:
690 arg = repr(d)
692 elif prop.optional:
693 arg = 'None'
695 else:
696 arg = '...'
698 r.append('%s=%s' % (prop.name, arg))
700 return '(%s)' % ', '.join(r)
702 @classmethod
703 def help(cls):
704 return cls.props_help_string()
707class ObjectMetaClass(type):
708 def __new__(meta, classname, bases, class_dict):
709 cls = type.__new__(meta, classname, bases, class_dict)
710 if classname != 'Object':
711 t_class_attr_name = '_%s__T' % classname
712 if not hasattr(cls, t_class_attr_name):
713 if hasattr(cls, 'T'):
714 class T(cls.T):
715 pass
716 else:
717 class T(TBase):
718 pass
720 setattr(cls, t_class_attr_name, T)
722 T = getattr(cls, t_class_attr_name)
724 if cls.dummy_for is not None:
725 T.cls = cls.dummy_for
726 else:
727 T.cls = cls
729 T.dummy_cls = cls
731 if hasattr(cls, 'xmltagname'):
732 T.xmltagname = cls.xmltagname
733 else:
734 T.xmltagname = classname
736 mod = sys.modules[cls.__module__]
738 if hasattr(cls, 'xmlns'):
739 T.xmlns = cls.xmlns
740 elif hasattr(mod, 'guts_xmlns'):
741 T.xmlns = mod.guts_xmlns
742 else:
743 T.xmlns = ''
745 if T.xmlns and hasattr(cls, 'guessable_xmlns'):
746 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns
748 if hasattr(mod, 'guts_prefix'):
749 if mod.guts_prefix:
750 T.tagname = mod.guts_prefix + '.' + classname
751 else:
752 T.tagname = classname
753 else:
754 if cls.__module__ != '__main__':
755 T.tagname = cls.__module__ + '.' + classname
756 else:
757 T.tagname = classname
759 T.classname = classname
761 T.init_propertystuff()
763 for k in dir(cls):
764 prop = getattr(cls, k)
766 if k.endswith('__'):
767 k = k[:-2]
769 if isinstance(prop, TBase):
770 if prop.deferred():
771 for defer in prop.deferred():
772 g_deferred_content.setdefault(
773 defer.classname[:-2], []).append((prop, defer))
774 g_deferred.setdefault(
775 defer.classname[:-2], []).append((T, k, prop))
777 else:
778 T.add_property(k, prop)
780 elif isinstance(prop, Defer):
781 g_deferred.setdefault(prop.classname[:-2], []).append(
782 (T, k, prop))
784 if classname in g_deferred_content:
785 for prop, defer in g_deferred_content[classname]:
786 prop.process_deferred(
787 defer, T(*defer.args, **defer.kwargs))
789 del g_deferred_content[classname]
791 if classname in g_deferred:
792 for (T_, k_, prop_) in g_deferred.get(classname, []):
793 if isinstance(prop_, Defer):
794 prop_ = T(*prop_.args, **prop_.kwargs)
796 if not prop_.deferred():
797 T_.add_property(k_, prop_)
799 del g_deferred[classname]
801 g_tagname_to_class[T.tagname] = cls
802 if hasattr(cls, 'xmltagname'):
803 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls
805 cls.T = T
806 T.instance = T()
808 cls.__doc_template__ = cls.__doc__
809 cls.__doc__ = T.class_help_string()
811 if cls.__doc__ is None:
812 cls.__doc__ = 'Undocumented.'
814 cls.__doc__ += '\n' + T.props_help_string()
816 return cls
819class ValidationError(Exception):
820 pass
823class ArgumentError(Exception):
824 pass
827def make_default(x):
828 if isinstance(x, DefaultMaker):
829 return x.make()
830 elif isinstance(x, Object):
831 return clone(x)
832 else:
833 return x
836class DefaultMaker(object):
837 def make(self):
838 raise NotImplementedError('Schould be implemented in subclass.')
841class ObjectDefaultMaker(DefaultMaker):
842 def __init__(self, cls, args, kwargs):
843 DefaultMaker.__init__(self)
844 self.cls = cls
845 self.args = args
846 self.kwargs = kwargs
847 self.instance = None
849 def make(self):
850 return self.cls(
851 *[make_default(x) for x in self.args],
852 **dict((k, make_default(v)) for (k, v) in self.kwargs.items()))
854 def __eq__(self, other):
855 if self.instance is None:
856 self.instance = self.make()
858 return self.instance == other
860 def __repr__(self):
861 sargs = []
862 for arg in self.args:
863 sargs.append(repr(arg))
865 for k, v in self.kwargs.items():
866 sargs.append('%s=%s' % (k, repr(v)))
868 return '%s(%s)' % (self.cls.__name__, ', '.join(sargs))
871class TimestampDefaultMaker(DefaultMaker):
872 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'):
873 DefaultMaker.__init__(self)
874 self._stime = s
875 self._format = format
877 def make(self):
878 return str_to_time(self._stime, self._format)
880 def __repr__(self):
881 return "str_to_time(%s)" % repr(self._stime)
884def with_metaclass(meta, *bases):
885 # inlined py2/py3 compat solution from python-future
886 class metaclass(meta):
887 __call__ = type.__call__
888 __init__ = type.__init__
890 def __new__(cls, name, this_bases, d):
891 if this_bases is None:
892 return type.__new__(cls, name, (), d)
893 return meta(name, bases, d)
895 return metaclass('temp', None, {})
898class Object(with_metaclass(ObjectMetaClass, object)):
899 dummy_for = None
900 dummy_for_description = None
902 def __init__(self, **kwargs):
903 if not kwargs.get('init_props', True):
904 return
906 for prop in self.T.properties:
907 k = prop.name
908 if k in kwargs:
909 setattr(self, k, kwargs.pop(k))
910 else:
911 if not prop.optional and not prop.has_default():
912 raise ArgumentError('Missing argument to %s: %s' % (
913 self.T.tagname, prop.name))
914 else:
915 setattr(self, k, prop.default())
917 if kwargs:
918 raise ArgumentError('Invalid argument to %s: %s' % (
919 self.T.tagname, ', '.join(list(kwargs.keys()))))
921 @classmethod
922 def D(cls, *args, **kwargs):
923 return ObjectDefaultMaker(cls, args, kwargs)
925 def validate(self, regularize=False, depth=-1):
926 self.T.instance.validate(self, regularize, depth)
928 def regularize(self, depth=-1):
929 self.validate(regularize=True, depth=depth)
931 def dump(self, stream=None, filename=None, header=False):
932 return dump(self, stream=stream, filename=filename, header=header)
934 def dump_xml(
935 self, stream=None, filename=None, header=False, ns_ignore=False):
936 return dump_xml(
937 self, stream=stream, filename=filename, header=header,
938 ns_ignore=ns_ignore)
940 @classmethod
941 def load(cls, stream=None, filename=None, string=None):
942 return load(stream=stream, filename=filename, string=string)
944 @classmethod
945 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None,
946 ns_ignore=False):
948 if ns_hints is None:
949 ns_hints = [cls.T.instance.get_xmlns()]
951 return load_xml(
952 stream=stream,
953 filename=filename,
954 string=string,
955 ns_hints=ns_hints,
956 ns_ignore=ns_ignore)
958 def __str__(self):
959 return self.dump()
962def to_dict(obj):
963 '''
964 Get dict of guts object attributes.
966 :param obj: :py:class`Object` object
967 '''
969 return dict(obj.T.inamevals(obj))
972class SObject(Object):
974 class __T(TBase):
975 def regularize_extra(self, val):
976 if isinstance(val, (str, newstr)):
977 return self.cls(val)
979 return val
981 def to_save(self, val):
982 return str(val)
984 def to_save_xml(self, val):
985 return str(val)
988class Any(Object):
990 class __T(TBase):
991 def validate(self, val, regularize=False, depth=-1):
992 if isinstance(val, Object):
993 val.validate(regularize, depth)
995 return val
998class Int(Object):
999 dummy_for = int
1001 class __T(TBase):
1002 strict = True
1004 def to_save_xml(self, value):
1005 return repr(value)
1008class Float(Object):
1009 dummy_for = float
1011 class __T(TBase):
1012 strict = True
1014 def to_save_xml(self, value):
1015 return repr(value)
1018class Complex(Object):
1019 dummy_for = complex
1021 class __T(TBase):
1022 strict = True
1024 def regularize_extra(self, val):
1026 if isinstance(val, list) or isinstance(val, tuple):
1027 assert len(val) == 2
1028 val = complex(*val)
1030 elif not isinstance(val, complex):
1031 val = complex(val)
1033 return val
1035 def to_save(self, value):
1036 return repr(value)
1038 def to_save_xml(self, value):
1039 return repr(value)
1042class Bool(Object):
1043 dummy_for = bool
1045 class __T(TBase):
1046 strict = True
1048 def regularize_extra(self, val):
1049 if isinstance(val, (str, newstr)):
1050 if val.lower().strip() in ('0', 'false'):
1051 return False
1053 return bool(val)
1055 def to_save_xml(self, value):
1056 return repr(bool(value)).lower()
1059class String(Object):
1060 dummy_for = str
1062 class __T(TBase):
1063 def __init__(self, *args, **kwargs):
1064 yamlstyle = kwargs.pop('yamlstyle', None)
1065 TBase.__init__(self, *args, **kwargs)
1066 self.style_cls = str_style_map[yamlstyle]
1068 def to_save(self, val):
1069 return self.style_cls(val)
1072class Unicode(Object):
1073 dummy_for = newstr
1075 class __T(TBase):
1076 def __init__(self, *args, **kwargs):
1077 yamlstyle = kwargs.pop('yamlstyle', None)
1078 TBase.__init__(self, *args, **kwargs)
1079 self.style_cls = unicode_style_map[yamlstyle]
1081 def to_save(self, val):
1082 return self.style_cls(val)
1085guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool)
1088class Dict(Object):
1089 dummy_for = dict
1091 class __T(TBase):
1092 multivalued = dict
1094 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs):
1095 TBase.__init__(self, *args, **kwargs)
1096 assert isinstance(key_t, TBase)
1097 assert isinstance(content_t, TBase)
1098 self.key_t = key_t
1099 self.content_t = content_t
1100 self.content_t.parent = self
1102 def default(self):
1103 if self._default is not None:
1104 return dict(
1105 (make_default(k), make_default(v))
1106 for (k, v) in self._default.items())
1108 if self.optional:
1109 return None
1110 else:
1111 return {}
1113 def has_default(self):
1114 return True
1116 def validate(self, val, regularize, depth):
1117 return TBase.validate(self, val, regularize, depth+1)
1119 def validate_children(self, val, regularize, depth):
1120 for key, ele in list(val.items()):
1121 newkey = self.key_t.validate(key, regularize, depth-1)
1122 newele = self.content_t.validate(ele, regularize, depth-1)
1123 if regularize:
1124 if newkey is not key or newele is not ele:
1125 del val[key]
1126 val[newkey] = newele
1128 return val
1130 def to_save(self, val):
1131 return dict((self.key_t.to_save(k), self.content_t.to_save(v))
1132 for (k, v) in val.items())
1134 def to_save_xml(self, val):
1135 raise NotImplementedError()
1137 def classname_for_help(self, strip_module=''):
1138 return '``dict`` of %s objects' % \
1139 self.content_t.classname_for_help(strip_module=strip_module)
1142class List(Object):
1143 dummy_for = list
1145 class __T(TBase):
1146 multivalued = list
1148 def __init__(self, content_t=Any.T(), *args, **kwargs):
1149 yamlstyle = kwargs.pop('yamlstyle', None)
1150 TBase.__init__(self, *args, **kwargs)
1151 assert isinstance(content_t, TBase) or isinstance(content_t, Defer)
1152 self.content_t = content_t
1153 self.content_t.parent = self
1154 self.style_cls = list_style_map[yamlstyle]
1156 def default(self):
1157 if self._default is not None:
1158 return [make_default(x) for x in self._default]
1159 if self.optional:
1160 return None
1161 else:
1162 return []
1164 def has_default(self):
1165 return True
1167 def validate(self, val, regularize, depth):
1168 return TBase.validate(self, val, regularize, depth+1)
1170 def validate_children(self, val, regularize, depth):
1171 for i, ele in enumerate(val):
1172 newele = self.content_t.validate(ele, regularize, depth-1)
1173 if regularize and newele is not ele:
1174 val[i] = newele
1176 return val
1178 def to_save(self, val):
1179 return self.style_cls(self.content_t.to_save(v) for v in val)
1181 def to_save_xml(self, val):
1182 return [self.content_t.to_save_xml(v) for v in val]
1184 def deferred(self):
1185 if isinstance(self.content_t, Defer):
1186 return [self.content_t]
1188 return []
1190 def process_deferred(self, defer, t_inst):
1191 if defer is self.content_t:
1192 self.content_t = t_inst
1194 def classname_for_help(self, strip_module=''):
1195 return '``list`` of %s objects' % \
1196 self.content_t.classname_for_help(strip_module=strip_module)
1199def make_typed_list_class(t):
1200 class TL(List):
1201 class __T(List.T):
1202 def __init__(self, *args, **kwargs):
1203 List.T.__init__(self, content_t=t.T(), *args, **kwargs)
1205 return TL
1208class Tuple(Object):
1209 dummy_for = tuple
1211 class __T(TBase):
1212 multivalued = tuple
1214 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs):
1215 TBase.__init__(self, *args, **kwargs)
1216 assert isinstance(content_t, TBase)
1217 self.content_t = content_t
1218 self.content_t.parent = self
1219 self.n = n
1221 def default(self):
1222 if self._default is not None:
1223 return tuple(
1224 make_default(x) for x in self._default)
1226 elif self.optional:
1227 return None
1228 else:
1229 if self.n is not None:
1230 return tuple(
1231 self.content_t.default() for x in range(self.n))
1232 else:
1233 return tuple()
1235 def has_default(self):
1236 return True
1238 def validate(self, val, regularize, depth):
1239 return TBase.validate(self, val, regularize, depth+1)
1241 def validate_extra(self, val):
1242 if self.n is not None and len(val) != self.n:
1243 raise ValidationError(
1244 '%s should have length %i' % (self.xname(), self.n))
1246 def validate_children(self, val, regularize, depth):
1247 if not regularize:
1248 for ele in val:
1249 self.content_t.validate(ele, regularize, depth-1)
1251 return val
1252 else:
1253 newval = []
1254 isnew = False
1255 for ele in val:
1256 newele = self.content_t.validate(ele, regularize, depth-1)
1257 newval.append(newele)
1258 if newele is not ele:
1259 isnew = True
1261 if isnew:
1262 return tuple(newval)
1263 else:
1264 return val
1266 def to_save(self, val):
1267 return tuple(self.content_t.to_save(v) for v in val)
1269 def to_save_xml(self, val):
1270 return [self.content_t.to_save_xml(v) for v in val]
1272 def classname_for_help(self, strip_module=''):
1273 if self.n is not None:
1274 return '``tuple`` of %i %s objects' % (
1275 self.n, self.content_t.classname_for_help(
1276 strip_module=strip_module))
1277 else:
1278 return '``tuple`` of %s objects' % (
1279 self.content_t.classname_for_help(
1280 strip_module=strip_module))
1283unit_factors = dict(
1284 s=1.0,
1285 m=60.0,
1286 h=3600.0,
1287 d=24*3600.0,
1288 y=365*24*3600.0)
1291class Duration(Object):
1292 dummy_for = float
1294 class __T(TBase):
1296 def regularize_extra(self, val):
1297 if isinstance(val, (str, newstr)):
1298 unit = val[-1]
1299 if unit in unit_factors:
1300 return float(val[:-1]) * unit_factors[unit]
1301 else:
1302 return float(val)
1304 return val
1307re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$')
1310class Timestamp(Object):
1311 dummy_for = (hpfloat, float)
1312 dummy_for_description = 'time_float'
1314 class __T(TBase):
1316 def regularize_extra(self, val):
1318 time_float = get_time_float()
1320 if isinstance(val, datetime.datetime):
1321 tt = val.utctimetuple()
1322 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1324 elif isinstance(val, datetime.date):
1325 tt = val.timetuple()
1326 val = time_float(calendar.timegm(tt))
1328 elif isinstance(val, (str, newstr)):
1329 val = val.strip()
1330 tz_offset = 0
1332 m = re_tz.search(val)
1333 if m:
1334 sh = m.group(2)
1335 sm = m.group(4)
1336 tz_offset = (int(sh)*3600 if sh else 0) \
1337 + (int(sm)*60 if sm else 0)
1339 val = re_tz.sub('', val)
1341 if len(val) > 10 and val[10] == 'T':
1342 val = val.replace('T', ' ', 1)
1344 try:
1345 val = str_to_time(val) - tz_offset
1346 except TimeStrError:
1347 raise ValidationError(
1348 '%s: cannot parse time/date: %s' % (self.xname(), val))
1350 elif isinstance(val, (int, float)):
1351 val = time_float(val)
1353 else:
1354 raise ValidationError(
1355 '%s: cannot convert "%s" to type %s' % (
1356 self.xname(), val, time_float))
1358 return val
1360 def to_save(self, val):
1361 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\
1362 .rstrip('0').rstrip('.')
1364 def to_save_xml(self, val):
1365 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\
1366 .rstrip('0').rstrip('.') + 'Z'
1368 @classmethod
1369 def D(self, s):
1370 return TimestampDefaultMaker(s)
1373class DateTimestamp(Object):
1374 dummy_for = (hpfloat, float)
1375 dummy_for_description = 'time_float'
1377 class __T(TBase):
1379 def regularize_extra(self, val):
1381 time_float = get_time_float()
1383 if isinstance(val, datetime.datetime):
1384 tt = val.utctimetuple()
1385 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6
1387 elif isinstance(val, datetime.date):
1388 tt = val.timetuple()
1389 val = time_float(calendar.timegm(tt))
1391 elif isinstance(val, (str, newstr)):
1392 val = str_to_time(val, format='%Y-%m-%d')
1394 elif isinstance(val, int):
1395 val = time_float(val)
1397 return val
1399 def to_save(self, val):
1400 return time_to_str(val, format='%Y-%m-%d')
1402 def to_save_xml(self, val):
1403 return time_to_str(val, format='%Y-%m-%d')
1405 @classmethod
1406 def D(self, s):
1407 return TimestampDefaultMaker(s, format='%Y-%m-%d')
1410class StringPattern(String):
1412 '''
1413 Any ``str`` matching pattern ``%(pattern)s``.
1414 '''
1416 dummy_for = str
1417 pattern = '.*'
1419 class __T(String.T):
1420 def __init__(self, pattern=None, *args, **kwargs):
1421 String.T.__init__(self, *args, **kwargs)
1423 if pattern is not None:
1424 self.pattern = pattern
1425 else:
1426 self.pattern = self.dummy_cls.pattern
1428 def validate_extra(self, val):
1429 pat = self.pattern
1430 if not re.search(pat, val):
1431 raise ValidationError('%s: "%s" does not match pattern %s' % (
1432 self.xname(), val, repr(pat)))
1434 @classmethod
1435 def class_help_string(cls):
1436 dcls = cls.dummy_cls
1437 doc = dcls.__doc_template__ or StringPattern.__doc_template__
1438 return doc % {'pattern': repr(dcls.pattern)}
1441class UnicodePattern(Unicode):
1443 '''
1444 Any ``unicode`` matching pattern ``%(pattern)s``.
1445 '''
1447 dummy_for = newstr
1448 pattern = '.*'
1450 class __T(TBase):
1451 def __init__(self, pattern=None, *args, **kwargs):
1452 TBase.__init__(self, *args, **kwargs)
1454 if pattern is not None:
1455 self.pattern = pattern
1456 else:
1457 self.pattern = self.dummy_cls.pattern
1459 def validate_extra(self, val):
1460 pat = self.pattern
1461 if not re.search(pat, val, flags=re.UNICODE):
1462 raise ValidationError('%s: "%s" does not match pattern %s' % (
1463 self.xname(), val, repr(pat)))
1465 @classmethod
1466 def class_help_string(cls):
1467 dcls = cls.dummy_cls
1468 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__
1469 return doc % {'pattern': repr(dcls.pattern)}
1472class StringChoice(String):
1474 '''
1475 Any ``str`` out of ``%(choices)s``.
1476 '''
1478 dummy_for = str
1479 choices = []
1480 ignore_case = False
1482 class __T(String.T):
1483 def __init__(self, choices=None, ignore_case=None, *args, **kwargs):
1484 String.T.__init__(self, *args, **kwargs)
1486 if choices is not None:
1487 self.choices = choices
1488 else:
1489 self.choices = self.dummy_cls.choices
1491 if ignore_case is not None:
1492 self.ignore_case = ignore_case
1493 else:
1494 self.ignore_case = self.dummy_cls.ignore_case
1496 if self.ignore_case:
1497 self.choices = [x.upper() for x in self.choices]
1499 def validate_extra(self, val):
1500 if self.ignore_case:
1501 val = val.upper()
1503 if val not in self.choices:
1504 raise ValidationError(
1505 '%s: "%s" is not a valid choice out of %s' % (
1506 self.xname(), val, repr(self.choices)))
1508 @classmethod
1509 def class_help_string(cls):
1510 dcls = cls.dummy_cls
1511 doc = dcls.__doc_template__ or StringChoice.__doc_template__
1512 return doc % {'choices': repr(dcls.choices)}
1515class IntChoice(Int):
1517 '''
1518 Any ``int`` out of ``%(choices)s``.
1519 '''
1521 dummy_for = int
1522 choices = []
1524 class __T(Int.T):
1525 def __init__(self, choices=None, *args, **kwargs):
1526 Int.T.__init__(self, *args, **kwargs)
1528 if choices is not None:
1529 self.choices = choices
1530 else:
1531 self.choices = self.dummy_cls.choices
1533 def validate_extra(self, val):
1534 if val not in self.choices:
1535 raise ValidationError(
1536 '%s: %i is not a valid choice out of %s' % (
1537 self.xname(), val, repr(self.choices)))
1539 @classmethod
1540 def class_help_string(cls):
1541 dcls = cls.dummy_cls
1542 doc = dcls.__doc_template__ or IntChoice.__doc_template__
1543 return doc % {'choices': repr(dcls.choices)}
1546# this will not always work...
1547class Union(Object):
1548 members = []
1549 dummy_for = str
1551 class __T(TBase):
1552 def __init__(self, members=None, *args, **kwargs):
1553 TBase.__init__(self, *args, **kwargs)
1554 if members is not None:
1555 self.members = members
1556 else:
1557 self.members = self.dummy_cls.members
1559 def validate(self, val, regularize=False, depth=-1):
1560 assert self.members
1561 e2 = None
1562 for member in self.members:
1563 try:
1564 return member.validate(val, regularize, depth=depth)
1565 except ValidationError as e:
1566 e2 = e
1568 raise e2
1571class Choice(Object):
1572 choices = []
1574 class __T(TBase):
1575 def __init__(self, choices=None, *args, **kwargs):
1576 TBase.__init__(self, *args, **kwargs)
1577 if choices is not None:
1578 self.choices = choices
1579 else:
1580 self.choices = self.dummy_cls.choices
1582 self.cls_to_xmltagname = dict(
1583 (t.cls, t.get_xmltagname()) for t in self.choices)
1585 def validate(self, val, regularize=False, depth=-1):
1586 if self.optional and val is None:
1587 return val
1589 t = None
1590 for tc in self.choices:
1591 is_derived = isinstance(val, tc.cls)
1592 is_exact = type(val) == tc.cls
1593 if not (not tc.strict and not is_derived or
1594 tc.strict and not is_exact):
1596 t = tc
1597 break
1599 if t is None:
1600 if regularize:
1601 ok = False
1602 for tc in self.choices:
1603 try:
1604 val = tc.regularize_extra(val)
1605 ok = True
1606 t = tc
1607 break
1608 except (ValidationError, ValueError):
1609 pass
1611 if not ok:
1612 raise ValidationError(
1613 '%s: could not convert "%s" to any type out of '
1614 '(%s)' % (self.xname(), val, ','.join(
1615 classnames(x.cls) for x in self.choices)))
1616 else:
1617 raise ValidationError(
1618 '%s: "%s" (type: %s) is not of any type out of '
1619 '(%s)' % (self.xname(), val, type(val), ','.join(
1620 classnames(x.cls) for x in self.choices)))
1622 validator = t
1624 if isinstance(t.cls, tuple):
1625 clss = t.cls
1626 else:
1627 clss = (t.cls,)
1629 for cls in clss:
1630 try:
1631 if type(val) != cls and isinstance(val, cls):
1632 validator = val.T.instance
1634 except AttributeError:
1635 pass
1637 validator.validate_extra(val)
1639 if depth != 0:
1640 val = validator.validate_children(val, regularize, depth)
1642 return val
1644 def extend_xmlelements(self, elems, v):
1645 elems.append((
1646 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v))
1649def _dump(
1650 object, stream,
1651 header=False,
1652 Dumper=GutsSafeDumper,
1653 _dump_function=yaml.dump):
1655 if not getattr(stream, 'encoding', None):
1656 enc = encode_utf8
1657 else:
1658 enc = no_encode
1660 if header:
1661 stream.write(enc(u'%YAML 1.1\n'))
1662 if isinstance(header, (str, newstr)):
1663 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
1664 stream.write(enc(banner))
1666 _dump_function(
1667 object,
1668 stream=stream,
1669 encoding='utf-8',
1670 explicit_start=True,
1671 Dumper=Dumper)
1674def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper):
1675 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all)
1678def _load(stream,
1679 Loader=GutsSafeLoader, allow_include=None, filename=None,
1680 included_files=None):
1682 class _Loader(Loader):
1683 _filename = filename
1684 _allow_include = allow_include
1685 _included_files = included_files or []
1687 return yaml.load(stream=stream, Loader=_Loader)
1690def _load_all(stream,
1691 Loader=GutsSafeLoader, allow_include=None, filename=None):
1693 class _Loader(Loader):
1694 _filename = filename
1695 _allow_include = allow_include
1697 return list(yaml.load_all(stream=stream, Loader=_Loader))
1700def _iload_all(stream,
1701 Loader=GutsSafeLoader, allow_include=None, filename=None):
1703 class _Loader(Loader):
1704 _filename = filename
1705 _allow_include = allow_include
1707 return yaml.load_all(stream=stream, Loader=_Loader)
1710def multi_representer(dumper, data):
1711 node = dumper.represent_mapping(
1712 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False)
1714 return node
1717# hack for compatibility with early GF Store versions
1718re_compatibility = re.compile(
1719 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.'
1720 r'(dummy|poel|qseis|qssp))\.'
1721)
1724def multi_constructor(loader, tag_suffix, node):
1725 tagname = str(tag_suffix)
1727 tagname = re_compatibility.sub('pf.', tagname)
1729 cls = g_tagname_to_class[tagname]
1730 kwargs = dict(iter(loader.construct_pairs(node, deep=True)))
1731 o = cls(**kwargs)
1732 o.validate(regularize=True, depth=1)
1733 return o
1736def include_constructor(loader, node):
1737 allow_include = loader._allow_include \
1738 if loader._allow_include is not None \
1739 else ALLOW_INCLUDE
1741 if not allow_include:
1742 raise EnvironmentError(
1743 'Not allowed to include YAML. Load with allow_include=True')
1745 if isinstance(node, yaml.nodes.ScalarNode):
1746 inc_file = loader.construct_scalar(node)
1747 else:
1748 raise TypeError('Unsupported YAML node %s' % repr(node))
1750 if loader._filename is not None and not op.isabs(inc_file):
1751 inc_file = op.join(op.dirname(loader._filename), inc_file)
1753 if not op.isfile(inc_file):
1754 raise FileNotFoundError(inc_file)
1756 included_files = list(loader._included_files)
1757 if loader._filename is not None:
1758 included_files.append(op.abspath(loader._filename))
1760 for included_file in loader._included_files:
1761 if op.samefile(inc_file, included_file):
1762 raise ImportError(
1763 'Circular import of file "%s". Include path: %s' % (
1764 op.abspath(inc_file),
1765 ' -> '.join('"%s"' % s for s in included_files)))
1767 with open(inc_file, 'rb') as f:
1768 return _load(
1769 f,
1770 Loader=loader.__class__, filename=inc_file,
1771 allow_include=True,
1772 included_files=included_files)
1775def dict_noflow_representer(dumper, data):
1776 return dumper.represent_mapping(
1777 'tag:yaml.org,2002:map', data, flow_style=False)
1780yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper)
1781yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader)
1782yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader)
1783yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper)
1786def newstr_representer(dumper, data):
1787 return dumper.represent_scalar(
1788 'tag:yaml.org,2002:str', unicode(data))
1791yaml.add_representer(newstr, newstr_representer, Dumper=GutsSafeDumper)
1794class Constructor(object):
1795 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None,
1796 ns_ignore=False):
1798 self.stack = []
1799 self.queue = []
1800 self.namespaces = defaultdict(list)
1801 self.add_namespace_maps = add_namespace_maps
1802 self.strict = strict
1803 self.ns_hints = ns_hints
1804 self.ns_ignore = ns_ignore
1806 def start_element(self, ns_name, attrs):
1807 if self.ns_ignore:
1808 ns_name = ns_name.split(' ')[-1]
1810 if -1 == ns_name.find(' '):
1811 if self.ns_hints is None and ns_name in g_guessable_xmlns:
1812 self.ns_hints = g_guessable_xmlns[ns_name]
1814 if self.ns_hints:
1815 ns_names = [
1816 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints]
1818 elif self.ns_hints is None:
1819 ns_names = [' ' + ns_name]
1821 else:
1822 ns_names = [ns_name]
1824 for ns_name in ns_names:
1825 if self.stack and self.stack[-1][1] is not None:
1826 cls = self.stack[-1][1].T.xmltagname_to_class.get(
1827 ns_name, None)
1829 if isinstance(cls, tuple):
1830 cls = None
1831 else:
1832 if cls is not None and (
1833 not issubclass(cls, Object)
1834 or issubclass(cls, SObject)):
1835 cls = None
1836 else:
1837 cls = g_xmltagname_to_class.get(ns_name, None)
1839 if cls:
1840 break
1842 self.stack.append((ns_name, cls, attrs, [], []))
1844 def end_element(self, _):
1845 ns_name, cls, attrs, content2, content1 = self.stack.pop()
1847 ns = ns_name.split(' ', 1)[0]
1849 if cls is not None:
1850 content2.extend(
1851 (ns + ' ' + k if -1 == k.find(' ') else k, v)
1852 for (k, v) in attrs.items())
1853 content2.append((None, ''.join(content1)))
1854 o = cls(**cls.T.translate_from_xml(content2, self.strict))
1855 o.validate(regularize=True, depth=1)
1856 if self.add_namespace_maps:
1857 o.namespace_map = self.get_current_namespace_map()
1859 if self.stack and not all(x[1] is None for x in self.stack):
1860 self.stack[-1][-2].append((ns_name, o))
1861 else:
1862 self.queue.append(o)
1863 else:
1864 content = [''.join(content1)]
1865 if self.stack:
1866 for c in content:
1867 self.stack[-1][-2].append((ns_name, c))
1869 def characters(self, char_content):
1870 if self.stack:
1871 self.stack[-1][-1].append(char_content)
1873 def start_namespace(self, ns, uri):
1874 self.namespaces[ns].append(uri)
1876 def end_namespace(self, ns):
1877 self.namespaces[ns].pop()
1879 def get_current_namespace_map(self):
1880 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v)
1882 def get_queued_elements(self):
1883 queue = self.queue
1884 self.queue = []
1885 return queue
1888def _iload_all_xml(
1889 stream,
1890 bufsize=100000,
1891 add_namespace_maps=False,
1892 strict=False,
1893 ns_hints=None,
1894 ns_ignore=False):
1896 from xml.parsers.expat import ParserCreate
1898 parser = ParserCreate('UTF-8', namespace_separator=' ')
1900 handler = Constructor(
1901 add_namespace_maps=add_namespace_maps,
1902 strict=strict,
1903 ns_hints=ns_hints,
1904 ns_ignore=ns_ignore)
1906 parser.StartElementHandler = handler.start_element
1907 parser.EndElementHandler = handler.end_element
1908 parser.CharacterDataHandler = handler.characters
1909 parser.StartNamespaceDeclHandler = handler.start_namespace
1910 parser.EndNamespaceDeclHandler = handler.end_namespace
1912 while True:
1913 data = stream.read(bufsize)
1914 parser.Parse(data, bool(not data))
1915 for element in handler.get_queued_elements():
1916 yield element
1918 if not data:
1919 break
1922def _load_all_xml(*args, **kwargs):
1923 return list(_iload_all_xml(*args, **kwargs))
1926def _load_xml(*args, **kwargs):
1927 g = _iload_all_xml(*args, **kwargs)
1928 return next(g)
1931def _dump_all_xml(objects, stream, root_element_name='root', header=True):
1933 if not getattr(stream, 'encoding', None):
1934 enc = encode_utf8
1935 else:
1936 enc = no_encode
1938 _dump_xml_header(stream, header)
1940 beg = u'<%s>\n' % root_element_name
1941 end = u'</%s>\n' % root_element_name
1943 stream.write(enc(beg))
1945 for ob in objects:
1946 _dump_xml(ob, stream=stream)
1948 stream.write(enc(end))
1951def _dump_xml_header(stream, banner=None):
1953 if not getattr(stream, 'encoding', None):
1954 enc = encode_utf8
1955 else:
1956 enc = no_encode
1958 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n'))
1959 if isinstance(banner, (str, newstr)):
1960 stream.write(enc(u'<!-- %s -->\n' % banner))
1963def _dump_xml(
1964 obj, stream, depth=0, ns_name=None, header=False, ns_map=[],
1965 ns_ignore=False):
1967 from xml.sax.saxutils import escape, quoteattr
1969 if not getattr(stream, 'encoding', None):
1970 enc = encode_utf8
1971 else:
1972 enc = no_encode
1974 if depth == 0 and header:
1975 _dump_xml_header(stream, header)
1977 indent = ' '*depth*2
1978 if ns_name is None:
1979 ns_name = obj.T.instance.get_xmltagname()
1981 if -1 != ns_name.find(' '):
1982 ns, name = ns_name.split(' ')
1983 else:
1984 ns, name = '', ns_name
1986 if isinstance(obj, Object):
1987 obj.validate(depth=1)
1988 attrs = []
1989 elems = []
1991 added_ns = False
1992 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns):
1993 attrs.append(('xmlns', ns))
1994 ns_map.append(ns)
1995 added_ns = True
1997 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True):
1998 if prop.xmlstyle == 'attribute':
1999 assert not prop.multivalued
2000 assert not isinstance(v, Object)
2001 attrs.append((prop.effective_xmltagname, v))
2003 elif prop.xmlstyle == 'content':
2004 assert not prop.multivalued
2005 assert not isinstance(v, Object)
2006 elems.append((None, v))
2008 else:
2009 prop.extend_xmlelements(elems, v)
2011 attr_str = ''
2012 if attrs:
2013 attr_str = ' ' + ' '.join(
2014 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v)))
2015 for (k, v) in attrs)
2017 if not elems:
2018 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str)))
2019 else:
2020 oneline = len(elems) == 1 and elems[0][0] is None
2021 stream.write(enc(u'%s<%s%s>%s' % (
2022 indent,
2023 name,
2024 attr_str,
2025 '' if oneline else '\n')))
2027 for (k, v) in elems:
2028 if k is None:
2029 stream.write(enc(escape(newstr(v), {'\0': '�'})))
2030 else:
2031 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore)
2033 stream.write(enc(u'%s</%s>\n' % (
2034 '' if oneline else indent, name)))
2036 if added_ns:
2037 ns_map.pop()
2039 else:
2040 stream.write(enc(u'%s<%s>%s</%s>\n' % (
2041 indent,
2042 name,
2043 escape(newstr(obj), {'\0': '�'}),
2044 name)))
2047def walk(x, typ=None, path=()):
2048 if typ is None or isinstance(x, typ):
2049 yield path, x
2051 if isinstance(x, Object):
2052 for (prop, val) in x.T.ipropvals(x):
2053 if prop.multivalued:
2054 if val is not None:
2055 for iele, ele in enumerate(val):
2056 for y in walk(ele, typ,
2057 path=path + ((prop.name, iele),)):
2058 yield y
2059 else:
2060 for y in walk(val, typ, path=path+(prop.name,)):
2061 yield y
2064def clone(x, pool=None):
2065 '''
2066 Clone guts object tree.
2068 Traverses guts object tree and recursively clones all guts attributes,
2069 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects
2070 deriving from :py:class:`Object` are instantiated using their respective
2071 init function. Multiply referenced objects in the source tree are multiply
2072 referenced also in the destination tree.
2074 This function can be used to clone guts objects ignoring any contained
2075 run-time state, i.e. any of their attributes not defined as a guts
2076 property.
2077 '''
2079 if pool is None:
2080 pool = {}
2082 if id(x) in pool:
2083 x_copy = pool[id(x)]
2085 else:
2086 if isinstance(x, SObject):
2087 x_copy = x.__class__(str(x))
2088 elif isinstance(x, Object):
2089 d = {}
2090 for (prop, y) in x.T.ipropvals(x):
2091 if y is not None:
2092 if not prop.multivalued:
2093 y_copy = clone(y, pool)
2094 elif prop.multivalued is dict:
2095 y_copy = dict(
2096 (clone(zk, pool), clone(zv, pool))
2097 for (zk, zv) in y.items())
2098 else:
2099 y_copy = type(y)(clone(z, pool) for z in y)
2100 else:
2101 y_copy = y
2103 d[prop.name] = y_copy
2105 x_copy = x.__class__(**d)
2107 else:
2108 x_copy = copy.deepcopy(x)
2110 pool[id(x)] = x_copy
2111 return x_copy
2114class YPathError(Exception):
2115 '''
2116 This exception is raised for invalid ypath specifications.
2117 '''
2118 pass
2121def _parse_yname(yname):
2122 ident = r'[a-zA-Z][a-zA-Z0-9_]*'
2123 rint = r'-?[0-9]+'
2124 m = re.match(
2125 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$'
2126 % (ident, rint, rint, rint), yname)
2128 if not m:
2129 raise YPathError('Syntax error in component: "%s"' % yname)
2131 d = dict(
2132 name=m.group(1))
2134 if m.group(2):
2135 if m.group(5):
2136 istart = iend = None
2137 if m.group(4):
2138 istart = int(m.group(4))
2139 if m.group(6):
2140 iend = int(m.group(6))
2142 d['slice'] = (istart, iend)
2143 else:
2144 d['index'] = int(m.group(7))
2146 return d
2149def _decend(obj, ynames):
2150 if ynames:
2151 for sobj in iter_elements(obj, ynames):
2152 yield sobj
2153 else:
2154 yield obj
2157def iter_elements(obj, ypath):
2158 '''
2159 Generator yielding elements matching a given ypath specification.
2161 :param obj: guts :py:class:`Object` instance
2162 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2163 To access list objects use slice notatation (e.g.
2164 'root.child[:].child[1:3].child[1]').
2166 Raises :py:exc:`YPathError` on failure.
2167 '''
2169 try:
2170 if isinstance(ypath, str):
2171 ynames = ypath.split('.')
2172 else:
2173 ynames = ypath
2175 yname = ynames[0]
2176 ynames = ynames[1:]
2177 d = _parse_yname(yname)
2178 if d['name'] not in obj.T.propnames:
2179 raise AttributeError(d['name'])
2181 obj = getattr(obj, d['name'])
2183 if 'index' in d:
2184 sobj = obj[d['index']]
2185 for ssobj in _decend(sobj, ynames):
2186 yield ssobj
2188 elif 'slice' in d:
2189 for i in range(*slice(*d['slice']).indices(len(obj))):
2190 sobj = obj[i]
2191 for ssobj in _decend(sobj, ynames):
2192 yield ssobj
2193 else:
2194 for sobj in _decend(obj, ynames):
2195 yield sobj
2197 except (AttributeError, IndexError) as e:
2198 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2201def get_elements(obj, ypath):
2202 '''
2203 Get all elements matching a given ypath specification.
2205 :param obj: guts :py:class:`Object` instance
2206 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2207 To access list objects use slice notatation (e.g.
2208 'root.child[:].child[1:3].child[1]').
2210 Raises :py:exc:`YPathError` on failure.
2211 '''
2212 return list(iter_elements(obj, ypath))
2215def set_elements(obj, ypath, value, validate=False, regularize=False):
2216 '''
2217 Set elements matching a given ypath specification.
2219 :param obj: guts :py:class:`Object` instance
2220 :param ypath: Dot-separated object path (e.g. 'root.child.child').
2221 To access list objects use slice notatation (e.g.
2222 'root.child[:].child[1:3].child[1]').
2223 :param value: All matching elements will be set to `value`.
2224 :param validate: Whether to validate affected subtrees.
2225 :param regularize: Whether to regularize affected subtrees.
2227 Raises :py:exc:`YPathError` on failure.
2228 '''
2230 ynames = ypath.split('.')
2231 try:
2232 d = _parse_yname(ynames[-1])
2233 for sobj in iter_elements(obj, ynames[:-1]):
2234 if d['name'] not in sobj.T.propnames:
2235 raise AttributeError(d['name'])
2237 if 'index' in d:
2238 ssobj = getattr(sobj, d['name'])
2239 ssobj[d['index']] = value
2240 elif 'slice' in d:
2241 ssobj = getattr(sobj, d['name'])
2242 for i in range(*slice(*d['slice']).indices(len(ssobj))):
2243 ssobj[i] = value
2244 else:
2245 setattr(sobj, d['name'], value)
2246 if regularize:
2247 sobj.regularize()
2248 if validate:
2249 sobj.validate()
2251 except (AttributeError, IndexError, YPathError) as e:
2252 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e)))
2255def zip_walk(x, typ=None, path=(), stack=()):
2256 if typ is None or isinstance(x, typ):
2257 yield path, stack + (x,)
2259 if isinstance(x, Object):
2260 for (prop, val) in x.T.ipropvals(x):
2261 if prop.multivalued:
2262 if val is not None:
2263 for iele, ele in enumerate(val):
2264 for y in zip_walk(
2265 ele, typ,
2266 path=path + ((prop.name, iele),),
2267 stack=stack + (x,)):
2269 yield y
2270 else:
2271 for y in zip_walk(val, typ,
2272 path=path+(prop.name,),
2273 stack=stack + (x,)):
2274 yield y
2277def path_element(x):
2278 if isinstance(x, tuple):
2279 return '%s[%i]' % x
2280 else:
2281 return x
2284def path_to_str(path):
2285 return '.'.join(path_element(x) for x in path)
2288@expand_stream_args('w')
2289def dump(*args, **kwargs):
2290 return _dump(*args, **kwargs)
2293@expand_stream_args('r')
2294def load(*args, **kwargs):
2295 return _load(*args, **kwargs)
2298def load_string(s, *args, **kwargs):
2299 return load(string=s, *args, **kwargs)
2302@expand_stream_args('w')
2303def dump_all(*args, **kwargs):
2304 return _dump_all(*args, **kwargs)
2307@expand_stream_args('r')
2308def load_all(*args, **kwargs):
2309 return _load_all(*args, **kwargs)
2312@expand_stream_args('r')
2313def iload_all(*args, **kwargs):
2314 return _iload_all(*args, **kwargs)
2317@expand_stream_args('w')
2318def dump_xml(*args, **kwargs):
2319 return _dump_xml(*args, **kwargs)
2322@expand_stream_args('r')
2323def load_xml(*args, **kwargs):
2324 kwargs.pop('filename', None)
2325 return _load_xml(*args, **kwargs)
2328def load_xml_string(s, *args, **kwargs):
2329 return load_xml(string=s, *args, **kwargs)
2332@expand_stream_args('w')
2333def dump_all_xml(*args, **kwargs):
2334 return _dump_all_xml(*args, **kwargs)
2337@expand_stream_args('r')
2338def load_all_xml(*args, **kwargs):
2339 kwargs.pop('filename', None)
2340 return _load_all_xml(*args, **kwargs)
2343@expand_stream_args('r')
2344def iload_all_xml(*args, **kwargs):
2345 kwargs.pop('filename', None)
2346 return _iload_all_xml(*args, **kwargs)
2349__all__ = guts_types + [
2350 'guts_types', 'TBase', 'ValidationError',
2351 'ArgumentError', 'Defer',
2352 'dump', 'load',
2353 'dump_all', 'load_all', 'iload_all',
2354 'dump_xml', 'load_xml',
2355 'dump_all_xml', 'load_all_xml', 'iload_all_xml',
2356 'load_string',
2357 'load_xml_string',
2358 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str'
2359]