1# http://pyrocko.org - GPLv3 

2# 

3# The Pyrocko Developers, 21st Century 

4# ---|P------/S----------~Lg---------- 

5''' 

6Lightweight declarative YAML and XML data binding for Python. 

7''' 

8 

9import datetime 

10import calendar 

11import re 

12import sys 

13import types 

14import copy 

15import os.path as op 

16from collections import defaultdict 

17 

18from io import BytesIO 

19 

20try: 

21 import numpy as num 

22except ImportError: 

23 num = None 

24 

25import yaml 

26try: 

27 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper 

28except ImportError: 

29 from yaml import SafeLoader, SafeDumper 

30 

31from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \ 

32 get_time_float 

33 

34 

35ALLOW_INCLUDE = False 

36 

37 

38class GutsSafeDumper(SafeDumper): 

39 pass 

40 

41 

42class GutsSafeLoader(SafeLoader): 

43 pass 

44 

45 

46g_iprop = 0 

47 

48g_deferred = {} 

49g_deferred_content = {} 

50 

51g_tagname_to_class = {} 

52g_xmltagname_to_class = {} 

53g_guessable_xmlns = {} 

54 

55guts_types = [ 

56 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float', 

57 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern', 

58 'UnicodePattern', 'StringChoice', 'IntChoice', 'List', 'Dict', 'Tuple', 

59 'Union', 'Choice', 'Any'] 

60 

61us_to_cc_regex = re.compile(r'([a-z])_([a-z])') 

62 

63 

64class literal(str): 

65 pass 

66 

67 

68class folded(str): 

69 pass 

70 

71 

72class singlequoted(str): 

73 pass 

74 

75 

76class doublequoted(str): 

77 pass 

78 

79 

80def make_str_presenter(style): 

81 def presenter(dumper, data): 

82 return dumper.represent_scalar( 

83 'tag:yaml.org,2002:str', str(data), style=style) 

84 

85 return presenter 

86 

87 

88str_style_map = { 

89 None: lambda x: x, 

90 '|': literal, 

91 '>': folded, 

92 "'": singlequoted, 

93 '"': doublequoted} 

94 

95for (style, cls) in str_style_map.items(): 

96 if style: 

97 GutsSafeDumper.add_representer(cls, make_str_presenter(style)) 

98 

99 

100class uliteral(str): 

101 pass 

102 

103 

104class ufolded(str): 

105 pass 

106 

107 

108class usinglequoted(str): 

109 pass 

110 

111 

112class udoublequoted(str): 

113 pass 

114 

115 

116def make_unicode_presenter(style): 

117 def presenter(dumper, data): 

118 return dumper.represent_scalar( 

119 'tag:yaml.org,2002:str', str(data), style=style) 

120 

121 return presenter 

122 

123 

124unicode_style_map = { 

125 None: lambda x: x, 

126 '|': literal, 

127 '>': folded, 

128 "'": singlequoted, 

129 '"': doublequoted} 

130 

131for (style, cls) in unicode_style_map.items(): 

132 if style: 

133 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style)) 

134 

135 

136class blist(list): 

137 pass 

138 

139 

140class flist(list): 

141 pass 

142 

143 

144list_style_map = { 

145 None: list, 

146 'block': blist, 

147 'flow': flist} 

148 

149 

150def make_list_presenter(flow_style): 

151 def presenter(dumper, data): 

152 return dumper.represent_sequence( 

153 'tag:yaml.org,2002:seq', data, flow_style=flow_style) 

154 

155 return presenter 

156 

157 

158GutsSafeDumper.add_representer(blist, make_list_presenter(False)) 

159GutsSafeDumper.add_representer(flist, make_list_presenter(True)) 

160 

161if num: 

162 def numpy_float_presenter(dumper, data): 

163 return dumper.represent_float(float(data)) 

164 

165 def numpy_int_presenter(dumper, data): 

166 return dumper.represent_int(int(data)) 

167 

168 for dtype in (num.float64, num.float32): 

169 GutsSafeDumper.add_representer(dtype, numpy_float_presenter) 

170 

171 for dtype in (num.int32, num.int64): 

172 GutsSafeDumper.add_representer(dtype, numpy_int_presenter) 

173 

174 

175def us_to_cc(s): 

176 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s) 

177 

178 

179cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)') 

180cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])') 

181 

182 

183def cc_to_us(s): 

184 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub( 

185 '\\1_\\2\\3', s)).lower() 

186 

187 

188re_frac = re.compile(r'\.[1-9]FRAC') 

189frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789']) 

190 

191 

192def encode_utf8(s): 

193 return s.encode('utf-8') 

194 

195 

196def no_encode(s): 

197 return s 

198 

199 

200def make_xmltagname_from_name(name): 

201 return us_to_cc(name) 

202 

203 

204def make_name_from_xmltagname(xmltagname): 

205 return cc_to_us(xmltagname) 

206 

207 

208def make_content_name(name): 

209 if name.endswith('_list'): 

210 return name[:-5] 

211 elif name.endswith('s'): 

212 return name[:-1] 

213 else: 

214 return name 

215 

216 

217def classnames(cls): 

218 if isinstance(cls, tuple): 

219 return '(%s)' % ', '.join(x.__name__ for x in cls) 

220 else: 

221 return cls.__name__ 

222 

223 

224def expand_stream_args(mode): 

225 def wrap(f): 

226 ''' 

227 Decorator to enhance functions taking stream objects. 

228 

229 Wraps a function f(..., stream, ...) so that it can also be called as 

230 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...). 

231 ''' 

232 

233 def g(*args, **kwargs): 

234 stream = kwargs.pop('stream', None) 

235 filename = kwargs.get('filename', None) 

236 if mode != 'r': 

237 filename = kwargs.pop('filename', None) 

238 string = kwargs.pop('string', None) 

239 

240 assert sum(x is not None for x in (stream, filename, string)) <= 1 

241 

242 if stream is not None: 

243 kwargs['stream'] = stream 

244 return f(*args, **kwargs) 

245 

246 elif filename is not None: 

247 stream = open(filename, mode+'b') 

248 kwargs['stream'] = stream 

249 retval = f(*args, **kwargs) 

250 if isinstance(retval, types.GeneratorType): 

251 def wrap_generator(gen): 

252 try: 

253 for x in gen: 

254 yield x 

255 

256 except GeneratorExit: 

257 pass 

258 

259 stream.close() 

260 

261 return wrap_generator(retval) 

262 

263 else: 

264 stream.close() 

265 return retval 

266 

267 elif string is not None: 

268 assert mode == 'r', \ 

269 'Keyword argument string=... cannot be used in dumper ' \ 

270 'function.' 

271 

272 kwargs['stream'] = BytesIO(string.encode('utf-8')) 

273 return f(*args, **kwargs) 

274 

275 else: 

276 assert mode == 'w', \ 

277 'Use keyword argument stream=... or filename=... in ' \ 

278 'loader function.' 

279 

280 sout = BytesIO() 

281 f(stream=sout, *args, **kwargs) 

282 return sout.getvalue().decode('utf-8') 

283 

284 return g 

285 

286 return wrap 

287 

288 

289class Defer(object): 

290 def __init__(self, classname, *args, **kwargs): 

291 global g_iprop 

292 if kwargs.get('position', None) is None: 

293 kwargs['position'] = g_iprop 

294 

295 g_iprop += 1 

296 

297 self.classname = classname 

298 self.args = args 

299 self.kwargs = kwargs 

300 

301 

302class TBase(object): 

303 

304 strict = False 

305 multivalued = None 

306 force_regularize = False 

307 propnames = [] 

308 

309 @classmethod 

310 def init_propertystuff(cls): 

311 cls.properties = [] 

312 cls.xmltagname_to_name = {} 

313 cls.xmltagname_to_name_multivalued = {} 

314 cls.xmltagname_to_class = {} 

315 cls.content_property = None 

316 

317 def __init__( 

318 self, 

319 default=None, 

320 optional=False, 

321 xmlstyle='element', 

322 xmltagname=None, 

323 xmlns=None, 

324 help=None, 

325 position=None): 

326 

327 global g_iprop 

328 if position is not None: 

329 self.position = position 

330 else: 

331 self.position = g_iprop 

332 

333 g_iprop += 1 

334 self._default = default 

335 

336 self.optional = optional 

337 self.name = None 

338 self._xmltagname = xmltagname 

339 self._xmlns = xmlns 

340 self.parent = None 

341 self.xmlstyle = xmlstyle 

342 self.help = help 

343 

344 def default(self): 

345 return make_default(self._default) 

346 

347 def is_default(self, val): 

348 if self._default is None: 

349 return val is None 

350 else: 

351 return self._default == val 

352 

353 def has_default(self): 

354 return self._default is not None 

355 

356 def xname(self): 

357 if self.name is not None: 

358 return self.name 

359 elif self.parent is not None: 

360 return 'element of %s' % self.parent.xname() 

361 else: 

362 return '?' 

363 

364 def set_xmlns(self, xmlns): 

365 if self._xmlns is None and not self.xmlns: 

366 self._xmlns = xmlns 

367 

368 if self.multivalued: 

369 self.content_t.set_xmlns(xmlns) 

370 

371 def get_xmlns(self): 

372 return self._xmlns or self.xmlns 

373 

374 def get_xmltagname(self): 

375 if self._xmltagname is not None: 

376 return self.get_xmlns() + ' ' + self._xmltagname 

377 elif self.name: 

378 return self.get_xmlns() + ' ' \ 

379 + make_xmltagname_from_name(self.name) 

380 elif self.xmltagname: 

381 return self.get_xmlns() + ' ' + self.xmltagname 

382 else: 

383 assert False 

384 

385 @classmethod 

386 def get_property(cls, name): 

387 for prop in cls.properties: 

388 if prop.name == name: 

389 return prop 

390 

391 raise ValueError() 

392 

393 @classmethod 

394 def remove_property(cls, name): 

395 

396 prop = cls.get_property(name) 

397 

398 if not prop.multivalued: 

399 del cls.xmltagname_to_class[prop.effective_xmltagname] 

400 del cls.xmltagname_to_name[prop.effective_xmltagname] 

401 else: 

402 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname] 

403 del cls.xmltagname_to_name_multivalued[ 

404 prop.content_t.effective_xmltagname] 

405 

406 if cls.content_property is prop: 

407 cls.content_property = None 

408 

409 cls.properties.remove(prop) 

410 cls.propnames.remove(name) 

411 

412 return prop 

413 

414 @classmethod 

415 def add_property(cls, name, prop): 

416 

417 prop.instance = prop 

418 prop.name = name 

419 prop.set_xmlns(cls.xmlns) 

420 

421 if isinstance(prop, Choice.T): 

422 for tc in prop.choices: 

423 tc.effective_xmltagname = tc.get_xmltagname() 

424 cls.xmltagname_to_class[tc.effective_xmltagname] = tc.cls 

425 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name 

426 elif not prop.multivalued: 

427 prop.effective_xmltagname = prop.get_xmltagname() 

428 cls.xmltagname_to_class[prop.effective_xmltagname] = prop.cls 

429 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name 

430 else: 

431 prop.content_t.name = make_content_name(prop.name) 

432 prop.content_t.effective_xmltagname = \ 

433 prop.content_t.get_xmltagname() 

434 cls.xmltagname_to_class[ 

435 prop.content_t.effective_xmltagname] = prop.content_t.cls 

436 cls.xmltagname_to_name_multivalued[ 

437 prop.content_t.effective_xmltagname] = prop.name 

438 

439 cls.properties.append(prop) 

440 

441 cls.properties.sort(key=lambda x: x.position) 

442 

443 cls.propnames = [p.name for p in cls.properties] 

444 

445 if prop.xmlstyle == 'content': 

446 cls.content_property = prop 

447 

448 @classmethod 

449 def ivals(cls, val): 

450 for prop in cls.properties: 

451 yield getattr(val, prop.name) 

452 

453 @classmethod 

454 def ipropvals(cls, val): 

455 for prop in cls.properties: 

456 yield prop, getattr(val, prop.name) 

457 

458 @classmethod 

459 def inamevals(cls, val): 

460 for prop in cls.properties: 

461 yield prop.name, getattr(val, prop.name) 

462 

463 @classmethod 

464 def ipropvals_to_save(cls, val, xmlmode=False): 

465 for prop in cls.properties: 

466 v = getattr(val, prop.name) 

467 if v is not None and ( 

468 not (prop.optional or (prop.multivalued and not v)) 

469 or (not prop.is_default(v))): 

470 

471 if xmlmode: 

472 yield prop, prop.to_save_xml(v) 

473 else: 

474 yield prop, prop.to_save(v) 

475 

476 @classmethod 

477 def inamevals_to_save(cls, val, xmlmode=False): 

478 for prop, v in cls.ipropvals_to_save(val, xmlmode): 

479 yield prop.name, v 

480 

481 @classmethod 

482 def translate_from_xml(cls, list_of_pairs, strict): 

483 d = {} 

484 for k, v in list_of_pairs: 

485 if k in cls.xmltagname_to_name_multivalued: 

486 k2 = cls.xmltagname_to_name_multivalued[k] 

487 if k2 not in d: 

488 d[k2] = [] 

489 

490 d[k2].append(v) 

491 elif k in cls.xmltagname_to_name: 

492 k2 = cls.xmltagname_to_name[k] 

493 if k2 in d: 

494 raise ArgumentError( 

495 'Unexpectedly found more than one child element "%s" ' 

496 'within "%s".' % (k, cls.tagname)) 

497 

498 d[k2] = v 

499 elif k is None: 

500 if cls.content_property: 

501 k2 = cls.content_property.name 

502 d[k2] = v 

503 else: 

504 if strict: 

505 raise ArgumentError( 

506 'Unexpected child element "%s" found within "%s".' % ( 

507 k, cls.tagname)) 

508 

509 return d 

510 

511 def validate(self, val, regularize=False, depth=-1): 

512 if self.optional and val is None: 

513 return val 

514 

515 is_derived = isinstance(val, self.cls) 

516 is_exact = type(val) == self.cls 

517 

518 not_ok = not self.strict and not is_derived or \ 

519 self.strict and not is_exact 

520 

521 if not_ok or self.force_regularize: 

522 if regularize: 

523 try: 

524 val = self.regularize_extra(val) 

525 except ValueError: 

526 raise ValidationError( 

527 '%s: could not convert "%s" to type %s' % ( 

528 self.xname(), val, classnames(self.cls))) 

529 else: 

530 raise ValidationError( 

531 '%s: "%s" (type: %s) is not of type %s' % ( 

532 self.xname(), val, type(val), classnames(self.cls))) 

533 

534 validator = self 

535 if isinstance(self.cls, tuple): 

536 clss = self.cls 

537 else: 

538 clss = (self.cls,) 

539 

540 for cls in clss: 

541 try: 

542 if type(val) != cls and isinstance(val, cls): 

543 validator = val.T.instance 

544 

545 except AttributeError: 

546 pass 

547 

548 validator.validate_extra(val) 

549 

550 if depth != 0: 

551 val = validator.validate_children(val, regularize, depth) 

552 

553 return val 

554 

555 def regularize_extra(self, val): 

556 return self.cls(val) 

557 

558 def validate_extra(self, val): 

559 pass 

560 

561 def validate_children(self, val, regularize, depth): 

562 for prop, propval in self.ipropvals(val): 

563 newpropval = prop.validate(propval, regularize, depth-1) 

564 if regularize and (newpropval is not propval): 

565 setattr(val, prop.name, newpropval) 

566 

567 return val 

568 

569 def to_save(self, val): 

570 return val 

571 

572 def to_save_xml(self, val): 

573 return self.to_save(val) 

574 

575 def extend_xmlelements(self, elems, v): 

576 if self.multivalued: 

577 for x in v: 

578 elems.append((self.content_t.effective_xmltagname, x)) 

579 else: 

580 elems.append((self.effective_xmltagname, v)) 

581 

582 def deferred(self): 

583 return [] 

584 

585 def classname_for_help(self, strip_module=''): 

586 

587 if self.dummy_cls is not self.cls: 

588 if self.dummy_cls.__module__ == strip_module: 

589 sadd = ' (:py:class:`%s`)' % ( 

590 self.dummy_cls.__name__) 

591 else: 

592 sadd = ' (:py:class:`%s.%s`)' % ( 

593 self.dummy_cls.__module__, self.dummy_cls.__name__) 

594 else: 

595 sadd = '' 

596 

597 if self.dummy_cls in guts_plain_dummy_types: 

598 return '``%s``' % self.cls.__name__ 

599 

600 elif self.dummy_cls.dummy_for_description: 

601 return '%s%s' % (self.dummy_cls.dummy_for_description, sadd) 

602 

603 else: 

604 def sclass(cls): 

605 mod = cls.__module__ 

606 clsn = cls.__name__ 

607 if mod == '__builtin__' or mod == 'builtins': 

608 return '``%s``' % clsn 

609 

610 elif mod == strip_module: 

611 return ':py:class:`%s`' % clsn 

612 

613 else: 

614 return ':py:class:`%s.%s`' % (mod, clsn) 

615 

616 if isinstance(self.cls, tuple): 

617 return '(%s)%s' % ( 

618 ' | '.join(sclass(cls) for cls in self.cls), sadd) 

619 else: 

620 return '%s%s' % (sclass(self.cls), sadd) 

621 

622 @classmethod 

623 def props_help_string(cls): 

624 baseprops = [] 

625 for base in cls.dummy_cls.__bases__: 

626 if hasattr(base, 'T'): 

627 baseprops.extend(base.T.properties) 

628 

629 hlp = [] 

630 hlp.append('') 

631 for prop in cls.properties: 

632 if prop in baseprops: 

633 continue 

634 

635 descr = [ 

636 prop.classname_for_help(strip_module=cls.dummy_cls.__module__)] 

637 

638 if prop.optional: 

639 descr.append('*optional*') 

640 

641 if isinstance(prop._default, DefaultMaker): 

642 descr.append('*default:* ``%s``' % repr(prop._default)) 

643 else: 

644 d = prop.default() 

645 if d is not None: 

646 descr.append('*default:* ``%s``' % repr(d)) 

647 

648 hlp.append(' .. py:gattribute:: %s' % prop.name) 

649 hlp.append('') 

650 hlp.append(' %s' % ', '.join(descr)) 

651 hlp.append(' ') 

652 if prop.help is not None: 

653 hlp.append(' %s' % prop.help) 

654 hlp.append('') 

655 

656 return '\n'.join(hlp) 

657 

658 @classmethod 

659 def class_help_string(cls): 

660 return cls.dummy_cls.__doc_template__ 

661 

662 @classmethod 

663 def class_signature(cls): 

664 r = [] 

665 for prop in cls.properties: 

666 d = prop.default() 

667 if d is not None: 

668 arg = repr(d) 

669 

670 elif prop.optional: 

671 arg = 'None' 

672 

673 else: 

674 arg = '...' 

675 

676 r.append('%s=%s' % (prop.name, arg)) 

677 

678 return '(%s)' % ', '.join(r) 

679 

680 @classmethod 

681 def help(cls): 

682 return cls.props_help_string() 

683 

684 

685class ObjectMetaClass(type): 

686 def __new__(meta, classname, bases, class_dict): 

687 cls = type.__new__(meta, classname, bases, class_dict) 

688 if classname != 'Object': 

689 t_class_attr_name = '_%s__T' % classname 

690 if not hasattr(cls, t_class_attr_name): 

691 if hasattr(cls, 'T'): 

692 class T(cls.T): 

693 pass 

694 else: 

695 class T(TBase): 

696 pass 

697 

698 setattr(cls, t_class_attr_name, T) 

699 

700 T = getattr(cls, t_class_attr_name) 

701 

702 if cls.dummy_for is not None: 

703 T.cls = cls.dummy_for 

704 else: 

705 T.cls = cls 

706 

707 T.dummy_cls = cls 

708 

709 if hasattr(cls, 'xmltagname'): 

710 T.xmltagname = cls.xmltagname 

711 else: 

712 T.xmltagname = classname 

713 

714 mod = sys.modules[cls.__module__] 

715 

716 if hasattr(cls, 'xmlns'): 

717 T.xmlns = cls.xmlns 

718 elif hasattr(mod, 'guts_xmlns'): 

719 T.xmlns = mod.guts_xmlns 

720 else: 

721 T.xmlns = '' 

722 

723 if T.xmlns and hasattr(cls, 'guessable_xmlns'): 

724 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns 

725 

726 if hasattr(mod, 'guts_prefix'): 

727 if mod.guts_prefix: 

728 T.tagname = mod.guts_prefix + '.' + classname 

729 else: 

730 T.tagname = classname 

731 else: 

732 if cls.__module__ != '__main__': 

733 T.tagname = cls.__module__ + '.' + classname 

734 else: 

735 T.tagname = classname 

736 

737 T.classname = classname 

738 

739 T.init_propertystuff() 

740 

741 for k in dir(cls): 

742 prop = getattr(cls, k) 

743 

744 if k.endswith('__'): 

745 k = k[:-2] 

746 

747 if isinstance(prop, TBase): 

748 if prop.deferred(): 

749 for defer in prop.deferred(): 

750 g_deferred_content.setdefault( 

751 defer.classname[:-2], []).append((prop, defer)) 

752 g_deferred.setdefault( 

753 defer.classname[:-2], []).append((T, k, prop)) 

754 

755 else: 

756 T.add_property(k, prop) 

757 

758 elif isinstance(prop, Defer): 

759 g_deferred.setdefault(prop.classname[:-2], []).append( 

760 (T, k, prop)) 

761 

762 if classname in g_deferred_content: 

763 for prop, defer in g_deferred_content[classname]: 

764 prop.process_deferred( 

765 defer, T(*defer.args, **defer.kwargs)) 

766 

767 del g_deferred_content[classname] 

768 

769 if classname in g_deferred: 

770 for (T_, k_, prop_) in g_deferred.get(classname, []): 

771 if isinstance(prop_, Defer): 

772 prop_ = T(*prop_.args, **prop_.kwargs) 

773 

774 if not prop_.deferred(): 

775 T_.add_property(k_, prop_) 

776 

777 del g_deferred[classname] 

778 

779 g_tagname_to_class[T.tagname] = cls 

780 if hasattr(cls, 'xmltagname'): 

781 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls 

782 

783 cls.T = T 

784 T.instance = T() 

785 

786 cls.__doc_template__ = cls.__doc__ 

787 cls.__doc__ = T.class_help_string() 

788 

789 if cls.__doc__ is None: 

790 cls.__doc__ = 'Undocumented.' 

791 

792 cls.__doc__ += '\n' + T.props_help_string() 

793 

794 return cls 

795 

796 

797class ValidationError(Exception): 

798 pass 

799 

800 

801class ArgumentError(Exception): 

802 pass 

803 

804 

805def make_default(x): 

806 if isinstance(x, DefaultMaker): 

807 return x.make() 

808 elif isinstance(x, Object): 

809 return clone(x) 

810 else: 

811 return x 

812 

813 

814class DefaultMaker(object): 

815 def make(self): 

816 raise NotImplementedError('Schould be implemented in subclass.') 

817 

818 

819class ObjectDefaultMaker(DefaultMaker): 

820 def __init__(self, cls, args, kwargs): 

821 DefaultMaker.__init__(self) 

822 self.cls = cls 

823 self.args = args 

824 self.kwargs = kwargs 

825 self.instance = None 

826 

827 def make(self): 

828 return self.cls( 

829 *[make_default(x) for x in self.args], 

830 **dict((k, make_default(v)) for (k, v) in self.kwargs.items())) 

831 

832 def __eq__(self, other): 

833 if self.instance is None: 

834 self.instance = self.make() 

835 

836 return self.instance == other 

837 

838 def __repr__(self): 

839 sargs = [] 

840 for arg in self.args: 

841 sargs.append(repr(arg)) 

842 

843 for k, v in self.kwargs.items(): 

844 sargs.append('%s=%s' % (k, repr(v))) 

845 

846 return '%s(%s)' % (self.cls.__name__, ', '.join(sargs)) 

847 

848 

849class TimestampDefaultMaker(DefaultMaker): 

850 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'): 

851 DefaultMaker.__init__(self) 

852 self._stime = s 

853 self._format = format 

854 

855 def make(self): 

856 return str_to_time(self._stime, self._format) 

857 

858 def __repr__(self): 

859 return "str_to_time(%s)" % repr(self._stime) 

860 

861 

862def with_metaclass(meta, *bases): 

863 # inlined py2/py3 compat solution from python-future 

864 class metaclass(meta): 

865 __call__ = type.__call__ 

866 __init__ = type.__init__ 

867 

868 def __new__(cls, name, this_bases, d): 

869 if this_bases is None: 

870 return type.__new__(cls, name, (), d) 

871 return meta(name, bases, d) 

872 

873 return metaclass('temp', None, {}) 

874 

875 

876class Object(with_metaclass(ObjectMetaClass, object)): 

877 dummy_for = None 

878 dummy_for_description = None 

879 

880 def __init__(self, **kwargs): 

881 if not kwargs.get('init_props', True): 

882 return 

883 

884 for prop in self.T.properties: 

885 k = prop.name 

886 if k in kwargs: 

887 setattr(self, k, kwargs.pop(k)) 

888 else: 

889 if not prop.optional and not prop.has_default(): 

890 raise ArgumentError('Missing argument to %s: %s' % ( 

891 self.T.tagname, prop.name)) 

892 else: 

893 setattr(self, k, prop.default()) 

894 

895 if kwargs: 

896 raise ArgumentError('Invalid argument to %s: %s' % ( 

897 self.T.tagname, ', '.join(list(kwargs.keys())))) 

898 

899 @classmethod 

900 def D(cls, *args, **kwargs): 

901 return ObjectDefaultMaker(cls, args, kwargs) 

902 

903 def validate(self, regularize=False, depth=-1): 

904 self.T.instance.validate(self, regularize, depth) 

905 

906 def regularize(self, depth=-1): 

907 self.validate(regularize=True, depth=depth) 

908 

909 def dump(self, stream=None, filename=None, header=False): 

910 return dump(self, stream=stream, filename=filename, header=header) 

911 

912 def dump_xml( 

913 self, stream=None, filename=None, header=False, ns_ignore=False): 

914 return dump_xml( 

915 self, stream=stream, filename=filename, header=header, 

916 ns_ignore=ns_ignore) 

917 

918 @classmethod 

919 def load(cls, stream=None, filename=None, string=None): 

920 return load(stream=stream, filename=filename, string=string) 

921 

922 @classmethod 

923 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None, 

924 ns_ignore=False): 

925 

926 if ns_hints is None: 

927 ns_hints = [cls.T.instance.get_xmlns()] 

928 

929 return load_xml( 

930 stream=stream, 

931 filename=filename, 

932 string=string, 

933 ns_hints=ns_hints, 

934 ns_ignore=ns_ignore) 

935 

936 def __str__(self): 

937 return self.dump() 

938 

939 

940def to_dict(obj): 

941 ''' 

942 Get dict of guts object attributes. 

943 

944 :param obj: :py:class`Object` object 

945 ''' 

946 

947 return dict(obj.T.inamevals(obj)) 

948 

949 

950class SObject(Object): 

951 

952 class __T(TBase): 

953 def regularize_extra(self, val): 

954 if isinstance(val, str): 

955 return self.cls(val) 

956 

957 return val 

958 

959 def to_save(self, val): 

960 return str(val) 

961 

962 def to_save_xml(self, val): 

963 return str(val) 

964 

965 

966class Any(Object): 

967 

968 class __T(TBase): 

969 def validate(self, val, regularize=False, depth=-1): 

970 if isinstance(val, Object): 

971 val.validate(regularize, depth) 

972 

973 return val 

974 

975 

976class Int(Object): 

977 dummy_for = int 

978 

979 class __T(TBase): 

980 strict = True 

981 

982 def to_save_xml(self, value): 

983 return repr(value) 

984 

985 

986class Float(Object): 

987 dummy_for = float 

988 

989 class __T(TBase): 

990 strict = True 

991 

992 def to_save_xml(self, value): 

993 return repr(value) 

994 

995 

996class Complex(Object): 

997 dummy_for = complex 

998 

999 class __T(TBase): 

1000 strict = True 

1001 

1002 def regularize_extra(self, val): 

1003 

1004 if isinstance(val, list) or isinstance(val, tuple): 

1005 assert len(val) == 2 

1006 val = complex(*val) 

1007 

1008 elif not isinstance(val, complex): 

1009 val = complex(val) 

1010 

1011 return val 

1012 

1013 def to_save(self, value): 

1014 return repr(value) 

1015 

1016 def to_save_xml(self, value): 

1017 return repr(value) 

1018 

1019 

1020class Bool(Object): 

1021 dummy_for = bool 

1022 

1023 class __T(TBase): 

1024 strict = True 

1025 

1026 def regularize_extra(self, val): 

1027 if isinstance(val, str): 

1028 if val.lower().strip() in ('0', 'false'): 

1029 return False 

1030 

1031 return bool(val) 

1032 

1033 def to_save_xml(self, value): 

1034 return repr(bool(value)).lower() 

1035 

1036 

1037class String(Object): 

1038 dummy_for = str 

1039 

1040 class __T(TBase): 

1041 def __init__(self, *args, **kwargs): 

1042 yamlstyle = kwargs.pop('yamlstyle', None) 

1043 TBase.__init__(self, *args, **kwargs) 

1044 self.style_cls = str_style_map[yamlstyle] 

1045 

1046 def to_save(self, val): 

1047 return self.style_cls(val) 

1048 

1049 

1050class Unicode(Object): 

1051 dummy_for = str 

1052 

1053 class __T(TBase): 

1054 def __init__(self, *args, **kwargs): 

1055 yamlstyle = kwargs.pop('yamlstyle', None) 

1056 TBase.__init__(self, *args, **kwargs) 

1057 self.style_cls = unicode_style_map[yamlstyle] 

1058 

1059 def to_save(self, val): 

1060 return self.style_cls(val) 

1061 

1062 

1063guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool) 

1064 

1065 

1066class Dict(Object): 

1067 dummy_for = dict 

1068 

1069 class __T(TBase): 

1070 multivalued = dict 

1071 

1072 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs): 

1073 TBase.__init__(self, *args, **kwargs) 

1074 assert isinstance(key_t, TBase) 

1075 assert isinstance(content_t, TBase) 

1076 self.key_t = key_t 

1077 self.content_t = content_t 

1078 self.content_t.parent = self 

1079 

1080 def default(self): 

1081 if self._default is not None: 

1082 return dict( 

1083 (make_default(k), make_default(v)) 

1084 for (k, v) in self._default.items()) 

1085 

1086 if self.optional: 

1087 return None 

1088 else: 

1089 return {} 

1090 

1091 def has_default(self): 

1092 return True 

1093 

1094 def validate(self, val, regularize, depth): 

1095 return TBase.validate(self, val, regularize, depth+1) 

1096 

1097 def validate_children(self, val, regularize, depth): 

1098 for key, ele in list(val.items()): 

1099 newkey = self.key_t.validate(key, regularize, depth-1) 

1100 newele = self.content_t.validate(ele, regularize, depth-1) 

1101 if regularize: 

1102 if newkey is not key or newele is not ele: 

1103 del val[key] 

1104 val[newkey] = newele 

1105 

1106 return val 

1107 

1108 def to_save(self, val): 

1109 return dict((self.key_t.to_save(k), self.content_t.to_save(v)) 

1110 for (k, v) in val.items()) 

1111 

1112 def to_save_xml(self, val): 

1113 raise NotImplementedError() 

1114 

1115 def classname_for_help(self, strip_module=''): 

1116 return '``dict`` of %s objects' % \ 

1117 self.content_t.classname_for_help(strip_module=strip_module) 

1118 

1119 

1120class List(Object): 

1121 dummy_for = list 

1122 

1123 class __T(TBase): 

1124 multivalued = list 

1125 

1126 def __init__(self, content_t=Any.T(), *args, **kwargs): 

1127 yamlstyle = kwargs.pop('yamlstyle', None) 

1128 TBase.__init__(self, *args, **kwargs) 

1129 assert isinstance(content_t, TBase) or isinstance(content_t, Defer) 

1130 self.content_t = content_t 

1131 self.content_t.parent = self 

1132 self.style_cls = list_style_map[yamlstyle] 

1133 

1134 def default(self): 

1135 if self._default is not None: 

1136 return [make_default(x) for x in self._default] 

1137 if self.optional: 

1138 return None 

1139 else: 

1140 return [] 

1141 

1142 def has_default(self): 

1143 return True 

1144 

1145 def validate(self, val, regularize, depth): 

1146 return TBase.validate(self, val, regularize, depth+1) 

1147 

1148 def validate_children(self, val, regularize, depth): 

1149 for i, ele in enumerate(val): 

1150 newele = self.content_t.validate(ele, regularize, depth-1) 

1151 if regularize and newele is not ele: 

1152 val[i] = newele 

1153 

1154 return val 

1155 

1156 def to_save(self, val): 

1157 return self.style_cls(self.content_t.to_save(v) for v in val) 

1158 

1159 def to_save_xml(self, val): 

1160 return [self.content_t.to_save_xml(v) for v in val] 

1161 

1162 def deferred(self): 

1163 if isinstance(self.content_t, Defer): 

1164 return [self.content_t] 

1165 

1166 return [] 

1167 

1168 def process_deferred(self, defer, t_inst): 

1169 if defer is self.content_t: 

1170 self.content_t = t_inst 

1171 

1172 def classname_for_help(self, strip_module=''): 

1173 return '``list`` of %s objects' % \ 

1174 self.content_t.classname_for_help(strip_module=strip_module) 

1175 

1176 

1177def make_typed_list_class(t): 

1178 class TL(List): 

1179 class __T(List.T): 

1180 def __init__(self, *args, **kwargs): 

1181 List.T.__init__(self, content_t=t.T(), *args, **kwargs) 

1182 

1183 return TL 

1184 

1185 

1186class Tuple(Object): 

1187 dummy_for = tuple 

1188 

1189 class __T(TBase): 

1190 multivalued = tuple 

1191 

1192 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs): 

1193 TBase.__init__(self, *args, **kwargs) 

1194 assert isinstance(content_t, TBase) 

1195 self.content_t = content_t 

1196 self.content_t.parent = self 

1197 self.n = n 

1198 

1199 def default(self): 

1200 if self._default is not None: 

1201 return tuple( 

1202 make_default(x) for x in self._default) 

1203 

1204 elif self.optional: 

1205 return None 

1206 else: 

1207 if self.n is not None: 

1208 return tuple( 

1209 self.content_t.default() for x in range(self.n)) 

1210 else: 

1211 return tuple() 

1212 

1213 def has_default(self): 

1214 return True 

1215 

1216 def validate(self, val, regularize, depth): 

1217 return TBase.validate(self, val, regularize, depth+1) 

1218 

1219 def validate_extra(self, val): 

1220 if self.n is not None and len(val) != self.n: 

1221 raise ValidationError( 

1222 '%s should have length %i' % (self.xname(), self.n)) 

1223 

1224 def validate_children(self, val, regularize, depth): 

1225 if not regularize: 

1226 for ele in val: 

1227 self.content_t.validate(ele, regularize, depth-1) 

1228 

1229 return val 

1230 else: 

1231 newval = [] 

1232 isnew = False 

1233 for ele in val: 

1234 newele = self.content_t.validate(ele, regularize, depth-1) 

1235 newval.append(newele) 

1236 if newele is not ele: 

1237 isnew = True 

1238 

1239 if isnew: 

1240 return tuple(newval) 

1241 else: 

1242 return val 

1243 

1244 def to_save(self, val): 

1245 return tuple(self.content_t.to_save(v) for v in val) 

1246 

1247 def to_save_xml(self, val): 

1248 return [self.content_t.to_save_xml(v) for v in val] 

1249 

1250 def classname_for_help(self, strip_module=''): 

1251 if self.n is not None: 

1252 return '``tuple`` of %i %s objects' % ( 

1253 self.n, self.content_t.classname_for_help( 

1254 strip_module=strip_module)) 

1255 else: 

1256 return '``tuple`` of %s objects' % ( 

1257 self.content_t.classname_for_help( 

1258 strip_module=strip_module)) 

1259 

1260 

1261unit_factors = dict( 

1262 s=1.0, 

1263 m=60.0, 

1264 h=3600.0, 

1265 d=24*3600.0, 

1266 y=365*24*3600.0) 

1267 

1268 

1269class Duration(Object): 

1270 dummy_for = float 

1271 

1272 class __T(TBase): 

1273 

1274 def regularize_extra(self, val): 

1275 if isinstance(val, str): 

1276 unit = val[-1] 

1277 if unit in unit_factors: 

1278 return float(val[:-1]) * unit_factors[unit] 

1279 else: 

1280 return float(val) 

1281 

1282 return val 

1283 

1284 

1285re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$') 

1286 

1287 

1288class Timestamp(Object): 

1289 dummy_for = (hpfloat, float) 

1290 dummy_for_description = 'time_float' 

1291 

1292 class __T(TBase): 

1293 

1294 def regularize_extra(self, val): 

1295 

1296 time_float = get_time_float() 

1297 

1298 if isinstance(val, datetime.datetime): 

1299 tt = val.utctimetuple() 

1300 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6 

1301 

1302 elif isinstance(val, datetime.date): 

1303 tt = val.timetuple() 

1304 val = time_float(calendar.timegm(tt)) 

1305 

1306 elif isinstance(val, str): 

1307 val = val.strip() 

1308 tz_offset = 0 

1309 

1310 m = re_tz.search(val) 

1311 if m: 

1312 sh = m.group(2) 

1313 sm = m.group(4) 

1314 tz_offset = (int(sh)*3600 if sh else 0) \ 

1315 + (int(sm)*60 if sm else 0) 

1316 

1317 val = re_tz.sub('', val) 

1318 

1319 if len(val) > 10 and val[10] == 'T': 

1320 val = val.replace('T', ' ', 1) 

1321 

1322 try: 

1323 val = str_to_time(val) - tz_offset 

1324 except TimeStrError: 

1325 raise ValidationError( 

1326 '%s: cannot parse time/date: %s' % (self.xname(), val)) 

1327 

1328 elif isinstance(val, (int, float)): 

1329 val = time_float(val) 

1330 

1331 else: 

1332 raise ValidationError( 

1333 '%s: cannot convert "%s" to type %s' % ( 

1334 self.xname(), val, time_float)) 

1335 

1336 return val 

1337 

1338 def to_save(self, val): 

1339 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\ 

1340 .rstrip('0').rstrip('.') 

1341 

1342 def to_save_xml(self, val): 

1343 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\ 

1344 .rstrip('0').rstrip('.') + 'Z' 

1345 

1346 @classmethod 

1347 def D(self, s): 

1348 return TimestampDefaultMaker(s) 

1349 

1350 

1351class DateTimestamp(Object): 

1352 dummy_for = (hpfloat, float) 

1353 dummy_for_description = 'time_float' 

1354 

1355 class __T(TBase): 

1356 

1357 def regularize_extra(self, val): 

1358 

1359 time_float = get_time_float() 

1360 

1361 if isinstance(val, datetime.datetime): 

1362 tt = val.utctimetuple() 

1363 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6 

1364 

1365 elif isinstance(val, datetime.date): 

1366 tt = val.timetuple() 

1367 val = time_float(calendar.timegm(tt)) 

1368 

1369 elif isinstance(val, str): 

1370 val = str_to_time(val, format='%Y-%m-%d') 

1371 

1372 elif isinstance(val, int): 

1373 val = time_float(val) 

1374 

1375 return val 

1376 

1377 def to_save(self, val): 

1378 return time_to_str(val, format='%Y-%m-%d') 

1379 

1380 def to_save_xml(self, val): 

1381 return time_to_str(val, format='%Y-%m-%d') 

1382 

1383 @classmethod 

1384 def D(self, s): 

1385 return TimestampDefaultMaker(s, format='%Y-%m-%d') 

1386 

1387 

1388class StringPattern(String): 

1389 

1390 ''' 

1391 Any ``str`` matching pattern ``%(pattern)s``. 

1392 ''' 

1393 

1394 dummy_for = str 

1395 pattern = '.*' 

1396 

1397 class __T(String.T): 

1398 def __init__(self, pattern=None, *args, **kwargs): 

1399 String.T.__init__(self, *args, **kwargs) 

1400 

1401 if pattern is not None: 

1402 self.pattern = pattern 

1403 else: 

1404 self.pattern = self.dummy_cls.pattern 

1405 

1406 def validate_extra(self, val): 

1407 pat = self.pattern 

1408 if not re.search(pat, val): 

1409 raise ValidationError('%s: "%s" does not match pattern %s' % ( 

1410 self.xname(), val, repr(pat))) 

1411 

1412 @classmethod 

1413 def class_help_string(cls): 

1414 dcls = cls.dummy_cls 

1415 doc = dcls.__doc_template__ or StringPattern.__doc_template__ 

1416 return doc % {'pattern': repr(dcls.pattern)} 

1417 

1418 

1419class UnicodePattern(Unicode): 

1420 

1421 ''' 

1422 Any ``str`` matching pattern ``%(pattern)s``. 

1423 ''' 

1424 

1425 dummy_for = str 

1426 pattern = '.*' 

1427 

1428 class __T(TBase): 

1429 def __init__(self, pattern=None, *args, **kwargs): 

1430 TBase.__init__(self, *args, **kwargs) 

1431 

1432 if pattern is not None: 

1433 self.pattern = pattern 

1434 else: 

1435 self.pattern = self.dummy_cls.pattern 

1436 

1437 def validate_extra(self, val): 

1438 pat = self.pattern 

1439 if not re.search(pat, val, flags=re.UNICODE): 

1440 raise ValidationError('%s: "%s" does not match pattern %s' % ( 

1441 self.xname(), val, repr(pat))) 

1442 

1443 @classmethod 

1444 def class_help_string(cls): 

1445 dcls = cls.dummy_cls 

1446 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__ 

1447 return doc % {'pattern': repr(dcls.pattern)} 

1448 

1449 

1450class StringChoice(String): 

1451 

1452 ''' 

1453 Any ``str`` out of ``%(choices)s``. 

1454 ''' 

1455 

1456 dummy_for = str 

1457 choices = [] 

1458 ignore_case = False 

1459 

1460 class __T(String.T): 

1461 def __init__(self, choices=None, ignore_case=None, *args, **kwargs): 

1462 String.T.__init__(self, *args, **kwargs) 

1463 

1464 if choices is not None: 

1465 self.choices = choices 

1466 else: 

1467 self.choices = self.dummy_cls.choices 

1468 

1469 if ignore_case is not None: 

1470 self.ignore_case = ignore_case 

1471 else: 

1472 self.ignore_case = self.dummy_cls.ignore_case 

1473 

1474 if self.ignore_case: 

1475 self.choices = [x.upper() for x in self.choices] 

1476 

1477 def validate_extra(self, val): 

1478 if self.ignore_case: 

1479 val = val.upper() 

1480 

1481 if val not in self.choices: 

1482 raise ValidationError( 

1483 '%s: "%s" is not a valid choice out of %s' % ( 

1484 self.xname(), val, repr(self.choices))) 

1485 

1486 @classmethod 

1487 def class_help_string(cls): 

1488 dcls = cls.dummy_cls 

1489 doc = dcls.__doc_template__ or StringChoice.__doc_template__ 

1490 return doc % {'choices': repr(dcls.choices)} 

1491 

1492 

1493class IntChoice(Int): 

1494 

1495 ''' 

1496 Any ``int`` out of ``%(choices)s``. 

1497 ''' 

1498 

1499 dummy_for = int 

1500 choices = [] 

1501 

1502 class __T(Int.T): 

1503 def __init__(self, choices=None, *args, **kwargs): 

1504 Int.T.__init__(self, *args, **kwargs) 

1505 

1506 if choices is not None: 

1507 self.choices = choices 

1508 else: 

1509 self.choices = self.dummy_cls.choices 

1510 

1511 def validate_extra(self, val): 

1512 if val not in self.choices: 

1513 raise ValidationError( 

1514 '%s: %i is not a valid choice out of %s' % ( 

1515 self.xname(), val, repr(self.choices))) 

1516 

1517 @classmethod 

1518 def class_help_string(cls): 

1519 dcls = cls.dummy_cls 

1520 doc = dcls.__doc_template__ or IntChoice.__doc_template__ 

1521 return doc % {'choices': repr(dcls.choices)} 

1522 

1523 

1524# this will not always work... 

1525class Union(Object): 

1526 members = [] 

1527 dummy_for = str 

1528 

1529 class __T(TBase): 

1530 def __init__(self, members=None, *args, **kwargs): 

1531 TBase.__init__(self, *args, **kwargs) 

1532 if members is not None: 

1533 self.members = members 

1534 else: 

1535 self.members = self.dummy_cls.members 

1536 

1537 def validate(self, val, regularize=False, depth=-1): 

1538 assert self.members 

1539 e2 = None 

1540 for member in self.members: 

1541 try: 

1542 return member.validate(val, regularize, depth=depth) 

1543 except ValidationError as e: 

1544 e2 = e 

1545 

1546 raise e2 

1547 

1548 

1549class Choice(Object): 

1550 choices = [] 

1551 

1552 class __T(TBase): 

1553 def __init__(self, choices=None, *args, **kwargs): 

1554 TBase.__init__(self, *args, **kwargs) 

1555 if choices is not None: 

1556 self.choices = choices 

1557 else: 

1558 self.choices = self.dummy_cls.choices 

1559 

1560 self.cls_to_xmltagname = dict( 

1561 (t.cls, t.get_xmltagname()) for t in self.choices) 

1562 

1563 def validate(self, val, regularize=False, depth=-1): 

1564 if self.optional and val is None: 

1565 return val 

1566 

1567 t = None 

1568 for tc in self.choices: 

1569 is_derived = isinstance(val, tc.cls) 

1570 is_exact = type(val) == tc.cls 

1571 if not (not tc.strict and not is_derived or 

1572 tc.strict and not is_exact): 

1573 

1574 t = tc 

1575 break 

1576 

1577 if t is None: 

1578 if regularize: 

1579 ok = False 

1580 for tc in self.choices: 

1581 try: 

1582 val = tc.regularize_extra(val) 

1583 ok = True 

1584 t = tc 

1585 break 

1586 except (ValidationError, ValueError): 

1587 pass 

1588 

1589 if not ok: 

1590 raise ValidationError( 

1591 '%s: could not convert "%s" to any type out of ' 

1592 '(%s)' % (self.xname(), val, ','.join( 

1593 classnames(x.cls) for x in self.choices))) 

1594 else: 

1595 raise ValidationError( 

1596 '%s: "%s" (type: %s) is not of any type out of ' 

1597 '(%s)' % (self.xname(), val, type(val), ','.join( 

1598 classnames(x.cls) for x in self.choices))) 

1599 

1600 validator = t 

1601 

1602 if isinstance(t.cls, tuple): 

1603 clss = t.cls 

1604 else: 

1605 clss = (t.cls,) 

1606 

1607 for cls in clss: 

1608 try: 

1609 if type(val) != cls and isinstance(val, cls): 

1610 validator = val.T.instance 

1611 

1612 except AttributeError: 

1613 pass 

1614 

1615 validator.validate_extra(val) 

1616 

1617 if depth != 0: 

1618 val = validator.validate_children(val, regularize, depth) 

1619 

1620 return val 

1621 

1622 def extend_xmlelements(self, elems, v): 

1623 elems.append(( 

1624 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v)) 

1625 

1626 

1627def _dump( 

1628 object, stream, 

1629 header=False, 

1630 Dumper=GutsSafeDumper, 

1631 _dump_function=yaml.dump): 

1632 

1633 if not getattr(stream, 'encoding', None): 

1634 enc = encode_utf8 

1635 else: 

1636 enc = no_encode 

1637 

1638 if header: 

1639 stream.write(enc(u'%YAML 1.1\n')) 

1640 if isinstance(header, str): 

1641 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n' 

1642 stream.write(enc(banner)) 

1643 

1644 _dump_function( 

1645 object, 

1646 stream=stream, 

1647 encoding='utf-8', 

1648 explicit_start=True, 

1649 Dumper=Dumper) 

1650 

1651 

1652def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper): 

1653 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all) 

1654 

1655 

1656def _load(stream, 

1657 Loader=GutsSafeLoader, allow_include=None, filename=None, 

1658 included_files=None): 

1659 

1660 class _Loader(Loader): 

1661 _filename = filename 

1662 _allow_include = allow_include 

1663 _included_files = included_files or [] 

1664 

1665 return yaml.load(stream=stream, Loader=_Loader) 

1666 

1667 

1668def _load_all(stream, 

1669 Loader=GutsSafeLoader, allow_include=None, filename=None): 

1670 

1671 class _Loader(Loader): 

1672 _filename = filename 

1673 _allow_include = allow_include 

1674 

1675 return list(yaml.load_all(stream=stream, Loader=_Loader)) 

1676 

1677 

1678def _iload_all(stream, 

1679 Loader=GutsSafeLoader, allow_include=None, filename=None): 

1680 

1681 class _Loader(Loader): 

1682 _filename = filename 

1683 _allow_include = allow_include 

1684 

1685 return yaml.load_all(stream=stream, Loader=_Loader) 

1686 

1687 

1688def multi_representer(dumper, data): 

1689 node = dumper.represent_mapping( 

1690 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False) 

1691 

1692 return node 

1693 

1694 

1695# hack for compatibility with early GF Store versions 

1696re_compatibility = re.compile( 

1697 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.' 

1698 r'(dummy|poel|qseis|qssp))\.' 

1699) 

1700 

1701 

1702def multi_constructor(loader, tag_suffix, node): 

1703 tagname = str(tag_suffix) 

1704 

1705 tagname = re_compatibility.sub('pf.', tagname) 

1706 

1707 cls = g_tagname_to_class[tagname] 

1708 kwargs = dict(iter(loader.construct_pairs(node, deep=True))) 

1709 o = cls(**kwargs) 

1710 o.validate(regularize=True, depth=1) 

1711 return o 

1712 

1713 

1714def include_constructor(loader, node): 

1715 allow_include = loader._allow_include \ 

1716 if loader._allow_include is not None \ 

1717 else ALLOW_INCLUDE 

1718 

1719 if not allow_include: 

1720 raise EnvironmentError( 

1721 'Not allowed to include YAML. Load with allow_include=True') 

1722 

1723 if isinstance(node, yaml.nodes.ScalarNode): 

1724 inc_file = loader.construct_scalar(node) 

1725 else: 

1726 raise TypeError('Unsupported YAML node %s' % repr(node)) 

1727 

1728 if loader._filename is not None and not op.isabs(inc_file): 

1729 inc_file = op.join(op.dirname(loader._filename), inc_file) 

1730 

1731 if not op.isfile(inc_file): 

1732 raise FileNotFoundError(inc_file) 

1733 

1734 included_files = list(loader._included_files) 

1735 if loader._filename is not None: 

1736 included_files.append(op.abspath(loader._filename)) 

1737 

1738 for included_file in loader._included_files: 

1739 if op.samefile(inc_file, included_file): 

1740 raise ImportError( 

1741 'Circular import of file "%s". Include path: %s' % ( 

1742 op.abspath(inc_file), 

1743 ' -> '.join('"%s"' % s for s in included_files))) 

1744 

1745 with open(inc_file, 'rb') as f: 

1746 return _load( 

1747 f, 

1748 Loader=loader.__class__, filename=inc_file, 

1749 allow_include=True, 

1750 included_files=included_files) 

1751 

1752 

1753def dict_noflow_representer(dumper, data): 

1754 return dumper.represent_mapping( 

1755 'tag:yaml.org,2002:map', data, flow_style=False) 

1756 

1757 

1758yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper) 

1759yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader) 

1760yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader) 

1761yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper) 

1762 

1763 

1764def str_representer(dumper, data): 

1765 return dumper.represent_scalar( 

1766 'tag:yaml.org,2002:str', str(data)) 

1767 

1768 

1769yaml.add_representer(str, str_representer, Dumper=GutsSafeDumper) 

1770 

1771 

1772class Constructor(object): 

1773 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None, 

1774 ns_ignore=False): 

1775 

1776 self.stack = [] 

1777 self.queue = [] 

1778 self.namespaces = defaultdict(list) 

1779 self.add_namespace_maps = add_namespace_maps 

1780 self.strict = strict 

1781 self.ns_hints = ns_hints 

1782 self.ns_ignore = ns_ignore 

1783 

1784 def start_element(self, ns_name, attrs): 

1785 if self.ns_ignore: 

1786 ns_name = ns_name.split(' ')[-1] 

1787 

1788 if -1 == ns_name.find(' '): 

1789 if self.ns_hints is None and ns_name in g_guessable_xmlns: 

1790 self.ns_hints = g_guessable_xmlns[ns_name] 

1791 

1792 if self.ns_hints: 

1793 ns_names = [ 

1794 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints] 

1795 

1796 elif self.ns_hints is None: 

1797 ns_names = [' ' + ns_name] 

1798 

1799 else: 

1800 ns_names = [ns_name] 

1801 

1802 for ns_name in ns_names: 

1803 if self.stack and self.stack[-1][1] is not None: 

1804 cls = self.stack[-1][1].T.xmltagname_to_class.get( 

1805 ns_name, None) 

1806 

1807 if isinstance(cls, tuple): 

1808 cls = None 

1809 else: 

1810 if cls is not None and ( 

1811 not issubclass(cls, Object) 

1812 or issubclass(cls, SObject)): 

1813 cls = None 

1814 else: 

1815 cls = g_xmltagname_to_class.get(ns_name, None) 

1816 

1817 if cls: 

1818 break 

1819 

1820 self.stack.append((ns_name, cls, attrs, [], [])) 

1821 

1822 def end_element(self, _): 

1823 ns_name, cls, attrs, content2, content1 = self.stack.pop() 

1824 

1825 ns = ns_name.split(' ', 1)[0] 

1826 

1827 if cls is not None: 

1828 content2.extend( 

1829 (ns + ' ' + k if -1 == k.find(' ') else k, v) 

1830 for (k, v) in attrs.items()) 

1831 content2.append((None, ''.join(content1))) 

1832 o = cls(**cls.T.translate_from_xml(content2, self.strict)) 

1833 o.validate(regularize=True, depth=1) 

1834 if self.add_namespace_maps: 

1835 o.namespace_map = self.get_current_namespace_map() 

1836 

1837 if self.stack and not all(x[1] is None for x in self.stack): 

1838 self.stack[-1][-2].append((ns_name, o)) 

1839 else: 

1840 self.queue.append(o) 

1841 else: 

1842 content = [''.join(content1)] 

1843 if self.stack: 

1844 for c in content: 

1845 self.stack[-1][-2].append((ns_name, c)) 

1846 

1847 def characters(self, char_content): 

1848 if self.stack: 

1849 self.stack[-1][-1].append(char_content) 

1850 

1851 def start_namespace(self, ns, uri): 

1852 self.namespaces[ns].append(uri) 

1853 

1854 def end_namespace(self, ns): 

1855 self.namespaces[ns].pop() 

1856 

1857 def get_current_namespace_map(self): 

1858 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v) 

1859 

1860 def get_queued_elements(self): 

1861 queue = self.queue 

1862 self.queue = [] 

1863 return queue 

1864 

1865 

1866def _iload_all_xml( 

1867 stream, 

1868 bufsize=100000, 

1869 add_namespace_maps=False, 

1870 strict=False, 

1871 ns_hints=None, 

1872 ns_ignore=False): 

1873 

1874 from xml.parsers.expat import ParserCreate 

1875 

1876 parser = ParserCreate('UTF-8', namespace_separator=' ') 

1877 

1878 handler = Constructor( 

1879 add_namespace_maps=add_namespace_maps, 

1880 strict=strict, 

1881 ns_hints=ns_hints, 

1882 ns_ignore=ns_ignore) 

1883 

1884 parser.StartElementHandler = handler.start_element 

1885 parser.EndElementHandler = handler.end_element 

1886 parser.CharacterDataHandler = handler.characters 

1887 parser.StartNamespaceDeclHandler = handler.start_namespace 

1888 parser.EndNamespaceDeclHandler = handler.end_namespace 

1889 

1890 while True: 

1891 data = stream.read(bufsize) 

1892 parser.Parse(data, bool(not data)) 

1893 for element in handler.get_queued_elements(): 

1894 yield element 

1895 

1896 if not data: 

1897 break 

1898 

1899 

1900def _load_all_xml(*args, **kwargs): 

1901 return list(_iload_all_xml(*args, **kwargs)) 

1902 

1903 

1904def _load_xml(*args, **kwargs): 

1905 g = _iload_all_xml(*args, **kwargs) 

1906 return next(g) 

1907 

1908 

1909def _dump_all_xml(objects, stream, root_element_name='root', header=True): 

1910 

1911 if not getattr(stream, 'encoding', None): 

1912 enc = encode_utf8 

1913 else: 

1914 enc = no_encode 

1915 

1916 _dump_xml_header(stream, header) 

1917 

1918 beg = u'<%s>\n' % root_element_name 

1919 end = u'</%s>\n' % root_element_name 

1920 

1921 stream.write(enc(beg)) 

1922 

1923 for ob in objects: 

1924 _dump_xml(ob, stream=stream) 

1925 

1926 stream.write(enc(end)) 

1927 

1928 

1929def _dump_xml_header(stream, banner=None): 

1930 

1931 if not getattr(stream, 'encoding', None): 

1932 enc = encode_utf8 

1933 else: 

1934 enc = no_encode 

1935 

1936 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n')) 

1937 if isinstance(banner, str): 

1938 stream.write(enc(u'<!-- %s -->\n' % banner)) 

1939 

1940 

1941def _dump_xml( 

1942 obj, stream, depth=0, ns_name=None, header=False, ns_map=[], 

1943 ns_ignore=False): 

1944 

1945 from xml.sax.saxutils import escape, quoteattr 

1946 

1947 if not getattr(stream, 'encoding', None): 

1948 enc = encode_utf8 

1949 else: 

1950 enc = no_encode 

1951 

1952 if depth == 0 and header: 

1953 _dump_xml_header(stream, header) 

1954 

1955 indent = ' '*depth*2 

1956 if ns_name is None: 

1957 ns_name = obj.T.instance.get_xmltagname() 

1958 

1959 if -1 != ns_name.find(' '): 

1960 ns, name = ns_name.split(' ') 

1961 else: 

1962 ns, name = '', ns_name 

1963 

1964 if isinstance(obj, Object): 

1965 obj.validate(depth=1) 

1966 attrs = [] 

1967 elems = [] 

1968 

1969 added_ns = False 

1970 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns): 

1971 attrs.append(('xmlns', ns)) 

1972 ns_map.append(ns) 

1973 added_ns = True 

1974 

1975 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True): 

1976 if prop.xmlstyle == 'attribute': 

1977 assert not prop.multivalued 

1978 assert not isinstance(v, Object) 

1979 attrs.append((prop.effective_xmltagname, v)) 

1980 

1981 elif prop.xmlstyle == 'content': 

1982 assert not prop.multivalued 

1983 assert not isinstance(v, Object) 

1984 elems.append((None, v)) 

1985 

1986 else: 

1987 prop.extend_xmlelements(elems, v) 

1988 

1989 attr_str = '' 

1990 if attrs: 

1991 attr_str = ' ' + ' '.join( 

1992 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v))) 

1993 for (k, v) in attrs) 

1994 

1995 if not elems: 

1996 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str))) 

1997 else: 

1998 oneline = len(elems) == 1 and elems[0][0] is None 

1999 stream.write(enc(u'%s<%s%s>%s' % ( 

2000 indent, 

2001 name, 

2002 attr_str, 

2003 '' if oneline else '\n'))) 

2004 

2005 for (k, v) in elems: 

2006 if k is None: 

2007 stream.write(enc(escape(str(v), {'\0': '&#00;'}))) 

2008 else: 

2009 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore) 

2010 

2011 stream.write(enc(u'%s</%s>\n' % ( 

2012 '' if oneline else indent, name))) 

2013 

2014 if added_ns: 

2015 ns_map.pop() 

2016 

2017 else: 

2018 stream.write(enc(u'%s<%s>%s</%s>\n' % ( 

2019 indent, 

2020 name, 

2021 escape(str(obj), {'\0': '&#00;'}), 

2022 name))) 

2023 

2024 

2025def walk(x, typ=None, path=()): 

2026 if typ is None or isinstance(x, typ): 

2027 yield path, x 

2028 

2029 if isinstance(x, Object): 

2030 for (prop, val) in x.T.ipropvals(x): 

2031 if prop.multivalued: 

2032 if val is not None: 

2033 for iele, ele in enumerate(val): 

2034 for y in walk(ele, typ, 

2035 path=path + ((prop.name, iele),)): 

2036 yield y 

2037 else: 

2038 for y in walk(val, typ, path=path+(prop.name,)): 

2039 yield y 

2040 

2041 

2042def clone(x, pool=None): 

2043 ''' 

2044 Clone guts object tree. 

2045 

2046 Traverses guts object tree and recursively clones all guts attributes, 

2047 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects 

2048 deriving from :py:class:`Object` are instantiated using their respective 

2049 init function. Multiply referenced objects in the source tree are multiply 

2050 referenced also in the destination tree. 

2051 

2052 This function can be used to clone guts objects ignoring any contained 

2053 run-time state, i.e. any of their attributes not defined as a guts 

2054 property. 

2055 ''' 

2056 

2057 if pool is None: 

2058 pool = {} 

2059 

2060 if id(x) in pool: 

2061 x_copy = pool[id(x)] 

2062 

2063 else: 

2064 if isinstance(x, SObject): 

2065 x_copy = x.__class__(str(x)) 

2066 elif isinstance(x, Object): 

2067 d = {} 

2068 for (prop, y) in x.T.ipropvals(x): 

2069 if y is not None: 

2070 if not prop.multivalued: 

2071 y_copy = clone(y, pool) 

2072 elif prop.multivalued is dict: 

2073 y_copy = dict( 

2074 (clone(zk, pool), clone(zv, pool)) 

2075 for (zk, zv) in y.items()) 

2076 else: 

2077 y_copy = type(y)(clone(z, pool) for z in y) 

2078 else: 

2079 y_copy = y 

2080 

2081 d[prop.name] = y_copy 

2082 

2083 x_copy = x.__class__(**d) 

2084 

2085 else: 

2086 x_copy = copy.deepcopy(x) 

2087 

2088 pool[id(x)] = x_copy 

2089 return x_copy 

2090 

2091 

2092class YPathError(Exception): 

2093 ''' 

2094 This exception is raised for invalid ypath specifications. 

2095 ''' 

2096 pass 

2097 

2098 

2099def _parse_yname(yname): 

2100 ident = r'[a-zA-Z][a-zA-Z0-9_]*' 

2101 rint = r'-?[0-9]+' 

2102 m = re.match( 

2103 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$' 

2104 % (ident, rint, rint, rint), yname) 

2105 

2106 if not m: 

2107 raise YPathError('Syntax error in component: "%s"' % yname) 

2108 

2109 d = dict( 

2110 name=m.group(1)) 

2111 

2112 if m.group(2): 

2113 if m.group(5): 

2114 istart = iend = None 

2115 if m.group(4): 

2116 istart = int(m.group(4)) 

2117 if m.group(6): 

2118 iend = int(m.group(6)) 

2119 

2120 d['slice'] = (istart, iend) 

2121 else: 

2122 d['index'] = int(m.group(7)) 

2123 

2124 return d 

2125 

2126 

2127def _decend(obj, ynames): 

2128 if ynames: 

2129 for sobj in iter_elements(obj, ynames): 

2130 yield sobj 

2131 else: 

2132 yield obj 

2133 

2134 

2135def iter_elements(obj, ypath): 

2136 ''' 

2137 Generator yielding elements matching a given ypath specification. 

2138 

2139 :param obj: guts :py:class:`Object` instance 

2140 :param ypath: Dot-separated object path (e.g. 'root.child.child'). 

2141 To access list objects use slice notatation (e.g. 

2142 'root.child[:].child[1:3].child[1]'). 

2143 

2144 Raises :py:exc:`YPathError` on failure. 

2145 ''' 

2146 

2147 try: 

2148 if isinstance(ypath, str): 

2149 ynames = ypath.split('.') 

2150 else: 

2151 ynames = ypath 

2152 

2153 yname = ynames[0] 

2154 ynames = ynames[1:] 

2155 d = _parse_yname(yname) 

2156 if d['name'] not in obj.T.propnames: 

2157 raise AttributeError(d['name']) 

2158 

2159 obj = getattr(obj, d['name']) 

2160 

2161 if 'index' in d: 

2162 sobj = obj[d['index']] 

2163 for ssobj in _decend(sobj, ynames): 

2164 yield ssobj 

2165 

2166 elif 'slice' in d: 

2167 for i in range(*slice(*d['slice']).indices(len(obj))): 

2168 sobj = obj[i] 

2169 for ssobj in _decend(sobj, ynames): 

2170 yield ssobj 

2171 else: 

2172 for sobj in _decend(obj, ynames): 

2173 yield sobj 

2174 

2175 except (AttributeError, IndexError) as e: 

2176 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e))) 

2177 

2178 

2179def get_elements(obj, ypath): 

2180 ''' 

2181 Get all elements matching a given ypath specification. 

2182 

2183 :param obj: guts :py:class:`Object` instance 

2184 :param ypath: Dot-separated object path (e.g. 'root.child.child'). 

2185 To access list objects use slice notatation (e.g. 

2186 'root.child[:].child[1:3].child[1]'). 

2187 

2188 Raises :py:exc:`YPathError` on failure. 

2189 ''' 

2190 return list(iter_elements(obj, ypath)) 

2191 

2192 

2193def set_elements(obj, ypath, value, validate=False, regularize=False): 

2194 ''' 

2195 Set elements matching a given ypath specification. 

2196 

2197 :param obj: guts :py:class:`Object` instance 

2198 :param ypath: Dot-separated object path (e.g. 'root.child.child'). 

2199 To access list objects use slice notatation (e.g. 

2200 'root.child[:].child[1:3].child[1]'). 

2201 :param value: All matching elements will be set to `value`. 

2202 :param validate: Whether to validate affected subtrees. 

2203 :param regularize: Whether to regularize affected subtrees. 

2204 

2205 Raises :py:exc:`YPathError` on failure. 

2206 ''' 

2207 

2208 ynames = ypath.split('.') 

2209 try: 

2210 d = _parse_yname(ynames[-1]) 

2211 for sobj in iter_elements(obj, ynames[:-1]): 

2212 if d['name'] not in sobj.T.propnames: 

2213 raise AttributeError(d['name']) 

2214 

2215 if 'index' in d: 

2216 ssobj = getattr(sobj, d['name']) 

2217 ssobj[d['index']] = value 

2218 elif 'slice' in d: 

2219 ssobj = getattr(sobj, d['name']) 

2220 for i in range(*slice(*d['slice']).indices(len(ssobj))): 

2221 ssobj[i] = value 

2222 else: 

2223 setattr(sobj, d['name'], value) 

2224 if regularize: 

2225 sobj.regularize() 

2226 if validate: 

2227 sobj.validate() 

2228 

2229 except (AttributeError, IndexError, YPathError) as e: 

2230 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e))) 

2231 

2232 

2233def zip_walk(x, typ=None, path=(), stack=()): 

2234 if typ is None or isinstance(x, typ): 

2235 yield path, stack + (x,) 

2236 

2237 if isinstance(x, Object): 

2238 for (prop, val) in x.T.ipropvals(x): 

2239 if prop.multivalued: 

2240 if val is not None: 

2241 for iele, ele in enumerate(val): 

2242 for y in zip_walk( 

2243 ele, typ, 

2244 path=path + ((prop.name, iele),), 

2245 stack=stack + (x,)): 

2246 

2247 yield y 

2248 else: 

2249 for y in zip_walk(val, typ, 

2250 path=path+(prop.name,), 

2251 stack=stack + (x,)): 

2252 yield y 

2253 

2254 

2255def path_element(x): 

2256 if isinstance(x, tuple): 

2257 return '%s[%i]' % x 

2258 else: 

2259 return x 

2260 

2261 

2262def path_to_str(path): 

2263 return '.'.join(path_element(x) for x in path) 

2264 

2265 

2266@expand_stream_args('w') 

2267def dump(*args, **kwargs): 

2268 return _dump(*args, **kwargs) 

2269 

2270 

2271@expand_stream_args('r') 

2272def load(*args, **kwargs): 

2273 return _load(*args, **kwargs) 

2274 

2275 

2276def load_string(s, *args, **kwargs): 

2277 return load(string=s, *args, **kwargs) 

2278 

2279 

2280@expand_stream_args('w') 

2281def dump_all(*args, **kwargs): 

2282 return _dump_all(*args, **kwargs) 

2283 

2284 

2285@expand_stream_args('r') 

2286def load_all(*args, **kwargs): 

2287 return _load_all(*args, **kwargs) 

2288 

2289 

2290@expand_stream_args('r') 

2291def iload_all(*args, **kwargs): 

2292 return _iload_all(*args, **kwargs) 

2293 

2294 

2295@expand_stream_args('w') 

2296def dump_xml(*args, **kwargs): 

2297 return _dump_xml(*args, **kwargs) 

2298 

2299 

2300@expand_stream_args('r') 

2301def load_xml(*args, **kwargs): 

2302 kwargs.pop('filename', None) 

2303 return _load_xml(*args, **kwargs) 

2304 

2305 

2306def load_xml_string(s, *args, **kwargs): 

2307 return load_xml(string=s, *args, **kwargs) 

2308 

2309 

2310@expand_stream_args('w') 

2311def dump_all_xml(*args, **kwargs): 

2312 return _dump_all_xml(*args, **kwargs) 

2313 

2314 

2315@expand_stream_args('r') 

2316def load_all_xml(*args, **kwargs): 

2317 kwargs.pop('filename', None) 

2318 return _load_all_xml(*args, **kwargs) 

2319 

2320 

2321@expand_stream_args('r') 

2322def iload_all_xml(*args, **kwargs): 

2323 kwargs.pop('filename', None) 

2324 return _iload_all_xml(*args, **kwargs) 

2325 

2326 

2327__all__ = guts_types + [ 

2328 'guts_types', 'TBase', 'ValidationError', 

2329 'ArgumentError', 'Defer', 

2330 'dump', 'load', 

2331 'dump_all', 'load_all', 'iload_all', 

2332 'dump_xml', 'load_xml', 

2333 'dump_all_xml', 'load_all_xml', 'iload_all_xml', 

2334 'load_string', 

2335 'load_xml_string', 

2336 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str' 

2337]