1# -*- coding: utf-8 -*- 

2# http://pyrocko.org - GPLv3 

3# 

4# The Pyrocko Developers, 21st Century 

5# ---|P------/S----------~Lg---------- 

6''' 

7Lightweight declarative YAML and XML data binding for Python. 

8''' 

9from __future__ import absolute_import, print_function 

10 

11import datetime 

12import calendar 

13import re 

14import sys 

15import types 

16import copy 

17import os.path as op 

18from collections import defaultdict 

19 

20from io import BytesIO 

21 

22try: 

23 import numpy as num 

24except ImportError: 

25 num = None 

26 

27import yaml 

28try: 

29 from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper 

30except ImportError: 

31 from yaml import SafeLoader, SafeDumper 

32 

33from .util import time_to_str, str_to_time, TimeStrError, hpfloat, \ 

34 get_time_float 

35 

36try: 

37 newstr = unicode 

38 range = xrange 

39except NameError: 

40 newstr = str 

41 

42try: 

43 # needed for py2/py3 compatibility to allow 

44 # from pyrocko.guts import FileNotFoundError 

45 FileNotFoundError = FileNotFoundError 

46except NameError: 

47 class FileNotFoundError(EnvironmentError): 

48 pass 

49 

50 

51ALLOW_INCLUDE = False 

52 

53 

54class GutsSafeDumper(SafeDumper): 

55 pass 

56 

57 

58class GutsSafeLoader(SafeLoader): 

59 pass 

60 

61 

62try: 

63 unicode 

64except NameError: 

65 unicode = str 

66 

67 

68g_iprop = 0 

69 

70g_deferred = {} 

71g_deferred_content = {} 

72 

73g_tagname_to_class = {} 

74g_xmltagname_to_class = {} 

75g_guessable_xmlns = {} 

76 

77guts_types = [ 

78 'Object', 'SObject', 'String', 'Unicode', 'Int', 'Float', 

79 'Complex', 'Bool', 'Timestamp', 'DateTimestamp', 'StringPattern', 

80 'UnicodePattern', 'StringChoice', 'List', 'Dict', 'Tuple', 'Union', 

81 'Choice', 'Any'] 

82 

83us_to_cc_regex = re.compile(r'([a-z])_([a-z])') 

84 

85 

86class literal(str): 

87 pass 

88 

89 

90class folded(str): 

91 pass 

92 

93 

94class singlequoted(str): 

95 pass 

96 

97 

98class doublequoted(str): 

99 pass 

100 

101 

102def make_str_presenter(style): 

103 def presenter(dumper, data): 

104 return dumper.represent_scalar( 

105 'tag:yaml.org,2002:str', str(data), style=style) 

106 

107 return presenter 

108 

109 

110str_style_map = { 

111 None: lambda x: x, 

112 '|': literal, 

113 '>': folded, 

114 "'": singlequoted, 

115 '"': doublequoted} 

116 

117for (style, cls) in str_style_map.items(): 

118 if style: 

119 GutsSafeDumper.add_representer(cls, make_str_presenter(style)) 

120 

121 

122class uliteral(unicode): 

123 pass 

124 

125 

126class ufolded(unicode): 

127 pass 

128 

129 

130class usinglequoted(unicode): 

131 pass 

132 

133 

134class udoublequoted(unicode): 

135 pass 

136 

137 

138def make_unicode_presenter(style): 

139 def presenter(dumper, data): 

140 return dumper.represent_scalar( 

141 'tag:yaml.org,2002:str', unicode(data), style=style) 

142 

143 return presenter 

144 

145 

146unicode_style_map = { 

147 None: lambda x: x, 

148 '|': literal, 

149 '>': folded, 

150 "'": singlequoted, 

151 '"': doublequoted} 

152 

153for (style, cls) in unicode_style_map.items(): 

154 if style: 

155 GutsSafeDumper.add_representer(cls, make_unicode_presenter(style)) 

156 

157 

158class blist(list): 

159 pass 

160 

161 

162class flist(list): 

163 pass 

164 

165 

166list_style_map = { 

167 None: list, 

168 'block': blist, 

169 'flow': flist} 

170 

171 

172def make_list_presenter(flow_style): 

173 def presenter(dumper, data): 

174 return dumper.represent_sequence( 

175 'tag:yaml.org,2002:seq', data, flow_style=flow_style) 

176 

177 return presenter 

178 

179 

180GutsSafeDumper.add_representer(blist, make_list_presenter(False)) 

181GutsSafeDumper.add_representer(flist, make_list_presenter(True)) 

182 

183if num: 

184 def numpy_float_presenter(dumper, data): 

185 return dumper.represent_float(float(data)) 

186 

187 def numpy_int_presenter(dumper, data): 

188 return dumper.represent_int(int(data)) 

189 

190 for dtype in (num.float64, num.float32): 

191 GutsSafeDumper.add_representer(dtype, numpy_float_presenter) 

192 

193 for dtype in (num.int32, num.int64): 

194 GutsSafeDumper.add_representer(dtype, numpy_int_presenter) 

195 

196 

197def us_to_cc(s): 

198 return us_to_cc_regex.sub(lambda pat: pat.group(1)+pat.group(2).upper(), s) 

199 

200 

201cc_to_us_regex1 = re.compile(r'([a-z])([A-Z]+)([a-z]|$)') 

202cc_to_us_regex2 = re.compile(r'([A-Z])([A-Z][a-z])') 

203 

204 

205def cc_to_us(s): 

206 return cc_to_us_regex2.sub('\\1_\\2', cc_to_us_regex1.sub( 

207 '\\1_\\2\\3', s)).lower() 

208 

209 

210re_frac = re.compile(r'\.[1-9]FRAC') 

211frac_formats = dict([('.%sFRAC' % x, '%.'+x+'f') for x in '123456789']) 

212 

213 

214def encode_utf8(s): 

215 return s.encode('utf-8') 

216 

217 

218def no_encode(s): 

219 return s 

220 

221 

222def make_xmltagname_from_name(name): 

223 return us_to_cc(name) 

224 

225 

226def make_name_from_xmltagname(xmltagname): 

227 return cc_to_us(xmltagname) 

228 

229 

230def make_content_name(name): 

231 if name.endswith('_list'): 

232 return name[:-5] 

233 elif name.endswith('s'): 

234 return name[:-1] 

235 else: 

236 return name 

237 

238 

239def classnames(cls): 

240 if isinstance(cls, tuple): 

241 return '(%s)' % ', '.join(x.__name__ for x in cls) 

242 else: 

243 return cls.__name__ 

244 

245 

246def expand_stream_args(mode): 

247 def wrap(f): 

248 ''' 

249 Decorator to enhance functions taking stream objects. 

250 

251 Wraps a function f(..., stream, ...) so that it can also be called as 

252 f(..., filename='myfilename', ...) or as f(..., string='mydata', ...). 

253 ''' 

254 

255 def g(*args, **kwargs): 

256 stream = kwargs.pop('stream', None) 

257 filename = kwargs.get('filename', None) 

258 if mode != 'r': 

259 filename = kwargs.pop('filename', None) 

260 string = kwargs.pop('string', None) 

261 

262 assert sum(x is not None for x in (stream, filename, string)) <= 1 

263 

264 if stream is not None: 

265 kwargs['stream'] = stream 

266 return f(*args, **kwargs) 

267 

268 elif filename is not None: 

269 stream = open(filename, mode+'b') 

270 kwargs['stream'] = stream 

271 retval = f(*args, **kwargs) 

272 if isinstance(retval, types.GeneratorType): 

273 def wrap_generator(gen): 

274 try: 

275 for x in gen: 

276 yield x 

277 

278 except GeneratorExit: 

279 pass 

280 

281 stream.close() 

282 

283 return wrap_generator(retval) 

284 

285 else: 

286 stream.close() 

287 return retval 

288 

289 elif string is not None: 

290 assert mode == 'r', \ 

291 'Keyword argument string=... cannot be used in dumper ' \ 

292 'function.' 

293 

294 kwargs['stream'] = BytesIO(string.encode('utf-8')) 

295 return f(*args, **kwargs) 

296 

297 else: 

298 assert mode == 'w', \ 

299 'Use keyword argument stream=... or filename=... in ' \ 

300 'loader function.' 

301 

302 sout = BytesIO() 

303 f(stream=sout, *args, **kwargs) 

304 return sout.getvalue().decode('utf-8') 

305 

306 return g 

307 

308 return wrap 

309 

310 

311class Defer(object): 

312 def __init__(self, classname, *args, **kwargs): 

313 global g_iprop 

314 if kwargs.get('position', None) is None: 

315 kwargs['position'] = g_iprop 

316 

317 g_iprop += 1 

318 

319 self.classname = classname 

320 self.args = args 

321 self.kwargs = kwargs 

322 

323 

324class TBase(object): 

325 

326 strict = False 

327 multivalued = None 

328 force_regularize = False 

329 propnames = [] 

330 

331 @classmethod 

332 def init_propertystuff(cls): 

333 cls.properties = [] 

334 cls.xmltagname_to_name = {} 

335 cls.xmltagname_to_name_multivalued = {} 

336 cls.xmltagname_to_class = {} 

337 cls.content_property = None 

338 

339 def __init__( 

340 self, 

341 default=None, 

342 optional=False, 

343 xmlstyle='element', 

344 xmltagname=None, 

345 xmlns=None, 

346 help=None, 

347 position=None): 

348 

349 global g_iprop 

350 if position is not None: 

351 self.position = position 

352 else: 

353 self.position = g_iprop 

354 

355 g_iprop += 1 

356 self._default = default 

357 

358 self.optional = optional 

359 self.name = None 

360 self._xmltagname = xmltagname 

361 self._xmlns = xmlns 

362 self.parent = None 

363 self.xmlstyle = xmlstyle 

364 self.help = help 

365 

366 def default(self): 

367 return make_default(self._default) 

368 

369 def is_default(self, val): 

370 if self._default is None: 

371 return val is None 

372 else: 

373 return self._default == val 

374 

375 def has_default(self): 

376 return self._default is not None 

377 

378 def xname(self): 

379 if self.name is not None: 

380 return self.name 

381 elif self.parent is not None: 

382 return 'element of %s' % self.parent.xname() 

383 else: 

384 return '?' 

385 

386 def set_xmlns(self, xmlns): 

387 if self._xmlns is None and not self.xmlns: 

388 self._xmlns = xmlns 

389 

390 if self.multivalued: 

391 self.content_t.set_xmlns(xmlns) 

392 

393 def get_xmlns(self): 

394 return self._xmlns or self.xmlns 

395 

396 def get_xmltagname(self): 

397 if self._xmltagname is not None: 

398 return self.get_xmlns() + ' ' + self._xmltagname 

399 elif self.name: 

400 return self.get_xmlns() + ' ' \ 

401 + make_xmltagname_from_name(self.name) 

402 elif self.xmltagname: 

403 return self.get_xmlns() + ' ' + self.xmltagname 

404 else: 

405 assert False 

406 

407 @classmethod 

408 def get_property(cls, name): 

409 for prop in cls.properties: 

410 if prop.name == name: 

411 return prop 

412 

413 raise ValueError() 

414 

415 @classmethod 

416 def remove_property(cls, name): 

417 

418 prop = cls.get_property(name) 

419 

420 if not prop.multivalued: 

421 del cls.xmltagname_to_class[prop.effective_xmltagname] 

422 del cls.xmltagname_to_name[prop.effective_xmltagname] 

423 else: 

424 del cls.xmltagname_to_class[prop.content_t.effective_xmltagname] 

425 del cls.xmltagname_to_name_multivalued[ 

426 prop.content_t.effective_xmltagname] 

427 

428 if cls.content_property is prop: 

429 cls.content_property = None 

430 

431 cls.properties.remove(prop) 

432 cls.propnames.remove(name) 

433 

434 return prop 

435 

436 @classmethod 

437 def add_property(cls, name, prop): 

438 

439 prop.instance = prop 

440 prop.name = name 

441 prop.set_xmlns(cls.xmlns) 

442 

443 if isinstance(prop, Choice.T): 

444 for tc in prop.choices: 

445 tc.effective_xmltagname = tc.get_xmltagname() 

446 cls.xmltagname_to_class[tc.effective_xmltagname] = tc.cls 

447 cls.xmltagname_to_name[tc.effective_xmltagname] = prop.name 

448 elif not prop.multivalued: 

449 prop.effective_xmltagname = prop.get_xmltagname() 

450 cls.xmltagname_to_class[prop.effective_xmltagname] = prop.cls 

451 cls.xmltagname_to_name[prop.effective_xmltagname] = prop.name 

452 else: 

453 prop.content_t.name = make_content_name(prop.name) 

454 prop.content_t.effective_xmltagname = \ 

455 prop.content_t.get_xmltagname() 

456 cls.xmltagname_to_class[ 

457 prop.content_t.effective_xmltagname] = prop.content_t.cls 

458 cls.xmltagname_to_name_multivalued[ 

459 prop.content_t.effective_xmltagname] = prop.name 

460 

461 cls.properties.append(prop) 

462 

463 cls.properties.sort(key=lambda x: x.position) 

464 

465 cls.propnames = [p.name for p in cls.properties] 

466 

467 if prop.xmlstyle == 'content': 

468 cls.content_property = prop 

469 

470 @classmethod 

471 def ivals(cls, val): 

472 for prop in cls.properties: 

473 yield getattr(val, prop.name) 

474 

475 @classmethod 

476 def ipropvals(cls, val): 

477 for prop in cls.properties: 

478 yield prop, getattr(val, prop.name) 

479 

480 @classmethod 

481 def inamevals(cls, val): 

482 for prop in cls.properties: 

483 yield prop.name, getattr(val, prop.name) 

484 

485 @classmethod 

486 def ipropvals_to_save(cls, val, xmlmode=False): 

487 for prop in cls.properties: 

488 v = getattr(val, prop.name) 

489 if v is not None and ( 

490 not (prop.optional or (prop.multivalued and not v)) 

491 or (not prop.is_default(v))): 

492 

493 if xmlmode: 

494 yield prop, prop.to_save_xml(v) 

495 else: 

496 yield prop, prop.to_save(v) 

497 

498 @classmethod 

499 def inamevals_to_save(cls, val, xmlmode=False): 

500 for prop, v in cls.ipropvals_to_save(val, xmlmode): 

501 yield prop.name, v 

502 

503 @classmethod 

504 def translate_from_xml(cls, list_of_pairs, strict): 

505 d = {} 

506 for k, v in list_of_pairs: 

507 if k in cls.xmltagname_to_name_multivalued: 

508 k2 = cls.xmltagname_to_name_multivalued[k] 

509 if k2 not in d: 

510 d[k2] = [] 

511 

512 d[k2].append(v) 

513 elif k in cls.xmltagname_to_name: 

514 k2 = cls.xmltagname_to_name[k] 

515 if k2 in d: 

516 raise ArgumentError( 

517 'Unexpectedly found more than one child element "%s" ' 

518 'within "%s".' % (k, cls.tagname)) 

519 

520 d[k2] = v 

521 elif k is None: 

522 if cls.content_property: 

523 k2 = cls.content_property.name 

524 d[k2] = v 

525 else: 

526 if strict: 

527 raise ArgumentError( 

528 'Unexpected child element "%s" found within "%s".' % ( 

529 k, cls.tagname)) 

530 

531 return d 

532 

533 def validate(self, val, regularize=False, depth=-1): 

534 if self.optional and val is None: 

535 return val 

536 

537 is_derived = isinstance(val, self.cls) 

538 is_exact = type(val) == self.cls 

539 

540 not_ok = not self.strict and not is_derived or \ 

541 self.strict and not is_exact 

542 

543 if not_ok or self.force_regularize: 

544 if regularize: 

545 try: 

546 val = self.regularize_extra(val) 

547 except ValueError: 

548 raise ValidationError( 

549 '%s: could not convert "%s" to type %s' % ( 

550 self.xname(), val, classnames(self.cls))) 

551 else: 

552 raise ValidationError( 

553 '%s: "%s" (type: %s) is not of type %s' % ( 

554 self.xname(), val, type(val), classnames(self.cls))) 

555 

556 validator = self 

557 if isinstance(self.cls, tuple): 

558 clss = self.cls 

559 else: 

560 clss = (self.cls,) 

561 

562 for cls in clss: 

563 try: 

564 if type(val) != cls and isinstance(val, cls): 

565 validator = val.T.instance 

566 

567 except AttributeError: 

568 pass 

569 

570 validator.validate_extra(val) 

571 

572 if depth != 0: 

573 val = validator.validate_children(val, regularize, depth) 

574 

575 return val 

576 

577 def regularize_extra(self, val): 

578 return self.cls(val) 

579 

580 def validate_extra(self, val): 

581 pass 

582 

583 def validate_children(self, val, regularize, depth): 

584 for prop, propval in self.ipropvals(val): 

585 newpropval = prop.validate(propval, regularize, depth-1) 

586 if regularize and (newpropval is not propval): 

587 setattr(val, prop.name, newpropval) 

588 

589 return val 

590 

591 def to_save(self, val): 

592 return val 

593 

594 def to_save_xml(self, val): 

595 return self.to_save(val) 

596 

597 def extend_xmlelements(self, elems, v): 

598 if self.multivalued: 

599 for x in v: 

600 elems.append((self.content_t.effective_xmltagname, x)) 

601 else: 

602 elems.append((self.effective_xmltagname, v)) 

603 

604 def deferred(self): 

605 return [] 

606 

607 def classname_for_help(self, strip_module=''): 

608 if self.dummy_cls in guts_plain_dummy_types: 

609 return '``%s``' % self.cls.__name__ 

610 

611 elif self.dummy_cls.dummy_for_description: 

612 return self.dummy_cls.dummy_for_description 

613 

614 else: 

615 if self.dummy_cls is not self.cls: 

616 if self.dummy_cls.__module__ == strip_module: 

617 sadd = ' (:py:class:`%s`)' % ( 

618 self.dummy_cls.__name__) 

619 else: 

620 sadd = ' (:py:class:`%s.%s`)' % ( 

621 self.dummy_cls.__module__, self.dummy_cls.__name__) 

622 else: 

623 sadd = '' 

624 

625 def sclass(cls): 

626 mod = cls.__module__ 

627 clsn = cls.__name__ 

628 if mod == '__builtin__' or mod == 'builtins': 

629 return '``%s``' % clsn 

630 

631 elif mod == strip_module: 

632 return ':py:class:`%s`' % clsn 

633 

634 else: 

635 return ':py:class:`%s.%s`' % (mod, clsn) 

636 

637 if isinstance(self.cls, tuple): 

638 return '(%s)%s' % ( 

639 ' | '.join(sclass(cls) for cls in self.cls), sadd) 

640 else: 

641 return '%s%s' % (sclass(cls), sadd) 

642 

643 @classmethod 

644 def props_help_string(cls): 

645 baseprops = [] 

646 for base in cls.dummy_cls.__bases__: 

647 if hasattr(base, 'T'): 

648 baseprops.extend(base.T.properties) 

649 

650 hlp = [] 

651 hlp.append('') 

652 for prop in cls.properties: 

653 if prop in baseprops: 

654 continue 

655 

656 descr = [ 

657 prop.classname_for_help(strip_module=cls.dummy_cls.__module__)] 

658 

659 if prop.optional: 

660 descr.append('*optional*') 

661 

662 if isinstance(prop._default, DefaultMaker): 

663 descr.append('*default:* ``%s``' % repr(prop._default)) 

664 else: 

665 d = prop.default() 

666 if d is not None: 

667 descr.append('*default:* ``%s``' % repr(d)) 

668 

669 hlp.append(' .. py:gattribute:: %s' % prop.name) 

670 hlp.append('') 

671 hlp.append(' %s' % ', '.join(descr)) 

672 hlp.append(' ') 

673 if prop.help is not None: 

674 hlp.append(' %s' % prop.help) 

675 hlp.append('') 

676 

677 return '\n'.join(hlp) 

678 

679 @classmethod 

680 def class_help_string(cls): 

681 return cls.dummy_cls.__doc_template__ 

682 

683 @classmethod 

684 def class_signature(cls): 

685 r = [] 

686 for prop in cls.properties: 

687 d = prop.default() 

688 if d is not None: 

689 arg = repr(d) 

690 

691 elif prop.optional: 

692 arg = 'None' 

693 

694 else: 

695 arg = '...' 

696 

697 r.append('%s=%s' % (prop.name, arg)) 

698 

699 return '(%s)' % ', '.join(r) 

700 

701 @classmethod 

702 def help(cls): 

703 return cls.props_help_string() 

704 

705 

706class ObjectMetaClass(type): 

707 def __new__(meta, classname, bases, class_dict): 

708 cls = type.__new__(meta, classname, bases, class_dict) 

709 if classname != 'Object': 

710 t_class_attr_name = '_%s__T' % classname 

711 if not hasattr(cls, t_class_attr_name): 

712 if hasattr(cls, 'T'): 

713 class T(cls.T): 

714 pass 

715 else: 

716 class T(TBase): 

717 pass 

718 

719 setattr(cls, t_class_attr_name, T) 

720 

721 T = getattr(cls, t_class_attr_name) 

722 

723 if cls.dummy_for is not None: 

724 T.cls = cls.dummy_for 

725 else: 

726 T.cls = cls 

727 

728 T.dummy_cls = cls 

729 

730 if hasattr(cls, 'xmltagname'): 

731 T.xmltagname = cls.xmltagname 

732 else: 

733 T.xmltagname = classname 

734 

735 mod = sys.modules[cls.__module__] 

736 

737 if hasattr(cls, 'xmlns'): 

738 T.xmlns = cls.xmlns 

739 elif hasattr(mod, 'guts_xmlns'): 

740 T.xmlns = mod.guts_xmlns 

741 else: 

742 T.xmlns = '' 

743 

744 if T.xmlns and hasattr(cls, 'guessable_xmlns'): 

745 g_guessable_xmlns[T.xmltagname] = cls.guessable_xmlns 

746 

747 if hasattr(mod, 'guts_prefix'): 

748 if mod.guts_prefix: 

749 T.tagname = mod.guts_prefix + '.' + classname 

750 else: 

751 T.tagname = classname 

752 else: 

753 if cls.__module__ != '__main__': 

754 T.tagname = cls.__module__ + '.' + classname 

755 else: 

756 T.tagname = classname 

757 

758 T.classname = classname 

759 

760 T.init_propertystuff() 

761 

762 for k in dir(cls): 

763 prop = getattr(cls, k) 

764 

765 if k.endswith('__'): 

766 k = k[:-2] 

767 

768 if isinstance(prop, TBase): 

769 if prop.deferred(): 

770 for defer in prop.deferred(): 

771 g_deferred_content.setdefault( 

772 defer.classname[:-2], []).append((prop, defer)) 

773 g_deferred.setdefault( 

774 defer.classname[:-2], []).append((T, k, prop)) 

775 

776 else: 

777 T.add_property(k, prop) 

778 

779 elif isinstance(prop, Defer): 

780 g_deferred.setdefault(prop.classname[:-2], []).append( 

781 (T, k, prop)) 

782 

783 if classname in g_deferred_content: 

784 for prop, defer in g_deferred_content[classname]: 

785 prop.process_deferred( 

786 defer, T(*defer.args, **defer.kwargs)) 

787 

788 del g_deferred_content[classname] 

789 

790 if classname in g_deferred: 

791 for (T_, k_, prop_) in g_deferred.get(classname, []): 

792 if isinstance(prop_, Defer): 

793 prop_ = T(*prop_.args, **prop_.kwargs) 

794 

795 if not prop_.deferred(): 

796 T_.add_property(k_, prop_) 

797 

798 del g_deferred[classname] 

799 

800 g_tagname_to_class[T.tagname] = cls 

801 if hasattr(cls, 'xmltagname'): 

802 g_xmltagname_to_class[T.xmlns + ' ' + T.xmltagname] = cls 

803 

804 cls.T = T 

805 T.instance = T() 

806 

807 cls.__doc_template__ = cls.__doc__ 

808 cls.__doc__ = T.class_help_string() 

809 

810 if cls.__doc__ is None: 

811 cls.__doc__ = 'Undocumented.' 

812 

813 cls.__doc__ += '\n' + T.props_help_string() 

814 

815 return cls 

816 

817 

818class ValidationError(Exception): 

819 pass 

820 

821 

822class ArgumentError(Exception): 

823 pass 

824 

825 

826def make_default(x): 

827 if isinstance(x, DefaultMaker): 

828 return x.make() 

829 elif isinstance(x, Object): 

830 return clone(x) 

831 else: 

832 return x 

833 

834 

835class DefaultMaker(object): 

836 def make(self): 

837 raise NotImplementedError('Schould be implemented in subclass.') 

838 

839 

840class ObjectDefaultMaker(DefaultMaker): 

841 def __init__(self, cls, args, kwargs): 

842 DefaultMaker.__init__(self) 

843 self.cls = cls 

844 self.args = args 

845 self.kwargs = kwargs 

846 self.instance = None 

847 

848 def make(self): 

849 return self.cls( 

850 *[make_default(x) for x in self.args], 

851 **dict((k, make_default(v)) for (k, v) in self.kwargs.items())) 

852 

853 def __eq__(self, other): 

854 if self.instance is None: 

855 self.instance = self.make() 

856 

857 return self.instance == other 

858 

859 def __repr__(self): 

860 sargs = [] 

861 for arg in self.args: 

862 sargs.append(repr(arg)) 

863 

864 for k, v in self.kwargs.items(): 

865 sargs.append('%s=%s' % (k, repr(v))) 

866 

867 return '%s(%s)' % (self.cls.__name__, ', '.join(sargs)) 

868 

869 

870class TimestampDefaultMaker(DefaultMaker): 

871 def __init__(self, s, format='%Y-%m-%d %H:%M:%S.OPTFRAC'): 

872 DefaultMaker.__init__(self) 

873 self._stime = s 

874 self._format = format 

875 

876 def make(self): 

877 return str_to_time(self._stime, self._format) 

878 

879 def __repr__(self): 

880 return "str_to_time(%s)" % repr(self._stime) 

881 

882 

883def with_metaclass(meta, *bases): 

884 # inlined py2/py3 compat solution from python-future 

885 class metaclass(meta): 

886 __call__ = type.__call__ 

887 __init__ = type.__init__ 

888 

889 def __new__(cls, name, this_bases, d): 

890 if this_bases is None: 

891 return type.__new__(cls, name, (), d) 

892 return meta(name, bases, d) 

893 

894 return metaclass('temp', None, {}) 

895 

896 

897class Object(with_metaclass(ObjectMetaClass, object)): 

898 dummy_for = None 

899 dummy_for_description = None 

900 

901 def __init__(self, **kwargs): 

902 if not kwargs.get('init_props', True): 

903 return 

904 

905 for prop in self.T.properties: 

906 k = prop.name 

907 if k in kwargs: 

908 setattr(self, k, kwargs.pop(k)) 

909 else: 

910 if not prop.optional and not prop.has_default(): 

911 raise ArgumentError('Missing argument to %s: %s' % ( 

912 self.T.tagname, prop.name)) 

913 else: 

914 setattr(self, k, prop.default()) 

915 

916 if kwargs: 

917 raise ArgumentError('Invalid argument to %s: %s' % ( 

918 self.T.tagname, ', '.join(list(kwargs.keys())))) 

919 

920 @classmethod 

921 def D(cls, *args, **kwargs): 

922 return ObjectDefaultMaker(cls, args, kwargs) 

923 

924 def validate(self, regularize=False, depth=-1): 

925 self.T.instance.validate(self, regularize, depth) 

926 

927 def regularize(self, depth=-1): 

928 self.validate(regularize=True, depth=depth) 

929 

930 def dump(self, stream=None, filename=None, header=False): 

931 return dump(self, stream=stream, filename=filename, header=header) 

932 

933 def dump_xml( 

934 self, stream=None, filename=None, header=False, ns_ignore=False): 

935 return dump_xml( 

936 self, stream=stream, filename=filename, header=header, 

937 ns_ignore=ns_ignore) 

938 

939 @classmethod 

940 def load(cls, stream=None, filename=None, string=None): 

941 return load(stream=stream, filename=filename, string=string) 

942 

943 @classmethod 

944 def load_xml(cls, stream=None, filename=None, string=None, ns_hints=None, 

945 ns_ignore=False): 

946 

947 if ns_hints is None: 

948 ns_hints = [cls.T.instance.get_xmlns()] 

949 

950 return load_xml( 

951 stream=stream, 

952 filename=filename, 

953 string=string, 

954 ns_hints=ns_hints, 

955 ns_ignore=ns_ignore) 

956 

957 def __str__(self): 

958 return self.dump() 

959 

960 

961def to_dict(obj): 

962 ''' 

963 Get dict of guts object attributes. 

964 

965 :param obj: :py:class`Object` object 

966 ''' 

967 

968 return dict(obj.T.inamevals(obj)) 

969 

970 

971class SObject(Object): 

972 

973 class __T(TBase): 

974 def regularize_extra(self, val): 

975 if isinstance(val, (str, newstr)): 

976 return self.cls(val) 

977 

978 return val 

979 

980 def to_save(self, val): 

981 return str(val) 

982 

983 def to_save_xml(self, val): 

984 return str(val) 

985 

986 

987class Any(Object): 

988 

989 class __T(TBase): 

990 def validate(self, val, regularize=False, depth=-1): 

991 if isinstance(val, Object): 

992 val.validate(regularize, depth) 

993 

994 return val 

995 

996 

997class Int(Object): 

998 dummy_for = int 

999 

1000 class __T(TBase): 

1001 strict = True 

1002 

1003 def to_save_xml(self, value): 

1004 return repr(value) 

1005 

1006 

1007class Float(Object): 

1008 dummy_for = float 

1009 

1010 class __T(TBase): 

1011 strict = True 

1012 

1013 def to_save_xml(self, value): 

1014 return repr(value) 

1015 

1016 

1017class Complex(Object): 

1018 dummy_for = complex 

1019 

1020 class __T(TBase): 

1021 strict = True 

1022 

1023 def regularize_extra(self, val): 

1024 

1025 if isinstance(val, list) or isinstance(val, tuple): 

1026 assert len(val) == 2 

1027 val = complex(*val) 

1028 

1029 elif not isinstance(val, complex): 

1030 val = complex(val) 

1031 

1032 return val 

1033 

1034 def to_save(self, value): 

1035 return repr(value) 

1036 

1037 def to_save_xml(self, value): 

1038 return repr(value) 

1039 

1040 

1041class Bool(Object): 

1042 dummy_for = bool 

1043 

1044 class __T(TBase): 

1045 strict = True 

1046 

1047 def regularize_extra(self, val): 

1048 if isinstance(val, (str, newstr)): 

1049 if val.lower().strip() in ('0', 'false'): 

1050 return False 

1051 

1052 return bool(val) 

1053 

1054 def to_save_xml(self, value): 

1055 return repr(bool(value)).lower() 

1056 

1057 

1058class String(Object): 

1059 dummy_for = str 

1060 

1061 class __T(TBase): 

1062 def __init__(self, *args, **kwargs): 

1063 yamlstyle = kwargs.pop('yamlstyle', None) 

1064 TBase.__init__(self, *args, **kwargs) 

1065 self.style_cls = str_style_map[yamlstyle] 

1066 

1067 def to_save(self, val): 

1068 return self.style_cls(val) 

1069 

1070 

1071class Unicode(Object): 

1072 dummy_for = newstr 

1073 

1074 class __T(TBase): 

1075 def __init__(self, *args, **kwargs): 

1076 yamlstyle = kwargs.pop('yamlstyle', None) 

1077 TBase.__init__(self, *args, **kwargs) 

1078 self.style_cls = unicode_style_map[yamlstyle] 

1079 

1080 def to_save(self, val): 

1081 return self.style_cls(val) 

1082 

1083 

1084guts_plain_dummy_types = (String, Unicode, Int, Float, Complex, Bool) 

1085 

1086 

1087class Dict(Object): 

1088 dummy_for = dict 

1089 

1090 class __T(TBase): 

1091 multivalued = dict 

1092 

1093 def __init__(self, key_t=Any.T(), content_t=Any.T(), *args, **kwargs): 

1094 TBase.__init__(self, *args, **kwargs) 

1095 assert isinstance(key_t, TBase) 

1096 assert isinstance(content_t, TBase) 

1097 self.key_t = key_t 

1098 self.content_t = content_t 

1099 self.content_t.parent = self 

1100 

1101 def default(self): 

1102 if self._default is not None: 

1103 return dict( 

1104 (make_default(k), make_default(v)) 

1105 for (k, v) in self._default.items()) 

1106 

1107 if self.optional: 

1108 return None 

1109 else: 

1110 return {} 

1111 

1112 def has_default(self): 

1113 return True 

1114 

1115 def validate(self, val, regularize, depth): 

1116 return TBase.validate(self, val, regularize, depth+1) 

1117 

1118 def validate_children(self, val, regularize, depth): 

1119 for key, ele in list(val.items()): 

1120 newkey = self.key_t.validate(key, regularize, depth-1) 

1121 newele = self.content_t.validate(ele, regularize, depth-1) 

1122 if regularize: 

1123 if newkey is not key or newele is not ele: 

1124 del val[key] 

1125 val[newkey] = newele 

1126 

1127 return val 

1128 

1129 def to_save(self, val): 

1130 return dict((self.key_t.to_save(k), self.content_t.to_save(v)) 

1131 for (k, v) in val.items()) 

1132 

1133 def to_save_xml(self, val): 

1134 raise NotImplementedError() 

1135 

1136 def classname_for_help(self, strip_module=''): 

1137 return '``dict`` of %s objects' % \ 

1138 self.content_t.classname_for_help(strip_module=strip_module) 

1139 

1140 

1141class List(Object): 

1142 dummy_for = list 

1143 

1144 class __T(TBase): 

1145 multivalued = list 

1146 

1147 def __init__(self, content_t=Any.T(), *args, **kwargs): 

1148 yamlstyle = kwargs.pop('yamlstyle', None) 

1149 TBase.__init__(self, *args, **kwargs) 

1150 assert isinstance(content_t, TBase) or isinstance(content_t, Defer) 

1151 self.content_t = content_t 

1152 self.content_t.parent = self 

1153 self.style_cls = list_style_map[yamlstyle] 

1154 

1155 def default(self): 

1156 if self._default is not None: 

1157 return [make_default(x) for x in self._default] 

1158 if self.optional: 

1159 return None 

1160 else: 

1161 return [] 

1162 

1163 def has_default(self): 

1164 return True 

1165 

1166 def validate(self, val, regularize, depth): 

1167 return TBase.validate(self, val, regularize, depth+1) 

1168 

1169 def validate_children(self, val, regularize, depth): 

1170 for i, ele in enumerate(val): 

1171 newele = self.content_t.validate(ele, regularize, depth-1) 

1172 if regularize and newele is not ele: 

1173 val[i] = newele 

1174 

1175 return val 

1176 

1177 def to_save(self, val): 

1178 return self.style_cls(self.content_t.to_save(v) for v in val) 

1179 

1180 def to_save_xml(self, val): 

1181 return [self.content_t.to_save_xml(v) for v in val] 

1182 

1183 def deferred(self): 

1184 if isinstance(self.content_t, Defer): 

1185 return [self.content_t] 

1186 

1187 return [] 

1188 

1189 def process_deferred(self, defer, t_inst): 

1190 if defer is self.content_t: 

1191 self.content_t = t_inst 

1192 

1193 def classname_for_help(self, strip_module=''): 

1194 return '``list`` of %s objects' % \ 

1195 self.content_t.classname_for_help(strip_module=strip_module) 

1196 

1197 

1198def make_typed_list_class(t): 

1199 class TL(List): 

1200 class __T(List.T): 

1201 def __init__(self, *args, **kwargs): 

1202 List.T.__init__(self, content_t=t.T(), *args, **kwargs) 

1203 

1204 return TL 

1205 

1206 

1207class Tuple(Object): 

1208 dummy_for = tuple 

1209 

1210 class __T(TBase): 

1211 multivalued = tuple 

1212 

1213 def __init__(self, n=None, content_t=Any.T(), *args, **kwargs): 

1214 TBase.__init__(self, *args, **kwargs) 

1215 assert isinstance(content_t, TBase) 

1216 self.content_t = content_t 

1217 self.content_t.parent = self 

1218 self.n = n 

1219 

1220 def default(self): 

1221 if self._default is not None: 

1222 return tuple( 

1223 make_default(x) for x in self._default) 

1224 

1225 elif self.optional: 

1226 return None 

1227 else: 

1228 if self.n is not None: 

1229 return tuple( 

1230 self.content_t.default() for x in range(self.n)) 

1231 else: 

1232 return tuple() 

1233 

1234 def has_default(self): 

1235 return True 

1236 

1237 def validate(self, val, regularize, depth): 

1238 return TBase.validate(self, val, regularize, depth+1) 

1239 

1240 def validate_extra(self, val): 

1241 if self.n is not None and len(val) != self.n: 

1242 raise ValidationError( 

1243 '%s should have length %i' % (self.xname(), self.n)) 

1244 

1245 def validate_children(self, val, regularize, depth): 

1246 if not regularize: 

1247 for ele in val: 

1248 self.content_t.validate(ele, regularize, depth-1) 

1249 

1250 return val 

1251 else: 

1252 newval = [] 

1253 isnew = False 

1254 for ele in val: 

1255 newele = self.content_t.validate(ele, regularize, depth-1) 

1256 newval.append(newele) 

1257 if newele is not ele: 

1258 isnew = True 

1259 

1260 if isnew: 

1261 return tuple(newval) 

1262 else: 

1263 return val 

1264 

1265 def to_save(self, val): 

1266 return tuple(self.content_t.to_save(v) for v in val) 

1267 

1268 def to_save_xml(self, val): 

1269 return [self.content_t.to_save_xml(v) for v in val] 

1270 

1271 def classname_for_help(self, strip_module=''): 

1272 if self.n is not None: 

1273 return '``tuple`` of %i %s objects' % ( 

1274 self.n, self.content_t.classname_for_help( 

1275 strip_module=strip_module)) 

1276 else: 

1277 return '``tuple`` of %s objects' % ( 

1278 self.content_t.classname_for_help( 

1279 strip_module=strip_module)) 

1280 

1281 

1282re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$') 

1283 

1284 

1285class Timestamp(Object): 

1286 dummy_for = (hpfloat, float) 

1287 dummy_for_description = 'time_float' 

1288 

1289 class __T(TBase): 

1290 

1291 def regularize_extra(self, val): 

1292 

1293 time_float = get_time_float() 

1294 

1295 if isinstance(val, datetime.datetime): 

1296 tt = val.utctimetuple() 

1297 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6 

1298 

1299 elif isinstance(val, datetime.date): 

1300 tt = val.timetuple() 

1301 val = time_float(calendar.timegm(tt)) 

1302 

1303 elif isinstance(val, (str, newstr)): 

1304 val = val.strip() 

1305 tz_offset = 0 

1306 

1307 m = re_tz.search(val) 

1308 if m: 

1309 sh = m.group(2) 

1310 sm = m.group(4) 

1311 tz_offset = (int(sh)*3600 if sh else 0) \ 

1312 + (int(sm)*60 if sm else 0) 

1313 

1314 val = re_tz.sub('', val) 

1315 

1316 if len(val) > 10 and val[10] == 'T': 

1317 val = val.replace('T', ' ', 1) 

1318 

1319 try: 

1320 val = str_to_time(val) - tz_offset 

1321 except TimeStrError: 

1322 raise ValidationError( 

1323 '%s: cannot parse time/date: %s' % (self.xname(), val)) 

1324 

1325 elif isinstance(val, (int, float)): 

1326 val = time_float(val) 

1327 

1328 else: 

1329 raise ValidationError( 

1330 '%s: cannot convert "%s" to type %s' % ( 

1331 self.xname(), val, time_float)) 

1332 

1333 return val 

1334 

1335 def to_save(self, val): 

1336 return time_to_str(val, format='%Y-%m-%d %H:%M:%S.9FRAC')\ 

1337 .rstrip('0').rstrip('.') 

1338 

1339 def to_save_xml(self, val): 

1340 return time_to_str(val, format='%Y-%m-%dT%H:%M:%S.9FRAC')\ 

1341 .rstrip('0').rstrip('.') + 'Z' 

1342 

1343 @classmethod 

1344 def D(self, s): 

1345 return TimestampDefaultMaker(s) 

1346 

1347 

1348class DateTimestamp(Object): 

1349 dummy_for = (hpfloat, float) 

1350 dummy_for_description = 'time_float' 

1351 

1352 class __T(TBase): 

1353 

1354 def regularize_extra(self, val): 

1355 

1356 time_float = get_time_float() 

1357 

1358 if isinstance(val, datetime.datetime): 

1359 tt = val.utctimetuple() 

1360 val = time_float(calendar.timegm(tt)) + val.microsecond * 1e-6 

1361 

1362 elif isinstance(val, datetime.date): 

1363 tt = val.timetuple() 

1364 val = time_float(calendar.timegm(tt)) 

1365 

1366 elif isinstance(val, (str, newstr)): 

1367 val = str_to_time(val, format='%Y-%m-%d') 

1368 

1369 elif isinstance(val, int): 

1370 val = time_float(val) 

1371 

1372 return val 

1373 

1374 def to_save(self, val): 

1375 return time_to_str(val, format='%Y-%m-%d') 

1376 

1377 def to_save_xml(self, val): 

1378 return time_to_str(val, format='%Y-%m-%d') 

1379 

1380 @classmethod 

1381 def D(self, s): 

1382 return TimestampDefaultMaker(s, format='%Y-%m-%d') 

1383 

1384 

1385class StringPattern(String): 

1386 

1387 ''' 

1388 Any ``str`` matching pattern ``%(pattern)s``. 

1389 ''' 

1390 

1391 dummy_for = str 

1392 pattern = '.*' 

1393 

1394 class __T(String.T): 

1395 def __init__(self, pattern=None, *args, **kwargs): 

1396 String.T.__init__(self, *args, **kwargs) 

1397 

1398 if pattern is not None: 

1399 self.pattern = pattern 

1400 else: 

1401 self.pattern = self.dummy_cls.pattern 

1402 

1403 def validate_extra(self, val): 

1404 pat = self.pattern 

1405 if not re.search(pat, val): 

1406 raise ValidationError('%s: "%s" does not match pattern %s' % ( 

1407 self.xname(), val, repr(pat))) 

1408 

1409 @classmethod 

1410 def class_help_string(cls): 

1411 dcls = cls.dummy_cls 

1412 doc = dcls.__doc_template__ or StringPattern.__doc_template__ 

1413 return doc % {'pattern': repr(dcls.pattern)} 

1414 

1415 

1416class UnicodePattern(Unicode): 

1417 

1418 ''' 

1419 Any ``unicode`` matching pattern ``%(pattern)s``. 

1420 ''' 

1421 

1422 dummy_for = newstr 

1423 pattern = '.*' 

1424 

1425 class __T(TBase): 

1426 def __init__(self, pattern=None, *args, **kwargs): 

1427 TBase.__init__(self, *args, **kwargs) 

1428 

1429 if pattern is not None: 

1430 self.pattern = pattern 

1431 else: 

1432 self.pattern = self.dummy_cls.pattern 

1433 

1434 def validate_extra(self, val): 

1435 pat = self.pattern 

1436 if not re.search(pat, val, flags=re.UNICODE): 

1437 raise ValidationError('%s: "%s" does not match pattern %s' % ( 

1438 self.xname(), val, repr(pat))) 

1439 

1440 @classmethod 

1441 def class_help_string(cls): 

1442 dcls = cls.dummy_cls 

1443 doc = dcls.__doc_template__ or UnicodePattern.__doc_template__ 

1444 return doc % {'pattern': repr(dcls.pattern)} 

1445 

1446 

1447class StringChoice(String): 

1448 

1449 ''' 

1450 Any ``str`` out of ``%(choices)s``. 

1451 ''' 

1452 

1453 dummy_for = str 

1454 choices = [] 

1455 ignore_case = False 

1456 

1457 class __T(String.T): 

1458 def __init__(self, choices=None, ignore_case=None, *args, **kwargs): 

1459 String.T.__init__(self, *args, **kwargs) 

1460 

1461 if choices is not None: 

1462 self.choices = choices 

1463 else: 

1464 self.choices = self.dummy_cls.choices 

1465 

1466 if ignore_case is not None: 

1467 self.ignore_case = ignore_case 

1468 else: 

1469 self.ignore_case = self.dummy_cls.ignore_case 

1470 

1471 if self.ignore_case: 

1472 self.choices = [x.upper() for x in self.choices] 

1473 

1474 def validate_extra(self, val): 

1475 if self.ignore_case: 

1476 val = val.upper() 

1477 

1478 if val not in self.choices: 

1479 raise ValidationError( 

1480 '%s: "%s" is not a valid choice out of %s' % ( 

1481 self.xname(), val, repr(self.choices))) 

1482 

1483 @classmethod 

1484 def class_help_string(cls): 

1485 dcls = cls.dummy_cls 

1486 doc = dcls.__doc_template__ or StringChoice.__doc_template__ 

1487 return doc % {'choices': repr(dcls.choices)} 

1488 

1489 

1490# this will not always work... 

1491class Union(Object): 

1492 members = [] 

1493 dummy_for = str 

1494 

1495 class __T(TBase): 

1496 def __init__(self, members=None, *args, **kwargs): 

1497 TBase.__init__(self, *args, **kwargs) 

1498 if members is not None: 

1499 self.members = members 

1500 else: 

1501 self.members = self.dummy_cls.members 

1502 

1503 def validate(self, val, regularize=False, depth=-1): 

1504 assert self.members 

1505 e2 = None 

1506 for member in self.members: 

1507 try: 

1508 return member.validate(val, regularize, depth=depth) 

1509 except ValidationError as e: 

1510 e2 = e 

1511 

1512 raise e2 

1513 

1514 

1515class Choice(Object): 

1516 choices = [] 

1517 

1518 class __T(TBase): 

1519 def __init__(self, choices=None, *args, **kwargs): 

1520 TBase.__init__(self, *args, **kwargs) 

1521 if choices is not None: 

1522 self.choices = choices 

1523 else: 

1524 self.choices = self.dummy_cls.choices 

1525 

1526 self.cls_to_xmltagname = dict( 

1527 (t.cls, t.get_xmltagname()) for t in self.choices) 

1528 

1529 def validate(self, val, regularize=False, depth=-1): 

1530 if self.optional and val is None: 

1531 return val 

1532 

1533 t = None 

1534 for tc in self.choices: 

1535 is_derived = isinstance(val, tc.cls) 

1536 is_exact = type(val) == tc.cls 

1537 if not (not tc.strict and not is_derived or 

1538 tc.strict and not is_exact): 

1539 

1540 t = tc 

1541 break 

1542 

1543 if t is None: 

1544 if regularize: 

1545 ok = False 

1546 for tc in self.choices: 

1547 try: 

1548 val = tc.regularize_extra(val) 

1549 ok = True 

1550 t = tc 

1551 break 

1552 except (ValidationError, ValueError): 

1553 pass 

1554 

1555 if not ok: 

1556 raise ValidationError( 

1557 '%s: could not convert "%s" to any type out of ' 

1558 '(%s)' % (self.xname(), val, ','.join( 

1559 classnames(x.cls) for x in self.choices))) 

1560 else: 

1561 raise ValidationError( 

1562 '%s: "%s" (type: %s) is not of any type out of ' 

1563 '(%s)' % (self.xname(), val, type(val), ','.join( 

1564 classnames(x.cls) for x in self.choices))) 

1565 

1566 validator = t 

1567 

1568 if isinstance(t.cls, tuple): 

1569 clss = t.cls 

1570 else: 

1571 clss = (t.cls,) 

1572 

1573 for cls in clss: 

1574 try: 

1575 if type(val) != cls and isinstance(val, cls): 

1576 validator = val.T.instance 

1577 

1578 except AttributeError: 

1579 pass 

1580 

1581 validator.validate_extra(val) 

1582 

1583 if depth != 0: 

1584 val = validator.validate_children(val, regularize, depth) 

1585 

1586 return val 

1587 

1588 def extend_xmlelements(self, elems, v): 

1589 elems.append(( 

1590 self.cls_to_xmltagname[type(v)].split(' ', 1)[-1], v)) 

1591 

1592 

1593def _dump( 

1594 object, stream, 

1595 header=False, 

1596 Dumper=GutsSafeDumper, 

1597 _dump_function=yaml.dump): 

1598 

1599 if not getattr(stream, 'encoding', None): 

1600 enc = encode_utf8 

1601 else: 

1602 enc = no_encode 

1603 

1604 if header: 

1605 stream.write(enc(u'%YAML 1.1\n')) 

1606 if isinstance(header, (str, newstr)): 

1607 banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n' 

1608 stream.write(enc(banner)) 

1609 

1610 _dump_function( 

1611 object, 

1612 stream=stream, 

1613 encoding='utf-8', 

1614 explicit_start=True, 

1615 Dumper=Dumper) 

1616 

1617 

1618def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper): 

1619 _dump(object, stream=stream, header=header, _dump_function=yaml.dump_all) 

1620 

1621 

1622def _load(stream, 

1623 Loader=GutsSafeLoader, allow_include=None, filename=None, 

1624 included_files=None): 

1625 

1626 class _Loader(Loader): 

1627 _filename = filename 

1628 _allow_include = allow_include 

1629 _included_files = included_files or [] 

1630 

1631 return yaml.load(stream=stream, Loader=_Loader) 

1632 

1633 

1634def _load_all(stream, 

1635 Loader=GutsSafeLoader, allow_include=None, filename=None): 

1636 

1637 class _Loader(Loader): 

1638 _filename = filename 

1639 _allow_include = allow_include 

1640 

1641 return list(yaml.load_all(stream=stream, Loader=_Loader)) 

1642 

1643 

1644def _iload_all(stream, 

1645 Loader=GutsSafeLoader, allow_include=None, filename=None): 

1646 

1647 class _Loader(Loader): 

1648 _filename = filename 

1649 _allow_include = allow_include 

1650 

1651 return yaml.load_all(stream=stream, Loader=_Loader) 

1652 

1653 

1654def multi_representer(dumper, data): 

1655 node = dumper.represent_mapping( 

1656 '!'+data.T.tagname, data.T.inamevals_to_save(data), flow_style=False) 

1657 

1658 return node 

1659 

1660 

1661# hack for compatibility with early GF Store versions 

1662re_compatibility = re.compile( 

1663 r'^pyrocko\.(trace|gf\.(meta|seismosizer)|fomosto\.' 

1664 r'(dummy|poel|qseis|qssp))\.' 

1665) 

1666 

1667 

1668def multi_constructor(loader, tag_suffix, node): 

1669 tagname = str(tag_suffix) 

1670 

1671 tagname = re_compatibility.sub('pf.', tagname) 

1672 

1673 cls = g_tagname_to_class[tagname] 

1674 kwargs = dict(iter(loader.construct_pairs(node, deep=True))) 

1675 o = cls(**kwargs) 

1676 o.validate(regularize=True, depth=1) 

1677 return o 

1678 

1679 

1680def include_constructor(loader, node): 

1681 allow_include = loader._allow_include \ 

1682 if loader._allow_include is not None \ 

1683 else ALLOW_INCLUDE 

1684 

1685 if not allow_include: 

1686 raise EnvironmentError( 

1687 'Not allowed to include YAML. Load with allow_include=True') 

1688 

1689 if isinstance(node, yaml.nodes.ScalarNode): 

1690 inc_file = loader.construct_scalar(node) 

1691 else: 

1692 raise TypeError('Unsupported YAML node %s' % repr(node)) 

1693 

1694 if loader._filename is not None and not op.isabs(inc_file): 

1695 inc_file = op.join(op.dirname(loader._filename), inc_file) 

1696 

1697 if not op.isfile(inc_file): 

1698 raise FileNotFoundError(inc_file) 

1699 

1700 included_files = list(loader._included_files) 

1701 if loader._filename is not None: 

1702 included_files.append(op.abspath(loader._filename)) 

1703 

1704 for included_file in loader._included_files: 

1705 if op.samefile(inc_file, included_file): 

1706 raise ImportError( 

1707 'Circular import of file "%s". Include path: %s' % ( 

1708 op.abspath(inc_file), 

1709 ' -> '.join('"%s"' % s for s in included_files))) 

1710 

1711 with open(inc_file) as f: 

1712 return _load( 

1713 f, 

1714 Loader=loader.__class__, filename=inc_file, 

1715 allow_include=True, 

1716 included_files=included_files) 

1717 

1718 

1719def dict_noflow_representer(dumper, data): 

1720 return dumper.represent_mapping( 

1721 'tag:yaml.org,2002:map', data, flow_style=False) 

1722 

1723 

1724yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper) 

1725yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader) 

1726yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader) 

1727yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper) 

1728 

1729 

1730def newstr_representer(dumper, data): 

1731 return dumper.represent_scalar( 

1732 'tag:yaml.org,2002:str', unicode(data)) 

1733 

1734 

1735yaml.add_representer(newstr, newstr_representer, Dumper=GutsSafeDumper) 

1736 

1737 

1738class Constructor(object): 

1739 def __init__(self, add_namespace_maps=False, strict=False, ns_hints=None, 

1740 ns_ignore=False): 

1741 

1742 self.stack = [] 

1743 self.queue = [] 

1744 self.namespaces = defaultdict(list) 

1745 self.add_namespace_maps = add_namespace_maps 

1746 self.strict = strict 

1747 self.ns_hints = ns_hints 

1748 self.ns_ignore = ns_ignore 

1749 

1750 def start_element(self, ns_name, attrs): 

1751 if self.ns_ignore: 

1752 ns_name = ns_name.split(' ')[-1] 

1753 

1754 if -1 == ns_name.find(' '): 

1755 if self.ns_hints is None and ns_name in g_guessable_xmlns: 

1756 self.ns_hints = g_guessable_xmlns[ns_name] 

1757 

1758 if self.ns_hints: 

1759 ns_names = [ 

1760 ns_hint + ' ' + ns_name for ns_hint in self.ns_hints] 

1761 

1762 elif self.ns_hints is None: 

1763 ns_names = [' ' + ns_name] 

1764 

1765 else: 

1766 ns_names = [ns_name] 

1767 

1768 for ns_name in ns_names: 

1769 if self.stack and self.stack[-1][1] is not None: 

1770 cls = self.stack[-1][1].T.xmltagname_to_class.get( 

1771 ns_name, None) 

1772 

1773 if isinstance(cls, tuple): 

1774 cls = None 

1775 else: 

1776 if cls is not None and ( 

1777 not issubclass(cls, Object) 

1778 or issubclass(cls, SObject)): 

1779 cls = None 

1780 else: 

1781 cls = g_xmltagname_to_class.get(ns_name, None) 

1782 

1783 if cls: 

1784 break 

1785 

1786 self.stack.append((ns_name, cls, attrs, [], [])) 

1787 

1788 def end_element(self, _): 

1789 ns_name, cls, attrs, content2, content1 = self.stack.pop() 

1790 

1791 ns = ns_name.split(' ', 1)[0] 

1792 

1793 if cls is not None: 

1794 content2.extend( 

1795 (ns + ' ' + k if -1 == k.find(' ') else k, v) 

1796 for (k, v) in attrs.items()) 

1797 content2.append((None, ''.join(content1))) 

1798 o = cls(**cls.T.translate_from_xml(content2, self.strict)) 

1799 o.validate(regularize=True, depth=1) 

1800 if self.add_namespace_maps: 

1801 o.namespace_map = self.get_current_namespace_map() 

1802 

1803 if self.stack and not all(x[1] is None for x in self.stack): 

1804 self.stack[-1][-2].append((ns_name, o)) 

1805 else: 

1806 self.queue.append(o) 

1807 else: 

1808 content = [''.join(content1)] 

1809 if self.stack: 

1810 for c in content: 

1811 self.stack[-1][-2].append((ns_name, c)) 

1812 

1813 def characters(self, char_content): 

1814 if self.stack: 

1815 self.stack[-1][-1].append(char_content) 

1816 

1817 def start_namespace(self, ns, uri): 

1818 self.namespaces[ns].append(uri) 

1819 

1820 def end_namespace(self, ns): 

1821 self.namespaces[ns].pop() 

1822 

1823 def get_current_namespace_map(self): 

1824 return dict((k, v[-1]) for (k, v) in self.namespaces.items() if v) 

1825 

1826 def get_queued_elements(self): 

1827 queue = self.queue 

1828 self.queue = [] 

1829 return queue 

1830 

1831 

1832def _iload_all_xml( 

1833 stream, 

1834 bufsize=100000, 

1835 add_namespace_maps=False, 

1836 strict=False, 

1837 ns_hints=None, 

1838 ns_ignore=False): 

1839 

1840 from xml.parsers.expat import ParserCreate 

1841 

1842 parser = ParserCreate('UTF-8', namespace_separator=' ') 

1843 

1844 handler = Constructor( 

1845 add_namespace_maps=add_namespace_maps, 

1846 strict=strict, 

1847 ns_hints=ns_hints, 

1848 ns_ignore=ns_ignore) 

1849 

1850 parser.StartElementHandler = handler.start_element 

1851 parser.EndElementHandler = handler.end_element 

1852 parser.CharacterDataHandler = handler.characters 

1853 parser.StartNamespaceDeclHandler = handler.start_namespace 

1854 parser.EndNamespaceDeclHandler = handler.end_namespace 

1855 

1856 while True: 

1857 data = stream.read(bufsize) 

1858 parser.Parse(data, bool(not data)) 

1859 for element in handler.get_queued_elements(): 

1860 yield element 

1861 

1862 if not data: 

1863 break 

1864 

1865 

1866def _load_all_xml(*args, **kwargs): 

1867 return list(_iload_all_xml(*args, **kwargs)) 

1868 

1869 

1870def _load_xml(*args, **kwargs): 

1871 g = _iload_all_xml(*args, **kwargs) 

1872 return next(g) 

1873 

1874 

1875def _dump_all_xml(objects, stream, root_element_name='root', header=True): 

1876 

1877 if not getattr(stream, 'encoding', None): 

1878 enc = encode_utf8 

1879 else: 

1880 enc = no_encode 

1881 

1882 _dump_xml_header(stream, header) 

1883 

1884 beg = u'<%s>\n' % root_element_name 

1885 end = u'</%s>\n' % root_element_name 

1886 

1887 stream.write(enc(beg)) 

1888 

1889 for ob in objects: 

1890 _dump_xml(ob, stream=stream) 

1891 

1892 stream.write(enc(end)) 

1893 

1894 

1895def _dump_xml_header(stream, banner=None): 

1896 

1897 if not getattr(stream, 'encoding', None): 

1898 enc = encode_utf8 

1899 else: 

1900 enc = no_encode 

1901 

1902 stream.write(enc(u'<?xml version="1.0" encoding="UTF-8" ?>\n')) 

1903 if isinstance(banner, (str, newstr)): 

1904 stream.write(enc(u'<!-- %s -->\n' % banner)) 

1905 

1906 

1907def _dump_xml( 

1908 obj, stream, depth=0, ns_name=None, header=False, ns_map=[], 

1909 ns_ignore=False): 

1910 

1911 from xml.sax.saxutils import escape, quoteattr 

1912 

1913 if not getattr(stream, 'encoding', None): 

1914 enc = encode_utf8 

1915 else: 

1916 enc = no_encode 

1917 

1918 if depth == 0 and header: 

1919 _dump_xml_header(stream, header) 

1920 

1921 indent = ' '*depth*2 

1922 if ns_name is None: 

1923 ns_name = obj.T.instance.get_xmltagname() 

1924 

1925 if -1 != ns_name.find(' '): 

1926 ns, name = ns_name.split(' ') 

1927 else: 

1928 ns, name = '', ns_name 

1929 

1930 if isinstance(obj, Object): 

1931 obj.validate(depth=1) 

1932 attrs = [] 

1933 elems = [] 

1934 

1935 added_ns = False 

1936 if not ns_ignore and ns and (not ns_map or ns_map[-1] != ns): 

1937 attrs.append(('xmlns', ns)) 

1938 ns_map.append(ns) 

1939 added_ns = True 

1940 

1941 for prop, v in obj.T.ipropvals_to_save(obj, xmlmode=True): 

1942 if prop.xmlstyle == 'attribute': 

1943 assert not prop.multivalued 

1944 assert not isinstance(v, Object) 

1945 attrs.append((prop.effective_xmltagname, v)) 

1946 

1947 elif prop.xmlstyle == 'content': 

1948 assert not prop.multivalued 

1949 assert not isinstance(v, Object) 

1950 elems.append((None, v)) 

1951 

1952 else: 

1953 prop.extend_xmlelements(elems, v) 

1954 

1955 attr_str = '' 

1956 if attrs: 

1957 attr_str = ' ' + ' '.join( 

1958 '%s=%s' % (k.split(' ')[-1], quoteattr(str(v))) 

1959 for (k, v) in attrs) 

1960 

1961 if not elems: 

1962 stream.write(enc(u'%s<%s%s />\n' % (indent, name, attr_str))) 

1963 else: 

1964 oneline = len(elems) == 1 and elems[0][0] is None 

1965 stream.write(enc(u'%s<%s%s>%s' % ( 

1966 indent, 

1967 name, 

1968 attr_str, 

1969 '' if oneline else '\n'))) 

1970 

1971 for (k, v) in elems: 

1972 if k is None: 

1973 stream.write(enc(escape(newstr(v), {'\0': '&#00;'}))) 

1974 else: 

1975 _dump_xml(v, stream, depth+1, k, False, ns_map, ns_ignore) 

1976 

1977 stream.write(enc(u'%s</%s>\n' % ( 

1978 '' if oneline else indent, name))) 

1979 

1980 if added_ns: 

1981 ns_map.pop() 

1982 

1983 else: 

1984 stream.write(enc(u'%s<%s>%s</%s>\n' % ( 

1985 indent, 

1986 name, 

1987 escape(newstr(obj), {'\0': '&#00;'}), 

1988 name))) 

1989 

1990 

1991def walk(x, typ=None, path=()): 

1992 if typ is None or isinstance(x, typ): 

1993 yield path, x 

1994 

1995 if isinstance(x, Object): 

1996 for (prop, val) in x.T.ipropvals(x): 

1997 if prop.multivalued: 

1998 if val is not None: 

1999 for iele, ele in enumerate(val): 

2000 for y in walk(ele, typ, 

2001 path=path + ((prop.name, iele),)): 

2002 yield y 

2003 else: 

2004 for y in walk(val, typ, path=path+(prop.name,)): 

2005 yield y 

2006 

2007 

2008def clone(x, pool=None): 

2009 ''' 

2010 Clone guts object tree. 

2011 

2012 Traverses guts object tree and recursively clones all guts attributes, 

2013 falling back to :py:func:`copy.deepcopy` for non-guts objects. Objects 

2014 deriving from :py:class:`Object` are instantiated using their respective 

2015 init function. Multiply referenced objects in the source tree are multiply 

2016 referenced also in the destination tree. 

2017 

2018 This function can be used to clone guts objects ignoring any contained 

2019 run-time state, i.e. any of their attributes not defined as a guts 

2020 property. 

2021 ''' 

2022 

2023 if pool is None: 

2024 pool = {} 

2025 

2026 if id(x) in pool: 

2027 x_copy = pool[id(x)] 

2028 

2029 else: 

2030 if isinstance(x, Object): 

2031 d = {} 

2032 for (prop, y) in x.T.ipropvals(x): 

2033 if y is not None: 

2034 if not prop.multivalued: 

2035 y_copy = clone(y, pool) 

2036 elif prop.multivalued is dict: 

2037 y_copy = dict( 

2038 (clone(zk, pool), clone(zv, pool)) 

2039 for (zk, zv) in y.items()) 

2040 else: 

2041 y_copy = type(y)(clone(z, pool) for z in y) 

2042 else: 

2043 y_copy = y 

2044 

2045 d[prop.name] = y_copy 

2046 

2047 x_copy = x.__class__(**d) 

2048 

2049 else: 

2050 x_copy = copy.deepcopy(x) 

2051 

2052 pool[id(x)] = x_copy 

2053 return x_copy 

2054 

2055 

2056class YPathError(Exception): 

2057 ''' 

2058 This exception is raised for invalid ypath specifications. 

2059 ''' 

2060 pass 

2061 

2062 

2063def _parse_yname(yname): 

2064 ident = r'[a-zA-Z][a-zA-Z0-9_]*' 

2065 rint = r'-?[0-9]+' 

2066 m = re.match( 

2067 r'^(%s)(\[((%s)?(:)(%s)?|(%s))\])?$' 

2068 % (ident, rint, rint, rint), yname) 

2069 

2070 if not m: 

2071 raise YPathError('Syntax error in component: "%s"' % yname) 

2072 

2073 d = dict( 

2074 name=m.group(1)) 

2075 

2076 if m.group(2): 

2077 if m.group(5): 

2078 istart = iend = None 

2079 if m.group(4): 

2080 istart = int(m.group(4)) 

2081 if m.group(6): 

2082 iend = int(m.group(6)) 

2083 

2084 d['slice'] = (istart, iend) 

2085 else: 

2086 d['index'] = int(m.group(7)) 

2087 

2088 return d 

2089 

2090 

2091def _decend(obj, ynames): 

2092 if ynames: 

2093 for sobj in iter_elements(obj, ynames): 

2094 yield sobj 

2095 else: 

2096 yield obj 

2097 

2098 

2099def iter_elements(obj, ypath): 

2100 ''' 

2101 Generator yielding elements matching a given ypath specification. 

2102 

2103 :param obj: guts :py:class:`Object` instance 

2104 :param ypath: Dot-separated object path (e.g. 'root.child.child'). 

2105 To access list objects use slice notatation (e.g. 

2106 'root.child[:].child[1:3].child[1]'). 

2107 

2108 Raises :py:exc:`YPathError` on failure. 

2109 ''' 

2110 

2111 try: 

2112 if isinstance(ypath, str): 

2113 ynames = ypath.split('.') 

2114 else: 

2115 ynames = ypath 

2116 

2117 yname = ynames[0] 

2118 ynames = ynames[1:] 

2119 d = _parse_yname(yname) 

2120 if d['name'] not in obj.T.propnames: 

2121 raise AttributeError(d['name']) 

2122 

2123 obj = getattr(obj, d['name']) 

2124 

2125 if 'index' in d: 

2126 sobj = obj[d['index']] 

2127 for ssobj in _decend(sobj, ynames): 

2128 yield ssobj 

2129 

2130 elif 'slice' in d: 

2131 for i in range(*slice(*d['slice']).indices(len(obj))): 

2132 sobj = obj[i] 

2133 for ssobj in _decend(sobj, ynames): 

2134 yield ssobj 

2135 else: 

2136 for sobj in _decend(obj, ynames): 

2137 yield sobj 

2138 

2139 except (AttributeError, IndexError) as e: 

2140 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e))) 

2141 

2142 

2143def get_elements(obj, ypath): 

2144 ''' 

2145 Get all elements matching a given ypath specification. 

2146 

2147 :param obj: guts :py:class:`Object` instance 

2148 :param ypath: Dot-separated object path (e.g. 'root.child.child'). 

2149 To access list objects use slice notatation (e.g. 

2150 'root.child[:].child[1:3].child[1]'). 

2151 

2152 Raises :py:exc:`YPathError` on failure. 

2153 ''' 

2154 return list(iter_elements(obj, ypath)) 

2155 

2156 

2157def set_elements(obj, ypath, value, validate=False, regularize=False): 

2158 ''' 

2159 Set elements matching a given ypath specification. 

2160 

2161 :param obj: guts :py:class:`Object` instance 

2162 :param ypath: Dot-separated object path (e.g. 'root.child.child'). 

2163 To access list objects use slice notatation (e.g. 

2164 'root.child[:].child[1:3].child[1]'). 

2165 :param value: All matching elements will be set to `value`. 

2166 :param validate: Whether to validate affected subtrees. 

2167 :param regularize: Whether to regularize affected subtrees. 

2168 

2169 Raises :py:exc:`YPathError` on failure. 

2170 ''' 

2171 

2172 ynames = ypath.split('.') 

2173 try: 

2174 d = _parse_yname(ynames[-1]) 

2175 for sobj in iter_elements(obj, ynames[:-1]): 

2176 if d['name'] not in sobj.T.propnames: 

2177 raise AttributeError(d['name']) 

2178 

2179 if 'index' in d: 

2180 ssobj = getattr(sobj, d['name']) 

2181 ssobj[d['index']] = value 

2182 elif 'slice' in d: 

2183 ssobj = getattr(sobj, d['name']) 

2184 for i in range(*slice(*d['slice']).indices(len(ssobj))): 

2185 ssobj[i] = value 

2186 else: 

2187 setattr(sobj, d['name'], value) 

2188 if regularize: 

2189 sobj.regularize() 

2190 if validate: 

2191 sobj.validate() 

2192 

2193 except (AttributeError, IndexError, YPathError) as e: 

2194 raise YPathError('Invalid ypath: "%s" (%s)' % (ypath, str(e))) 

2195 

2196 

2197def zip_walk(x, typ=None, path=(), stack=()): 

2198 if typ is None or isinstance(x, typ): 

2199 yield path, stack + (x,) 

2200 

2201 if isinstance(x, Object): 

2202 for (prop, val) in x.T.ipropvals(x): 

2203 if prop.multivalued: 

2204 if val is not None: 

2205 for iele, ele in enumerate(val): 

2206 for y in zip_walk( 

2207 ele, typ, 

2208 path=path + ((prop.name, iele),), 

2209 stack=stack + (x,)): 

2210 

2211 yield y 

2212 else: 

2213 for y in zip_walk(val, typ, 

2214 path=path+(prop.name,), 

2215 stack=stack + (x,)): 

2216 yield y 

2217 

2218 

2219def path_element(x): 

2220 if isinstance(x, tuple): 

2221 return '%s[%i]' % x 

2222 else: 

2223 return x 

2224 

2225 

2226def path_to_str(path): 

2227 return '.'.join(path_element(x) for x in path) 

2228 

2229 

2230@expand_stream_args('w') 

2231def dump(*args, **kwargs): 

2232 return _dump(*args, **kwargs) 

2233 

2234 

2235@expand_stream_args('r') 

2236def load(*args, **kwargs): 

2237 return _load(*args, **kwargs) 

2238 

2239 

2240def load_string(s, *args, **kwargs): 

2241 return load(string=s, *args, **kwargs) 

2242 

2243 

2244@expand_stream_args('w') 

2245def dump_all(*args, **kwargs): 

2246 return _dump_all(*args, **kwargs) 

2247 

2248 

2249@expand_stream_args('r') 

2250def load_all(*args, **kwargs): 

2251 return _load_all(*args, **kwargs) 

2252 

2253 

2254@expand_stream_args('r') 

2255def iload_all(*args, **kwargs): 

2256 return _iload_all(*args, **kwargs) 

2257 

2258 

2259@expand_stream_args('w') 

2260def dump_xml(*args, **kwargs): 

2261 return _dump_xml(*args, **kwargs) 

2262 

2263 

2264@expand_stream_args('r') 

2265def load_xml(*args, **kwargs): 

2266 kwargs.pop('filename', None) 

2267 return _load_xml(*args, **kwargs) 

2268 

2269 

2270def load_xml_string(s, *args, **kwargs): 

2271 return load_xml(string=s, *args, **kwargs) 

2272 

2273 

2274@expand_stream_args('w') 

2275def dump_all_xml(*args, **kwargs): 

2276 return _dump_all_xml(*args, **kwargs) 

2277 

2278 

2279@expand_stream_args('r') 

2280def load_all_xml(*args, **kwargs): 

2281 kwargs.pop('filename', None) 

2282 return _load_all_xml(*args, **kwargs) 

2283 

2284 

2285@expand_stream_args('r') 

2286def iload_all_xml(*args, **kwargs): 

2287 kwargs.pop('filename', None) 

2288 return _iload_all_xml(*args, **kwargs) 

2289 

2290 

2291__all__ = guts_types + [ 

2292 'guts_types', 'TBase', 'ValidationError', 

2293 'ArgumentError', 'Defer', 

2294 'dump', 'load', 

2295 'dump_all', 'load_all', 'iload_all', 

2296 'dump_xml', 'load_xml', 

2297 'dump_all_xml', 'load_all_xml', 'iload_all_xml', 

2298 'load_string', 

2299 'load_xml_string', 

2300 'make_typed_list_class', 'walk', 'zip_walk', 'path_to_str' 

2301]