Coverage for /usr/local/lib/python3.11/dist-packages/grond/report/base.py: 44%

1# https://pyrocko.org/grond - GPLv3 

2# 

3# The Grond Developers, 21st Century 

4import logging 

5import os.path as op 

6import shutil 

7import os 

8import tarfile 

9import threading 

10import signal 

11import time 

12 

13from http.server import HTTPServer, SimpleHTTPRequestHandler 

14 

15from pyrocko import guts, util 

16from pyrocko.model import Event 

17from pyrocko.guts import Object, String, Unicode, Bool 

18 

19from grond.meta import HasPaths, Path, expand_template, GrondError 

20 

21from grond import core, environment 

22from grond.problems import ProblemInfoNotAvailable, ProblemDataNotAvailable 

23from grond.version import __version__ 

24from grond import info 

25from grond.plot import PlotConfigCollection, get_all_plot_classes 

26from grond.run_info import RunInfo 

27 

28guts_prefix = 'grond' 

29logger = logging.getLogger('grond.report') 

30 

31 

32class ReportIndexEntry(Object): 

33 path = String.T() 

34 problem_name = String.T() 

35 event_reference = Event.T(optional=True) 

36 event_best = Event.T(optional=True) 

37 grond_version = String.T(optional=True) 

38 run_info = RunInfo.T(optional=True) 

39 

40 

41class ReportConfig(HasPaths): 

42 report_base_path = Path.T(default='report') 

43 entries_sub_path = String.T( 

44 default='${event_name}/${problem_name}') 

45 title = Unicode.T( 

46 default=u'Grond Report', 

47 help='Title shown on report overview page.') 

48 authors = Unicode.T( 

49 optional=True, 

50 help='Authors shown on report overview page.') 

51 description = Unicode.T( 

52 default=u'This interactive document aggregates earthquake source ' 

53 u'inversion results from optimisations performed with Grond.', 

54 help='Description shown on report overview page.') 

55 plot_config_collection = PlotConfigCollection.T( 

56 help='Configurations for plots to be included in the report.') 

57 make_archive = Bool.T( 

58 default=False, 

59 help='Set to `false` to prevent creation of compressed archive.') 

60 

61 

62class ReportInfo(Object): 

63 title = Unicode.T(optional=True) 

64 authors = Unicode.T(optional=True) 

65 description = Unicode.T(optional=True) 

66 version_info = info.VersionInfo.T() 

67 have_archive = Bool.T(optional=True) 

68 

69 

70def read_config(path): 

71 get_all_plot_classes() # make sure all plot modules are imported 

72 try: 

73 config = guts.load(filename=path) 

74 except OSError: 

75 raise GrondError( 

76 'Cannot read Grond report configuration file: %s' % path) 

77 

78 if not isinstance(config, ReportConfig): 

79 raise GrondError( 

80 'Invalid Grond report configuration in file "%s".' % path) 

81 

82 config.set_basepath(op.dirname(path) or '.') 

83 return config 

84 

85 

86def write_config(config, path): 

87 try: 

88 basepath = config.get_basepath() 

89 dirname = op.dirname(path) or '.' 

90 config.change_basepath(dirname) 

91 guts.dump( 

92 config, 

93 filename=path, 

94 header='Grond report configuration file, version %s' % __version__) 

95 

96 config.change_basepath(basepath) 

97 

98 except OSError: 

99 raise GrondError( 

100 'Cannot write Grond report configuration file: %s' % path) 

101 

102 

103def iter_report_entry_dirs(report_base_path): 

104 for path, dirnames, filenames in os.walk(report_base_path): 

105 for dirname in dirnames: 

106 dirpath = op.join(path, dirname) 

107 stats_path = op.join(dirpath, 'problem.yaml') 

108 if op.exists(stats_path): 

109 yield dirpath 

110 

111 

112def copytree(src, dst): 

113 names = os.listdir(src) 

114 if not op.exists(dst): 

115 os.makedirs(dst) 

116 

117 for name in names: 

118 srcname = op.join(src, name) 

119 dstname = op.join(dst, name) 

120 if op.isdir(srcname): 

121 copytree(srcname, dstname) 

122 else: 

123 shutil.copy(srcname, dstname) 

124 

125 

126def report(env, report_config=None, update_without_plotting=False, 

127 make_index=True, make_archive=True): 

128 

129 if report_config is None: 

130 report_config = ReportConfig() 

131 report_config.set_basepath('.') 

132 

133 event_name = env.get_current_event_name() 

134 problem = env.get_problem() 

135 logger.info('Creating report entry for run "%s"...' % problem.name) 

136 

137 fp = report_config.expand_path 

138 entry_path = expand_template( 

139 op.join( 

140 fp(report_config.report_base_path), 

141 report_config.entries_sub_path), 

142 dict( 

143 event_name=event_name, 

144 problem_name=problem.name)) 

145 

146 if op.exists(entry_path) and not update_without_plotting: 

147 shutil.rmtree(entry_path) 

148 

149 try: 

150 problem.dump_problem_info(entry_path) 

151 

152 guts.dump(env.get_config(), 

153 filename=op.join(entry_path, 'config.yaml'), 

154 header=True) 

155 

156 util.ensuredir(entry_path) 

157 plots_dir_out = op.join(entry_path, 'plots') 

158 util.ensuredir(plots_dir_out) 

159 

160 event = env.get_dataset().get_event() 

161 guts.dump(event, filename=op.join(entry_path, 'event.reference.yaml')) 

162 

163 try: 

164 rundir_path = env.get_rundir_path() 

165 

166 core.export( 

167 'stats', [rundir_path], 

168 filename=op.join(entry_path, 'stats.yaml')) 

169 

170 core.export( 

171 'best', [rundir_path], 

172 filename=op.join(entry_path, 'event.solution.best.yaml'), 

173 type='event-yaml') 

174 

175 core.export( 

176 'mean', [rundir_path], 

177 filename=op.join(entry_path, 'event.solution.mean.yaml'), 

178 type='event-yaml') 

179 

180 core.export( 

181 'ensemble', [rundir_path], 

182 filename=op.join(entry_path, 'event.solution.ensemble.yaml'), 

183 type='event-yaml') 

184 

185 except (environment.NoRundirAvailable, ProblemInfoNotAvailable, 

186 ProblemDataNotAvailable): 

187 

188 pass 

189 

190 if not update_without_plotting: 

191 from grond import plot 

192 pcc = report_config.plot_config_collection.get_weeded(env) 

193 plot.make_plots( 

194 env, 

195 plots_path=op.join(entry_path, 'plots'), 

196 plot_config_collection=pcc) 

197 

198 try: 

199 run_info = env.get_run_info() 

200 except environment.NoRundirAvailable: 

201 run_info = None 

202 

203 rie = ReportIndexEntry( 

204 path='.', 

205 problem_name=problem.name, 

206 grond_version=problem.grond_version, 

207 run_info=run_info) 

208 

209 fn = op.join(entry_path, 'event.solution.best.yaml') 

210 if op.exists(fn): 

211 rie.event_best = guts.load(filename=fn) 

212 

213 fn = op.join(entry_path, 'event.reference.yaml') 

214 if op.exists(fn): 

215 rie.event_reference = guts.load(filename=fn) 

216 

217 fn = op.join(entry_path, 'index.yaml') 

218 guts.dump(rie, filename=fn) 

219 

220 except Exception as e: 

221 logger.warning( 

222 'Failed to create report entry, removing incomplete subdirectory: ' 

223 '%s' % entry_path) 

224 raise e 

225 

226 if op.exists(entry_path): 

227 shutil.rmtree(entry_path) 

228 

229 logger.info('Done creating report entry for run "%s".' % problem.name) 

230 

231 if make_index: 

232 report_index(report_config) 

233 

234 if make_archive: 

235 report_archive(report_config) 

236 

237 

238def report_index(report_config=None): 

239 if report_config is None: 

240 report_config = ReportConfig() 

241 

242 report_base_path = report_config.report_base_path 

243 entries = [] 

244 for entry_path in iter_report_entry_dirs(report_base_path): 

245 

246 fn = op.join(entry_path, 'index.yaml') 

247 if not os.path.exists(fn): 

248 logger.warning( 

249 'Skipping indexing of incomplete report entry: %s' 

250 % entry_path) 

251 

252 continue 

253 

254 logger.info('Indexing %s...' % entry_path) 

255 

256 rie = guts.load(filename=fn) 

257 report_relpath = op.relpath(entry_path, report_base_path) 

258 rie.path = report_relpath 

259 entries.append(rie) 

260 

261 guts.dump_all( 

262 entries, 

263 filename=op.join(report_base_path, 'report_list.yaml')) 

264 

265 guts.dump( 

266 ReportInfo( 

267 title=report_config.title, 

268 authors=report_config.authors, 

269 description=report_config.description, 

270 version_info=info.version_info(), 

271 have_archive=report_config.make_archive), 

272 filename=op.join(report_base_path, 'info.yaml')) 

273 

274 app_dir = op.join(op.split(__file__)[0], 'app') 

275 copytree(app_dir, report_base_path) 

276 update_guts_registry(op.join(report_base_path, 'js', 'guts_registry.js')) 

277 

278 logger.info('Created report in %s/index.html' % report_base_path) 

279 

280 

281def update_guts_registry(path): 

282 tags = ['!' + s for s in guts.g_tagname_to_class.keys()] 

283 js_data = 'GUTS_TYPES = [%s];\n' % ', '.join("'%s'" % tag for tag in tags) 

284 with open(path, 'w') as out: 

285 out.write(js_data) 

286 

287 

288def report_archive(report_config): 

289 if report_config is None: 

290 report_config = ReportConfig() 

291 

292 if not report_config.make_archive: 

293 return 

294 

295 report_base_path = report_config.report_base_path 

296 

297 logger.info('Generating report\'s archive...') 

298 with tarfile.open(op.join(report_base_path, 'grond-report.tar.gz'), 

299 mode='w:gz') as tar: 

300 tar.add(report_base_path, arcname='grond-report') 

301 

302 

303def serve_ip(host): 

304 if host == 'localhost': 

305 ip = '127.0.0.1' 

306 elif host == 'default': 

307 import socket 

308 ip = [ 

309 (s.connect(('4.4.4.4', 80)), s.getsockname()[0], s.close()) 

310 for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1] 

311 elif host == '*': 

312 ip = '' 

313 else: 

314 ip = host 

315 

316 return ip 

317 

318 

319class ReportHandler(SimpleHTTPRequestHandler): 

320 

321 def _log_error(self, fmt, *args): 

322 logger.error(fmt % args) 

323 

324 def _log_message(self, fmt, *args): 

325 logger.debug(fmt % args) 

326 

327 def end_headers(self): 

328 self.send_header('Cache-Control', 'no-cache') 

329 SimpleHTTPRequestHandler.end_headers(self) 

330 

331 

332ReportHandler.extensions_map.update({ 

333 '.yaml': 'application/x-yaml', 

334 '.yml': 'application/x-yaml'}) 

335 

336 

337g_terminate = False 

338 

339 

340def serve_report( 

341 addr=('127.0.0.1', 8383), 

342 report_config=None, 

343 fixed_port=False, 

344 open=False): 

345 

346 if report_config is None: 

347 report_config = ReportConfig() 

348 

349 path = report_config.expand_path(report_config.report_base_path) 

350 os.chdir(path) 

351 

352 host, port = addr 

353 if fixed_port: 

354 ports = [port] 

355 else: 

356 ports = range(port, port+20) 

357 

358 httpd = None 

359 for port in ports: 

360 try: 

361 httpd = HTTPServer((host, port), ReportHandler) 

362 break 

363 except OSError as e: 

364 logger.warning(str(e)) 

365 

366 if httpd: 

367 logger.info( 

368 'Starting report web service at http://%s:%d' % (host, port)) 

369 

370 thread = threading.Thread(None, httpd.serve_forever) 

371 thread.start() 

372 

373 if open: 

374 import webbrowser 

375 if open: 

376 webbrowser.open('http://%s:%d' % (host, port)) 

377 

378 def handler(signum, frame): 

379 global g_terminate 

380 g_terminate = True 

381 

382 signal.signal(signal.SIGINT, handler) 

383 signal.signal(signal.SIGTERM, handler) 

384 

385 while not g_terminate: 

386 time.sleep(0.1) 

387 

388 signal.signal(signal.SIGINT, signal.SIG_DFL) 

389 signal.signal(signal.SIGTERM, signal.SIG_DFL) 

390 

391 logger.info('Stopping report web service...') 

392 

393 httpd.shutdown() 

394 thread.join() 

395 

396 logger.info('... done') 

397 

398 else: 

399 logger.error('Failed to start web service.') 

400 

401 

402__all__ = ''' 

403 report 

404 report_index 

405 report_archive 

406 ReportConfig 

407 ReportIndexEntry 

408 ReportInfo 

409 serve_ip 

410 serve_report 

411 read_config 

412 write_config 

413'''.split()