Coverage for /usr/local/lib/python3.11/dist-packages/grond/report/base.py: 45%

1from __future__ import print_function 

2import logging 

3import os.path as op 

4import shutil 

5import os 

6import tarfile 

7import threading 

8import signal 

9import time 

10 

11from http.server import HTTPServer, SimpleHTTPRequestHandler 

12 

13from pyrocko import guts, util 

14from pyrocko.model import Event 

15from pyrocko.guts import Object, String, Unicode, Bool 

16 

17from grond.meta import HasPaths, Path, expand_template, GrondError 

18 

19from grond import core, environment 

20from grond.problems import ProblemInfoNotAvailable, ProblemDataNotAvailable 

21from grond.version import __version__ 

22from grond import info 

23from grond.plot import PlotConfigCollection, get_all_plot_classes 

24from grond.run_info import RunInfo 

25 

26guts_prefix = 'grond' 

27logger = logging.getLogger('grond.report') 

28 

29 

30class ReportIndexEntry(Object): 

31 path = String.T() 

32 problem_name = String.T() 

33 event_reference = Event.T(optional=True) 

34 event_best = Event.T(optional=True) 

35 grond_version = String.T(optional=True) 

36 run_info = RunInfo.T(optional=True) 

37 

38 

39class ReportConfig(HasPaths): 

40 report_base_path = Path.T(default='report') 

41 entries_sub_path = String.T( 

42 default='${event_name}/${problem_name}') 

43 title = Unicode.T( 

44 default=u'Grond Report', 

45 help='Title shown on report overview page.') 

46 description = Unicode.T( 

47 default=u'This interactive document aggregates earthquake source ' 

48 u'inversion results from optimisations performed with Grond.', 

49 help='Description shown on report overview page.') 

50 plot_config_collection = PlotConfigCollection.T( 

51 help='Configurations for plots to be included in the report.') 

52 make_archive = Bool.T( 

53 default=True, 

54 help='Set to `false` to prevent creation of compressed archive.') 

55 

56 

57class ReportInfo(Object): 

58 title = Unicode.T(optional=True) 

59 description = Unicode.T(optional=True) 

60 version_info = info.VersionInfo.T() 

61 have_archive = Bool.T(optional=True) 

62 

63 

64def read_config(path): 

65 get_all_plot_classes() # make sure all plot modules are imported 

66 try: 

67 config = guts.load(filename=path) 

68 except OSError: 

69 raise GrondError( 

70 'Cannot read Grond report configuration file: %s' % path) 

71 

72 if not isinstance(config, ReportConfig): 

73 raise GrondError( 

74 'Invalid Grond report configuration in file "%s".' % path) 

75 

76 config.set_basepath(op.dirname(path) or '.') 

77 return config 

78 

79 

80def write_config(config, path): 

81 try: 

82 basepath = config.get_basepath() 

83 dirname = op.dirname(path) or '.' 

84 config.change_basepath(dirname) 

85 guts.dump( 

86 config, 

87 filename=path, 

88 header='Grond report configuration file, version %s' % __version__) 

89 

90 config.change_basepath(basepath) 

91 

92 except OSError: 

93 raise GrondError( 

94 'Cannot write Grond report configuration file: %s' % path) 

95 

96 

97def iter_report_entry_dirs(report_base_path): 

98 for path, dirnames, filenames in os.walk(report_base_path): 

99 for dirname in dirnames: 

100 dirpath = op.join(path, dirname) 

101 stats_path = op.join(dirpath, 'problem.yaml') 

102 if op.exists(stats_path): 

103 yield dirpath 

104 

105 

106def copytree(src, dst): 

107 names = os.listdir(src) 

108 if not op.exists(dst): 

109 os.makedirs(dst) 

110 

111 for name in names: 

112 srcname = op.join(src, name) 

113 dstname = op.join(dst, name) 

114 if op.isdir(srcname): 

115 copytree(srcname, dstname) 

116 else: 

117 shutil.copy(srcname, dstname) 

118 

119 

120def report(env, report_config=None, update_without_plotting=False, 

121 make_index=True, make_archive=True): 

122 

123 if report_config is None: 

124 report_config = ReportConfig() 

125 report_config.set_basepath('.') 

126 

127 event_name = env.get_current_event_name() 

128 problem = env.get_problem() 

129 logger.info('Creating report entry for run "%s"...' % problem.name) 

130 

131 fp = report_config.expand_path 

132 entry_path = expand_template( 

133 op.join( 

134 fp(report_config.report_base_path), 

135 report_config.entries_sub_path), 

136 dict( 

137 event_name=event_name, 

138 problem_name=problem.name)) 

139 

140 if op.exists(entry_path) and not update_without_plotting: 

141 shutil.rmtree(entry_path) 

142 

143 try: 

144 problem.dump_problem_info(entry_path) 

145 

146 guts.dump(env.get_config(), 

147 filename=op.join(entry_path, 'config.yaml'), 

148 header=True) 

149 

150 util.ensuredir(entry_path) 

151 plots_dir_out = op.join(entry_path, 'plots') 

152 util.ensuredir(plots_dir_out) 

153 

154 event = env.get_dataset().get_event() 

155 guts.dump(event, filename=op.join(entry_path, 'event.reference.yaml')) 

156 

157 try: 

158 rundir_path = env.get_rundir_path() 

159 

160 core.export( 

161 'stats', [rundir_path], 

162 filename=op.join(entry_path, 'stats.yaml')) 

163 

164 core.export( 

165 'best', [rundir_path], 

166 filename=op.join(entry_path, 'event.solution.best.yaml'), 

167 type='event-yaml') 

168 

169 core.export( 

170 'mean', [rundir_path], 

171 filename=op.join(entry_path, 'event.solution.mean.yaml'), 

172 type='event-yaml') 

173 

174 core.export( 

175 'ensemble', [rundir_path], 

176 filename=op.join(entry_path, 'event.solution.ensemble.yaml'), 

177 type='event-yaml') 

178 

179 except (environment.NoRundirAvailable, ProblemInfoNotAvailable, 

180 ProblemDataNotAvailable): 

181 

182 pass 

183 

184 if not update_without_plotting: 

185 from grond import plot 

186 pcc = report_config.plot_config_collection.get_weeded(env) 

187 plot.make_plots( 

188 env, 

189 plots_path=op.join(entry_path, 'plots'), 

190 plot_config_collection=pcc) 

191 

192 try: 

193 run_info = env.get_run_info() 

194 except environment.NoRundirAvailable: 

195 run_info = None 

196 

197 rie = ReportIndexEntry( 

198 path='.', 

199 problem_name=problem.name, 

200 grond_version=problem.grond_version, 

201 run_info=run_info) 

202 

203 fn = op.join(entry_path, 'event.solution.best.yaml') 

204 if op.exists(fn): 

205 rie.event_best = guts.load(filename=fn) 

206 

207 fn = op.join(entry_path, 'event.reference.yaml') 

208 if op.exists(fn): 

209 rie.event_reference = guts.load(filename=fn) 

210 

211 fn = op.join(entry_path, 'index.yaml') 

212 guts.dump(rie, filename=fn) 

213 

214 except Exception as e: 

215 logger.warning( 

216 'Failed to create report entry, removing incomplete subdirectory: ' 

217 '%s' % entry_path) 

218 raise e 

219 

220 if op.exists(entry_path): 

221 shutil.rmtree(entry_path) 

222 

223 logger.info('Done creating report entry for run "%s".' % problem.name) 

224 

225 if make_index: 

226 report_index(report_config) 

227 

228 if make_archive: 

229 report_archive(report_config) 

230 

231 

232def report_index(report_config=None): 

233 if report_config is None: 

234 report_config = ReportConfig() 

235 

236 report_base_path = report_config.report_base_path 

237 entries = [] 

238 for entry_path in iter_report_entry_dirs(report_base_path): 

239 

240 fn = op.join(entry_path, 'index.yaml') 

241 if not os.path.exists(fn): 

242 logger.warning( 

243 'Skipping indexing of incomplete report entry: %s' 

244 % entry_path) 

245 

246 continue 

247 

248 logger.info('Indexing %s...' % entry_path) 

249 

250 rie = guts.load(filename=fn) 

251 report_relpath = op.relpath(entry_path, report_base_path) 

252 rie.path = report_relpath 

253 entries.append(rie) 

254 

255 guts.dump_all( 

256 entries, 

257 filename=op.join(report_base_path, 'report_list.yaml')) 

258 

259 guts.dump( 

260 ReportInfo( 

261 title=report_config.title, 

262 description=report_config.description, 

263 version_info=info.version_info(), 

264 have_archive=report_config.make_archive), 

265 filename=op.join(report_base_path, 'info.yaml')) 

266 

267 app_dir = op.join(op.split(__file__)[0], 'app') 

268 copytree(app_dir, report_base_path) 

269 update_guts_registry(op.join(report_base_path, 'js', 'guts_registry.js')) 

270 

271 logger.info('Created report in %s/index.html' % report_base_path) 

272 

273 

274def update_guts_registry(path): 

275 tags = ['!' + s for s in guts.g_tagname_to_class.keys()] 

276 js_data = 'GUTS_TYPES = [%s];\n' % ', '.join("'%s'" % tag for tag in tags) 

277 with open(path, 'w') as out: 

278 out.write(js_data) 

279 

280 

281def report_archive(report_config): 

282 if report_config is None: 

283 report_config = ReportConfig() 

284 

285 if not report_config.make_archive: 

286 return 

287 

288 report_base_path = report_config.report_base_path 

289 

290 logger.info('Generating report\'s archive...') 

291 with tarfile.open(op.join(report_base_path, 'grond-report.tar.gz'), 

292 mode='w:gz') as tar: 

293 tar.add(report_base_path, arcname='grond-report') 

294 

295 

296def serve_ip(host): 

297 if host == 'localhost': 

298 ip = '127.0.0.1' 

299 elif host == 'default': 

300 import socket 

301 ip = [ 

302 (s.connect(('4.4.4.4', 80)), s.getsockname()[0], s.close()) 

303 for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1] 

304 elif host == '*': 

305 ip = '' 

306 else: 

307 ip = host 

308 

309 return ip 

310 

311 

312class ReportHandler(SimpleHTTPRequestHandler): 

313 

314 def _log_error(self, fmt, *args): 

315 logger.error(fmt % args) 

316 

317 def _log_message(self, fmt, *args): 

318 logger.debug(fmt % args) 

319 

320 def end_headers(self): 

321 self.send_header('Cache-Control', 'no-cache') 

322 SimpleHTTPRequestHandler.end_headers(self) 

323 

324 

325ReportHandler.extensions_map.update({ 

326 '.yaml': 'application/x-yaml', 

327 '.yml': 'application/x-yaml'}) 

328 

329 

330g_terminate = False 

331 

332 

333def serve_report( 

334 addr=('127.0.0.1', 8383), 

335 report_config=None, 

336 fixed_port=False, 

337 open=False): 

338 

339 if report_config is None: 

340 report_config = ReportConfig() 

341 

342 path = report_config.expand_path(report_config.report_base_path) 

343 os.chdir(path) 

344 

345 host, port = addr 

346 if fixed_port: 

347 ports = [port] 

348 else: 

349 ports = range(port, port+20) 

350 

351 httpd = None 

352 for port in ports: 

353 try: 

354 httpd = HTTPServer((host, port), ReportHandler) 

355 break 

356 except OSError as e: 

357 logger.warning(str(e)) 

358 

359 if httpd: 

360 logger.info( 

361 'Starting report web service at http://%s:%d' % (host, port)) 

362 

363 thread = threading.Thread(None, httpd.serve_forever) 

364 thread.start() 

365 

366 if open: 

367 import webbrowser 

368 if open: 

369 webbrowser.open('http://%s:%d' % (host, port)) 

370 

371 def handler(signum, frame): 

372 global g_terminate 

373 g_terminate = True 

374 

375 signal.signal(signal.SIGINT, handler) 

376 signal.signal(signal.SIGTERM, handler) 

377 

378 while not g_terminate: 

379 time.sleep(0.1) 

380 

381 signal.signal(signal.SIGINT, signal.SIG_DFL) 

382 signal.signal(signal.SIGTERM, signal.SIG_DFL) 

383 

384 logger.info('Stopping report web service...') 

385 

386 httpd.shutdown() 

387 thread.join() 

388 

389 logger.info('... done') 

390 

391 else: 

392 logger.error('Failed to start web service.') 

393 

394 

395__all__ = ''' 

396 report 

397 report_index 

398 report_archive 

399 ReportConfig 

400 ReportIndexEntry 

401 ReportInfo 

402 serve_ip 

403 serve_report 

404 read_config 

405 write_config 

406'''.split()