Coverage for /usr/local/lib/python3.11/dist-packages/grond/report/base.py: 63%

1from __future__ import print_function 

2import logging 

3import os.path as op 

4import shutil 

5import os 

6import tarfile 

7import threading 

8import signal 

9import time 

10 

11from http.server import HTTPServer, SimpleHTTPRequestHandler 

12 

13from pyrocko import guts, util 

14from pyrocko.model import Event 

15from pyrocko.guts import Object, String, Unicode, Bool 

16 

17from grond.meta import HasPaths, Path, expand_template, GrondError 

18 

19from grond import core, environment 

20from grond.problems import ProblemInfoNotAvailable, ProblemDataNotAvailable 

21from grond.version import __version__ 

22from grond import info 

23from grond.plot import PlotConfigCollection, get_all_plot_classes 

24from grond.run_info import RunInfo 

25 

26guts_prefix = 'grond' 

27logger = logging.getLogger('grond.report') 

28 

29 

30class ReportIndexEntry(Object): 

31 path = String.T() 

32 problem_name = String.T() 

33 event_reference = Event.T(optional=True) 

34 event_best = Event.T(optional=True) 

35 grond_version = String.T(optional=True) 

36 run_info = RunInfo.T(optional=True) 

37 

38 

39class ReportConfig(HasPaths): 

40 report_base_path = Path.T(default='report') 

41 entries_sub_path = String.T( 

42 default='${event_name}/${problem_name}') 

43 title = Unicode.T( 

44 default=u'Grond Report', 

45 help='Title shown on report overview page.') 

46 description = Unicode.T( 

47 default=u'This interactive document aggregates earthquake source ' 

48 u'inversion results from optimisations performed with Grond.', 

49 help='Description shown on report overview page.') 

50 plot_config_collection = PlotConfigCollection.T( 

51 help='Configurations for plots to be included in the report.') 

52 make_archive = Bool.T( 

53 default=True, 

54 help='Set to `false` to prevent creation of compressed archive.') 

55 

56 

57class ReportInfo(Object): 

58 title = Unicode.T(optional=True) 

59 description = Unicode.T(optional=True) 

60 version_info = info.VersionInfo.T() 

61 have_archive = Bool.T(optional=True) 

62 

63 

64def read_config(path): 

65 get_all_plot_classes() # make sure all plot modules are imported 

66 try: 

67 config = guts.load(filename=path) 

68 except OSError: 

69 raise GrondError( 

70 'Cannot read Grond report configuration file: %s' % path) 

71 

72 if not isinstance(config, ReportConfig): 

73 raise GrondError( 

74 'Invalid Grond report configuration in file "%s".' % path) 

75 

76 config.set_basepath(op.dirname(path) or '.') 

77 return config 

78 

79 

80def write_config(config, path): 

81 try: 

82 basepath = config.get_basepath() 

83 dirname = op.dirname(path) or '.' 

84 config.change_basepath(dirname) 

85 guts.dump( 

86 config, 

87 filename=path, 

88 header='Grond report configuration file, version %s' % __version__) 

89 

90 config.change_basepath(basepath) 

91 

92 except OSError: 

93 raise GrondError( 

94 'Cannot write Grond report configuration file: %s' % path) 

95 

96 

97def iter_report_entry_dirs(report_base_path): 

98 for path, dirnames, filenames in os.walk(report_base_path): 

99 for dirname in dirnames: 

100 dirpath = op.join(path, dirname) 

101 stats_path = op.join(dirpath, 'problem.yaml') 

102 if op.exists(stats_path): 

103 yield dirpath 

104 

105 

106def copytree(src, dst): 

107 names = os.listdir(src) 

108 if not op.exists(dst): 

109 os.makedirs(dst) 

110 

111 for name in names: 

112 srcname = op.join(src, name) 

113 dstname = op.join(dst, name) 

114 if op.isdir(srcname): 

115 copytree(srcname, dstname) 

116 else: 

117 shutil.copy(srcname, dstname) 

118 

119 

120def report(env, report_config=None, update_without_plotting=False, 

121 make_index=True, make_archive=True, nthreads=0): 

122 

123 if report_config is None: 

124 report_config = ReportConfig() 

125 report_config.set_basepath('.') 

126 

127 event_name = env.get_current_event_name() 

128 problem = env.get_problem() 

129 logger.info('Creating report entry for run "%s"...' % problem.name) 

130 

131 optimiser = env.get_optimiser() 

132 optimiser.set_nthreads(nthreads) 

133 

134 fp = report_config.expand_path 

135 entry_path = expand_template( 

136 op.join( 

137 fp(report_config.report_base_path), 

138 report_config.entries_sub_path), 

139 dict( 

140 event_name=event_name, 

141 problem_name=problem.name)) 

142 

143 if op.exists(entry_path) and not update_without_plotting: 

144 shutil.rmtree(entry_path) 

145 

146 try: 

147 problem.dump_problem_info(entry_path) 

148 

149 guts.dump(env.get_config(), 

150 filename=op.join(entry_path, 'config.yaml'), 

151 header=True) 

152 

153 util.ensuredir(entry_path) 

154 plots_dir_out = op.join(entry_path, 'plots') 

155 util.ensuredir(plots_dir_out) 

156 

157 event = env.get_dataset().get_event() 

158 guts.dump(event, filename=op.join(entry_path, 'event.reference.yaml')) 

159 

160 try: 

161 rundir_path = env.get_rundir_path() 

162 

163 core.export( 

164 'stats', [rundir_path], 

165 filename=op.join(entry_path, 'stats.yaml')) 

166 

167 core.export( 

168 'best', [rundir_path], 

169 filename=op.join(entry_path, 'event.solution.best.yaml'), 

170 type='event-yaml') 

171 

172 core.export( 

173 'mean', [rundir_path], 

174 filename=op.join(entry_path, 'event.solution.mean.yaml'), 

175 type='event-yaml') 

176 

177 core.export( 

178 'ensemble', [rundir_path], 

179 filename=op.join(entry_path, 'event.solution.ensemble.yaml'), 

180 type='event-yaml') 

181 

182 except (environment.NoRundirAvailable, ProblemInfoNotAvailable, 

183 ProblemDataNotAvailable): 

184 

185 pass 

186 

187 if not update_without_plotting: 

188 from grond import plot 

189 pcc = report_config.plot_config_collection.get_weeded(env) 

190 plot.make_plots( 

191 env, 

192 plots_path=op.join(entry_path, 'plots'), 

193 plot_config_collection=pcc) 

194 

195 try: 

196 run_info = env.get_run_info() 

197 except environment.NoRundirAvailable: 

198 run_info = None 

199 

200 rie = ReportIndexEntry( 

201 path='.', 

202 problem_name=problem.name, 

203 grond_version=problem.grond_version, 

204 run_info=run_info) 

205 

206 fn = op.join(entry_path, 'event.solution.best.yaml') 

207 if op.exists(fn): 

208 rie.event_best = guts.load(filename=fn) 

209 

210 fn = op.join(entry_path, 'event.reference.yaml') 

211 if op.exists(fn): 

212 rie.event_reference = guts.load(filename=fn) 

213 

214 fn = op.join(entry_path, 'index.yaml') 

215 guts.dump(rie, filename=fn) 

216 

217 except Exception as e: 

218 logger.warning( 

219 'Failed to create report entry, removing incomplete subdirectory: ' 

220 '%s' % entry_path) 

221 raise e 

222 

223 if op.exists(entry_path): 

224 shutil.rmtree(entry_path) 

225 

226 logger.info('Done creating report entry for run "%s".' % problem.name) 

227 

228 if make_index: 

229 report_index(report_config) 

230 

231 if make_archive: 

232 report_archive(report_config) 

233 

234 

235def report_index(report_config=None): 

236 if report_config is None: 

237 report_config = ReportConfig() 

238 

239 report_base_path = report_config.report_base_path 

240 entries = [] 

241 for entry_path in iter_report_entry_dirs(report_base_path): 

242 

243 fn = op.join(entry_path, 'index.yaml') 

244 if not os.path.exists(fn): 

245 logger.warning( 

246 'Skipping indexing of incomplete report entry: %s' 

247 % entry_path) 

248 

249 continue 

250 

251 logger.info('Indexing %s...' % entry_path) 

252 

253 rie = guts.load(filename=fn) 

254 report_relpath = op.relpath(entry_path, report_base_path) 

255 rie.path = report_relpath 

256 entries.append(rie) 

257 

258 guts.dump_all( 

259 entries, 

260 filename=op.join(report_base_path, 'report_list.yaml')) 

261 

262 guts.dump( 

263 ReportInfo( 

264 title=report_config.title, 

265 description=report_config.description, 

266 version_info=info.version_info(), 

267 have_archive=report_config.make_archive), 

268 filename=op.join(report_base_path, 'info.yaml')) 

269 

270 app_dir = op.join(op.split(__file__)[0], 'app') 

271 copytree(app_dir, report_base_path) 

272 update_guts_registry(op.join(report_base_path, 'js', 'guts_registry.js')) 

273 

274 logger.info('Created report in %s/index.html' % report_base_path) 

275 

276 

277def update_guts_registry(path): 

278 tags = ['!' + s for s in guts.g_tagname_to_class.keys()] 

279 js_data = 'GUTS_TYPES = [%s];\n' % ', '.join("'%s'" % tag for tag in tags) 

280 with open(path, 'w') as out: 

281 out.write(js_data) 

282 

283 

284def report_archive(report_config): 

285 if report_config is None: 

286 report_config = ReportConfig() 

287 

288 if not report_config.make_archive: 

289 return 

290 

291 report_base_path = report_config.report_base_path 

292 

293 logger.info('Generating report\'s archive...') 

294 with tarfile.open(op.join(report_base_path, 'grond-report.tar.gz'), 

295 mode='w:gz') as tar: 

296 tar.add(report_base_path, arcname='grond-report') 

297 

298 

299def serve_ip(host): 

300 if host == 'localhost': 

301 ip = '127.0.0.1' 

302 elif host == 'default': 

303 import socket 

304 ip = [ 

305 (s.connect(('4.4.4.4', 80)), s.getsockname()[0], s.close()) 

306 for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1] 

307 elif host == '*': 

308 ip = '' 

309 else: 

310 ip = host 

311 

312 return ip 

313 

314 

315class ReportHandler(SimpleHTTPRequestHandler): 

316 

317 def _log_error(self, fmt, *args): 

318 logger.error(fmt % args) 

319 

320 def _log_message(self, fmt, *args): 

321 logger.debug(fmt % args) 

322 

323 def end_headers(self): 

324 self.send_header('Cache-Control', 'no-cache') 

325 SimpleHTTPRequestHandler.end_headers(self) 

326 

327 

328ReportHandler.extensions_map.update({ 

329 '.yaml': 'application/x-yaml', 

330 '.yml': 'application/x-yaml'}) 

331 

332 

333g_terminate = False 

334 

335 

336def serve_report( 

337 addr=('127.0.0.1', 8383), 

338 report_config=None, 

339 fixed_port=False, 

340 open=False): 

341 

342 if report_config is None: 

343 report_config = ReportConfig() 

344 

345 path = report_config.expand_path(report_config.report_base_path) 

346 os.chdir(path) 

347 

348 host, port = addr 

349 if fixed_port: 

350 ports = [port] 

351 else: 

352 ports = range(port, port+20) 

353 

354 httpd = None 

355 for port in ports: 

356 try: 

357 httpd = HTTPServer((host, port), ReportHandler) 

358 break 

359 except OSError as e: 

360 logger.warning(str(e)) 

361 

362 if httpd: 

363 logger.info( 

364 'Starting report web service at http://%s:%d' % (host, port)) 

365 

366 thread = threading.Thread(None, httpd.serve_forever) 

367 thread.start() 

368 

369 if open: 

370 import webbrowser 

371 if open: 

372 webbrowser.open('http://%s:%d' % (host, port)) 

373 

374 def handler(signum, frame): 

375 global g_terminate 

376 g_terminate = True 

377 

378 signal.signal(signal.SIGINT, handler) 

379 signal.signal(signal.SIGTERM, handler) 

380 

381 while not g_terminate: 

382 time.sleep(0.1) 

383 

384 signal.signal(signal.SIGINT, signal.SIG_DFL) 

385 signal.signal(signal.SIGTERM, signal.SIG_DFL) 

386 

387 logger.info('Stopping report web service...') 

388 

389 httpd.shutdown() 

390 thread.join() 

391 

392 logger.info('... done') 

393 

394 else: 

395 logger.error('Failed to start web service.') 

396 

397 

398__all__ = ''' 

399 report 

400 report_index 

401 report_archive 

402 ReportConfig 

403 ReportIndexEntry 

404 ReportInfo 

405 serve_ip 

406 serve_report 

407 read_config 

408 write_config 

409'''.split()