1from __future__ import print_function
2import logging
3import os.path as op
4import shutil
5import os
6import tarfile
7import threading
8import signal
9import time
11from http.server import HTTPServer, SimpleHTTPRequestHandler
13from pyrocko import guts, util
14from pyrocko.model import Event
15from pyrocko.guts import Object, String, Unicode, Bool
17from grond.meta import HasPaths, Path, expand_template, GrondError
19from grond import core, environment
20from grond.problems import ProblemInfoNotAvailable, ProblemDataNotAvailable
21from grond.version import __version__
22from grond import info
23from grond.plot import PlotConfigCollection, get_all_plot_classes
24from grond.run_info import RunInfo
26guts_prefix = 'grond'
27logger = logging.getLogger('grond.report')
30class ReportIndexEntry(Object):
31 path = String.T()
32 problem_name = String.T()
33 event_reference = Event.T(optional=True)
34 event_best = Event.T(optional=True)
35 grond_version = String.T(optional=True)
36 run_info = RunInfo.T(optional=True)
39class ReportConfig(HasPaths):
40 report_base_path = Path.T(default='report')
41 entries_sub_path = String.T(
42 default='${event_name}/${problem_name}')
43 title = Unicode.T(
44 default=u'Grond Report',
45 help='Title shown on report overview page.')
46 description = Unicode.T(
47 default=u'This interactive document aggregates earthquake source '
48 u'inversion results from optimisations performed with Grond.',
49 help='Description shown on report overview page.')
50 plot_config_collection = PlotConfigCollection.T(
51 help='Configurations for plots to be included in the report.')
52 make_archive = Bool.T(
53 default=True,
54 help='Set to `false` to prevent creation of compressed archive.')
57class ReportInfo(Object):
58 title = Unicode.T(optional=True)
59 description = Unicode.T(optional=True)
60 version_info = info.VersionInfo.T()
61 have_archive = Bool.T(optional=True)
64def read_config(path):
65 get_all_plot_classes() # make sure all plot modules are imported
66 try:
67 config = guts.load(filename=path)
68 except OSError:
69 raise GrondError(
70 'Cannot read Grond report configuration file: %s' % path)
72 if not isinstance(config, ReportConfig):
73 raise GrondError(
74 'Invalid Grond report configuration in file "%s".' % path)
76 config.set_basepath(op.dirname(path) or '.')
77 return config
80def write_config(config, path):
81 try:
82 basepath = config.get_basepath()
83 dirname = op.dirname(path) or '.'
84 config.change_basepath(dirname)
85 guts.dump(
86 config,
87 filename=path,
88 header='Grond report configuration file, version %s' % __version__)
90 config.change_basepath(basepath)
92 except OSError:
93 raise GrondError(
94 'Cannot write Grond report configuration file: %s' % path)
97def iter_report_entry_dirs(report_base_path):
98 for path, dirnames, filenames in os.walk(report_base_path):
99 for dirname in dirnames:
100 dirpath = op.join(path, dirname)
101 stats_path = op.join(dirpath, 'problem.yaml')
102 if op.exists(stats_path):
103 yield dirpath
106def copytree(src, dst):
107 names = os.listdir(src)
108 if not op.exists(dst):
109 os.makedirs(dst)
111 for name in names:
112 srcname = op.join(src, name)
113 dstname = op.join(dst, name)
114 if op.isdir(srcname):
115 copytree(srcname, dstname)
116 else:
117 shutil.copy(srcname, dstname)
120def report(env, report_config=None, update_without_plotting=False,
121 make_index=True, make_archive=True, nthreads=0):
123 if report_config is None:
124 report_config = ReportConfig()
125 report_config.set_basepath('.')
127 event_name = env.get_current_event_name()
128 problem = env.get_problem()
129 logger.info('Creating report entry for run "%s"...' % problem.name)
131 optimiser = env.get_optimiser()
132 optimiser.set_nthreads(nthreads)
134 fp = report_config.expand_path
135 entry_path = expand_template(
136 op.join(
137 fp(report_config.report_base_path),
138 report_config.entries_sub_path),
139 dict(
140 event_name=event_name,
141 problem_name=problem.name))
143 if op.exists(entry_path) and not update_without_plotting:
144 shutil.rmtree(entry_path)
146 try:
147 problem.dump_problem_info(entry_path)
149 guts.dump(env.get_config(),
150 filename=op.join(entry_path, 'config.yaml'),
151 header=True)
153 util.ensuredir(entry_path)
154 plots_dir_out = op.join(entry_path, 'plots')
155 util.ensuredir(plots_dir_out)
157 event = env.get_dataset().get_event()
158 guts.dump(event, filename=op.join(entry_path, 'event.reference.yaml'))
160 try:
161 rundir_path = env.get_rundir_path()
163 core.export(
164 'stats', [rundir_path],
165 filename=op.join(entry_path, 'stats.yaml'))
167 core.export(
168 'best', [rundir_path],
169 filename=op.join(entry_path, 'event.solution.best.yaml'),
170 type='event-yaml')
172 core.export(
173 'mean', [rundir_path],
174 filename=op.join(entry_path, 'event.solution.mean.yaml'),
175 type='event-yaml')
177 core.export(
178 'ensemble', [rundir_path],
179 filename=op.join(entry_path, 'event.solution.ensemble.yaml'),
180 type='event-yaml')
182 except (environment.NoRundirAvailable, ProblemInfoNotAvailable,
183 ProblemDataNotAvailable):
185 pass
187 if not update_without_plotting:
188 from grond import plot
189 pcc = report_config.plot_config_collection.get_weeded(env)
190 plot.make_plots(
191 env,
192 plots_path=op.join(entry_path, 'plots'),
193 plot_config_collection=pcc)
195 try:
196 run_info = env.get_run_info()
197 except environment.NoRundirAvailable:
198 run_info = None
200 rie = ReportIndexEntry(
201 path='.',
202 problem_name=problem.name,
203 grond_version=problem.grond_version,
204 run_info=run_info)
206 fn = op.join(entry_path, 'event.solution.best.yaml')
207 if op.exists(fn):
208 rie.event_best = guts.load(filename=fn)
210 fn = op.join(entry_path, 'event.reference.yaml')
211 if op.exists(fn):
212 rie.event_reference = guts.load(filename=fn)
214 fn = op.join(entry_path, 'index.yaml')
215 guts.dump(rie, filename=fn)
217 except Exception as e:
218 logger.warning(
219 'Failed to create report entry, removing incomplete subdirectory: '
220 '%s' % entry_path)
221 raise e
223 if op.exists(entry_path):
224 shutil.rmtree(entry_path)
226 logger.info('Done creating report entry for run "%s".' % problem.name)
228 if make_index:
229 report_index(report_config)
231 if make_archive:
232 report_archive(report_config)
235def report_index(report_config=None):
236 if report_config is None:
237 report_config = ReportConfig()
239 report_base_path = report_config.report_base_path
240 entries = []
241 for entry_path in iter_report_entry_dirs(report_base_path):
243 fn = op.join(entry_path, 'index.yaml')
244 if not os.path.exists(fn):
245 logger.warning(
246 'Skipping indexing of incomplete report entry: %s'
247 % entry_path)
249 continue
251 logger.info('Indexing %s...' % entry_path)
253 rie = guts.load(filename=fn)
254 report_relpath = op.relpath(entry_path, report_base_path)
255 rie.path = report_relpath
256 entries.append(rie)
258 guts.dump_all(
259 entries,
260 filename=op.join(report_base_path, 'report_list.yaml'))
262 guts.dump(
263 ReportInfo(
264 title=report_config.title,
265 description=report_config.description,
266 version_info=info.version_info(),
267 have_archive=report_config.make_archive),
268 filename=op.join(report_base_path, 'info.yaml'))
270 app_dir = op.join(op.split(__file__)[0], 'app')
271 copytree(app_dir, report_base_path)
272 update_guts_registry(op.join(report_base_path, 'js', 'guts_registry.js'))
274 logger.info('Created report in %s/index.html' % report_base_path)
277def update_guts_registry(path):
278 tags = ['!' + s for s in guts.g_tagname_to_class.keys()]
279 js_data = 'GUTS_TYPES = [%s];\n' % ', '.join("'%s'" % tag for tag in tags)
280 with open(path, 'w') as out:
281 out.write(js_data)
284def report_archive(report_config):
285 if report_config is None:
286 report_config = ReportConfig()
288 if not report_config.make_archive:
289 return
291 report_base_path = report_config.report_base_path
293 logger.info('Generating report\'s archive...')
294 with tarfile.open(op.join(report_base_path, 'grond-report.tar.gz'),
295 mode='w:gz') as tar:
296 tar.add(report_base_path, arcname='grond-report')
299def serve_ip(host):
300 if host == 'localhost':
301 ip = '127.0.0.1'
302 elif host == 'default':
303 import socket
304 ip = [
305 (s.connect(('4.4.4.4', 80)), s.getsockname()[0], s.close())
306 for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]
307 elif host == '*':
308 ip = ''
309 else:
310 ip = host
312 return ip
315class ReportHandler(SimpleHTTPRequestHandler):
317 def _log_error(self, fmt, *args):
318 logger.error(fmt % args)
320 def _log_message(self, fmt, *args):
321 logger.debug(fmt % args)
323 def end_headers(self):
324 self.send_header('Cache-Control', 'no-cache')
325 SimpleHTTPRequestHandler.end_headers(self)
328ReportHandler.extensions_map.update({
329 '.yaml': 'application/x-yaml',
330 '.yml': 'application/x-yaml'})
333g_terminate = False
336def serve_report(
337 addr=('127.0.0.1', 8383),
338 report_config=None,
339 fixed_port=False,
340 open=False):
342 if report_config is None:
343 report_config = ReportConfig()
345 path = report_config.expand_path(report_config.report_base_path)
346 os.chdir(path)
348 host, port = addr
349 if fixed_port:
350 ports = [port]
351 else:
352 ports = range(port, port+20)
354 httpd = None
355 for port in ports:
356 try:
357 httpd = HTTPServer((host, port), ReportHandler)
358 break
359 except OSError as e:
360 logger.warning(str(e))
362 if httpd:
363 logger.info(
364 'Starting report web service at http://%s:%d' % (host, port))
366 thread = threading.Thread(None, httpd.serve_forever)
367 thread.start()
369 if open:
370 import webbrowser
371 if open:
372 webbrowser.open('http://%s:%d' % (host, port))
374 def handler(signum, frame):
375 global g_terminate
376 g_terminate = True
378 signal.signal(signal.SIGINT, handler)
379 signal.signal(signal.SIGTERM, handler)
381 while not g_terminate:
382 time.sleep(0.1)
384 signal.signal(signal.SIGINT, signal.SIG_DFL)
385 signal.signal(signal.SIGTERM, signal.SIG_DFL)
387 logger.info('Stopping report web service...')
389 httpd.shutdown()
390 thread.join()
392 logger.info('... done')
394 else:
395 logger.error('Failed to start web service.')
398__all__ = '''
399 report
400 report_index
401 report_archive
402 ReportConfig
403 ReportIndexEntry
404 ReportInfo
405 serve_ip
406 serve_report
407 read_config
408 write_config
409'''.split()