Coverage for /usr/local/lib/python3.7/dist-packages/pyrocko/util.py : 78%

# http://pyrocko.org - GPLv3 

# 

# The Pyrocko Developers, 21st Century 

# ---|P------/S----------~Lg---------- 

'''Utility functions for Pyrocko.''' 

from __future__ import division, print_function 

import time 

import logging 

import os 

import sys 

import re 

import calendar 

import math 

import fnmatch 

import fcntl 

import shlex 

import optparse 

import os.path as op 

import platform 

import errno 

import numpy as num 

from scipy import signal 

from pyrocko import dummy_progressbar 

try: 

from urllib.parse import urlencode, quote, unquote # noqa 

from urllib.request import ( 

Request, build_opener, HTTPDigestAuthHandler, urlopen) # noqa 

from urllib.error import HTTPError, URLError # noqa 

except ImportError: 

from urllib import urlencode, quote, unquote # noqa 

from urllib2 import (Request, build_opener, HTTPDigestAuthHandler, # noqa 

HTTPError, URLError, urlopen) # noqa 

class URLErrorSSL(URLError): 

def __init__(self, urlerror): 

self.__dict__ = urlerror.__dict__.copy() 

def __str__(self): 

return ( 

'Requesting web resource failed and the problem could be ' 

'related to SSL. Python standard libraries on some older ' 

'systems (like Ubuntu 14.04) are known to have trouble ' 

'with some SSL setups of today\'s servers: %s' 

% URLError.__str__(self)) 

def urlopen_ssl_check(*args, **kwargs): 

try: 

return urlopen(*args, **kwargs) 

except URLError as e: 

if str(e).find('SSL') != -1: 

raise URLErrorSSL(e) 

else: 

raise 

try: 

long 

except NameError: 

long = int 

force_dummy_progressbar = False 

if platform.system() != 'Darwin': 

from pyrocko import util_ext 

else: 

util_ext = None 

logger = logging.getLogger('pyrocko.util') 

try: 

import progressbar as progressbar_mod 

except ImportError: 

from pyrocko import dummy_progressbar as progressbar_mod 

try: 

num_full = num.full 

except AttributeError: 

def num_full(shape, fill_value, dtype=None, order='C'): 

a = num.empty(shape, dtype=dtype, order=order) 

a.fill(fill_value) 

return a 

try: 

num_full_like = num.full_like 

except AttributeError: 

def num_full_like(arr, fill_value, dtype=None, order='K', subok=True): 

a = num.empty_like(arr, dtype=dtype, order=order, subok=subok) 

a.fill(fill_value) 

return a 

def progressbar_module(): 

return progressbar_mod 

g_setup_logging_args = 'pyrocko', 'warning' 

def setup_logging(programname='pyrocko', levelname='warning'): 

''' 

Initialize logging. 

:param programname: program name to be written in log 

:param levelname: string indicating the logging level ('debug', 'info', 

'warning', 'error', 'critical') 

This function is called at startup by most pyrocko programs to set up a 

consistent logging format. This is simply a shortcut to a call to 

:py:func:`logging.basicConfig()`. 

''' 

global g_setup_logging_args 

g_setup_logging_args = (programname, levelname) 

levels = {'debug': logging.DEBUG, 

'info': logging.INFO, 

'warning': logging.WARNING, 

'error': logging.ERROR, 

'critical': logging.CRITICAL} 

logging.basicConfig( 

level=levels[levelname], 

format=programname+':%(name)-20s - %(levelname)-8s - %(message)s') 

def subprocess_setup_logging_args(): 

''' 

Get arguments from previous call to setup_logging. 

These can be sent down to a worker process so it can setup its logging 

in the same way as the main process. 

''' 

return g_setup_logging_args 

def data_file(fn): 

return os.path.join(os.path.split(__file__)[0], 'data', fn) 

class DownloadError(Exception): 

pass 

class PathExists(DownloadError): 

pass 

def _download(url, fpath, username=None, password=None, 

force=False, method='download', stats=None, 

status_callback=None, entries_wanted=None, 

recursive=False, header=None): 

import requests 

from requests.auth import HTTPBasicAuth 

from requests.exceptions import HTTPError as req_HTTPError 

requests.adapters.DEFAULT_RETRIES = 5 

urljoin = requests.compat.urljoin 

session = requests.Session() 

session.header = header 

session.auth = None if username is None\ 

else HTTPBasicAuth(username, password) 

status = { 

'ntotal_files': 0, 

'nread_files': 0, 

'ntotal_bytes_all_files': 0, 

'nread_bytes_all_files': 0, 

'ntotal_bytes_current_file': 0, 

'nread_bytes_current_file': 0, 

'finished': False 

} 

try: 

url_to_size = {} 

if callable(status_callback): 

status_callback(status) 

if not recursive and url.endswith('/'): 

raise DownloadError( 

'URL: %s appears to be a directory' 

' but recurvise download is False' % url) 

if recursive and not url.endswith('/'): 

url += '/' 

def parse_directory_tree(url, subdir=''): 

r = session.get(urljoin(url, subdir)) 

r.raise_for_status() 

entries = re.findall(r'href="([a-zA-Z0-9_.-]+/?)"', r.text) 

files = sorted(set(subdir + fn for fn in entries 

if not fn.endswith('/'))) 

if entries_wanted is not None: 

files = [fn for fn in files 

if (fn in wanted for wanted in entries_wanted)] 

status['ntotal_files'] += len(files) 

dirs = sorted(set(subdir + dn for dn in entries 

if dn.endswith('/') 

and dn not in ('./', '../'))) 

for dn in dirs: 

files.extend(parse_directory_tree( 

url, subdir=dn)) 

return files 

def get_content_length(url): 

if url not in url_to_size: 

r = session.head(url, headers={'Accept-Encoding': ''}) 

content_length = r.headers.get('content-length', None) 

if content_length is None: 

logger.debug('Could not get HTTP header ' 

'Content-Length for %s' % url) 

content_length = None 

else: 

content_length = int(content_length) 

status['ntotal_bytes_all_files'] += content_length 

if callable(status_callback): 

status_callback(status) 

url_to_size[url] = content_length 

return url_to_size[url] 

def download_file(url, fn): 

logger.info('starting download of %s...' % url) 

ensuredirs(fn) 

fsize = get_content_length(url) 

status['ntotal_bytes_current_file'] = fsize 

status['nread_bytes_current_file'] = 0 

if callable(status_callback): 

status_callback(status) 

r = session.get(url, stream=True, timeout=5) 

r.raise_for_status() 

frx = 0 

fn_tmp = fn + '.%i.temp' % os.getpid() 

with open(fn_tmp, 'wb') as f: 

for d in r.iter_content(chunk_size=1024): 

f.write(d) 

frx += len(d) 

status['nread_bytes_all_files'] += len(d) 

status['nread_bytes_current_file'] += len(d) 

if callable(status_callback): 

status_callback(status) 

os.rename(fn_tmp, fn) 

if fsize is not None and frx != fsize: 

logger.warning( 

'HTTP header Content-Length: %i bytes does not match ' 

'download size %i bytes' % (fsize, frx)) 

logger.info('finished download of %s' % url) 

status['nread_files'] += 1 

if callable(status_callback): 

status_callback(status) 

if recursive: 

if op.exists(fpath) and not force: 

raise PathExists('path %s already exists' % fpath) 

files = parse_directory_tree(url) 

dsize = 0 

for fn in files: 

file_url = urljoin(url, fn) 

dsize += get_content_length(file_url) or 0 

if method == 'calcsize': 

return dsize 

else: 

for fn in files: 

file_url = urljoin(url, fn) 

download_file(file_url, op.join(fpath, fn)) 

else: 

status['ntotal_files'] += 1 

if callable(status_callback): 

status_callback(status) 

fsize = get_content_length(url) 

if method == 'calcsize': 

return fsize 

else: 

download_file(url, fpath) 

except req_HTTPError as e: 

logging.warn("http error: %s" % e) 

raise DownloadError('could not download file(s) from: %s' % url) 

finally: 

status['finished'] = True 

if callable(status_callback): 

status_callback(status) 

session.close() 

def download_file( 

url, fpath, username=None, password=None, status_callback=None, 

**kwargs): 

return _download( 

url, fpath, username, password, 

recursive=False, 

status_callback=status_callback, 

**kwargs) 

def download_dir( 

url, fpath, username=None, password=None, status_callback=None, 

**kwargs): 

return _download( 

url, fpath, username, password, 

recursive=True, 

status_callback=status_callback, 

**kwargs) 

if hasattr(num, 'float128'): 

hpfloat = num.float128 

elif hasattr(num, 'float96'): 

hpfloat = num.float96 

else: 

def hpfloat(x): 

raise Exception( 

'NumPy lacks support for float128 or float96 data type on this ' 

'platform.') 

class Stopwatch(object): 

''' 

Simple stopwatch to measure elapsed wall clock time. 

Usage:: 

s = Stopwatch() 

time.sleep(1) 

print s() 

time.sleep(1) 

print s() 

''' 

def __init__(self): 

self.start = time.time() 

def __call__(self): 

return time.time() - self.start 

def wrap(text, line_length=80): 

''' 

Paragraph and list-aware wrapping of text. 

''' 

text = text.strip('\n') 

at_lineend = re.compile(r' *\n') 

at_para = re.compile(r'((^|(\n\s*)?\n)(\s+[*] )|\n\s*\n)') 

paragraphs = at_para.split(text)[::5] 

listindents = at_para.split(text)[4::5] 

newlist = at_para.split(text)[3::5] 

listindents[0:0] = [None] 

listindents.append(True) 

newlist.append(None) 

det_indent = re.compile(r'^ *') 

outlines = [] 

for ip, p in enumerate(paragraphs): 

if not p: 

continue 

if listindents[ip] is None: 

_indent = det_indent.findall(p)[0] 

findent = _indent 

else: 

findent = listindents[ip] 

_indent = ' ' * len(findent) 

ll = line_length - len(_indent) 

llf = ll 

oldlines = [s.strip() for s in at_lineend.split(p.rstrip())] 

p1 = ' '.join(oldlines) 

possible = re.compile(r'(^.{1,%i}|.{1,%i})( |$)' % (llf, ll)) 

for imatch, match in enumerate(possible.finditer(p1)): 

parout = match.group(1) 

if imatch == 0: 

outlines.append(findent + parout) 

else: 

outlines.append(_indent + parout) 

if ip != len(paragraphs)-1 and ( 

listindents[ip] is None 

or newlist[ip] is not None 

or listindents[ip+1] is None): 

outlines.append('') 

return outlines 

class BetterHelpFormatter(optparse.IndentedHelpFormatter): 

def __init__(self, *args, **kwargs): 

optparse.IndentedHelpFormatter.__init__(self, *args, **kwargs) 

def format_option(self, option): 

''' 

From IndentedHelpFormatter but using a different wrap method. 

''' 

help_text_position = 4 + self.current_indent 

help_text_width = self.width - help_text_position 

opts = self.option_strings[option] 

opts = "%*s%s" % (self.current_indent, "", opts) 

if option.help: 

help_text = self.expand_default(option) 

if self.help_position + len(help_text) + 1 <= self.width: 

lines = [ 

'', 

'%-*s %s' % (self.help_position, opts, help_text), 

''] 

else: 

lines = [''] 

lines.append(opts) 

lines.append('') 

if option.help: 

help_lines = wrap(help_text, help_text_width) 

lines.extend(["%*s%s" % (help_text_position, "", line) 

for line in help_lines]) 

lines.append('') 

return "\n".join(lines) 

def format_description(self, description): 

if not description: 

return '' 

if self.current_indent == 0: 

lines = [] 

else: 

lines = [''] 

lines.extend(wrap(description, self.width - self.current_indent)) 

if self.current_indent == 0: 

lines.append('\n') 

return '\n'.join( 

['%*s%s' % (self.current_indent, '', line) for line in lines]) 

def progressbar(label, maxval): 

progressbar_mod = progressbar_module() 

if force_dummy_progressbar: 

progressbar_mod = dummy_progressbar 

widgets = [ 

label, ' ', 

progressbar_mod.Bar(marker='-', left='[', right=']'), ' ', 

progressbar_mod.Percentage(), ' ', 

progressbar_mod.ETA()] 

pbar = progressbar_mod.ProgressBar(widgets=widgets, maxval=maxval).start() 

return pbar 

def progress_beg(label): 

''' 

Notify user that an operation has started. 

:param label: name of the operation 

To be used in conjuction with :py:func:`progress_end`. 

''' 

sys.stderr.write(label) 

sys.stderr.flush() 

def progress_end(label=''): 

''' 

Notify user that an operation has ended. 

:param label: name of the operation 

To be used in conjuction with :py:func:`progress_beg`. 

''' 

sys.stderr.write(' done. %s\n' % label) 

sys.stderr.flush() 

class ArangeError(Exception): 

pass 

def arange2(start, stop, step, dtype=num.float, epsilon=1e-6, error='raise'): 

''' 

Return evenly spaced numbers over a specified interval. 

Like :py:func:`numpy.arange` but returning floating point numbers by 

default and with defined behaviour when stepsize is inconsistent with 

interval bounds. It is considered inconsistent if the difference between 

the closest multiple of ``step`` and ``stop`` is larger than ``epsilon * 

step``. Inconsistencies are handled according to the ``error`` parameter. 

If it is set to ``'raise'`` an exception of type :py:exc:`ArangeError` is 

raised. If it is set to ``'round'``, ``'floor'``, or ``'ceil'``, ``stop`` 

is silently changed to the closest, the next smaller, or next larger 

multiple of ``step``, respectively. 

''' 

assert error in ('raise', 'round', 'floor', 'ceil') 

start = dtype(start) 

stop = dtype(stop) 

step = dtype(step) 

rnd = {'floor': math.floor, 'ceil': math.ceil}.get(error, round) 

n = int(rnd((stop - start) / step)) + 1 

stop_check = start + (n-1) * step 

if error == 'raise' and abs(stop_check - stop) > step * epsilon: 

raise ArangeError( 

'inconsistent range specification: start=%g, stop=%g, step=%g' 

% (start, stop, step)) 

x = num.arange(n, dtype=dtype)