|
""" A place for internal code
Some things are more easily handled Python.
"""
except ImportError: ctypes = None
else: _nbo = b'>'
allfields = [] fnames = list(adict.keys()) for fname in fnames: obj = adict[fname] n = len(obj) if not isinstance(obj, tuple) or n not in [2, 3]: raise ValueError("entry not a 2- or 3- tuple") if (n > 2) and (obj[2] == fname): continue num = int(obj[1]) if (num < 0): raise ValueError("invalid offset.") format = dtype(obj[0], align=align) if (n > 2): title = obj[2] else: title = None allfields.append((fname, format, num, title)) # sort by offsets allfields.sort(key=lambda x: x[2]) names = [x[0] for x in allfields] formats = [x[1] for x in allfields] offsets = [x[2] for x in allfields] titles = [x[3] for x in allfields]
return names, formats, offsets, titles
# Called in PyArray_DescrConverter function when # a dictionary without "names" and "formats" # fields is used as a data-type descriptor. try: names = adict[-1] except KeyError: names = None if names is None: names, formats, offsets, titles = _makenames_list(adict, align) else: formats = [] offsets = [] titles = [] for name in names: res = adict[name] formats.append(res[0]) offsets.append(res[1]) if (len(res) > 2): titles.append(res[2]) else: titles.append(None)
return dtype({"names": names, "formats": formats, "offsets": offsets, "titles": titles}, align)
# construct an array_protocol descriptor list # from the fields attribute of a descriptor # This calls itself recursively but should eventually hit # a descriptor that has no fields and then return # a simple typestring
fields = descriptor.fields if fields is None: subdtype = descriptor.subdtype if subdtype is None: if descriptor.metadata is None: return descriptor.str else: new = descriptor.metadata.copy() if new: return (descriptor.str, new) else: return descriptor.str else: return (_array_descr(subdtype[0]), subdtype[1])
names = descriptor.names ordered_fields = [fields[x] + (x,) for x in names] result = [] offset = 0 for field in ordered_fields: if field[1] > offset: num = field[1] - offset result.append(('', '|V%d' % num)) offset += num elif field[1] < offset: raise ValueError( "dtype.descr is not defined for types with overlapping or " "out-of-order fields") if len(field) > 3: name = (field[2], field[3]) else: name = field[2] if field[0].subdtype: tup = (name, _array_descr(field[0].subdtype[0]), field[0].subdtype[1]) else: tup = (name, _array_descr(field[0])) offset += field[0].itemsize result.append(tup)
if descriptor.itemsize > offset: num = descriptor.itemsize - offset result.append(('', '|V%d' % num))
return result
# Build a new array from the information in a pickle. # Note that the name numpy.core._internal._reconstruct is embedded in # pickles of ndarrays made with NumPy before release 1.0 # so don't remove the name here, or you'll # break backward compatibility. return ndarray.__new__(subtype, shape, dtype)
# format_re was originally from numarray by J. Todd Miller
br'(?P<repeats> *[(]?[ ,0-9L]*[)]? *)' br'(?P<order2>[<>|=]?)' br'(?P<dtype>[A-Za-z0-9.?]*(?:\[[a-zA-Z0-9,.]+\])?)')
# astr is a string (perhaps comma separated)
startindex = 0 result = [] while startindex < len(astr): mo = format_re.match(astr, pos=startindex) try: (order1, repeats, order2, dtype) = mo.groups() except (TypeError, AttributeError): raise ValueError('format number %d of "%s" is not recognized' % (len(result)+1, astr)) startindex = mo.end() # Separator or ending padding if startindex < len(astr): if space_re.match(astr, pos=startindex): startindex = len(astr) else: mo = sep_re.match(astr, pos=startindex) if not mo: raise ValueError( 'format number %d of "%s" is not recognized' % (len(result)+1, astr)) startindex = mo.end()
if order2 == b'': order = order1 elif order1 == b'': order = order2 else: order1 = _convorder.get(order1, order1) order2 = _convorder.get(order2, order2) if (order1 != order2): raise ValueError( 'inconsistent byte-order specification %s and %s' % (order1, order2)) order = order1
if order in [b'|', b'=', _nbo]: order = b'' dtype = order + dtype if (repeats == b''): newitem = dtype else: newitem = (dtype, eval(repeats)) result.append(newitem)
return result
self._cls = cls return self return self._cls(other) return self._cls == other._cls return self._cls != other._cls
return val import numpy as np val = dummy_ctype(np.intp) else: val = ctypes.c_int elif char == 'q': val = ctypes.c_longlong else: val = ctypes.c_long
# Used for .ctypes attribute of ndarray
return num.value
self.value = ptr
""" Helper to allow viewing an array as a ctypes pointer to the first element
This avoids: * dealing with strides * `.view` rejecting object-containing arrays * `memoryview` not supporting overlapping fields """ self.base = arr
def __array_interface__(self): i = dict( shape=(), typestr='|V0', data=(self.base.__array_interface__['data'][0], False), strides=(), version=3, ) return i
""" Get a `ctypes.c_void_p` to arr.data, that keeps a reference to the array """ import numpy as np # convert to a 0d array that has a data pointer referrign to the start # of arr. This holds a reference to arr. simple_arr = np.asarray(_unsafe_first_element_pointer(arr))
# create a `char[0]` using the same memory. c_arr = (ctypes.c_char * 0).from_buffer(simple_arr)
# finally cast to void* return ctypes.cast(ctypes.pointer(c_arr), ctypes.c_void_p)
self._arr = array
if ctypes: self._ctypes = ctypes # get a void pointer to the buffer, which keeps the array alive self._data = _get_void_ptr(array) assert self._data.value == ptr else: # fake a pointer-like object that holds onto the reference self._ctypes = _missing_ctypes() self._data = self._ctypes.c_void_p(ptr) self._data._objects = array
if self._arr.ndim == 0: self._zerod = True else: self._zerod = False
""" Return the data pointer cast to a particular c-types object. For example, calling ``self._as_parameter_`` is equivalent to ``self.data_as(ctypes.c_void_p)``. Perhaps you want to use the data as a pointer to a ctypes array of floating-point data: ``self.data_as(ctypes.POINTER(ctypes.c_double))``.
The returned pointer will keep a reference to the array. """ return self._ctypes.cast(self._data, obj)
""" Return the shape tuple as an array of some other c-types type. For example: ``self.shape_as(ctypes.c_short)``. """ if self._zerod: return None return (obj*self._arr.ndim)(*self._arr.shape)
""" Return the strides tuple as an array of some other c-types type. For example: ``self.strides_as(ctypes.c_longlong)``. """ if self._zerod: return None return (obj*self._arr.ndim)(*self._arr.strides)
def data(self): """ A pointer to the memory area of the array as a Python integer. This memory area may contain data that is not aligned, or not in correct byte-order. The memory area may not even be writeable. The array flags and data-type of this array should be respected when passing this attribute to arbitrary C-code to avoid trouble that can include Python crashing. User Beware! The value of this attribute is exactly the same as ``self._array_interface_['data'][0]``.
Note that unlike `data_as`, a reference will not be kept to the array: code like ``ctypes.c_void_p((a + b).ctypes.data)`` will result in a pointer to a deallocated array, and should be spelt ``(a + b).ctypes.data_as(ctypes.c_void_p)`` """ return self._data.value
def shape(self): """ (c_intp*self.ndim): A ctypes array of length self.ndim where the basetype is the C-integer corresponding to ``dtype('p')`` on this platform. This base-type could be `ctypes.c_int`, `ctypes.c_long`, or `ctypes.c_longlong` depending on the platform. The c_intp type is defined accordingly in `numpy.ctypeslib`. The ctypes array contains the shape of the underlying array. """ return self.shape_as(_getintp_ctype())
def strides(self): """ (c_intp*self.ndim): A ctypes array of length self.ndim where the basetype is the same as for the shape attribute. This ctypes array contains the strides information from the underlying array. This strides information is important for showing how many bytes must be jumped to get to the next element in the array. """ return self.strides_as(_getintp_ctype())
def _as_parameter_(self): """ Overrides the ctypes semi-magic method
Enables `c_func(some_array.ctypes)` """ return self._data
# kept for compatibility
""" Given a datatype and an order object, return a new names tuple, with the order indicated """ oldnames = datatype.names nameslist = list(oldnames) if isinstance(order, (str, unicode)): order = [order] seen = set() if isinstance(order, (list, tuple)): for name in order: try: nameslist.remove(name) except ValueError: if name in seen: raise ValueError("duplicate field name: %s" % (name,)) else: raise ValueError("unknown field name: %s" % (name,)) seen.add(name) return tuple(list(order) + nameslist) raise ValueError("unsupported order value: %s" % (order,))
"""Return copy of structured array with padding between fields removed.
Parameters ---------- ary : ndarray Structured array from which to remove padding bytes
Returns ------- ary_copy : ndarray Copy of ary with padding bytes removed """ dt = ary.dtype copy_dtype = {'names': dt.names, 'formats': [dt.fields[name][0] for name in dt.names]} return array(ary, dtype=copy_dtype, copy=True)
""" Checks safety of getfield for object arrays.
As in _view_is_safe, we need to check that memory containing objects is not reinterpreted as a non-object datatype and vice versa.
Parameters ---------- oldtype : data-type Data type of the original ndarray. newtype : data-type Data type of the field being accessed by ndarray.getfield offset : int Offset of the field being accessed by ndarray.getfield
Raises ------ TypeError If the field access is invalid
""" if newtype.hasobject or oldtype.hasobject: if offset == 0 and newtype == oldtype: return if oldtype.names: for name in oldtype.names: if (oldtype.fields[name][1] == offset and oldtype.fields[name][0] == newtype): return raise TypeError("Cannot get/set field of an object array") return
""" Checks safety of a view involving object arrays, for example when doing::
np.zeros(10, dtype=oldtype).view(newtype)
Parameters ---------- oldtype : data-type Data type of original ndarray newtype : data-type Data type of the view
Raises ------ TypeError If the new type is incompatible with the old type.
"""
# if the types are equivalent, there is no problem. # for example: dtype((np.record, 'i4,i4')) == dtype((np.void, 'i4,i4')) if oldtype == newtype: return
if newtype.hasobject or oldtype.hasobject: raise TypeError("Cannot change data-type for object array.") return
# Given a string containing a PEP 3118 format specifier, # construct a NumPy dtype
'?': '?', 'c': 'S1', 'b': 'b', 'B': 'B', 'h': 'h', 'H': 'H', 'i': 'i', 'I': 'I', 'l': 'l', 'L': 'L', 'q': 'q', 'Q': 'Q', 'e': 'e', 'f': 'f', 'd': 'd', 'g': 'g', 'Zf': 'F', 'Zd': 'D', 'Zg': 'G', 's': 'S', 'w': 'U', 'O': 'O', 'x': 'V', # padding }
'?': '?', 'c': 'S1', 'b': 'b', 'B': 'B', 'h': 'i2', 'H': 'u2', 'i': 'i4', 'I': 'u4', 'l': 'i4', 'L': 'u4', 'q': 'i8', 'Q': 'u8', 'e': 'f2', 'f': 'f', 'd': 'd', 'Zf': 'F', 'Zd': 'D', 's': 'S', 'w': 'U', 'O': 'O', 'x': 'V', # padding }
'u': 'UCS-2 strings', '&': 'pointers', 't': 'bitfields', 'X': 'function pointers', }
self.s = s self.byteorder = '@'
res = self.s[:n] self.s = self.s[n:] return res
if self.s[:len(c)] == c: self.advance(len(c)) return True return False
if callable(c): i = 0 while i < len(self.s) and not c(self.s[i]): i = i + 1 return self.advance(i) else: i = self.s.index(c) res = self.advance(i) self.advance(len(c)) return res
def next(self): return self.s[0]
return bool(self.s)
stream = _Stream(spec) dtype, align = __dtype_from_pep3118(stream, is_subdtype=False) return dtype
field_spec = dict( names=[], formats=[], offsets=[], itemsize=0 ) offset = 0 common_alignment = 1 is_padding = False
# Parse spec while stream: value = None
# End of structure, bail out to upper level if stream.consume('}'): break
# Sub-arrays (1) shape = None if stream.consume('('): shape = stream.consume_until(')') shape = tuple(map(int, shape.split(',')))
# Byte order if stream.next in ('@', '=', '<', '>', '^', '!'): byteorder = stream.advance(1) if byteorder == '!': byteorder = '>' stream.byteorder = byteorder
# Byte order characters also control native vs. standard type sizes if stream.byteorder in ('@', '^'): type_map = _pep3118_native_map type_map_chars = _pep3118_native_typechars else: type_map = _pep3118_standard_map type_map_chars = _pep3118_standard_typechars
# Item sizes itemsize_str = stream.consume_until(lambda c: not c.isdigit()) if itemsize_str: itemsize = int(itemsize_str) else: itemsize = 1
# Data types is_padding = False
if stream.consume('T{'): value, align = __dtype_from_pep3118( stream, is_subdtype=True) elif stream.next in type_map_chars: if stream.next == 'Z': typechar = stream.advance(2) else: typechar = stream.advance(1)
is_padding = (typechar == 'x') dtypechar = type_map[typechar] if dtypechar in 'USV': dtypechar += '%d' % itemsize itemsize = 1 numpy_byteorder = {'@': '=', '^': '='}.get( stream.byteorder, stream.byteorder) value = dtype(numpy_byteorder + dtypechar) align = value.alignment elif stream.next in _pep3118_unsupported_map: desc = _pep3118_unsupported_map[stream.next] raise NotImplementedError( "Unrepresentable PEP 3118 data type {!r} ({})" .format(stream.next, desc)) else: raise ValueError("Unknown PEP 3118 data type specifier %r" % stream.s)
# # Native alignment may require padding # # Here we assume that the presence of a '@' character implicitly implies # that the start of the array is *already* aligned. # extra_offset = 0 if stream.byteorder == '@': start_padding = (-offset) % align intra_padding = (-value.itemsize) % align
offset += start_padding
if intra_padding != 0: if itemsize > 1 or (shape is not None and _prod(shape) > 1): # Inject internal padding to the end of the sub-item value = _add_trailing_padding(value, intra_padding) else: # We can postpone the injection of internal padding, # as the item appears at most once extra_offset += intra_padding
# Update common alignment common_alignment = _lcm(align, common_alignment)
# Convert itemsize to sub-array if itemsize != 1: value = dtype((value, (itemsize,)))
# Sub-arrays (2) if shape is not None: value = dtype((value, shape))
# Field name if stream.consume(':'): name = stream.consume_until(':') else: name = None
if not (is_padding and name is None): if name is not None and name in field_spec['names']: raise RuntimeError("Duplicate field name '%s' in PEP3118 format" % name) field_spec['names'].append(name) field_spec['formats'].append(value) field_spec['offsets'].append(offset)
offset += value.itemsize offset += extra_offset
field_spec['itemsize'] = offset
# extra final padding for aligned types if stream.byteorder == '@': field_spec['itemsize'] += (-offset) % common_alignment
# Check if this was a simple 1-item type, and unwrap it if (field_spec['names'] == [None] and field_spec['offsets'][0] == 0 and field_spec['itemsize'] == field_spec['formats'][0].itemsize and not is_subdtype): ret = field_spec['formats'][0] else: _fix_names(field_spec) ret = dtype(field_spec)
# Finished return ret, common_alignment
""" Replace names which are None with the next unused f%d name """ names = field_spec['names'] for i, name in enumerate(names): if name is not None: continue
j = 0 while True: name = 'f{}'.format(j) if name not in names: break j = j + 1 names[i] = name
"""Inject the specified number of padding bytes at the end of a dtype""" if value.fields is None: field_spec = dict( names=['f0'], formats=[value], offsets=[0], itemsize=value.itemsize ) else: fields = value.fields names = value.names field_spec = dict( names=names, formats=[fields[name][0] for name in names], offsets=[fields[name][1] for name in names], itemsize=value.itemsize )
field_spec['itemsize'] += padding return dtype(field_spec)
p = 1 for x in a: p *= x return p
"""Calculate the greatest common divisor of a and b""" while b: a, b = b, a % b return a
return a // _gcd(a, b) * b
# Exception used in shares_memory()
""" Axis supplied was invalid. """ # single-argument form just delegates to base class if ndim is None and msg_prefix is None: msg = axis
# do the string formatting here, to save work in the C code else: msg = ("axis {} is out of bounds for array of dimension {}" .format(axis, ndim)) if msg_prefix is not None: msg = "{}: {}".format(msg_prefix, msg)
super(AxisError, self).__init__(msg)
""" Format the error message for when __array_ufunc__ gives up. """ args_string = ', '.join(['{!r}'.format(arg) for arg in inputs] + ['{}={!r}'.format(k, v) for k, v in kwargs.items()]) args = inputs + kwargs.get('out', ()) types_string = ', '.join(repr(type(arg).__name__) for arg in args) return ('operand type(s) all returned NotImplemented from ' '__array_ufunc__({!r}, {!r}, {}): {}' .format(ufunc, method, args_string, types_string))
""" Format the error message for when __array_ufunc__ gives up. """ func_name = '{}.{}'.format(public_api.__module__, public_api.__name__) return ("no implementation found for '{}' on types that implement " '__array_function__: {}'.format(func_name, list(types)))
""" Builds a signature string which resembles PEP 457
This is used to construct the first line of the docstring """
# input arguments are simple else:
# output arguments are both keyword or positional out_args = ', /, out=()' else: out_args = '[, {positional}], / [, out={default}]'.format( positional=', '.join( 'out{}'.format(i+1) for i in range(ufunc.nout)), default=repr((None,)*ufunc.nout) )
# keyword only args depend on whether this is a gufunc ", casting='same_kind'" ", order='K'" ", dtype=None" ", subok=True" "[, signature" ", extobj]" )
# join all the parts together name=ufunc.__name__, in_args=in_args, out_args=out_args, kwargs=kwargs )
# determine if a class comes from ctypes, in order to work around # a bug in the buffer protocol for those objects, bpo-10746 # ctypes class are new-style, so have an __mro__. This probably fails # for ctypes classes with multiple inheritance. # right now, they're part of the _ctypes module
''' A decorator class for recursive nested functions. Naive recursive nested functions hold a reference to themselves:
def outer(*args): def stringify_leaky(arg0, *arg1): if len(arg1) > 0: return stringify_leaky(*arg1) # <- HERE return str(arg0) stringify_leaky(*args)
This design pattern creates a reference cycle that is difficult for a garbage collector to resolve. The decorator class prevents the cycle by passing the nested function in as an argument `self`:
def outer(*args): @recursive def stringify(self, arg0, *arg1): if len(arg1) > 0: return self(*arg1) return str(arg0) stringify(*args)
'''
|