mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-28 03:58:12 +08:00
anndata X indexing & version compatibility improvements (#1157)
* revert MatrixProxy; replace with correct use of adata slicing * work around 0.6 adata slicing bug * fix incorrect var slice * simplify slicing of X * add warning about performance impact of anndata<=0.7 * lint and remove unused code * improve comment * lint * correctly parse versions * temp files should preserve file suffix if possible - anndata 0.7 compat * update anndata dependency to 0.6.20 * resolve PR review comments
This commit is contained in:
Bruce Martin
@@ -1,47 +0,0 @@
|
||||
from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView
|
||||
|
||||
"""
|
||||
AnnData/h5py are inconsistent in the API supported by various types of
|
||||
X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse
|
||||
matrix, sometimes h5py proxies with a subset of our needed interfaces.
|
||||
|
||||
This glue code paves over all of that, providing the core set of methods
|
||||
that the cellxgene ScanPy driver assumes are in existance. Put another
|
||||
way, all of the non-portable assumptions are here.
|
||||
"""
|
||||
|
||||
|
||||
class ArrayProxyView_anndata_h5py(ArrayProxyView):
|
||||
"""
|
||||
override to handle sparse getitem semantics, which differ
|
||||
from numpy.
|
||||
"""
|
||||
|
||||
def toarray(self):
|
||||
""" sadly, sparse indexing doesn't drop dimensions like numpy! """
|
||||
arr = self.m[self._index[0], self._index[1]]
|
||||
if self._vdim == 0:
|
||||
arr = arr.transpose()
|
||||
return arr.toarray()[0]
|
||||
|
||||
|
||||
class MatrixProxy_anndata_h5py(MatrixProxyView):
|
||||
"""
|
||||
AnnData sparse array stored in H5AD, or proxies for backed data.
|
||||
None of these handle indexing very well, so we plop a proxy on top.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def __supports__(cls):
|
||||
return (
|
||||
"anndata.h5py.h5sparse.SparseDataset",
|
||||
"anndata.h5py.h5sparse.backed_csc_matrix",
|
||||
"anndata.h5py.h5sparse.backed_csr_matrix",
|
||||
"h5py._hl.dataset.Dataset",
|
||||
"anndata._core.sparse_dataset.backed_csr_matrix",
|
||||
"anndata._core.sparse_dataset.backed_csc_matrix",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create_array(cls, *args, **kwargs):
|
||||
return ArrayProxyView_anndata_h5py(*args, **kwargs)
|
||||
@@ -5,6 +5,7 @@ from datetime import datetime
|
||||
import os.path
|
||||
from hashlib import blake2b
|
||||
import base64
|
||||
from packaging import version
|
||||
|
||||
import numpy as np
|
||||
import pandas
|
||||
@@ -26,8 +27,15 @@ from server.app.util.utils import jsonify_scanpy, requires_data
|
||||
from server.app.scanpy_engine.diffexp import diffexp_ttest
|
||||
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
|
||||
from server.app.scanpy_engine.labels import read_labels, write_labels
|
||||
import server.app.scanpy_engine.matrix_proxy # noqa: F401
|
||||
from server.app.util.matrix_proxy import MatrixProxy
|
||||
|
||||
|
||||
anndata_version = version.parse(str(anndata.__version__)).release
|
||||
|
||||
|
||||
def anndata_version_is_pre_070():
|
||||
major = anndata_version[0]
|
||||
minor = anndata_version[1] if len(anndata_version) > 1 else 0
|
||||
return major == 0 and minor < 7
|
||||
|
||||
|
||||
def has_method(o, name):
|
||||
@@ -303,6 +311,10 @@ class ScanpyEngine(CXGDriver):
|
||||
|
||||
@requires_data
|
||||
def _validate_and_initialize(self):
|
||||
if anndata_version_is_pre_070() and self.config['backed']:
|
||||
warnings.warn(f"Use of --backed mode with anndata versions older than 0.7 will have serious "
|
||||
"performance issues. Please update to at least anndata 0.7 or later.")
|
||||
|
||||
# var and obs column names must be unique
|
||||
if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
|
||||
raise KeyError(f"All annotation column names must be unique.")
|
||||
@@ -370,7 +382,14 @@ class ScanpyEngine(CXGDriver):
|
||||
|
||||
@requires_data
|
||||
def _validate_data_types(self):
|
||||
if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X):
|
||||
# The backed API does not support interrogation of the underlying sparsity or sparse matrix type
|
||||
# Fake it by asking for a small subarray and testing it. NOTE: if the user has ignored our
|
||||
# anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset,
|
||||
# this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7
|
||||
# which will load the entire X matrix to fullfill any slicing request if X is sparse. See
|
||||
# user warning in _load_data().
|
||||
X0 = self.data.X[0, 0:1]
|
||||
if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0):
|
||||
warnings.warn(
|
||||
f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. "
|
||||
f"Performance may be improved by using CSC."
|
||||
@@ -577,8 +596,9 @@ class ScanpyEngine(CXGDriver):
|
||||
raise FilterError("filtering on obs unsupported")
|
||||
|
||||
# Currently only handles VAR dimension
|
||||
X = MatrixProxy.create(self.data.X if var_selector is None else self.data.X[:, var_selector])
|
||||
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None)
|
||||
X = self.data.X[:, slice(None) if var_selector is None else var_selector]
|
||||
col_idx = np.nonzero([] if var_selector is None else var_selector)[0]
|
||||
return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
|
||||
|
||||
@requires_data
|
||||
def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None):
|
||||
|
||||
@@ -81,8 +81,11 @@ class DataLocator:
|
||||
return LocalFilePath(self.path)
|
||||
|
||||
# if not local, create a tmp file system object to contain the data,
|
||||
# and clean it up when done.
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp:
|
||||
# and clean it up when done. If the path has a suffix/extension,
|
||||
# do our best to create a file with the same.
|
||||
ext = os.path.splitext(self.path)
|
||||
suffix = None if ext[1] == '' else ext[1]
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(src.read())
|
||||
tmp.close()
|
||||
src.close()
|
||||
|
||||
@@ -12,7 +12,6 @@ import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
|
||||
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
|
||||
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
|
||||
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
|
||||
from server.app.util.matrix_proxy import MatrixProxy
|
||||
|
||||
|
||||
# Placeholder until recent enhancements to flatbuffers Python
|
||||
@@ -92,7 +91,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
|
||||
as_json = arr.to_json(orient="records")
|
||||
arr = np.array(bytearray(as_json, "utf-8"))
|
||||
else:
|
||||
if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr):
|
||||
if sparse.issparse(arr):
|
||||
arr = arr.toarray()
|
||||
elif isinstance(arr, pd.Series):
|
||||
arr = arr.to_numpy()
|
||||
|
||||
@@ -1,425 +0,0 @@
|
||||
import abc
|
||||
from itertools import zip_longest
|
||||
from copy import copy
|
||||
import numpy as np
|
||||
|
||||
"""
|
||||
cellxgene deals with a variety of matrix data types, many of which do
|
||||
not support a consistent API. This framework allows proxies to be created
|
||||
to pave over some of this. Most significantly, AnnData.X does not guarantee
|
||||
that much of its API (eg. .X.T) will work.
|
||||
"""
|
||||
|
||||
INT_TYPES = (int, np.integer)
|
||||
|
||||
|
||||
class _ArrayProxyBase(abc.ABC):
|
||||
"""
|
||||
Private base class for array or matrix proxy. This summarizes
|
||||
the interface used by the rest of cellxgene.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def dtype(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def ndim(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def shape(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def T(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def __iter__(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def __getitem__(self, args):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def toarray():
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class MatrixProxy(_ArrayProxyBase):
|
||||
"""
|
||||
Abstract class - all interfaces we need, plus a factory method
|
||||
to create a proxy based upon actual matrix type.
|
||||
|
||||
This class primarily provides the factory method and related support.
|
||||
All other functionality is delegated to subclasses.
|
||||
"""
|
||||
|
||||
"""
|
||||
Registry of types to proxy class, where values are:
|
||||
* None: unsupported
|
||||
* True: self-supported
|
||||
* string: proxy class
|
||||
Sub-classes automatically register.
|
||||
"""
|
||||
base_proxy_registry = {
|
||||
"pandas.core.frame.DataFrame": True,
|
||||
"numpy.ndarray": True,
|
||||
"scipy.sparse.csc.csc_matrix": True,
|
||||
"scipy.sparse.csr.csr_matrix": True,
|
||||
"anndata._core.sparse_dataset.backed_csr_matrix": "MatrixProxy_anndata_h5py",
|
||||
"anndata._core.sparse_dataset.backed_csc_matrix": "MatrixProxy_anndata_h5py",
|
||||
}
|
||||
proxy_registry = None
|
||||
last_cache_token = None
|
||||
|
||||
@staticmethod
|
||||
def _register_subclasses(subclasses, registry):
|
||||
for c in subclasses:
|
||||
names = c.__supports__()
|
||||
for name in names:
|
||||
registry[name] = c
|
||||
MatrixProxy._register_subclasses(c.__subclasses__(), registry)
|
||||
|
||||
@classmethod
|
||||
def build_proxy_registry(cls):
|
||||
if cls.proxy_registry and abc.get_cache_token() == cls.last_cache_token:
|
||||
return
|
||||
|
||||
cls.last_cache_token = abc.get_cache_token()
|
||||
registry = copy(cls.base_proxy_registry)
|
||||
MatrixProxy._register_subclasses(cls.__subclasses__(), registry)
|
||||
cls.proxy_registry = registry
|
||||
|
||||
@classmethod
|
||||
def create(cls, matrix):
|
||||
"""
|
||||
Factory - call with a matrix and it will create a proxy if needed.
|
||||
If the type already supports the necessary API, it is just returned
|
||||
directly.
|
||||
"""
|
||||
cls.build_proxy_registry()
|
||||
t = type(matrix)
|
||||
fqtn = t.__module__ + "." + t.__name__
|
||||
proxy_cls = cls.proxy_registry.get(fqtn, None)
|
||||
if proxy_cls is None:
|
||||
raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.")
|
||||
if proxy_cls is True:
|
||||
return matrix
|
||||
return proxy_cls(matrix)
|
||||
|
||||
def __init__(self, m):
|
||||
self.m = m
|
||||
|
||||
@classmethod
|
||||
@abc.abstractmethod
|
||||
def __supports__(cls):
|
||||
raise NotImplementedError()
|
||||
|
||||
@classmethod
|
||||
def ismatrixproxy(cls, m):
|
||||
return isinstance(m, _ArrayProxyBase)
|
||||
|
||||
|
||||
class MatrixProxyView(MatrixProxy):
|
||||
"""
|
||||
2D matrix view to a 2D matrix
|
||||
"""
|
||||
|
||||
def __init__(self, arg1, shape=None, index=(), transposed=False, copy=False):
|
||||
if not copy:
|
||||
m = arg1
|
||||
super().__init__(m)
|
||||
|
||||
if shape is None:
|
||||
shape = m.shape
|
||||
assert len(shape) == 2
|
||||
|
||||
index = tuple(map(lambda s_i: slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], zip_longest(shape, index)))
|
||||
|
||||
self._shape = shape
|
||||
self._index = index
|
||||
self.transposed = transposed
|
||||
|
||||
else: # copy mode
|
||||
super().__init__(arg1.m)
|
||||
self._shape = arg1._shape
|
||||
self._index = arg1._index
|
||||
self.transposed = arg1.transposed
|
||||
|
||||
@classmethod
|
||||
def create_array(cls, *args, **kwargs):
|
||||
""" override if you use a different 1D array proxy """
|
||||
return ArrayProxyView(*args, **kwargs)
|
||||
|
||||
def copy(self):
|
||||
""" override if you need additional behaviors """
|
||||
return self.__class__(self, copy=True)
|
||||
|
||||
@classmethod
|
||||
def __supports__(cls):
|
||||
return ()
|
||||
|
||||
def _swap(self, x):
|
||||
return (x[1], x[0]) if self.transposed else x
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return self.m.dtype
|
||||
|
||||
@property
|
||||
def ndim(self):
|
||||
return len(self._shape)
|
||||
|
||||
@property
|
||||
def shape(self):
|
||||
return self._swap(self._shape)
|
||||
|
||||
@property
|
||||
def T(self):
|
||||
m = self.copy()
|
||||
m.transposed = not m.transposed
|
||||
return m
|
||||
|
||||
def __iter__(self):
|
||||
M = self._swap(self._shape)[0]
|
||||
s = self._swap((self._index))[0]
|
||||
start, stop, step = s.indices(M)
|
||||
for d in range(start, stop, step):
|
||||
yield self[d]
|
||||
|
||||
def __getitem__(self, args):
|
||||
"""
|
||||
decompose into the indexing patterns we use, throw for the rest.
|
||||
Subclassses implement specialized access.
|
||||
"""
|
||||
row, col = self._swap(_unpack_index(args, self.shape))
|
||||
M, N = self._shape
|
||||
allM, allN = self.m.shape
|
||||
|
||||
if isinstance(row, INT_TYPES):
|
||||
row += self._index[0].start
|
||||
elif isinstance(row, slice):
|
||||
row = _slice_slice(self._index[0], allM, row, M)
|
||||
if isinstance(col, INT_TYPES):
|
||||
col += self._index[1].start
|
||||
elif isinstance(col, slice):
|
||||
col = _slice_slice(self._index[1], allN, col, N)
|
||||
|
||||
if isinstance(row, INT_TYPES):
|
||||
if isinstance(col, INT_TYPES):
|
||||
return self._getitem_intXint(row, col)
|
||||
elif isinstance(col, slice):
|
||||
return self._getitem_intXslice(row, col)
|
||||
elif isinstance(row, slice):
|
||||
if isinstance(col, INT_TYPES):
|
||||
return self._getitem_sliceXint(row, col)
|
||||
elif isinstance(col, slice):
|
||||
return self._getitem_sliceXslice(row, col)
|
||||
|
||||
raise IndexError("unsupported column index types")
|
||||
|
||||
"""
|
||||
These getitem signatures are separate so that they may be
|
||||
overridden by subclasses as necessary. We don't do much
|
||||
with them by default other than the obvious sub-slicing.
|
||||
|
||||
NOTE: these follow the numpy rules for dimensionality reduction
|
||||
when an integer index is specified.
|
||||
"""
|
||||
|
||||
def _getitem_intXint(self, row, col):
|
||||
return self.m[row, col]
|
||||
|
||||
def _getitem_intXslice(self, row, col):
|
||||
shape = (_slice_length(col, self.m.shape[1]),)
|
||||
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
|
||||
|
||||
def _getitem_sliceXint(self, row, col):
|
||||
shape = (_slice_length(row, self.m.shape[0]),)
|
||||
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
|
||||
|
||||
def _getitem_sliceXslice(self, row, col):
|
||||
shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1]))
|
||||
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed)
|
||||
|
||||
def toarray(self):
|
||||
arr = self.m[self._index]
|
||||
if self.transposed:
|
||||
arr = arr.transpose()
|
||||
return arr
|
||||
|
||||
|
||||
class ArrayProxyView(_ArrayProxyBase):
|
||||
"""
|
||||
1D array view to a 2D matrix
|
||||
"""
|
||||
|
||||
def __init__(self, arg1, shape=None, index=None, copy=False):
|
||||
super().__init__()
|
||||
if not copy:
|
||||
m = arg1
|
||||
|
||||
# one index MUST be an integer and the other MUST be a slice
|
||||
assert len(index) == 2
|
||||
assert all(isinstance(idx, INT_TYPES + (slice,)) for idx in index)
|
||||
assert isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)
|
||||
|
||||
if shape is None:
|
||||
if isinstance(index[0], INT_TYPES):
|
||||
shape = (m.shape[0],)
|
||||
else:
|
||||
shape = (m.shape[1],)
|
||||
assert len(shape) == 1
|
||||
|
||||
self._shape = shape
|
||||
self.m = m
|
||||
self._index = index
|
||||
self._vdim = 1 if isinstance(index[0], INT_TYPES) else 0
|
||||
|
||||
else:
|
||||
self.m = arg1.m
|
||||
self._shape = arg1._shape
|
||||
self._index = arg1._index
|
||||
self.fixed = arg1.fixed
|
||||
|
||||
def copy(self):
|
||||
return self.__class__(self, copy=True)
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return self.m.dtype
|
||||
|
||||
@property
|
||||
def ndim(self):
|
||||
return len(self._shape)
|
||||
|
||||
@property
|
||||
def shape(self):
|
||||
return self._shape
|
||||
|
||||
@property
|
||||
def T(self):
|
||||
return self.copy()
|
||||
|
||||
def __iter__(self):
|
||||
_vdim = self._vdim
|
||||
M = self._shape[0]
|
||||
for d in range(*self._index[_vdim].indices(M)):
|
||||
yield self[d]
|
||||
|
||||
def __getitem__(self, args):
|
||||
_vdim = self._vdim
|
||||
index = _unpack_index(args, self.shape)[0]
|
||||
M = self._shape[0]
|
||||
allM = self.m.shape[_vdim]
|
||||
|
||||
if isinstance(index, INT_TYPES):
|
||||
index += self._index[_vdim].start
|
||||
elif isinstance(index, slice):
|
||||
index = _slice_slice(self._index[_vdim], allM, index, M)
|
||||
|
||||
if _vdim == 0:
|
||||
row, col = index, self._index[1]
|
||||
else:
|
||||
row, col = self._index[0], index
|
||||
|
||||
if isinstance(row, INT_TYPES):
|
||||
if isinstance(col, INT_TYPES):
|
||||
return self._getitem_intXint(row, col)
|
||||
elif isinstance(col, slice):
|
||||
return self._getitem_intXslice(row, col)
|
||||
elif isinstance(row, slice):
|
||||
assert isinstance(col, INT_TYPES)
|
||||
return self._getitem_sliceXint(row, col)
|
||||
|
||||
raise IndexError("unsupported column index types")
|
||||
|
||||
def _getitem_intXint(self, row, col):
|
||||
return self.m[row, col]
|
||||
|
||||
def _getitem_intXslice(self, row, col):
|
||||
shape = (_slice_length(col, self.m.shape[1]),)
|
||||
return self.__class__(self.m, shape=shape, index=(row, col))
|
||||
|
||||
def _getitem_sliceXint(self, row, col):
|
||||
shape = (_slice_length(row, self.m.shape[0]),)
|
||||
return self.__class__(self.m, shape=shape, index=(row, col))
|
||||
|
||||
def toarray(self):
|
||||
return self.m[self._index]
|
||||
|
||||
|
||||
def _unpack_index(index, shape):
|
||||
if not isinstance(index, tuple):
|
||||
index = (index,)
|
||||
if len(shape) < len(index):
|
||||
raise IndexError("invalid index dimensionality - must be 2")
|
||||
|
||||
unpacked = ()
|
||||
for shp, idx in zip_longest(shape, index):
|
||||
idx = slice(None) if idx is None else idx
|
||||
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
|
||||
unpacked += (idx,)
|
||||
|
||||
return unpacked
|
||||
|
||||
|
||||
def _slice_slice(outer, outer_len, inner, inner_len):
|
||||
"""
|
||||
slice a slice - we take advantage of Python 3 range's support
|
||||
for indexing.
|
||||
"""
|
||||
assert outer_len >= inner_len
|
||||
outer_rng = range(*outer.indices(outer_len))
|
||||
rng = outer_rng[inner]
|
||||
start, stop, step = rng.start, rng.stop, rng.step
|
||||
if step < 0 and stop < 0:
|
||||
stop = None
|
||||
return slice(start, stop, step)
|
||||
|
||||
|
||||
def _range_length(start, stop, step):
|
||||
""" return length of range """
|
||||
assert step != 0
|
||||
assert start is not None and stop is not None and step is not None
|
||||
if step > 0 and start < stop:
|
||||
return 1 + (stop - 1 - start) // step
|
||||
elif step < 0 and start > stop:
|
||||
return 1 + (start - 1 - stop) // -step
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def _slice_length(s, length):
|
||||
""" return slice length """
|
||||
return _range_length(*s.indices(length))
|
||||
|
||||
|
||||
def _slice_defaults(s, length):
|
||||
""" apply slice defaulting conventions """
|
||||
assert length >= 0
|
||||
|
||||
step = 1 if s.step is None else s.step
|
||||
|
||||
if s.start is not None:
|
||||
start = s.start
|
||||
if start < 0:
|
||||
start += length
|
||||
else:
|
||||
start = 0 if step > 0 else (length - 1)
|
||||
|
||||
if s.stop is not None:
|
||||
stop = s.stop
|
||||
if stop < 0:
|
||||
stop += length
|
||||
else:
|
||||
stop = length if step > 0 else -length - 1
|
||||
|
||||
return slice(start, stop, step)
|
||||
+16
-17
@@ -1,17 +1,16 @@
|
||||
anndata>=0.6.2
|
||||
click>=6.7
|
||||
fastobo>=0.6.1
|
||||
Flask>=1.0.2
|
||||
Flask-Caching>=1.4.0
|
||||
Flask-Compress>=1.4.0
|
||||
Flask-Cors>=3.0.6
|
||||
Flask-RESTful>=0.3.6
|
||||
flatbuffers>=1.10.0
|
||||
fsspec>=0.4.4
|
||||
numpy>=1.15.2
|
||||
pandas>=0.24.2
|
||||
scipy>=1.3.0
|
||||
tables==3.5.1
|
||||
# TEMP workaround for https://github.com/theislab/scanpy/issues/832 aka h5py regression
|
||||
h5py==2.9.0
|
||||
requests>=2.22.0
|
||||
anndata>=0.6.20
|
||||
click>=6.7
|
||||
fastobo>=0.6.1
|
||||
Flask>=1.0.2
|
||||
Flask-Caching>=1.4.0
|
||||
Flask-Compress>=1.4.0
|
||||
Flask-Cors>=3.0.6
|
||||
Flask-RESTful>=0.3.6
|
||||
flatbuffers>=1.10.0
|
||||
fsspec>=0.4.4
|
||||
numpy>=1.15.2
|
||||
packaging>=20.0
|
||||
pandas>=0.24.2
|
||||
scipy>=1.3.0
|
||||
tables==3.5.1
|
||||
requests>=2.22.0
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
import unittest
|
||||
import numpy as np
|
||||
from server.app.util.matrix_proxy import MatrixProxyView, MatrixProxy
|
||||
|
||||
|
||||
class NdArrayProxyView(MatrixProxyView):
|
||||
"""
|
||||
Fake test class for matrix proxy - wraps ndarray
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def __supports__(cls):
|
||||
return ("numpy.ndarray",)
|
||||
|
||||
|
||||
class MatrixProxyViewTest(unittest.TestCase):
|
||||
def test_ismatrixproxy(self):
|
||||
n = np.zeros((2, 4))
|
||||
mp = MatrixProxy.create(n)
|
||||
|
||||
self.assertIsNotNone(n)
|
||||
self.assertIsNotNone(mp)
|
||||
self.assertTrue(isinstance(mp, NdArrayProxyView))
|
||||
self.assertFalse(MatrixProxy.ismatrixproxy(n))
|
||||
self.assertTrue(MatrixProxy.ismatrixproxy(mp))
|
||||
|
||||
def test_params(self):
|
||||
n = np.arange(15, dtype=np.float32).reshape((3, 5))
|
||||
mp = MatrixProxy.create(n)
|
||||
|
||||
self.assertTrue(MatrixProxy.ismatrixproxy(mp))
|
||||
self.assertEqual(mp.ndim, 2)
|
||||
self.assertEqual(mp.shape, (3, 5))
|
||||
self.assertEqual(mp.dtype, np.float32)
|
||||
|
||||
mpt = mp.T
|
||||
self.assertTrue(MatrixProxy.ismatrixproxy(mpt))
|
||||
self.assertEqual(mpt.ndim, 2)
|
||||
self.assertEqual(mpt.shape, (5, 3))
|
||||
self.assertEqual(mpt.dtype, np.float32)
|
||||
|
||||
def test_toarray(self):
|
||||
n = np.arange(15, dtype=np.float32).reshape((3, 5))
|
||||
mp = MatrixProxy.create(n)
|
||||
self.assertTrue(
|
||||
np.all(
|
||||
mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]]
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
np.all(
|
||||
mp.T.toarray()
|
||||
== [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]]
|
||||
)
|
||||
)
|
||||
|
||||
def test_indexing(self):
|
||||
"""
|
||||
[[ 0., 1., 2., 3., 4.],
|
||||
[ 5., 6., 7., 8., 9.],
|
||||
[10., 11., 12., 13., 14.]]
|
||||
"""
|
||||
n = np.arange(15, dtype=np.float32).reshape((3, 5))
|
||||
mp = MatrixProxy.create(n)
|
||||
|
||||
# should support int or slice for one or both dimensions
|
||||
|
||||
# int, int
|
||||
|
||||
self.assertEqual(mp[0, 0], 0)
|
||||
self.assertEqual(mp.T[0, 0], 0)
|
||||
self.assertEqual(mp[2, 4], 14)
|
||||
self.assertEqual(mp.T[4, 2], 14)
|
||||
self.assertEqual(mp[1, 3], mp.T[3, 1])
|
||||
|
||||
# int, slice
|
||||
|
||||
self.assertTrue(np.all(mp[0].toarray() == [0, 1, 2, 3, 4]))
|
||||
self.assertTrue(np.all(mp[1, 1:].toarray() == [6, 7, 8, 9]))
|
||||
self.assertTrue(np.all(mp[2, 1::-1].toarray() == [11, 10]))
|
||||
|
||||
self.assertTrue(np.all(mp.T[0].toarray() == [0, 5, 10]))
|
||||
self.assertTrue(np.all(mp.T[1, 1:].toarray() == [6, 11]))
|
||||
self.assertTrue(np.all(mp.T[2, 1::-1].toarray() == [7, 2]))
|
||||
|
||||
# slice, int
|
||||
|
||||
self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
|
||||
self.assertTrue(np.all(mp[1:, 1].toarray() == [6, 11]))
|
||||
self.assertTrue(np.all(mp[1::-1, 2].toarray() == [7, 2]))
|
||||
|
||||
self.assertTrue(np.all(mp.T[:, 0].toarray() == [0, 1, 2, 3, 4]))
|
||||
self.assertTrue(np.all(mp.T[1:, 1].toarray() == [6, 7, 8, 9]))
|
||||
self.assertTrue(np.all(mp.T[1::-1, 2].toarray() == [11, 10]))
|
||||
|
||||
# slice, slice
|
||||
|
||||
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]]))
|
||||
self.assertTrue(
|
||||
np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]])
|
||||
)
|
||||
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]]))
|
||||
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]]))
|
||||
|
||||
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]]))
|
||||
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]]))
|
||||
self.assertTrue(
|
||||
np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]])
|
||||
)
|
||||
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]]))
|
||||
|
||||
def test_repeated_indexing(self):
|
||||
"""
|
||||
[[ 0., 1., 2., 3., 4.],
|
||||
[ 5., 6., 7., 8., 9.],
|
||||
[10., 11., 12., 13., 14.]]
|
||||
"""
|
||||
n = np.arange(15, dtype=np.float32).reshape((3, 5))
|
||||
mp = MatrixProxy.create(n)
|
||||
|
||||
self.assertEqual(mp[0][1], 1)
|
||||
self.assertEqual(mp.T[0][1], 5)
|
||||
|
||||
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1]))
|
||||
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3]))
|
||||
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2]))
|
||||
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2]))
|
||||
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3]))
|
||||
|
||||
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1]))
|
||||
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10]))
|
||||
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10]))
|
||||
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == []))
|
||||
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == []))
|
||||
|
||||
def test_dimension_drop(self):
|
||||
"""
|
||||
[[ 0., 1., 2., 3., 4.],
|
||||
[ 5., 6., 7., 8., 9.],
|
||||
[10., 11., 12., 13., 14.]]
|
||||
"""
|
||||
n = np.arange(15, dtype=np.float32).reshape((3, 5))
|
||||
mp = MatrixProxy.create(n)
|
||||
|
||||
# drop both dimensions, to a scalar
|
||||
self.assertEqual(mp[0, 0], 0)
|
||||
|
||||
# drop 1 dimension, to an array
|
||||
self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4]))
|
||||
self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
|
||||
|
||||
# with .T
|
||||
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]]))
|
||||
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9]))
|
||||
|
||||
def test_iter(self):
|
||||
"""
|
||||
check that __iter__ is doing what we expect
|
||||
"""
|
||||
n = np.arange(15, dtype=np.float32).reshape((3, 5))
|
||||
mp = MatrixProxy.create(n)
|
||||
|
||||
rows = [r for r in mp]
|
||||
self.assertEqual(len(rows), 3)
|
||||
self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4]))
|
||||
for i, r in enumerate(rows):
|
||||
self.assertTrue(np.all(mp[i].toarray() == r.toarray()))
|
||||
|
||||
cols = [c for c in mp.T]
|
||||
self.assertEqual(len(cols), 5)
|
||||
self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10]))
|
||||
for i, c in enumerate(cols):
|
||||
self.assertTrue(np.all(mp.T[i].toarray() == c.toarray()))
|
||||
|
||||
e = [e for e in mp[0]]
|
||||
self.assertEqual(len(e), 5)
|
||||
self.assertEqual(e, [0, 1, 2, 3, 4])
|
||||
Reference in New Issue
Block a user