From 349c413d8b1e75519e7cca5321413f73e599c876 Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Wed, 19 Feb 2020 09:57:51 -0700 Subject: [PATCH] anndata X indexing & version compatibility improvements (#1157) * revert MatrixProxy; replace with correct use of adata slicing * work around 0.6 adata slicing bug * fix incorrect var slice * simplify slicing of X * add warning about performance impact of anndata<=0.7 * lint and remove unused code * improve comment * lint * correctly parse versions * temp files should preserve file suffix if possible - anndata 0.7 compat * update anndata dependency to 0.6.20 * resolve PR review comments --- server/app/scanpy_engine/matrix_proxy.py | 47 --- server/app/scanpy_engine/scanpy_engine.py | 30 +- server/app/util/data_locator.py | 7 +- server/app/util/fbs/matrix.py | 3 +- server/app/util/matrix_proxy.py | 425 ---------------------- server/requirements.txt | 33 +- server/test/test_matrix_proxy.py | 177 --------- 7 files changed, 47 insertions(+), 675 deletions(-) delete mode 100644 server/app/scanpy_engine/matrix_proxy.py delete mode 100644 server/app/util/matrix_proxy.py delete mode 100644 server/test/test_matrix_proxy.py diff --git a/server/app/scanpy_engine/matrix_proxy.py b/server/app/scanpy_engine/matrix_proxy.py deleted file mode 100644 index c244d4b6..00000000 --- a/server/app/scanpy_engine/matrix_proxy.py +++ /dev/null @@ -1,47 +0,0 @@ -from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView - -""" -AnnData/h5py are inconsistent in the API supported by various types of -X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse -matrix, sometimes h5py proxies with a subset of our needed interfaces. - -This glue code paves over all of that, providing the core set of methods -that the cellxgene ScanPy driver assumes are in existance. Put another -way, all of the non-portable assumptions are here. -""" - - -class ArrayProxyView_anndata_h5py(ArrayProxyView): - """ - override to handle sparse getitem semantics, which differ - from numpy. - """ - - def toarray(self): - """ sadly, sparse indexing doesn't drop dimensions like numpy! """ - arr = self.m[self._index[0], self._index[1]] - if self._vdim == 0: - arr = arr.transpose() - return arr.toarray()[0] - - -class MatrixProxy_anndata_h5py(MatrixProxyView): - """ - AnnData sparse array stored in H5AD, or proxies for backed data. - None of these handle indexing very well, so we plop a proxy on top. - """ - - @classmethod - def __supports__(cls): - return ( - "anndata.h5py.h5sparse.SparseDataset", - "anndata.h5py.h5sparse.backed_csc_matrix", - "anndata.h5py.h5sparse.backed_csr_matrix", - "h5py._hl.dataset.Dataset", - "anndata._core.sparse_dataset.backed_csr_matrix", - "anndata._core.sparse_dataset.backed_csc_matrix", - ) - - @classmethod - def create_array(cls, *args, **kwargs): - return ArrayProxyView_anndata_h5py(*args, **kwargs) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 845f8e3b..63f03830 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -5,6 +5,7 @@ from datetime import datetime import os.path from hashlib import blake2b import base64 +from packaging import version import numpy as np import pandas @@ -26,8 +27,15 @@ from server.app.util.utils import jsonify_scanpy, requires_data from server.app.scanpy_engine.diffexp import diffexp_ttest from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs from server.app.scanpy_engine.labels import read_labels, write_labels -import server.app.scanpy_engine.matrix_proxy # noqa: F401 -from server.app.util.matrix_proxy import MatrixProxy + + +anndata_version = version.parse(str(anndata.__version__)).release + + +def anndata_version_is_pre_070(): + major = anndata_version[0] + minor = anndata_version[1] if len(anndata_version) > 1 else 0 + return major == 0 and minor < 7 def has_method(o, name): @@ -303,6 +311,10 @@ class ScanpyEngine(CXGDriver): @requires_data def _validate_and_initialize(self): + if anndata_version_is_pre_070() and self.config['backed']: + warnings.warn(f"Use of --backed mode with anndata versions older than 0.7 will have serious " + "performance issues. Please update to at least anndata 0.7 or later.") + # var and obs column names must be unique if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique: raise KeyError(f"All annotation column names must be unique.") @@ -370,7 +382,14 @@ class ScanpyEngine(CXGDriver): @requires_data def _validate_data_types(self): - if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X): + # The backed API does not support interrogation of the underlying sparsity or sparse matrix type + # Fake it by asking for a small subarray and testing it. NOTE: if the user has ignored our + # anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset, + # this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7 + # which will load the entire X matrix to fullfill any slicing request if X is sparse. See + # user warning in _load_data(). + X0 = self.data.X[0, 0:1] + if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0): warnings.warn( f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. " f"Performance may be improved by using CSC." @@ -577,8 +596,9 @@ class ScanpyEngine(CXGDriver): raise FilterError("filtering on obs unsupported") # Currently only handles VAR dimension - X = MatrixProxy.create(self.data.X if var_selector is None else self.data.X[:, var_selector]) - return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) + X = self.data.X[:, slice(None) if var_selector is None else var_selector] + col_idx = np.nonzero([] if var_selector is None else var_selector)[0] + return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None) @requires_data def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None): diff --git a/server/app/util/data_locator.py b/server/app/util/data_locator.py index 2b7a8150..c16eb40a 100644 --- a/server/app/util/data_locator.py +++ b/server/app/util/data_locator.py @@ -81,8 +81,11 @@ class DataLocator: return LocalFilePath(self.path) # if not local, create a tmp file system object to contain the data, - # and clean it up when done. - with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp: + # and clean it up when done. If the path has a suffix/extension, + # do our best to create a file with the same. + ext = os.path.splitext(self.path) + suffix = None if ext[1] == '' else ext[1] + with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp: tmp.write(src.read()) tmp.close() src.close() diff --git a/server/app/util/fbs/matrix.py b/server/app/util/fbs/matrix.py index 637c39d7..250f31b4 100644 --- a/server/app/util/fbs/matrix.py +++ b/server/app/util/fbs/matrix.py @@ -12,7 +12,6 @@ import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array import server.app.util.fbs.NetEncoding.Float32Array as Float32Array import server.app.util.fbs.NetEncoding.Float64Array as Float64Array import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray -from server.app.util.matrix_proxy import MatrixProxy # Placeholder until recent enhancements to flatbuffers Python @@ -92,7 +91,7 @@ def serialize_typed_array(builder, source_array, encoding_info): as_json = arr.to_json(orient="records") arr = np.array(bytearray(as_json, "utf-8")) else: - if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr): + if sparse.issparse(arr): arr = arr.toarray() elif isinstance(arr, pd.Series): arr = arr.to_numpy() diff --git a/server/app/util/matrix_proxy.py b/server/app/util/matrix_proxy.py deleted file mode 100644 index 988d6f75..00000000 --- a/server/app/util/matrix_proxy.py +++ /dev/null @@ -1,425 +0,0 @@ -import abc -from itertools import zip_longest -from copy import copy -import numpy as np - -""" -cellxgene deals with a variety of matrix data types, many of which do -not support a consistent API. This framework allows proxies to be created -to pave over some of this. Most significantly, AnnData.X does not guarantee -that much of its API (eg. .X.T) will work. -""" - -INT_TYPES = (int, np.integer) - - -class _ArrayProxyBase(abc.ABC): - """ - Private base class for array or matrix proxy. This summarizes - the interface used by the rest of cellxgene. - """ - - @property - @abc.abstractmethod - def dtype(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def ndim(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def shape(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def T(self): - raise NotImplementedError() - - @abc.abstractmethod - def __iter__(self): - raise NotImplementedError() - - @abc.abstractmethod - def __getitem__(self, args): - raise NotImplementedError() - - @abc.abstractmethod - def toarray(): - raise NotImplementedError() - - -class MatrixProxy(_ArrayProxyBase): - """ - Abstract class - all interfaces we need, plus a factory method - to create a proxy based upon actual matrix type. - - This class primarily provides the factory method and related support. - All other functionality is delegated to subclasses. - """ - - """ - Registry of types to proxy class, where values are: - * None: unsupported - * True: self-supported - * string: proxy class - Sub-classes automatically register. - """ - base_proxy_registry = { - "pandas.core.frame.DataFrame": True, - "numpy.ndarray": True, - "scipy.sparse.csc.csc_matrix": True, - "scipy.sparse.csr.csr_matrix": True, - "anndata._core.sparse_dataset.backed_csr_matrix": "MatrixProxy_anndata_h5py", - "anndata._core.sparse_dataset.backed_csc_matrix": "MatrixProxy_anndata_h5py", - } - proxy_registry = None - last_cache_token = None - - @staticmethod - def _register_subclasses(subclasses, registry): - for c in subclasses: - names = c.__supports__() - for name in names: - registry[name] = c - MatrixProxy._register_subclasses(c.__subclasses__(), registry) - - @classmethod - def build_proxy_registry(cls): - if cls.proxy_registry and abc.get_cache_token() == cls.last_cache_token: - return - - cls.last_cache_token = abc.get_cache_token() - registry = copy(cls.base_proxy_registry) - MatrixProxy._register_subclasses(cls.__subclasses__(), registry) - cls.proxy_registry = registry - - @classmethod - def create(cls, matrix): - """ - Factory - call with a matrix and it will create a proxy if needed. - If the type already supports the necessary API, it is just returned - directly. - """ - cls.build_proxy_registry() - t = type(matrix) - fqtn = t.__module__ + "." + t.__name__ - proxy_cls = cls.proxy_registry.get(fqtn, None) - if proxy_cls is None: - raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.") - if proxy_cls is True: - return matrix - return proxy_cls(matrix) - - def __init__(self, m): - self.m = m - - @classmethod - @abc.abstractmethod - def __supports__(cls): - raise NotImplementedError() - - @classmethod - def ismatrixproxy(cls, m): - return isinstance(m, _ArrayProxyBase) - - -class MatrixProxyView(MatrixProxy): - """ - 2D matrix view to a 2D matrix - """ - - def __init__(self, arg1, shape=None, index=(), transposed=False, copy=False): - if not copy: - m = arg1 - super().__init__(m) - - if shape is None: - shape = m.shape - assert len(shape) == 2 - - index = tuple(map(lambda s_i: slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], zip_longest(shape, index))) - - self._shape = shape - self._index = index - self.transposed = transposed - - else: # copy mode - super().__init__(arg1.m) - self._shape = arg1._shape - self._index = arg1._index - self.transposed = arg1.transposed - - @classmethod - def create_array(cls, *args, **kwargs): - """ override if you use a different 1D array proxy """ - return ArrayProxyView(*args, **kwargs) - - def copy(self): - """ override if you need additional behaviors """ - return self.__class__(self, copy=True) - - @classmethod - def __supports__(cls): - return () - - def _swap(self, x): - return (x[1], x[0]) if self.transposed else x - - @property - def dtype(self): - return self.m.dtype - - @property - def ndim(self): - return len(self._shape) - - @property - def shape(self): - return self._swap(self._shape) - - @property - def T(self): - m = self.copy() - m.transposed = not m.transposed - return m - - def __iter__(self): - M = self._swap(self._shape)[0] - s = self._swap((self._index))[0] - start, stop, step = s.indices(M) - for d in range(start, stop, step): - yield self[d] - - def __getitem__(self, args): - """ - decompose into the indexing patterns we use, throw for the rest. - Subclassses implement specialized access. - """ - row, col = self._swap(_unpack_index(args, self.shape)) - M, N = self._shape - allM, allN = self.m.shape - - if isinstance(row, INT_TYPES): - row += self._index[0].start - elif isinstance(row, slice): - row = _slice_slice(self._index[0], allM, row, M) - if isinstance(col, INT_TYPES): - col += self._index[1].start - elif isinstance(col, slice): - col = _slice_slice(self._index[1], allN, col, N) - - if isinstance(row, INT_TYPES): - if isinstance(col, INT_TYPES): - return self._getitem_intXint(row, col) - elif isinstance(col, slice): - return self._getitem_intXslice(row, col) - elif isinstance(row, slice): - if isinstance(col, INT_TYPES): - return self._getitem_sliceXint(row, col) - elif isinstance(col, slice): - return self._getitem_sliceXslice(row, col) - - raise IndexError("unsupported column index types") - - """ - These getitem signatures are separate so that they may be - overridden by subclasses as necessary. We don't do much - with them by default other than the obvious sub-slicing. - - NOTE: these follow the numpy rules for dimensionality reduction - when an integer index is specified. - """ - - def _getitem_intXint(self, row, col): - return self.m[row, col] - - def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]),) - return self.__class__.create_array(self.m, shape=shape, index=(row, col)) - - def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]),) - return self.__class__.create_array(self.m, shape=shape, index=(row, col)) - - def _getitem_sliceXslice(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1])) - return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed) - - def toarray(self): - arr = self.m[self._index] - if self.transposed: - arr = arr.transpose() - return arr - - -class ArrayProxyView(_ArrayProxyBase): - """ - 1D array view to a 2D matrix - """ - - def __init__(self, arg1, shape=None, index=None, copy=False): - super().__init__() - if not copy: - m = arg1 - - # one index MUST be an integer and the other MUST be a slice - assert len(index) == 2 - assert all(isinstance(idx, INT_TYPES + (slice,)) for idx in index) - assert isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES) - - if shape is None: - if isinstance(index[0], INT_TYPES): - shape = (m.shape[0],) - else: - shape = (m.shape[1],) - assert len(shape) == 1 - - self._shape = shape - self.m = m - self._index = index - self._vdim = 1 if isinstance(index[0], INT_TYPES) else 0 - - else: - self.m = arg1.m - self._shape = arg1._shape - self._index = arg1._index - self.fixed = arg1.fixed - - def copy(self): - return self.__class__(self, copy=True) - - @property - def dtype(self): - return self.m.dtype - - @property - def ndim(self): - return len(self._shape) - - @property - def shape(self): - return self._shape - - @property - def T(self): - return self.copy() - - def __iter__(self): - _vdim = self._vdim - M = self._shape[0] - for d in range(*self._index[_vdim].indices(M)): - yield self[d] - - def __getitem__(self, args): - _vdim = self._vdim - index = _unpack_index(args, self.shape)[0] - M = self._shape[0] - allM = self.m.shape[_vdim] - - if isinstance(index, INT_TYPES): - index += self._index[_vdim].start - elif isinstance(index, slice): - index = _slice_slice(self._index[_vdim], allM, index, M) - - if _vdim == 0: - row, col = index, self._index[1] - else: - row, col = self._index[0], index - - if isinstance(row, INT_TYPES): - if isinstance(col, INT_TYPES): - return self._getitem_intXint(row, col) - elif isinstance(col, slice): - return self._getitem_intXslice(row, col) - elif isinstance(row, slice): - assert isinstance(col, INT_TYPES) - return self._getitem_sliceXint(row, col) - - raise IndexError("unsupported column index types") - - def _getitem_intXint(self, row, col): - return self.m[row, col] - - def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]),) - return self.__class__(self.m, shape=shape, index=(row, col)) - - def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]),) - return self.__class__(self.m, shape=shape, index=(row, col)) - - def toarray(self): - return self.m[self._index] - - -def _unpack_index(index, shape): - if not isinstance(index, tuple): - index = (index,) - if len(shape) < len(index): - raise IndexError("invalid index dimensionality - must be 2") - - unpacked = () - for shp, idx in zip_longest(shape, index): - idx = slice(None) if idx is None else idx - idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx - unpacked += (idx,) - - return unpacked - - -def _slice_slice(outer, outer_len, inner, inner_len): - """ - slice a slice - we take advantage of Python 3 range's support - for indexing. - """ - assert outer_len >= inner_len - outer_rng = range(*outer.indices(outer_len)) - rng = outer_rng[inner] - start, stop, step = rng.start, rng.stop, rng.step - if step < 0 and stop < 0: - stop = None - return slice(start, stop, step) - - -def _range_length(start, stop, step): - """ return length of range """ - assert step != 0 - assert start is not None and stop is not None and step is not None - if step > 0 and start < stop: - return 1 + (stop - 1 - start) // step - elif step < 0 and start > stop: - return 1 + (start - 1 - stop) // -step - else: - return 0 - - -def _slice_length(s, length): - """ return slice length """ - return _range_length(*s.indices(length)) - - -def _slice_defaults(s, length): - """ apply slice defaulting conventions """ - assert length >= 0 - - step = 1 if s.step is None else s.step - - if s.start is not None: - start = s.start - if start < 0: - start += length - else: - start = 0 if step > 0 else (length - 1) - - if s.stop is not None: - stop = s.stop - if stop < 0: - stop += length - else: - stop = length if step > 0 else -length - 1 - - return slice(start, stop, step) diff --git a/server/requirements.txt b/server/requirements.txt index 085c1cb1..4c64e693 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,17 +1,16 @@ -anndata>=0.6.2 -click>=6.7 -fastobo>=0.6.1 -Flask>=1.0.2 -Flask-Caching>=1.4.0 -Flask-Compress>=1.4.0 -Flask-Cors>=3.0.6 -Flask-RESTful>=0.3.6 -flatbuffers>=1.10.0 -fsspec>=0.4.4 -numpy>=1.15.2 -pandas>=0.24.2 -scipy>=1.3.0 -tables==3.5.1 -# TEMP workaround for https://github.com/theislab/scanpy/issues/832 aka h5py regression -h5py==2.9.0 -requests>=2.22.0 +anndata>=0.6.20 +click>=6.7 +fastobo>=0.6.1 +Flask>=1.0.2 +Flask-Caching>=1.4.0 +Flask-Compress>=1.4.0 +Flask-Cors>=3.0.6 +Flask-RESTful>=0.3.6 +flatbuffers>=1.10.0 +fsspec>=0.4.4 +numpy>=1.15.2 +packaging>=20.0 +pandas>=0.24.2 +scipy>=1.3.0 +tables==3.5.1 +requests>=2.22.0 diff --git a/server/test/test_matrix_proxy.py b/server/test/test_matrix_proxy.py deleted file mode 100644 index 2c3ad5d9..00000000 --- a/server/test/test_matrix_proxy.py +++ /dev/null @@ -1,177 +0,0 @@ -import unittest -import numpy as np -from server.app.util.matrix_proxy import MatrixProxyView, MatrixProxy - - -class NdArrayProxyView(MatrixProxyView): - """ - Fake test class for matrix proxy - wraps ndarray - """ - - @classmethod - def __supports__(cls): - return ("numpy.ndarray",) - - -class MatrixProxyViewTest(unittest.TestCase): - def test_ismatrixproxy(self): - n = np.zeros((2, 4)) - mp = MatrixProxy.create(n) - - self.assertIsNotNone(n) - self.assertIsNotNone(mp) - self.assertTrue(isinstance(mp, NdArrayProxyView)) - self.assertFalse(MatrixProxy.ismatrixproxy(n)) - self.assertTrue(MatrixProxy.ismatrixproxy(mp)) - - def test_params(self): - n = np.arange(15, dtype=np.float32).reshape((3, 5)) - mp = MatrixProxy.create(n) - - self.assertTrue(MatrixProxy.ismatrixproxy(mp)) - self.assertEqual(mp.ndim, 2) - self.assertEqual(mp.shape, (3, 5)) - self.assertEqual(mp.dtype, np.float32) - - mpt = mp.T - self.assertTrue(MatrixProxy.ismatrixproxy(mpt)) - self.assertEqual(mpt.ndim, 2) - self.assertEqual(mpt.shape, (5, 3)) - self.assertEqual(mpt.dtype, np.float32) - - def test_toarray(self): - n = np.arange(15, dtype=np.float32).reshape((3, 5)) - mp = MatrixProxy.create(n) - self.assertTrue( - np.all( - mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]] - ) - ) - self.assertTrue( - np.all( - mp.T.toarray() - == [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]] - ) - ) - - def test_indexing(self): - """ - [[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.], - [10., 11., 12., 13., 14.]] - """ - n = np.arange(15, dtype=np.float32).reshape((3, 5)) - mp = MatrixProxy.create(n) - - # should support int or slice for one or both dimensions - - # int, int - - self.assertEqual(mp[0, 0], 0) - self.assertEqual(mp.T[0, 0], 0) - self.assertEqual(mp[2, 4], 14) - self.assertEqual(mp.T[4, 2], 14) - self.assertEqual(mp[1, 3], mp.T[3, 1]) - - # int, slice - - self.assertTrue(np.all(mp[0].toarray() == [0, 1, 2, 3, 4])) - self.assertTrue(np.all(mp[1, 1:].toarray() == [6, 7, 8, 9])) - self.assertTrue(np.all(mp[2, 1::-1].toarray() == [11, 10])) - - self.assertTrue(np.all(mp.T[0].toarray() == [0, 5, 10])) - self.assertTrue(np.all(mp.T[1, 1:].toarray() == [6, 11])) - self.assertTrue(np.all(mp.T[2, 1::-1].toarray() == [7, 2])) - - # slice, int - - self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10])) - self.assertTrue(np.all(mp[1:, 1].toarray() == [6, 11])) - self.assertTrue(np.all(mp[1::-1, 2].toarray() == [7, 2])) - - self.assertTrue(np.all(mp.T[:, 0].toarray() == [0, 1, 2, 3, 4])) - self.assertTrue(np.all(mp.T[1:, 1].toarray() == [6, 7, 8, 9])) - self.assertTrue(np.all(mp.T[1::-1, 2].toarray() == [11, 10])) - - # slice, slice - - self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]])) - self.assertTrue( - np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]]) - ) - self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]])) - self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]])) - - self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]])) - self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]])) - self.assertTrue( - np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]]) - ) - self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]])) - - def test_repeated_indexing(self): - """ - [[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.], - [10., 11., 12., 13., 14.]] - """ - n = np.arange(15, dtype=np.float32).reshape((3, 5)) - mp = MatrixProxy.create(n) - - self.assertEqual(mp[0][1], 1) - self.assertEqual(mp.T[0][1], 5) - - self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3])) - - self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10])) - self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == [])) - self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [])) - - def test_dimension_drop(self): - """ - [[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.], - [10., 11., 12., 13., 14.]] - """ - n = np.arange(15, dtype=np.float32).reshape((3, 5)) - mp = MatrixProxy.create(n) - - # drop both dimensions, to a scalar - self.assertEqual(mp[0, 0], 0) - - # drop 1 dimension, to an array - self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4])) - self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10])) - - # with .T - self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]])) - self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9])) - - def test_iter(self): - """ - check that __iter__ is doing what we expect - """ - n = np.arange(15, dtype=np.float32).reshape((3, 5)) - mp = MatrixProxy.create(n) - - rows = [r for r in mp] - self.assertEqual(len(rows), 3) - self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4])) - for i, r in enumerate(rows): - self.assertTrue(np.all(mp[i].toarray() == r.toarray())) - - cols = [c for c in mp.T] - self.assertEqual(len(cols), 5) - self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10])) - for i, c in enumerate(cols): - self.assertTrue(np.all(mp.T[i].toarray() == c.toarray())) - - e = [e for e in mp[0]] - self.assertEqual(len(e), 5) - self.assertEqual(e, [0, 1, 2, 3, 4])