fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-08 21:08:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

anndata X indexing & version compatibility improvements (#1157)

Browse Source 
* revert MatrixProxy; replace with correct use of adata slicing

* work around 0.6 adata slicing bug

* fix incorrect var slice

* simplify slicing of X

* add warning about performance impact of anndata<=0.7

* lint and remove unused code

* improve comment

* lint

* correctly parse versions

* temp files should preserve file suffix if possible - anndata 0.7 compat

* update anndata dependency to 0.6.20

* resolve PR review comments
This commit is contained in:
Bruce Martin
2020-02-19 09:57:51 -07:00
committed by GitHub
parent c630be33df
commit 349c413d8b
7 changed files with 47 additions and 675 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/scanpy_engine/matrix_proxy.py
-47
View File
@@ -1,47 +0,0 @@
from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView
"""
AnnData/h5py are inconsistent in the API supported by various types of
X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse
matrix, sometimes h5py proxies with a subset of our needed interfaces.
This glue code paves over all of that, providing the core set of methods
that the cellxgene ScanPy driver assumes are in existance. Put another
way, all of the non-portable assumptions are here.
"""
class ArrayProxyView_anndata_h5py(ArrayProxyView):
"""
override to handle sparse getitem semantics, which differ
from numpy.
"""
def toarray(self):
""" sadly, sparse indexing doesn't drop dimensions like numpy! """
arr = self.m[self._index[0], self._index[1]]
if self._vdim == 0:
arr = arr.transpose()
return arr.toarray()[0]
class MatrixProxy_anndata_h5py(MatrixProxyView):
"""
AnnData sparse array stored in H5AD, or proxies for backed data.
None of these handle indexing very well, so we plop a proxy on top.
"""
@classmethod
def __supports__(cls):
return (
"anndata.h5py.h5sparse.SparseDataset",
"anndata.h5py.h5sparse.backed_csc_matrix",
"anndata.h5py.h5sparse.backed_csr_matrix",
"h5py._hl.dataset.Dataset",
"anndata._core.sparse_dataset.backed_csr_matrix",
"anndata._core.sparse_dataset.backed_csc_matrix",
)
@classmethod
def create_array(cls, *args, **kwargs):
return ArrayProxyView_anndata_h5py(*args, **kwargs)
server/app/scanpy_engine/scanpy_engine.py
+25 -5
View File
@@ -5,6 +5,7 @@ from datetime import datetime
import os.path import os.path
from hashlib import blake2b from hashlib import blake2b
import base64 import base64
from packaging import version
import numpy as np import numpy as np
import pandas import pandas
@@ -26,8 +27,15 @@ from server.app.util.utils import jsonify_scanpy, requires_data
from server.app.scanpy_engine.diffexp import diffexp_ttest from server.app.scanpy_engine.diffexp import diffexp_ttest
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
from server.app.scanpy_engine.labels import read_labels, write_labels from server.app.scanpy_engine.labels import read_labels, write_labels
import server.app.scanpy_engine.matrix_proxy # noqa: F401
from server.app.util.matrix_proxy import MatrixProxy
anndata_version = version.parse(str(anndata.__version__)).release
def anndata_version_is_pre_070():
major = anndata_version[0]
minor = anndata_version[1] if len(anndata_version) > 1 else 0
return major == 0 and minor < 7
def has_method(o, name): def has_method(o, name):
@@ -303,6 +311,10 @@ class ScanpyEngine(CXGDriver):
@requires_data @requires_data
def _validate_and_initialize(self): def _validate_and_initialize(self):
if anndata_version_is_pre_070() and self.config['backed']:
warnings.warn(f"Use of --backed mode with anndata versions older than 0.7 will have serious "
"performance issues. Please update to at least anndata 0.7 or later.")
# var and obs column names must be unique # var and obs column names must be unique
if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique: if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
raise KeyError(f"All annotation column names must be unique.") raise KeyError(f"All annotation column names must be unique.")
@@ -370,7 +382,14 @@ class ScanpyEngine(CXGDriver):
@requires_data @requires_data
def _validate_data_types(self): def _validate_data_types(self):
if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X): # The backed API does not support interrogation of the underlying sparsity or sparse matrix type
# Fake it by asking for a small subarray and testing it. NOTE: if the user has ignored our
# anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset,
# this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7
# which will load the entire X matrix to fullfill any slicing request if X is sparse. See
# user warning in _load_data().
X0 = self.data.X[0, 0:1]
if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0):
warnings.warn( warnings.warn(
f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. " f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. "
f"Performance may be improved by using CSC." f"Performance may be improved by using CSC."
@@ -577,8 +596,9 @@ class ScanpyEngine(CXGDriver):
raise FilterError("filtering on obs unsupported") raise FilterError("filtering on obs unsupported")
# Currently only handles VAR dimension # Currently only handles VAR dimension
X = MatrixProxy.create(self.data.X if var_selector is None else self.data.X[:, var_selector]) X = self.data.X[:, slice(None) if var_selector is None else var_selector]
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) col_idx = np.nonzero([] if var_selector is None else var_selector)[0]
return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
@requires_data @requires_data
def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None): def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None):
server/app/util/data_locator.py
+5 -2
View File
@@ -81,8 +81,11 @@ class DataLocator:
return LocalFilePath(self.path) return LocalFilePath(self.path)
# if not local, create a tmp file system object to contain the data, # if not local, create a tmp file system object to contain the data,
# and clean it up when done. # and clean it up when done. If the path has a suffix/extension,
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp: # do our best to create a file with the same.
ext = os.path.splitext(self.path)
suffix = None if ext[1] == '' else ext[1]
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
tmp.write(src.read()) tmp.write(src.read())
tmp.close() tmp.close()
src.close() src.close()
server/app/util/fbs/matrix.py
+1 -2
View File
@@ -12,7 +12,6 @@ import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
from server.app.util.matrix_proxy import MatrixProxy
# Placeholder until recent enhancements to flatbuffers Python # Placeholder until recent enhancements to flatbuffers Python
@@ -92,7 +91,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
as_json = arr.to_json(orient="records") as_json = arr.to_json(orient="records")
arr = np.array(bytearray(as_json, "utf-8")) arr = np.array(bytearray(as_json, "utf-8"))
else: else:
if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr): if sparse.issparse(arr):
arr = arr.toarray() arr = arr.toarray()
elif isinstance(arr, pd.Series): elif isinstance(arr, pd.Series):
arr = arr.to_numpy() arr = arr.to_numpy()
server/app/util/matrix_proxy.py
-425
View File
@@ -1,425 +0,0 @@
import abc
from itertools import zip_longest
from copy import copy
import numpy as np
"""
cellxgene deals with a variety of matrix data types, many of which do
not support a consistent API. This framework allows proxies to be created
to pave over some of this. Most significantly, AnnData.X does not guarantee
that much of its API (eg. .X.T) will work.
"""
INT_TYPES = (int, np.integer)
class _ArrayProxyBase(abc.ABC):
"""
Private base class for array or matrix proxy. This summarizes
the interface used by the rest of cellxgene.
"""
@property
@abc.abstractmethod
def dtype(self):
raise NotImplementedError()
@property
@abc.abstractmethod
def ndim(self):
raise NotImplementedError()
@property
@abc.abstractmethod
def shape(self):
raise NotImplementedError()
@property
@abc.abstractmethod
def T(self):
raise NotImplementedError()
@abc.abstractmethod
def __iter__(self):
raise NotImplementedError()
@abc.abstractmethod
def __getitem__(self, args):
raise NotImplementedError()
@abc.abstractmethod
def toarray():
raise NotImplementedError()
class MatrixProxy(_ArrayProxyBase):
"""
Abstract class - all interfaces we need, plus a factory method
to create a proxy based upon actual matrix type.
This class primarily provides the factory method and related support.
All other functionality is delegated to subclasses.
"""
"""
Registry of types to proxy class, where values are:
* None: unsupported
* True: self-supported
* string: proxy class
Sub-classes automatically register.
"""
base_proxy_registry = {
"pandas.core.frame.DataFrame": True,
"numpy.ndarray": True,
"scipy.sparse.csc.csc_matrix": True,
"scipy.sparse.csr.csr_matrix": True,
"anndata._core.sparse_dataset.backed_csr_matrix": "MatrixProxy_anndata_h5py",
"anndata._core.sparse_dataset.backed_csc_matrix": "MatrixProxy_anndata_h5py",
}
proxy_registry = None
last_cache_token = None
@staticmethod
def _register_subclasses(subclasses, registry):
for c in subclasses:
names = c.__supports__()
for name in names:
registry[name] = c
MatrixProxy._register_subclasses(c.__subclasses__(), registry)
@classmethod
def build_proxy_registry(cls):
if cls.proxy_registry and abc.get_cache_token() == cls.last_cache_token:
return
cls.last_cache_token = abc.get_cache_token()
registry = copy(cls.base_proxy_registry)
MatrixProxy._register_subclasses(cls.__subclasses__(), registry)
cls.proxy_registry = registry
@classmethod
def create(cls, matrix):
"""
Factory - call with a matrix and it will create a proxy if needed.
If the type already supports the necessary API, it is just returned
directly.
"""
cls.build_proxy_registry()
t = type(matrix)
fqtn = t.__module__ + "." + t.__name__
proxy_cls = cls.proxy_registry.get(fqtn, None)
if proxy_cls is None:
raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.")
if proxy_cls is True:
return matrix
return proxy_cls(matrix)
def __init__(self, m):
self.m = m
@classmethod
@abc.abstractmethod
def __supports__(cls):
raise NotImplementedError()
@classmethod
def ismatrixproxy(cls, m):
return isinstance(m, _ArrayProxyBase)
class MatrixProxyView(MatrixProxy):
"""
2D matrix view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=(), transposed=False, copy=False):
if not copy:
m = arg1
super().__init__(m)
if shape is None:
shape = m.shape
assert len(shape) == 2
index = tuple(map(lambda s_i: slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], zip_longest(shape, index)))
self._shape = shape
self._index = index
self.transposed = transposed
else: # copy mode
super().__init__(arg1.m)
self._shape = arg1._shape
self._index = arg1._index
self.transposed = arg1.transposed
@classmethod
def create_array(cls, *args, **kwargs):
""" override if you use a different 1D array proxy """
return ArrayProxyView(*args, **kwargs)
def copy(self):
""" override if you need additional behaviors """
return self.__class__(self, copy=True)
@classmethod
def __supports__(cls):
return ()
def _swap(self, x):
return (x[1], x[0]) if self.transposed else x
@property
def dtype(self):
return self.m.dtype
@property
def ndim(self):
return len(self._shape)
@property
def shape(self):
return self._swap(self._shape)
@property
def T(self):
m = self.copy()
m.transposed = not m.transposed
return m
def __iter__(self):
M = self._swap(self._shape)[0]
s = self._swap((self._index))[0]
start, stop, step = s.indices(M)
for d in range(start, stop, step):
yield self[d]
def __getitem__(self, args):
"""
decompose into the indexing patterns we use, throw for the rest.
Subclassses implement specialized access.
"""
row, col = self._swap(_unpack_index(args, self.shape))
M, N = self._shape
allM, allN = self.m.shape
if isinstance(row, INT_TYPES):
row += self._index[0].start
elif isinstance(row, slice):
row = _slice_slice(self._index[0], allM, row, M)
if isinstance(col, INT_TYPES):
col += self._index[1].start
elif isinstance(col, slice):
col = _slice_slice(self._index[1], allN, col, N)
if isinstance(row, INT_TYPES):
if isinstance(col, INT_TYPES):
return self._getitem_intXint(row, col)
elif isinstance(col, slice):
return self._getitem_intXslice(row, col)
elif isinstance(row, slice):
if isinstance(col, INT_TYPES):
return self._getitem_sliceXint(row, col)
elif isinstance(col, slice):
return self._getitem_sliceXslice(row, col)
raise IndexError("unsupported column index types")
"""
These getitem signatures are separate so that they may be
overridden by subclasses as necessary. We don't do much
with them by default other than the obvious sub-slicing.
NOTE: these follow the numpy rules for dimensionality reduction
when an integer index is specified.
"""
def _getitem_intXint(self, row, col):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
def _getitem_sliceXslice(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1]))
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed)
def toarray(self):
arr = self.m[self._index]
if self.transposed:
arr = arr.transpose()
return arr
class ArrayProxyView(_ArrayProxyBase):
"""
1D array view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=None, copy=False):
super().__init__()
if not copy:
m = arg1
# one index MUST be an integer and the other MUST be a slice
assert len(index) == 2
assert all(isinstance(idx, INT_TYPES + (slice,)) for idx in index)
assert isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)
if shape is None:
if isinstance(index[0], INT_TYPES):
shape = (m.shape[0],)
else:
shape = (m.shape[1],)
assert len(shape) == 1
self._shape = shape
self.m = m
self._index = index
self._vdim = 1 if isinstance(index[0], INT_TYPES) else 0
else:
self.m = arg1.m
self._shape = arg1._shape
self._index = arg1._index
self.fixed = arg1.fixed
def copy(self):
return self.__class__(self, copy=True)
@property
def dtype(self):
return self.m.dtype
@property
def ndim(self):
return len(self._shape)
@property
def shape(self):
return self._shape
@property
def T(self):
return self.copy()
def __iter__(self):
_vdim = self._vdim
M = self._shape[0]
for d in range(*self._index[_vdim].indices(M)):
yield self[d]
def __getitem__(self, args):
_vdim = self._vdim
index = _unpack_index(args, self.shape)[0]
M = self._shape[0]
allM = self.m.shape[_vdim]
if isinstance(index, INT_TYPES):
index += self._index[_vdim].start
elif isinstance(index, slice):
index = _slice_slice(self._index[_vdim], allM, index, M)
if _vdim == 0:
row, col = index, self._index[1]
else:
row, col = self._index[0], index
if isinstance(row, INT_TYPES):
if isinstance(col, INT_TYPES):
return self._getitem_intXint(row, col)
elif isinstance(col, slice):
return self._getitem_intXslice(row, col)
elif isinstance(row, slice):
assert isinstance(col, INT_TYPES)
return self._getitem_sliceXint(row, col)
raise IndexError("unsupported column index types")
def _getitem_intXint(self, row, col):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def toarray(self):
return self.m[self._index]
def _unpack_index(index, shape):
if not isinstance(index, tuple):
index = (index,)
if len(shape) < len(index):
raise IndexError("invalid index dimensionality - must be 2")
unpacked = ()
for shp, idx in zip_longest(shape, index):
idx = slice(None) if idx is None else idx
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
unpacked += (idx,)
return unpacked
def _slice_slice(outer, outer_len, inner, inner_len):
"""
slice a slice - we take advantage of Python 3 range's support
for indexing.
"""
assert outer_len >= inner_len
outer_rng = range(*outer.indices(outer_len))
rng = outer_rng[inner]
start, stop, step = rng.start, rng.stop, rng.step
if step < 0 and stop < 0:
stop = None
return slice(start, stop, step)
def _range_length(start, stop, step):
""" return length of range """
assert step != 0
assert start is not None and stop is not None and step is not None
if step > 0 and start < stop:
return 1 + (stop - 1 - start) // step
elif step < 0 and start > stop:
return 1 + (start - 1 - stop) // -step
else:
return 0
def _slice_length(s, length):
""" return slice length """
return _range_length(*s.indices(length))
def _slice_defaults(s, length):
""" apply slice defaulting conventions """
assert length >= 0
step = 1 if s.step is None else s.step
if s.start is not None:
start = s.start
if start < 0:
start += length
else:
start = 0 if step > 0 else (length - 1)
if s.stop is not None:
stop = s.stop
if stop < 0:
stop += length
else:
stop = length if step > 0 else -length - 1
return slice(start, stop, step)
server/requirements.txt
+16 -17
View File
@@ -1,17 +1,16 @@
anndata>=0.6.2 anndata>=0.6.20
click>=6.7 click>=6.7
fastobo>=0.6.1 fastobo>=0.6.1
Flask>=1.0.2 Flask>=1.0.2
Flask-Caching>=1.4.0 Flask-Caching>=1.4.0
Flask-Compress>=1.4.0 Flask-Compress>=1.4.0
Flask-Cors>=3.0.6 Flask-Cors>=3.0.6
Flask-RESTful>=0.3.6 Flask-RESTful>=0.3.6
flatbuffers>=1.10.0 flatbuffers>=1.10.0
fsspec>=0.4.4 fsspec>=0.4.4
numpy>=1.15.2 numpy>=1.15.2
pandas>=0.24.2 packaging>=20.0
scipy>=1.3.0 pandas>=0.24.2
tables==3.5.1 scipy>=1.3.0
# TEMP workaround for https://github.com/theislab/scanpy/issues/832 aka h5py regression tables==3.5.1
h5py==2.9.0 requests>=2.22.0
requests>=2.22.0
server/test/test_matrix_proxy.py
-177
View File
@@ -1,177 +0,0 @@
import unittest
import numpy as np
from server.app.util.matrix_proxy import MatrixProxyView, MatrixProxy
class NdArrayProxyView(MatrixProxyView):
"""
Fake test class for matrix proxy - wraps ndarray
"""
@classmethod
def __supports__(cls):
return ("numpy.ndarray",)
class MatrixProxyViewTest(unittest.TestCase):
def test_ismatrixproxy(self):
n = np.zeros((2, 4))
mp = MatrixProxy.create(n)
self.assertIsNotNone(n)
self.assertIsNotNone(mp)
self.assertTrue(isinstance(mp, NdArrayProxyView))
self.assertFalse(MatrixProxy.ismatrixproxy(n))
self.assertTrue(MatrixProxy.ismatrixproxy(mp))
def test_params(self):
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
self.assertTrue(MatrixProxy.ismatrixproxy(mp))
self.assertEqual(mp.ndim, 2)
self.assertEqual(mp.shape, (3, 5))
self.assertEqual(mp.dtype, np.float32)
mpt = mp.T
self.assertTrue(MatrixProxy.ismatrixproxy(mpt))
self.assertEqual(mpt.ndim, 2)
self.assertEqual(mpt.shape, (5, 3))
self.assertEqual(mpt.dtype, np.float32)
def test_toarray(self):
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
self.assertTrue(
np.all(
mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]]
)
)
self.assertTrue(
np.all(
mp.T.toarray()
== [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]]
)
)
def test_indexing(self):
"""
[[ 0., 1., 2., 3., 4.],
[ 5., 6., 7., 8., 9.],
[10., 11., 12., 13., 14.]]
"""
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
# should support int or slice for one or both dimensions
# int, int
self.assertEqual(mp[0, 0], 0)
self.assertEqual(mp.T[0, 0], 0)
self.assertEqual(mp[2, 4], 14)
self.assertEqual(mp.T[4, 2], 14)
self.assertEqual(mp[1, 3], mp.T[3, 1])
# int, slice
self.assertTrue(np.all(mp[0].toarray() == [0, 1, 2, 3, 4]))
self.assertTrue(np.all(mp[1, 1:].toarray() == [6, 7, 8, 9]))
self.assertTrue(np.all(mp[2, 1::-1].toarray() == [11, 10]))
self.assertTrue(np.all(mp.T[0].toarray() == [0, 5, 10]))
self.assertTrue(np.all(mp.T[1, 1:].toarray() == [6, 11]))
self.assertTrue(np.all(mp.T[2, 1::-1].toarray() == [7, 2]))
# slice, int
self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
self.assertTrue(np.all(mp[1:, 1].toarray() == [6, 11]))
self.assertTrue(np.all(mp[1::-1, 2].toarray() == [7, 2]))
self.assertTrue(np.all(mp.T[:, 0].toarray() == [0, 1, 2, 3, 4]))
self.assertTrue(np.all(mp.T[1:, 1].toarray() == [6, 7, 8, 9]))
self.assertTrue(np.all(mp.T[1::-1, 2].toarray() == [11, 10]))
# slice, slice
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]]))
self.assertTrue(
np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]])
)
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]]))
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]]))
self.assertTrue(
np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]])
)
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]]))
def test_repeated_indexing(self):
"""
[[ 0., 1., 2., 3., 4.],
[ 5., 6., 7., 8., 9.],
[10., 11., 12., 13., 14.]]
"""
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
self.assertEqual(mp[0][1], 1)
self.assertEqual(mp.T[0][1], 5)
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == []))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == []))
def test_dimension_drop(self):
"""
[[ 0., 1., 2., 3., 4.],
[ 5., 6., 7., 8., 9.],
[10., 11., 12., 13., 14.]]
"""
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
# drop both dimensions, to a scalar
self.assertEqual(mp[0, 0], 0)
# drop 1 dimension, to an array
self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4]))
self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
# with .T
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9]))
def test_iter(self):
"""
check that __iter__ is doing what we expect
"""
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
rows = [r for r in mp]
self.assertEqual(len(rows), 3)
self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4]))
for i, r in enumerate(rows):
self.assertTrue(np.all(mp[i].toarray() == r.toarray()))
cols = [c for c in mp.T]
self.assertEqual(len(cols), 5)
self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10]))
for i, c in enumerate(cols):
self.assertTrue(np.all(mp.T[i].toarray() == c.toarray()))
e = [e for e in mp[0]]
self.assertEqual(len(e), 5)
self.assertEqual(e, [0, 1, 2, 3, 4])