mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-30 01:58:11 +08:00
Apply yapf to python files
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
DEFAULT_TOP_N = 10
|
||||
|
||||
|
||||
class AugmentedEnum(Enum):
|
||||
|
||||
def __hash__(self):
|
||||
return self.value.__hash__()
|
||||
|
||||
|
||||
@@ -27,7 +27,8 @@ class DataLocator():
|
||||
|
||||
def __init__(self, uri_or_path):
|
||||
self.uri_or_path = uri_or_path
|
||||
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
|
||||
self.protocol, self.path = DataLocator._get_protocol_and_path(
|
||||
uri_or_path)
|
||||
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
|
||||
self.cname = self.path if self.protocol == 'file' else self.uri_or_path
|
||||
# will throw RuntimeError if the protocol is unsupported
|
||||
@@ -82,7 +83,8 @@ class DataLocator():
|
||||
|
||||
# if not local, create a tmp file system object to contain the data,
|
||||
# and clean it up when done.
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp:
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(
|
||||
prefix="cellxgene_", delete=False) as tmp:
|
||||
tmp.write(src.read())
|
||||
tmp.close()
|
||||
src.close()
|
||||
@@ -91,6 +93,7 @@ class DataLocator():
|
||||
|
||||
|
||||
class LocalFilePath():
|
||||
|
||||
def __init__(self, tmp_path, delete=False):
|
||||
self.tmp_path = tmp_path
|
||||
self.delete = delete
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import flatbuffers
|
||||
|
||||
|
||||
class Column(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@@ -22,7 +23,8 @@ class Column(object):
|
||||
def UType(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
|
||||
o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Column
|
||||
@@ -35,7 +37,19 @@ class Column(object):
|
||||
return obj
|
||||
return None
|
||||
|
||||
def ColumnStart(builder): builder.StartObject(2)
|
||||
def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
|
||||
def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
|
||||
def ColumnEnd(builder): return builder.EndObject()
|
||||
|
||||
def ColumnStart(builder):
|
||||
builder.StartObject(2)
|
||||
|
||||
|
||||
def ColumnAddUType(builder, uType):
|
||||
builder.PrependUint8Slot(0, uType, 0)
|
||||
|
||||
|
||||
def ColumnAddU(builder, u):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
|
||||
|
||||
|
||||
def ColumnEnd(builder):
|
||||
return builder.EndObject()
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import flatbuffers
|
||||
|
||||
|
||||
class Float32Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@@ -23,14 +24,17 @@ class Float32Array(object):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return self._tab.Get(
|
||||
flatbuffers.number_types.Float32Flags,
|
||||
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return 0
|
||||
|
||||
# Float32Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
|
||||
return self._tab.GetVectorAsNumpy(
|
||||
flatbuffers.number_types.Float32Flags, o)
|
||||
return 0
|
||||
|
||||
# Float32Array
|
||||
@@ -40,7 +44,19 @@ class Float32Array(object):
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Float32ArrayStart(builder): builder.StartObject(1)
|
||||
def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def Float32ArrayEnd(builder): return builder.EndObject()
|
||||
|
||||
def Float32ArrayStart(builder):
|
||||
builder.StartObject(1)
|
||||
|
||||
|
||||
def Float32ArrayAddData(builder, data):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
|
||||
|
||||
def Float32ArrayStartDataVector(builder, numElems):
|
||||
return builder.StartVector(4, numElems, 4)
|
||||
|
||||
|
||||
def Float32ArrayEnd(builder):
|
||||
return builder.EndObject()
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import flatbuffers
|
||||
|
||||
|
||||
class Float64Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@@ -23,14 +24,17 @@ class Float64Array(object):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
|
||||
return self._tab.Get(
|
||||
flatbuffers.number_types.Float64Flags,
|
||||
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
|
||||
return 0
|
||||
|
||||
# Float64Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
|
||||
return self._tab.GetVectorAsNumpy(
|
||||
flatbuffers.number_types.Float64Flags, o)
|
||||
return 0
|
||||
|
||||
# Float64Array
|
||||
@@ -40,7 +44,19 @@ class Float64Array(object):
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Float64ArrayStart(builder): builder.StartObject(1)
|
||||
def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
|
||||
def Float64ArrayEnd(builder): return builder.EndObject()
|
||||
|
||||
def Float64ArrayStart(builder):
|
||||
builder.StartObject(1)
|
||||
|
||||
|
||||
def Float64ArrayAddData(builder, data):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
|
||||
|
||||
def Float64ArrayStartDataVector(builder, numElems):
|
||||
return builder.StartVector(8, numElems, 8)
|
||||
|
||||
|
||||
def Float64ArrayEnd(builder):
|
||||
return builder.EndObject()
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import flatbuffers
|
||||
|
||||
|
||||
class Int32Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@@ -23,14 +24,17 @@ class Int32Array(object):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return self._tab.Get(
|
||||
flatbuffers.number_types.Int32Flags,
|
||||
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return 0
|
||||
|
||||
# Int32Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
|
||||
return self._tab.GetVectorAsNumpy(
|
||||
flatbuffers.number_types.Int32Flags, o)
|
||||
return 0
|
||||
|
||||
# Int32Array
|
||||
@@ -40,7 +44,19 @@ class Int32Array(object):
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Int32ArrayStart(builder): builder.StartObject(1)
|
||||
def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def Int32ArrayEnd(builder): return builder.EndObject()
|
||||
|
||||
def Int32ArrayStart(builder):
|
||||
builder.StartObject(1)
|
||||
|
||||
|
||||
def Int32ArrayAddData(builder, data):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
|
||||
|
||||
def Int32ArrayStartDataVector(builder, numElems):
|
||||
return builder.StartVector(4, numElems, 4)
|
||||
|
||||
|
||||
def Int32ArrayEnd(builder):
|
||||
return builder.EndObject()
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import flatbuffers
|
||||
|
||||
|
||||
class JSONEncodedArray(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@@ -23,14 +24,17 @@ class JSONEncodedArray(object):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
|
||||
return self._tab.Get(
|
||||
flatbuffers.number_types.Uint8Flags,
|
||||
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
|
||||
return 0
|
||||
|
||||
# JSONEncodedArray
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
|
||||
return self._tab.GetVectorAsNumpy(
|
||||
flatbuffers.number_types.Uint8Flags, o)
|
||||
return 0
|
||||
|
||||
# JSONEncodedArray
|
||||
@@ -40,7 +44,19 @@ class JSONEncodedArray(object):
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def JSONEncodedArrayStart(builder): builder.StartObject(1)
|
||||
def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
|
||||
def JSONEncodedArrayEnd(builder): return builder.EndObject()
|
||||
|
||||
def JSONEncodedArrayStart(builder):
|
||||
builder.StartObject(1)
|
||||
|
||||
|
||||
def JSONEncodedArrayAddData(builder, data):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
|
||||
|
||||
def JSONEncodedArrayStartDataVector(builder, numElems):
|
||||
return builder.StartVector(1, numElems, 1)
|
||||
|
||||
|
||||
def JSONEncodedArrayEnd(builder):
|
||||
return builder.EndObject()
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import flatbuffers
|
||||
|
||||
|
||||
class Matrix(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@@ -22,14 +23,16 @@ class Matrix(object):
|
||||
def NRows(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags,
|
||||
o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
def NCols(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags,
|
||||
o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
@@ -56,7 +59,8 @@ class Matrix(object):
|
||||
def ColIndexType(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
|
||||
o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
@@ -73,7 +77,8 @@ class Matrix(object):
|
||||
def RowIndexType(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
|
||||
o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
@@ -86,13 +91,45 @@ class Matrix(object):
|
||||
return obj
|
||||
return None
|
||||
|
||||
def MatrixStart(builder): builder.StartObject(7)
|
||||
def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
|
||||
def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
|
||||
def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
|
||||
def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
|
||||
def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
|
||||
def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
|
||||
def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
|
||||
def MatrixEnd(builder): return builder.EndObject()
|
||||
|
||||
def MatrixStart(builder):
|
||||
builder.StartObject(7)
|
||||
|
||||
|
||||
def MatrixAddNRows(builder, nRows):
|
||||
builder.PrependUint32Slot(0, nRows, 0)
|
||||
|
||||
|
||||
def MatrixAddNCols(builder, nCols):
|
||||
builder.PrependUint32Slot(1, nCols, 0)
|
||||
|
||||
|
||||
def MatrixAddColumns(builder, columns):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
|
||||
|
||||
|
||||
def MatrixStartColumnsVector(builder, numElems):
|
||||
return builder.StartVector(4, numElems, 4)
|
||||
|
||||
|
||||
def MatrixAddColIndexType(builder, colIndexType):
|
||||
builder.PrependUint8Slot(3, colIndexType, 0)
|
||||
|
||||
|
||||
def MatrixAddColIndex(builder, colIndex):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
|
||||
|
||||
|
||||
def MatrixAddRowIndexType(builder, rowIndexType):
|
||||
builder.PrependUint8Slot(5, rowIndexType, 0)
|
||||
|
||||
|
||||
def MatrixAddRowIndex(builder, rowIndex):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
|
||||
|
||||
|
||||
def MatrixEnd(builder):
|
||||
return builder.EndObject()
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
|
||||
class TypedArray(object):
|
||||
NONE = 0
|
||||
Float32Array = 1
|
||||
@@ -9,4 +10,3 @@ class TypedArray(object):
|
||||
Uint32Array = 3
|
||||
Float64Array = 4
|
||||
JSONEncodedArray = 5
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import flatbuffers
|
||||
|
||||
|
||||
class Uint32Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@@ -23,14 +24,17 @@ class Uint32Array(object):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return self._tab.Get(
|
||||
flatbuffers.number_types.Uint32Flags,
|
||||
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return 0
|
||||
|
||||
# Uint32Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
|
||||
return self._tab.GetVectorAsNumpy(
|
||||
flatbuffers.number_types.Uint32Flags, o)
|
||||
return 0
|
||||
|
||||
# Uint32Array
|
||||
@@ -40,7 +44,19 @@ class Uint32Array(object):
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Uint32ArrayStart(builder): builder.StartObject(1)
|
||||
def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def Uint32ArrayEnd(builder): return builder.EndObject()
|
||||
|
||||
def Uint32ArrayStart(builder):
|
||||
builder.StartObject(1)
|
||||
|
||||
|
||||
def Uint32ArrayAddData(builder, data):
|
||||
builder.PrependUOffsetTRelativeSlot(
|
||||
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
|
||||
|
||||
def Uint32ArrayStartDataVector(builder, numElems):
|
||||
return builder.StartVector(4, numElems, 4)
|
||||
|
||||
|
||||
def Uint32ArrayEnd(builder):
|
||||
return builder.EndObject()
|
||||
|
||||
@@ -25,7 +25,8 @@ def CreateNumpyVector(builder, x):
|
||||
"""CreateNumpyVector writes a numpy array into the buffer."""
|
||||
|
||||
if not isinstance(x, np.ndarray):
|
||||
raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
|
||||
raise TypeError(
|
||||
f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
|
||||
|
||||
if x.dtype.kind not in ['b', 'i', 'u', 'f']:
|
||||
raise TypeError("numpy-ndarray holds elements of unsupported datatype")
|
||||
@@ -46,7 +47,8 @@ def CreateNumpyVector(builder, x):
|
||||
builder.head = int(builder.Head() - len)
|
||||
|
||||
# tobytes ensures c_contiguous ordering
|
||||
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C')
|
||||
builder.Bytes[builder.Head():builder.Head() +
|
||||
len] = x_little_endian.tobytes(order='C')
|
||||
|
||||
return builder.EndVector(x.size)
|
||||
|
||||
@@ -119,12 +121,10 @@ column_encoding_type_map = {
|
||||
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
|
||||
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
|
||||
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
|
||||
|
||||
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
|
||||
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
@@ -141,7 +141,6 @@ index_encoding_type_map = {
|
||||
# array protocol string: ( array_type, as_type )
|
||||
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
|
||||
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
|
||||
}
|
||||
@@ -192,7 +191,8 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
|
||||
columns = []
|
||||
for cidx in range(n_cols - 1, -1, -1):
|
||||
# serialize the typed array
|
||||
col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx]
|
||||
col = matrix.iloc[:, cidx] if isinstance(
|
||||
matrix, pd.DataFrame) else matrix[:, cidx]
|
||||
typed_arr = serialize_typed_array(builder, col, column_encoding)
|
||||
|
||||
# serialize the Column union
|
||||
@@ -218,12 +218,18 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
|
||||
|
||||
def deserialize_typed_array(tarr):
|
||||
type_map = {
|
||||
TypedArray.TypedArray.NONE: None,
|
||||
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
|
||||
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
|
||||
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
|
||||
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
|
||||
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
|
||||
TypedArray.TypedArray.NONE:
|
||||
None,
|
||||
TypedArray.TypedArray.Uint32Array:
|
||||
Uint32Array.Uint32Array,
|
||||
TypedArray.TypedArray.Int32Array:
|
||||
Int32Array.Int32Array,
|
||||
TypedArray.TypedArray.Float32Array:
|
||||
Float32Array.Float32Array,
|
||||
TypedArray.TypedArray.Float64Array:
|
||||
Float64Array.Float64Array,
|
||||
TypedArray.TypedArray.JSONEncodedArray:
|
||||
JSONEncodedArray.JSONEncodedArray
|
||||
}
|
||||
(u_type, u) = tarr
|
||||
if u_type is TypedArray.TypedArray.NONE:
|
||||
@@ -257,13 +263,16 @@ def decode_matrix_fbs(fbs):
|
||||
|
||||
columns_length = matrix.ColumnsLength()
|
||||
|
||||
columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
|
||||
columns_index = deserialize_typed_array(
|
||||
(matrix.ColIndexType(), matrix.ColIndex()))
|
||||
if columns_index is None:
|
||||
columns_index = range(0, n_cols)
|
||||
|
||||
# sanity checks
|
||||
if len(columns_index) != n_cols or columns_length != n_cols:
|
||||
raise ValueError("FBS column count does not match number of columns in underlying matrix")
|
||||
raise ValueError(
|
||||
"FBS column count does not match number of columns in underlying matrix"
|
||||
)
|
||||
|
||||
columns_data = {}
|
||||
columns_type = {}
|
||||
@@ -277,7 +286,8 @@ def decode_matrix_fbs(fbs):
|
||||
if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
|
||||
columns_type[columns_index[col_idx]] = "category"
|
||||
|
||||
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
|
||||
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type,
|
||||
copy=False)
|
||||
|
||||
# more sanity checks
|
||||
if not df.columns.is_unique or len(df.columns) != n_cols:
|
||||
|
||||
@@ -2,7 +2,6 @@ import abc
|
||||
from itertools import zip_longest
|
||||
from copy import copy
|
||||
import numpy as np
|
||||
|
||||
"""
|
||||
cellxgene deals with a variety of matrix data types, many of which do
|
||||
not support a consistent API. This framework allows proxies to be created
|
||||
@@ -18,6 +17,7 @@ class _ArrayProxyBase(abc.ABC):
|
||||
Private base class for array or matrix proxy. This summarizes
|
||||
the interface used by the rest of cellxgene.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def dtype(self):
|
||||
@@ -59,7 +59,6 @@ class MatrixProxy(_ArrayProxyBase):
|
||||
This class primarily provides the factory method and related support.
|
||||
All other functionality is delegated to subclasses.
|
||||
"""
|
||||
|
||||
"""
|
||||
Registry of types to proxy class, where values are:
|
||||
* None: unsupported
|
||||
@@ -128,21 +127,25 @@ class MatrixProxyView(MatrixProxy):
|
||||
"""
|
||||
2D matrix view to a 2D matrix
|
||||
"""
|
||||
def __init__(self, arg1, shape=None, index=(),
|
||||
transposed=False, copy=False):
|
||||
|
||||
def __init__(self,
|
||||
arg1,
|
||||
shape=None,
|
||||
index=(),
|
||||
transposed=False,
|
||||
copy=False):
|
||||
if not copy:
|
||||
m = arg1
|
||||
super().__init__(m)
|
||||
|
||||
if shape is None:
|
||||
shape = m.shape
|
||||
assert(len(shape) == 2)
|
||||
assert (len(shape) == 2)
|
||||
|
||||
index = tuple(
|
||||
map(lambda s_i:
|
||||
slice(0, s_i[0], 1) if s_i[1] is None else s_i[1],
|
||||
zip_longest(shape, index))
|
||||
)
|
||||
map(
|
||||
lambda s_i: slice(0, s_i[0], 1)
|
||||
if s_i[1] is None else s_i[1], zip_longest(shape, index)))
|
||||
|
||||
self._shape = shape
|
||||
self._index = index
|
||||
@@ -234,21 +237,29 @@ class MatrixProxyView(MatrixProxy):
|
||||
NOTE: these follow the numpy rules for dimensionality reduction
|
||||
when an integer index is specified.
|
||||
"""
|
||||
|
||||
def _getitem_intXint(self, row, col):
|
||||
return self.m[row, col]
|
||||
|
||||
def _getitem_intXslice(self, row, col):
|
||||
shape = (_slice_length(col, self.m.shape[1]), )
|
||||
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
|
||||
shape = (_slice_length(col, self.m.shape[1]),)
|
||||
return self.__class__.create_array(self.m,
|
||||
shape=shape,
|
||||
index=(row, col))
|
||||
|
||||
def _getitem_sliceXint(self, row, col):
|
||||
shape = (_slice_length(row, self.m.shape[0]), )
|
||||
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
|
||||
shape = (_slice_length(row, self.m.shape[0]),)
|
||||
return self.__class__.create_array(self.m,
|
||||
shape=shape,
|
||||
index=(row, col))
|
||||
|
||||
def _getitem_sliceXslice(self, row, col):
|
||||
shape = (_slice_length(row, self.m.shape[0]),
|
||||
_slice_length(col, self.m.shape[1]))
|
||||
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed)
|
||||
return self.__class__(self.m,
|
||||
shape=shape,
|
||||
index=(row, col),
|
||||
transposed=self.transposed)
|
||||
|
||||
def toarray(self):
|
||||
arr = self.m[self._index]
|
||||
@@ -261,22 +272,24 @@ class ArrayProxyView(_ArrayProxyBase):
|
||||
"""
|
||||
1D array view to a 2D matrix
|
||||
"""
|
||||
|
||||
def __init__(self, arg1, shape=None, index=None, copy=False):
|
||||
super().__init__()
|
||||
if not copy:
|
||||
m = arg1
|
||||
|
||||
# one index MUST be an integer and the other MUST be a slice
|
||||
assert(len(index) == 2)
|
||||
assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index))
|
||||
assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES))
|
||||
assert (len(index) == 2)
|
||||
assert (all(isinstance(idx, INT_TYPES + (slice,)) for idx in index))
|
||||
assert (isinstance(index[0], INT_TYPES) != isinstance(
|
||||
index[1], INT_TYPES))
|
||||
|
||||
if shape is None:
|
||||
if isinstance(index[0], INT_TYPES):
|
||||
shape = (m.shape[0], )
|
||||
shape = (m.shape[0],)
|
||||
else:
|
||||
shape = (m.shape[1], )
|
||||
assert(len(shape) == 1)
|
||||
shape = (m.shape[1],)
|
||||
assert (len(shape) == 1)
|
||||
|
||||
self._shape = shape
|
||||
self.m = m
|
||||
@@ -336,7 +349,7 @@ class ArrayProxyView(_ArrayProxyBase):
|
||||
elif isinstance(col, slice):
|
||||
return self._getitem_intXslice(row, col)
|
||||
elif isinstance(row, slice):
|
||||
assert(isinstance(col, INT_TYPES))
|
||||
assert (isinstance(col, INT_TYPES))
|
||||
return self._getitem_sliceXint(row, col)
|
||||
|
||||
raise IndexError("unsupported column index types")
|
||||
@@ -345,11 +358,11 @@ class ArrayProxyView(_ArrayProxyBase):
|
||||
return self.m[row, col]
|
||||
|
||||
def _getitem_intXslice(self, row, col):
|
||||
shape = (_slice_length(col, self.m.shape[1]), )
|
||||
shape = (_slice_length(col, self.m.shape[1]),)
|
||||
return self.__class__(self.m, shape=shape, index=(row, col))
|
||||
|
||||
def _getitem_sliceXint(self, row, col):
|
||||
shape = (_slice_length(row, self.m.shape[0]), )
|
||||
shape = (_slice_length(row, self.m.shape[0]),)
|
||||
return self.__class__(self.m, shape=shape, index=(row, col))
|
||||
|
||||
def toarray(self):
|
||||
@@ -358,7 +371,7 @@ class ArrayProxyView(_ArrayProxyBase):
|
||||
|
||||
def _unpack_index(index, shape):
|
||||
if not isinstance(index, tuple):
|
||||
index = (index, )
|
||||
index = (index,)
|
||||
if len(shape) < len(index):
|
||||
raise IndexError("invalid index dimensionality - must be 2")
|
||||
|
||||
@@ -366,7 +379,7 @@ def _unpack_index(index, shape):
|
||||
for shp, idx in zip_longest(shape, index):
|
||||
idx = slice(None) if idx is None else idx
|
||||
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
|
||||
unpacked += (idx, )
|
||||
unpacked += (idx,)
|
||||
|
||||
return unpacked
|
||||
|
||||
@@ -376,7 +389,7 @@ def _slice_slice(outer, outer_len, inner, inner_len):
|
||||
slice a slice - we take advantage of Python 3 range's support
|
||||
for indexing.
|
||||
"""
|
||||
assert(outer_len >= inner_len)
|
||||
assert (outer_len >= inner_len)
|
||||
outer_rng = range(*outer.indices(outer_len))
|
||||
rng = outer_rng[inner]
|
||||
start, stop, step = rng.start, rng.stop, rng.step
|
||||
@@ -387,8 +400,8 @@ def _slice_slice(outer, outer_len, inner, inner_len):
|
||||
|
||||
def _range_length(start, stop, step):
|
||||
""" return length of range """
|
||||
assert(step != 0)
|
||||
assert(start is not None and stop is not None and step is not None)
|
||||
assert (step != 0)
|
||||
assert (start is not None and stop is not None and step is not None)
|
||||
if step > 0 and start < stop:
|
||||
return 1 + (stop - 1 - start) // step
|
||||
elif step < 0 and start > stop:
|
||||
@@ -404,7 +417,7 @@ def _slice_length(s, length):
|
||||
|
||||
def _slice_defaults(s, length):
|
||||
""" apply slice defaulting conventions """
|
||||
assert(length >= 0)
|
||||
assert (length >= 0)
|
||||
|
||||
step = 1 if s.step is None else s.step
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from server.app.util.errors import DriverError
|
||||
|
||||
|
||||
class Float32JSONEncoder(json.JSONEncoder):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
NaN/Infinities are illegal in standard JSON. Python extends JSON with
|
||||
@@ -35,9 +36,12 @@ def jsonify_scanpy(data):
|
||||
|
||||
|
||||
def requires_data(func):
|
||||
|
||||
@wraps(func)
|
||||
def wrapped_function(self, *args, **kwargs):
|
||||
if self.data is None:
|
||||
raise DriverError(f"error data must be loaded before you call {func.__name__}")
|
||||
raise DriverError(
|
||||
f"error data must be loaded before you call {func.__name__}")
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
return wrapped_function
|
||||
|
||||
Reference in New Issue
Block a user