fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-06 05:28:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

clean up type inferencing (#2332)

Browse Source 
* unit tests for 64 bit conversion

* clean up type handling

* type inference tests

* more type inference fixes

* use schema to determine user intent for data typing

* stop using deprecated API

* fbs type encoding test

* add missing test

* add more tests

* correctly infer X type for CXG adaptor

* lint

* fix typo

* ts migration

* cleanup from PR review

* lint

* PR review changes
This commit is contained in:
Bruce Martin
2021-07-28 15:10:12 -07:00
committed by GitHub
parent 1140676106
commit 32f60a1547
15 changed files with 730 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/common/fbs/matrix.py
+7 -3
View File
@@ -5,6 +5,8 @@ import pandas as pd
from flatbuffers import Builder
from scipy import sparse
from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array
import backend.common.fbs.NetEncoding.Column as Column
import backend.common.fbs.NetEncoding.Float32Array as Float32Array
import backend.common.fbs.NetEncoding.Float64Array as Float64Array
@@ -14,6 +16,7 @@ import backend.common.fbs.NetEncoding.Matrix as Matrix
import backend.common.fbs.NetEncoding.TypedArray as TypedArray
import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
# Serialization helper
def serialize_column(builder, typed_arr):
""" Serialize NetEncoding.Column """
@@ -84,7 +87,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
def column_encoding(arr):
column_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64),
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
@@ -98,7 +101,8 @@ def column_encoding(arr):
}
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
encoding_dtype = np.dtype(get_encoding_dtype_of_array(arr))
return column_encoding_type_map.get(encoding_dtype.str, column_encoding_default)
def index_encoding(arr):
@@ -198,7 +202,7 @@ def deserialize_typed_array(tarr):
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode("utf-8"))
narr = json.loads(narr.tobytes().decode("utf-8"))
return narr