fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-30 17:48:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Makefile modularity, test targets, and auto-formatting (#1070)

Browse Source 
* Fix Makefile whitespace and .PHONY use

* Fix Makefile filename

* Modularize Makefile into client and server Makefiles

Part of the reason that the Makefile in the root directory is a bit
complicated is that it tries to handle tasks that can be handled
separately in the client and server modules.

This commit pushes some of the make logic specific to each module into
their own makefiles and calls out to those makefiles from that in the
project root.

* Add auto-formatting to client and server modules

One thing that can make linting faster is auto-formatting. This commit
adds the yapf auto-formatting tool to the server module and uses
eslint's "fix" functionality to speed up the linting/formatting process.

* Add yapf for automatic code formatting

* Add a root test target that calls sub-tests

* Apply yapf to python files

* Do not duplicate npm commands, simply pass through

* Update documentation

* Do not shadow reserved word len

* Add general test target

* Fix make call in dev-env

* Use black instead of yapf

* Run flake8 from the root directory

* Revert "Apply yapf to python files"

This reverts commit cdca128a01.

* Apply black to python code

* Resolve lint errors resulting from black format

* Add explanation of server unit tests in dev guidelines
This commit is contained in:
Matt Weiden
2019-12-27 14:43:37 -08:00
committed by GitHub
parent ec79995be8
commit f3015cb9df
37 changed files with 738 additions and 806 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/util/fbs/matrix.py
+14 -17
View File
@@ -27,7 +27,7 @@ def CreateNumpyVector(builder, x):
if not isinstance(x, np.ndarray):
raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
if x.dtype.kind not in ['b', 'i', 'u', 'f']:
if x.dtype.kind not in ["b", "i", "u", "f"]:
raise TypeError("numpy-ndarray holds elements of unsupported datatype")
if x.ndim > 1:
@@ -42,11 +42,11 @@ def CreateNumpyVector(builder, x):
x_little_endian = x.byteswap(inplace=False)
# Calculate total length
len = int(x_little_endian.itemsize * x_little_endian.size)
builder.head = int(builder.Head() - len)
length = int(x_little_endian.itemsize * x_little_endian.size)
builder.head = int(builder.Head() - length)
# tobytes ensures c_contiguous ordering
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C')
builder.Bytes[builder.Head() : builder.Head() + length] = x_little_endian.tobytes(order="C")
return builder.EndVector(x.size)
@@ -88,9 +88,9 @@ def serialize_typed_array(builder, source_array, encoding_info):
arr = arr.to_series()
# convert to a simple ndarray
if as_type == 'json':
as_json = arr.to_json(orient='records')
arr = np.array(bytearray(as_json, 'utf-8'))
if as_type == "json":
as_json = arr.to_json(orient="records")
arr = np.array(bytearray(as_json, "utf-8"))
else:
if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr):
arr = arr.toarray()
@@ -119,18 +119,16 @@ column_encoding_type_map = {
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
}
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
def column_encoding(arr):
@@ -141,11 +139,10 @@ index_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
}
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
def index_encoding(arr):
@@ -163,7 +160,7 @@ def guess_at_mem_needed(matrix):
guess = 1
# round up to nearest 1024 bytes
guess = (guess + 0x400) & (~0x3ff)
guess = (guess + 0x400) & (~0x3FF)
return guess
@@ -223,7 +220,7 @@ def deserialize_typed_array(tarr):
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
}
(u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE:
@@ -237,7 +234,7 @@ def deserialize_typed_array(tarr):
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8'))
narr = json.loads(narr.tostring().decode("utf-8"))
return narr