fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-30 05:48:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Experimental - manual annotations (#837)

Browse Source 
* icons, partway

* redux for values

* onChange

* cancel

* annotations lifecycle for category names

* copy categorical

* edit category

* add Dataframe.withColsFrom

* render user annotations; default add/delete annotation category

* add label name to actions

* category name edit

* error checking improvements

* change schema field isUserAnnotation to writable

* always have an unassigned label; implement delete label

* implement add new label and edit label name

* label current cell selection

* fix select exact bug in crossfilter

* clean up categorical reducer

* fix tests

* remove debugging printf

* implement subset/reset for user annotations

* undo redo support for user annotations

* remove duplicate button from categories

* add modal

* remove obsolete duplicate annotation reducers

* remove old debugging printf

* connect modal to annotation create and dup

* initial full-stack wiring

* finish up end-to-end wiring

* fix existing unit tests

* fix pytests to match new schema API

* remove debugging printfs

* add label file rotation

* remove obsolete comment

* add fbs encode/decode tests

* add tests for writable annotations

* simplify code

* fix hashing bug with FBS encoding

* lint

* fix smoke tests

* improve error checking in Dataframe.withColsFrom

* add unit test for Dataframe.withColsFrom

* add unit test for Dataframe.columns and Dataframe.renameCol

* fix bug in FBS encode, add better error checks, refactor

* add FBS encode/decode test

* add clarifying comment

* clean up action type names; fix state inconsistency in crossfilter update

* change autosave timer to 2.5sec

* sort categorical metadata render order so it remains consistent

* add temporary autogenerated label for add-new-label operation

* fix hover-over label menu interference with cell highlighting

* remove debugging code

* add missing reducer cases & fix typo

* make dataframe memoize more general purpose

* add dev mode for annos

* fix error on select duplicate

* handle zero occupancy categories

* correctly maintain unclipped AND clipped world

* correctly handle zero length FBS matrix and label files

* ensure all writable categorical schema contains an unassigned category

* handle case where building occupancy stack for category with no members

* dialog for creating label, disable button if duplicate or empty

* visually separate writeable

* edit category

* fix edit category name

* remove debugging code

* fix edit annotation label

* visually define unassigned, change options

* Pull in requirements.txt from `master`

* label currently selected cells

* duplicate label

* lint

* fix pytest merge issues

* rename --label-file to --experimental-label-file

* remove debugging console log

* spelling error fix; fix bug found in PR review.

* lint
This commit is contained in:
Bruce Martin
2019-09-18 07:33:41 -04:00
committed by Colin Megill
parent ab2c423006
commit 3660a6cc27
51 changed files with 2823 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/driver/driver.py
+14
View File
@@ -45,6 +45,13 @@ class CXGDriver(metaclass=ABCMeta):
features["layout"]["obs"] = {"available": True, "interactiveLimit": 50000}
return features
@abstractmethod
def get_schema(self):
"""
Return current schema
"""
pass
@abstractmethod
def _load_data(self, data_locator):
pass
@@ -59,6 +66,13 @@ class CXGDriver(metaclass=ABCMeta):
"""
pass
@abstractmethod
def annotation_put_fbs(self, axis, fbs):
"""
Put/save FBS as user-defined labels
"""
pass
@abstractmethod
def data_frame_to_fbs_matrix(self, filter, axis):
pass
server/app/rest_api/rest.py
+20 -1
View File
@@ -1,5 +1,6 @@
from http import HTTPStatus
import warnings
from os.path import basename
from flask import Blueprint, current_app, jsonify, make_response, request
from flask_restful import Api, Resource
@@ -16,6 +17,7 @@ from server.app.util.errors import (
InteractiveError,
JSONEncodingValueError,
PrepareError,
DisabledFeatureError,
)
"""
@@ -29,7 +31,7 @@ Sort order for routes
class SchemaAPI(Resource):
def get(self):
return make_response(
jsonify({"schema": current_app.data.schema}), HTTPStatus.OK
jsonify({"schema": current_app.data.get_schema()}), HTTPStatus.OK
)
@@ -72,6 +74,11 @@ class ConfigAPI(Resource):
}
}
}
label_file = current_app.data.config["label_file"]
if label_file:
config["config"]["parameters"]["label_file"] = basename(label_file)
return make_response(jsonify(config), HTTPStatus.OK)
@@ -93,6 +100,18 @@ class AnnotationsObsAPI(Resource):
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
def put(self):
try:
fbs = request.get_data()
res = current_app.data.annotation_put_fbs("obs", fbs)
return make_response(
res, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except (ValueError, DisabledFeatureError, KeyError) as e:
return make_response(str(e), HTTPStatus.BAD_REQUEST)
except Exception as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class AnnotationsVarAPI(Resource):
def get(self):
server/app/scanpy_engine/labels.py
+49
View File
@@ -0,0 +1,49 @@
"""
Helpers for user annotations / label_file parameter
"""
from os.path import exists, splitext, getsize
from os import remove, rename
import pandas as pd
def read_labels(fname):
if exists(fname) and getsize(fname) > 0:
return pd.read_csv(fname, dtype='category')
else:
return pd.DataFrame()
def write_labels(fname, df):
rotate_fname(fname)
if not df.empty:
df.to_csv(fname, index=False)
else:
open(fname, 'a').close()
def rotate_fname(fname):
"""
save N backups of file.
fname -> fname-0
fname-0 -> fname->1
...
fname-(N-1) -> fname-N
"""
def rotate(src, dst):
if exists(src):
if exists(dst):
remove(dst)
rename(src, dst)
rotation_size = 9 # rotation size
name, ext = splitext(fname)
# rotate existing files
for i in range(rotation_size - 1, 0, -1):
src = f"{name}-{i}{ext}"
tgt = f"{name}-{i+1}{ext}"
rotate(src, tgt)
tgt = f"{name}-1{ext}"
rotate(fname, tgt)
server/app/scanpy_engine/scanpy_engine.py
+111 -21
View File
@@ -1,4 +1,6 @@
import warnings
import copy
import threading
import numpy as np
import pandas
@@ -13,10 +15,12 @@ from server.app.util.errors import (
JSONEncodingValueError,
PrepareError,
ScanpyFileError,
DisabledFeatureError,
)
from server.app.util.utils import jsonify_scanpy, requires_data
from server.app.scanpy_engine.diffexp import diffexp_ttest
from server.app.util.fbs.matrix import encode_matrix_fbs
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
from server.app.scanpy_engine.labels import read_labels, write_labels
"""
Sort order for methods
@@ -31,6 +35,8 @@ Sort order for methods
class ScanpyEngine(CXGDriver):
def __init__(self, data=None, args={}):
super().__init__(data, args)
# lock used to protect label file write ops
self.label_lock = threading.Lock()
if self.data:
self._validate_and_initialize()
@@ -47,6 +53,7 @@ class ScanpyEngine(CXGDriver):
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"label_file": None,
}
@staticmethod
@@ -125,6 +132,29 @@ class ScanpyEngine(CXGDriver):
return True
return False
@staticmethod
def _get_col_type(col):
dtype = col.dtype
data_kind = dtype.kind
schema = {}
if ScanpyEngine._can_cast_to_float32(col):
schema["type"] = "float32"
elif ScanpyEngine._can_cast_to_int32(col):
schema["type"] = "int32"
elif dtype == np.bool_:
schema["type"] = "boolean"
elif data_kind == "O" and dtype == "object":
schema["type"] = "string"
elif data_kind == "O" and dtype == "category":
schema["type"] = "categorical"
schema["categories"] = dtype.categories.tolist()
else:
raise TypeError(
f"Annotations of type {dtype} are unsupported by cellxgene."
)
return schema
@requires_data
def _create_schema(self):
self.schema = {
@@ -148,25 +178,8 @@ class ScanpyEngine(CXGDriver):
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
ann_schema = {"name": ann}
dtype = curr_axis[ann].dtype
data_kind = dtype.kind
if self._can_cast_to_float32(curr_axis[ann]):
ann_schema["type"] = "float32"
elif self._can_cast_to_int32(curr_axis[ann]):
ann_schema["type"] = "int32"
elif dtype == np.bool_:
ann_schema["type"] = "boolean"
elif data_kind == "O" and dtype == "object":
ann_schema["type"] = "string"
elif data_kind == "O" and dtype == "category":
ann_schema["type"] = "categorical"
ann_schema["categories"] = curr_axis[ann].dtype.categories.tolist()
else:
raise TypeError(
f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
)
ann_schema = {"name": ann, "writable": False}
ann_schema.update(self._get_col_type(curr_axis[ann]))
self.schema["annotations"][ax]["columns"].append(ann_schema)
for layout in self.config['layout']:
@@ -177,7 +190,24 @@ class ScanpyEngine(CXGDriver):
}
self.schema["layout"]["obs"].append(layout_schema)
@requires_data
def get_schema(self):
schema = self.schema # base schema
# add label obs annotations as needed
if self.labels is not None:
schema = copy.deepcopy(schema)
for col in self.labels.columns:
col_schema = {
"name": col,
"writable": True,
}
col_schema.update(self._get_col_type(self.labels[col]))
schema["annotations"]["obs"]["columns"].append(col_schema)
return schema
def _load_data(self, data_locator):
# as of AnnData 0.6.19, backed mode performs initial load fast, but at the
# cost of significantly slower access to X data.
try:
# there is no guarantee data_locator indicates a local file. The AnnData
# API will only consume local file objects. If we get a non-local object,
@@ -203,6 +233,17 @@ class ScanpyEngine(CXGDriver):
f"Please check your input and try again."
)
if self.config["label_file"]:
try:
self.labels = read_labels(self.config["label_file"])
except Exception as e:
raise ScanpyFileError(
f"Error while loading label file: {e}, File must be in the .csv format, please check "
f"your input and try again."
)
else:
self.labels = None
@requires_data
def _validate_and_initialize(self):
# var and obs column names must be unique
@@ -214,6 +255,7 @@ class ScanpyEngine(CXGDriver):
self.cell_count = self.data.shape[0]
self.gene_count = self.data.shape[1]
self._default_and_validate_layouts()
self._validate_label_file()
self._create_schema()
@requires_data
@@ -297,6 +339,26 @@ class ScanpyEngine(CXGDriver):
f"annotations with more than 500 categories in the UI"
)
@requires_data
def _validate_label_file(self):
"""
labels is None if disabled, empty if enabled by no data
"""
if self.labels is None or self.labels.empty:
return
# all lables must have a name, which must be unique and not used in obs column names
if not self.labels.columns.is_unique:
raise KeyError(f"All column names specified in {self.config['label_file']} must be unique.")
duplicate_columns = list(set(self.labels.columns) & set(self.data.obs.columns))
if len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# labels must have same count as obs annotations
if self.labels.shape[0] != self.data.obs.shape[0]:
raise ValueError("Labels file must have same number of rows as h5ad file.")
@staticmethod
def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool)
@@ -364,13 +426,41 @@ class ScanpyEngine(CXGDriver):
@requires_data
def annotation_to_fbs_matrix(self, axis, fields=None):
if axis == Axis.OBS:
df = self.data.obs
if self.labels is not None and not self.labels.empty:
df = pandas.concat([self.data.obs, self.labels], axis=1, join_axes=[self.data.obs.index], copy=False)
else:
df = self.data.obs
else:
df = self.data.var
if fields is not None and len(fields) > 0:
df = df[fields]
return encode_matrix_fbs(df, col_idx=df.columns)
@requires_data
def annotation_put_fbs(self, axis, fbs):
fname = self.config["label_file"]
if not fname or self.labels is None:
raise DisabledFeatureError("Writable annotations are not enabled")
if axis != Axis.OBS:
raise ValueError("Only OBS dimension access is supported")
new_label_df = decode_matrix_fbs(fbs)
# if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))
if not new_label_df.columns.is_unique or len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section critical section.
with self.label_lock:
self.labels = new_label_df
write_labels(fname, self.labels)
return jsonify_scanpy({"status": "OK"})
@staticmethod
def slice_columns(X, var_mask):
"""
server/app/util/errors.py
+9
View File
@@ -59,3 +59,12 @@ class DriverError(Exception):
def __init__(self, message):
self.message = message
class DisabledFeatureError(Exception):
"""
Raised when an attempt to use a disabled feature occurs
"""
def __init__(self, message):
self.message = message
server/app/util/fbs/matrix.py
+106 -26
View File
@@ -2,10 +2,16 @@ import flatbuffers
import numpy as np
from scipy import sparse
import pandas as pd
import json
import server.app.util.fbs.NetEncoding.Column as Column
import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
import server.app.util.fbs.NetEncoding.Matrix as Matrix
import server.app.util.fbs.NetEncoding.Int32Array as Int32Array
import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
# Placeholder until recent enhancements to flatbuffers Python
@@ -104,38 +110,42 @@ def serialize_typed_array(builder, source_array, encoding_info):
return (array_type, array_value)
column_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
def column_encoding(arr):
type_map = {
# dtype: ( array_type, as_type )
np.float64: (TypedArray.TypedArray.Float32Array, np.float32),
np.float32: (TypedArray.TypedArray.Float32Array, np.float32),
np.float16: (TypedArray.TypedArray.Float32Array, np.float32),
return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
np.int8: (TypedArray.TypedArray.Int32Array, np.int32),
np.int16: (TypedArray.TypedArray.Int32Array, np.int32),
np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
np.uint8: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint16: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
return type_map.get(arr.dtype.type, type_map_default)
index_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
def index_encoding(arr):
type_map = {
# dtype: ( array_type, as_type )
np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
return type_map.get(arr.dtype.type, type_map_default)
return index_encoding_type_map.get(arr.dtype.str, index_encoding_default)
def guess_at_mem_needed(matrix):
@@ -205,3 +215,73 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
builder.Finish(matrix)
return builder.Output()
def deserialize_typed_array(tarr):
type_map = {
TypedArray.TypedArray.NONE: None,
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
}
(u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE:
return None
TarType = type_map.get(u_type, None)
if TarType is None:
raise TypeError(f"FBS contains unknown data type: {u_type}")
arr = TarType()
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8'))
return narr
def decode_matrix_fbs(fbs):
"""
Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data
and indices.
"""
matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0)
n_rows = matrix.NRows()
n_cols = matrix.NCols()
if n_rows == 0 or n_cols == 0:
return pd.DataFrame()
if matrix.RowIndexType() is not TypedArray.TypedArray.NONE:
raise ValueError("row indexing not supported for FBS Matrix")
columns_length = matrix.ColumnsLength()
columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
if columns_index is None:
columns_index = range(0, n_cols)
# sanity checks
if len(columns_index) != n_cols or columns_length != n_cols:
raise ValueError("FBS column count does not match number of columns in underlying matrix")
columns_data = {}
columns_type = {}
for col_idx in range(0, columns_length):
col = matrix.Columns(col_idx)
tarr = (col.UType(), col.U())
data = deserialize_typed_array(tarr)
columns_data[columns_index[col_idx]] = data
if len(data) != n_rows:
raise ValueError("FBS column length does not match number of rows")
if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
columns_type[columns_index[col_idx]] = "category"
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
# more sanity checks
if not df.columns.is_unique or len(df.columns) != n_cols:
raise KeyError("FBS column indices are not unique")
return df
server/cli/launch.py
+19 -3
View File
@@ -47,19 +47,28 @@ def common_args(func):
show_default=True,
help="Relative expression cutoff used when selecting top N differentially expressed genes",
)
@click.option(
"--experimental-label-file",
default=None,
show_default=True,
multiple=False,
metavar="<user labels CSV file>",
help="CSV file containing user annotations; will be overwritten. Created if does not exist.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff):
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, experimental_label_file):
return {
"layout": embedding,
"max_category_items": max_category_items,
"diffexp_lfc_cutoff": diffexp_lfc_cutoff,
"obs_names": obs_names,
"var_names": var_names,
"label_file": experimental_label_file,
}
@@ -107,7 +116,8 @@ def launch(
max_category_items,
diffexp_lfc_cutoff,
title,
scripts
scripts,
experimental_label_file
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -122,7 +132,8 @@ def launch(
> cellxgene launch <url>"""
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff)
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items,
diffexp_lfc_cutoff, experimental_label_file)
try:
data_locator = DataLocator(data)
except RuntimeError as re:
@@ -181,6 +192,11 @@ def launch(
else:
port = find_available_port(host)
if experimental_label_file:
lf_name, lf_ext = splitext(experimental_label_file)
if lf_ext and lf_ext != ".csv":
raise click.FileError(basename(experimental_label_file), hint="label file type must be .csv")
# Setup app
cellxgene_url = f"http://{host}:{port}"
server/test/schema.json
+14 -7
View File
@@ -10,19 +10,23 @@
"columns": [
{
"name": "name_0",
"type": "string"
"type": "string",
"writable": false
},
{
"name": "n_genes",
"type": "int32"
"type": "int32",
"writable": false
},
{
"name": "percent_mito",
"type": "float32"
"type": "float32",
"writable": false
},
{
"name": "n_counts",
"type": "float32"
"type": "float32",
"writable": false
},
{
"name": "louvain",
@@ -36,7 +40,8 @@
"FCGR3A+ Monocytes",
"Dendritic cells",
"Megakaryocytes"
]
],
"writable": false
}
]
},
@@ -45,11 +50,13 @@
"columns": [
{
"name": "name_0",
"type": "string"
"type": "string",
"writable": false
},
{
"name": "n_cells",
"type": "int32"
"type": "int32",
"writable": false
}
]
}
server/test/test_fbs.py
+94
View File
@@ -0,0 +1,94 @@
import unittest
import pandas as pd
import numpy as np
from scipy import sparse
import decode_fbs
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
class FbsTests(unittest.TestCase):
"""Test Case for Matrix FBS data encode/decode """
def test_encode_boundary(self):
""" test various boundary checks """
# row indexing is unsupported
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
# matrix must be 2D
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.ones((10,)))
def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
d = decode_fbs.decode_matrix_FBS(fbs)
print(d)
self.assertEqual(d["n_rows"], dims[0])
self.assertEqual(d["n_cols"], dims[1])
self.assertIsNone(d["row_idx"])
self.assertEqual(len(d["columns"]), dims[1])
for i in range(0, len(d["columns"])):
self.assertEqual(len(d["columns"][i]), dims[0])
self.assertIsInstance(d["columns"][i], expected_types[i][0])
if (expected_types[i][1] is not None):
self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
if expected_column_idx is not None:
self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
def test_encode_DataFrame(self):
df = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.int32),
(np.ndarray, np.uint32),
(list, None)
)
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd'])
def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32)
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.float32),
(np.ndarray, np.float32)
)
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = (
(np.ndarray, np.int32),
(np.ndarray, np.int32),
(np.ndarray, np.int32)
)
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self):
dfSrc = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
for c in dfSrc.columns:
self.assertTrue(c in dfDst.columns)
if isinstance(dfSrc[c], pd.Series):
self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
else:
self.assertEqual(dfSrc[c], dfDst[c])
server/test/test_scanpy_engine.py
+156 -16
View File
@@ -1,15 +1,18 @@
import json
from os import path
from os import path, listdir
import pytest
import time
import unittest
import decode_fbs
import tempfile
import shutil
import numpy as np
from pandas import Series
import pandas as pd
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import FilterError
from server.app.util.errors import FilterError, DisabledFeatureError
from server.app.util.fbs.matrix import encode_matrix_fbs
from server.app.util.data_locator import DataLocator
@@ -22,6 +25,7 @@ class EngineTest(unittest.TestCase):
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"layout_file": None,
}
self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
@@ -32,10 +36,10 @@ class EngineTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self):
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertIn(obs_index_col_name, self.data.data.obs)
self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertIn(var_index_col_name, self.data.data.var)
self.assertEqual(list(self.data.data.var.index), list(range(1838)))
@@ -73,16 +77,16 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 91)
def test_obs_and_var_names(self):
self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
def test_schema(self):
def test_get_schema(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh:
schema = json.load(fh)
self.assertEqual(self.data.schema, schema)
self.assertEqual(self.data.get_schema(), schema)
def test_schema_produces_error(self):
self.data.data.obs["time"] = Series(
self.data.data.obs["time"] = pd.Series(
list([time.time() for i in range(self.data.cell_count)]),
dtype="datetime64[ns]",
)
@@ -111,7 +115,7 @@ class EngineTest(unittest.TestCase):
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
@@ -121,7 +125,7 @@ class EngineTest(unittest.TestCase):
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
def test_annotation_fields(self):
@@ -130,12 +134,16 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 1)
def test_annotation_put(self):
with self.assertRaises(DisabledFeatureError):
self.data.annotation_put_fbs(None, "obs")
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
@@ -169,7 +177,7 @@ class EngineTest(unittest.TestCase):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self):
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
@@ -192,5 +200,137 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 3)
self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
if __name__ == "__main__":
unittest.main()
class WritableAnnotationTest(unittest.TestCase):
def setUp(self):
self.tmpDir = tempfile.mkdtemp()
self.label_file = path.join(self.tmpDir, "labels.csv")
args = {
"layout": ["umap"],
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"label_file": self.label_file
}
self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
def tearDown(self):
shutil.rmtree(self.tmpDir)
def make_fbs(self, data):
df = pd.DataFrame(data)
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
def test_error_checks(self):
# verify that the expected errors are generated
n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({
'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category')
})
# ensure attempt to change VAR annotation
with self.assertRaises(ValueError):
self.data.annotation_put_fbs("var", fbs_bad)
# ensure we catch attempt to overwrite non-writable data
with self.assertRaises(KeyError):
self.data.annotation_put_fbs("obs", fbs_bad)
def test_write_to_file(self):
# verify the file is written as expected
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.label_file))
df = pd.read_csv(self.label_file)
self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
# verify complete overwrite on second attempt, AND rotation occurs
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'),
'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category')
})
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.label_file))
df = pd.read_csv(self.label_file)
self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))
# rotation
name, ext = path.splitext(self.label_file)
self.assertTrue(path.exists(f"{name}-1{ext}"))
def test_file_rotation_to_max_9(self):
# verify we stop rotation at 9
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
name, ext = path.splitext(self.label_file)
expected_files = [self.label_file] + [f"{name}-{i}{ext}" for i in range(1, 10)]
found_files = [path.join(self.tmpDir, p) for p in listdir(self.tmpDir)]
self.assertEqual(set(expected_files), set(found_files))
def test_put_get_roundtrip(self):
# verify that OBS PUTs (annotation_put_fbs) are accessible via
# GET (annotation_to_fbs_matrix)
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
# put
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# get
fbsAll = self.data.annotation_to_fbs_matrix("obs")
schema = self.data.get_schema()
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
obs_index_col_name = schema["annotations"]["obs"]["index"]
self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"
])
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [
'label_A' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [
'label_B' for l in range(0, n_rows)
])
# verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(all_col_schema["cat_A"], {
"name": "cat_A",
"type": "categorical",
"categories": ["label_A"],
"writable": True
})
self.assertEqual(all_col_schema["cat_B"], {
"name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})
server/test/test_scanpy_engine_data_load.py
+1
View File
@@ -24,6 +24,7 @@ class DataLoadEngineTest(unittest.TestCase):
"obs_names": "foo",
"var_names": "bar",
"diffexp_lfc_cutoff": 0.1,
"label_file": None,
}
self.data.update(args=args)
self.assertEqual(args, self.data.config)