diff --git a/makefile b/makefile index 46cb6540..b880c50b 100644 --- a/makefile +++ b/makefile @@ -15,6 +15,7 @@ build-server : build-client cp -r server/* $(SERVERBUILD) cp -r client/build/ $(CLIENTBUILD) mkdir -p $(SERVERBUILD)/app/web/static/img + mkdir -p $(SERVERBUILD)/app/web/templates/ cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/ cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/ cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img @@ -28,6 +29,8 @@ build-client : # If you are actively developing in the server folder use this, dirties the source tree build-for-server-dev : clean-server build-client mkdir -p server/app/web/static/img + mkdir -p server/app/web/static/js + mkdir -p server/app/web/templates/ cp client/build/index.html server/app/web/templates/ cp -r client/build/static server/app/web/ cp client/build/favicon.png server/app/web/static/img diff --git a/server/app/app.py b/server/app/app.py index ffb6f1ae..f620b277 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -4,7 +4,6 @@ from flask import Flask from flask_caching import Cache from flask_compress import Compress from flask_cors import CORS -from flask_restful_swagger_2 import get_swagger_blueprint from .rest_api.rest import get_api_resources from .util.utils import Float32JSONEncoder @@ -26,21 +25,7 @@ app.config.update(SECRET_KEY=SECRET_KEY) # Application Data data = None -# A list of swagger document objects -docs = [] resources = get_api_resources() -docs.append(resources.get_swagger_doc()) - app.register_blueprint(webapp.bp) app.register_blueprint(resources.blueprint) -app.register_blueprint( - get_swagger_blueprint( - docs, - "/api/swagger", - produces=["application/json"], - title="cellxgene rest api", - description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene", - ) -) - app.add_url_rule("/", endpoint="index") diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index 09ef3fc3..bc4965cf 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -42,45 +42,12 @@ class CXGDriver(metaclass=ABCMeta): pass @abstractmethod - def filter_dataframe(self, filter): - """ - Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with - indexing and filtering by annotation value. Filters are combined with the and operator. - See REST specs for info on filter format: - https://github.com/chanzuckerberg/cellxgene/blob/master/docs/REST_API.md - - :param filter: dictionary with filter params - :return: View into scanpy object with cells/genes filtered - """ - pass - - @abstractmethod - def annotation(self, filter, axis, fields=None): + def annotation_to_fbs_matrix(self, axis, field=None): """ Gets annotation value for each observation - :param filter: filter: dictionary with filter params :param axis: string obs or var :param fields: list of keys for annotation to return, returns all annotation values if not set. - :return: dict: names - list of fields in order, data - list of lists or metadata - [observation ids, val1, val2...] - """ - pass - - @abstractmethod - def annotation_to_fbs_matrix(self, axis, field=None): - """ Same as annotation(), except returns a flatbuffer, and does not support filtering. """ - pass - - @abstractmethod - def data_frame(self, filter, axis): - """ - Retrieves data for each variable for observations in data frame - :param filter: filter: dictionary with filter params - :param axis: string obs or var - :return: { - "var": list of variable ids, - "obs": [cellid, var1 expression, var2 expression, ...], - } + :return: flatbuffer: in fbs/matrix.fbs encoding """ pass @@ -104,16 +71,6 @@ class CXGDriver(metaclass=ABCMeta): """ pass - @abstractmethod - def layout(self, filter, interactive_limit=None): - """ - Computes a n-d layout for cells through dimensionality reduction. - :param filter: filter: dictionary with filter params - :param interactive_limit: -- don't compute if total # genes in dataframes are larger than this - :return: [cellid, x, y, ...] - """ - pass - @abstractmethod def layout_to_fbs_matrix(self, filter): """ same as layout, except returns a flatbuffer """ diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index 1904359e..9540f067 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -3,22 +3,17 @@ import pkg_resources import warnings from flask import Blueprint, current_app, jsonify, make_response, request -from flask_restful_swagger_2 import Api, swagger, Resource -from werkzeug.datastructures import ImmutableMultiDict +from flask_restful import Api, Resource from server.app.util.constants import ( Axis, DiffExpMode, JSON_NaN_to_num_warning_msg, ) -from server.app.util.filter import parse_filter, QueryStringError -from server.app.util.models import FilterModel -from server.app.util.utils import get_mime_type from server.app.util.errors import ( FilterError, InteractiveError, JSONEncodingValueError, - MimeTypeError, PrepareError, ) @@ -31,47 +26,6 @@ Sort order for routes class SchemaAPI(Resource): - @swagger.doc( - { - "summary": "get schema for dataframe and annotations", - "tags": ["initialize"], - "parameters": [], - "responses": { - "200": { - "description": "schema", - "examples": { - "application/json": { - "schema": { - "dataframe": { - "nObs": 383, - "nVar": 19944, - "type": "float32", - }, - "annotations": { - "obs": [ - {"name": "name", "type": "string"}, - {"name": "tissue_type", "type": "string"}, - {"name": "num_reads", "type": "int32"}, - {"name": "sample_name", "type": "string"}, - { - "name": "clusters", - "type": "categorical", - "categories": [99, 1, "unknown cluster"], - }, - {"name": "QScore", "type": "float32"}, - ], - "var": [ - {"name": "name", "type": "string"}, - {"name": "gene", "type": "string"}, - ], - }, - } - } - }, - } - }, - } - ) def get(self): return make_response( jsonify({"schema": current_app.data.schema}), HTTPStatus.OK @@ -79,47 +33,6 @@ class SchemaAPI(Resource): class ConfigAPI(Resource): - @swagger.doc( - { - "summary": "Configuration information to assist in front-end adaptation" - " to underlying engine, available functionality, interactive time limits, etc", - "tags": ["initialize"], - "parameters": [], - "responses": { - "200": { - "description": "schema", - "examples": { - "application/json": { - "config": { - "features": [ - { - "method": "POST", - "path": "/cluster/", - "available": False, - }, - { - "method": "POST", - "path": "/layout/obs", - "available": True, - "interactiveLimit": 10000, - }, - { - "method": "POST", - "path": "/layout/var", - "available": False, - }, - ], - "displayNames": { - "engine": "ScanPy version 1.33", - "dataset": "/home/joe/mouse/blorth.csv", - }, - } - } - }, - } - }, - } - ) def get(self): config = { "config": { @@ -158,51 +71,13 @@ class ConfigAPI(Resource): class AnnotationsObsAPI(Resource): - @swagger.doc( - { - "summary": "Fetch annotations (metadata) for all observations.", - "tags": ["annotations"], - "parameters": [ - { - "in": "query", - "name": "annotation-name", - "type": "string", - "description": "list of 1 or more annotation names", - } - ], - "responses": { - "200": { - "description": "annotations", - "examples": { - "application/json": { - "names": ["tissue_type", "sex", "num_reads", "clusters"], - "data": [ - [0, "lung", "F", 39844, 99], - [1, "heart", "M", 83, 1], - [49, "spleen", None, 2, "unknown cluster"], - ], - } - }, - }, - "400": { - "description": "one or more of the annotation-name identifiers were not associated with an " - "annotation name" - }, - }, - } - ) def get(self): fields = request.args.getlist("annotation-name", None) preferred_mimetype = request.accept_mimetypes.best_match( - ["application/json", "application/octet-stream"], - "application/json" + ["application/octet-stream"] ) try: - if preferred_mimetype == "application/json": - return make_response( - current_app.data.annotation({}, "obs", fields), HTTPStatus.OK, {"Content-Type": "application/json"} - ) - elif preferred_mimetype == "application/octet-stream": + if preferred_mimetype == "application/octet-stream": return make_response(current_app.data.annotation_to_fbs_matrix("obs", fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) @@ -210,119 +85,18 @@ class AnnotationsObsAPI(Resource): return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except KeyError: return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - except ValueError as e: - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - - @swagger.doc( - { - "summary": "Fetch annotations (metadata) for filtered subset of observations.", - "tags": ["annotations"], - "parameters": [ - { - "in": "query", - "name": "annotation-name", - "type": "string", - "description": "list of 1 or more annotation names", - }, - { - "name": "filter", - "description": "Complex Filter", - "in": "body", - "schema": FilterModel, - }, - ], - "responses": { - "200": { - "description": "annotations", - "examples": { - "application/json": { - "names": ["tissue_type", "sex", "num_reads", "clusters"], - "data": [ - [0, "lung", "F", 39844, 99], - [1, "heart", "M", 83, 1], - [49, "spleen", None, 2, "unknown cluster"], - ], - } - }, - }, - "400": { - "description": "malformed filter or one or more of the annotation-name identifiers were" - "not associated with an annotation name" - }, - }, - } - ) - def put(self): - fields = request.args.getlist("annotation-name", None) - try: - annotation_response = current_app.data.annotation( - request.get_json()["filter"], "obs", fields - ) - return make_response( - annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"} - ) - except KeyError: - return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) - except FilterError as e: - return make_response(e.message, HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) except ValueError as e: return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) class AnnotationsVarAPI(Resource): - @swagger.doc( - { - "summary": "Fetch annotations (metadata) for all variables.", - "tags": ["annotations"], - "parameters": [ - { - "in": "query", - "name": "annotation-name", - "type": "string", - "description": "list of 1 or more annotation names", - } - ], - "responses": { - "200": { - "description": "annotations", - "examples": { - "application/json": { - "names": ["name", "category"], - "data": [ - [0, "ATAD3C", 1], - [1, "RER1", None], - [49, "S100B", 6], - ], - } - }, - }, - "400": { - "description": "one or more of the annotation-name identifiers were not associated with an" - " annotation name" - }, - }, - } - ) def get(self): fields = request.args.getlist("annotation-name", None) preferred_mimetype = request.accept_mimetypes.best_match( - ["application/json", "application/octet-stream"], - "application/json" + ["application/octet-stream"] ) try: - if preferred_mimetype == "application/json": - return make_response(current_app.data.annotation({}, "var", fields), - HTTPStatus.OK, - {"Content-Type": "application/json"}) - elif preferred_mimetype == "application/octet-stream": + if preferred_mimetype == "application/octet-stream": return make_response(current_app.data.annotation_to_fbs_matrix("var", fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) @@ -330,317 +104,22 @@ class AnnotationsVarAPI(Resource): return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except KeyError: return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - except ValueError as e: - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - - @swagger.doc( - { - "summary": "Fetch annotations (metadata) for filtered subset of variables.", - "tags": ["annotations"], - "parameters": [ - { - "in": "query", - "name": "annotation-name", - "type": "string", - "description": "list of 1 or more annotation names", - }, - { - "name": "filter", - "description": "Complex Filter", - "in": "body", - "schema": FilterModel, - }, - ], - "responses": { - "200": { - "description": "annotations", - "examples": { - "application/json": { - "names": ["name", "category"], - "data": [ - [0, "ATAD3C", 1], - [1, "RER1", None], - [49, "S100B", 6], - ], - } - }, - }, - "400": { - "description": "malformed filter or one or more of the annotation-name identifiers were" - "not associated with an annotation name" - }, - }, - } - ) - def put(self): - fields = request.args.getlist("annotation-name", None) - try: - annotation_response = current_app.data.annotation( - request.get_json()["filter"], "var", fields - ) - return make_response( - annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"} - ) - except KeyError: - return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) - except FilterError: - return make_response("Malformed filter", HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - except ValueError as e: - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - - -class DataObsAPI(Resource): - @swagger.doc( - { - "summary": "Get data (expression values) from the dataframe.", - "tags": ["data"], - "parameters": [ - { - "in": "query", - "name": "filter", - "type": "string", - "description": "axis:key:value", - }, - { - "in": "query", - "name": "accept-type", - "type": "string", - "description": "MIME type", - }, - ], - "responses": { - "200": { - "description": "expression", - "examples": { - "application/json": { - "var": [0, 20000], - "obs": [[1, 39483, 3902, 203, 0, 0, 28]], - } - }, - }, - "400": {"description": "Malformed filter"}, - "406": {"description": "Unacceptable MIME type"}, - }, - } - ) - def get(self): - accept_type = request.args.get("accept-type", None) - # request.args is immutable - args = request.args.copy() - args.pop("accept-type", None) - try: - filter_ = parse_filter( - ImmutableMultiDict(args), current_app.data.schema["annotations"] - ) - except QueryStringError as e: - return make_response(e.message, HTTPStatus.BAD_REQUEST) - # TODO support CSV - try: - # TODO store mime_type when more than one is supported - get_mime_type( - acceptable_types=["application/json"], - query_param=accept_type, - header=request.accept_mimetypes, - ) - except MimeTypeError as e: - return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) - try: - return make_response( - current_app.data.data_frame(filter_, axis=Axis.OBS), - HTTPStatus.OK, - {"Content-Type": "application/json"}, - ) - except FilterError as e: - return make_response(e.message, HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - except ValueError as e: - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - - @swagger.doc( - { - "summary": "Get data (expression values) from the dataframe.", - "tags": ["data"], - "parameters": [ - { - "name": "filter", - "description": "Complex Filter", - "in": "body", - "schema": FilterModel, - } - ], - "responses": { - "200": { - "description": "expression", - "examples": { - "application/json": { - "var": [0, 20000], - "obs": [[1, 39483, 3902, 203, 0, 0, 28]], - } - }, - }, - "400": {"description": "Malformed filter"}, - "406": {"description": "Unacceptable MIME type"}, - }, - } - ) - def put(self): - if not request.accept_mimetypes.best_match(["application/json", "text/csv"]): - return make_response( - f"Unsupported MIME type '{request.accept_mimetypes}'", - HTTPStatus.NOT_ACCEPTABLE, - ) - try: - get_mime_type( - acceptable_types=["application/json"], header=request.accept_mimetypes - ) - except MimeTypeError as e: - return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) - try: - return make_response( - ( - current_app.data.data_frame( - request.get_json()["filter"], axis=Axis.OBS - ) - ), - HTTPStatus.OK, - {"Content-Type": "application/json"}, - ) - except FilterError as e: - return make_response(e.message, HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) except ValueError as e: return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) class DataVarAPI(Resource): - @swagger.doc( - { - "summary": "Get data (expression values) from the dataframe.", - "tags": ["data"], - "parameters": [ - { - "in": "query", - "name": "filter", - "type": "string", - "description": "axis:key:value", - }, - { - "in": "query", - "name": "accept-type", - "type": "string", - "description": "MIME type", - }, - ], - "responses": { - "200": { - "description": "expression", - "examples": { - "application/json": { - "obs": [0, 20000], - "var": [[1, 39483, 3902, 203, 0, 0, 28]], - } - }, - }, - "400": {"description": "Malformed filter"}, - "406": {"description": "Unacceptable MIME type"}, - }, - } - ) - def get(self): - accept_type = request.args.get("accept-type", None) - # request.args is immutable - args = request.args.copy() - args.pop("accept-type", None) - try: - filter_ = parse_filter( - ImmutableMultiDict(args), current_app.data.schema["annotations"] - ) - except QueryStringError as e: - return make_response(e.message, HTTPStatus.BAD_REQUEST) - try: - get_mime_type( - acceptable_types=["application/json"], - query_param=accept_type, - header=request.accept_mimetypes, - ) - except MimeTypeError as e: - return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE) - try: - return make_response( - current_app.data.data_frame(filter_, axis=Axis.VAR), - HTTPStatus.OK, - {"Content-Type": "application/json"}, - ) - except FilterError as e: - return make_response(e.message, HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - except ValueError as e: - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - - @swagger.doc( - { - "summary": "Get data (expression values) from the dataframe.", - "tags": ["data"], - "parameters": [ - { - "name": "filter", - "description": "Complex Filter", - "in": "body", - "schema": FilterModel, - } - ], - "responses": { - "200": { - "description": "expression", - "examples": { - "application/json": { - "obs": [0, 20000], - "var": [[1, 39483, 3902, 203, 0, 0, 28]], - } - }, - }, - "400": {"description": "Malformed filter"}, - "406": {"description": "Unacceptable MIME type"}, - }, - } - ) def put(self): preferred_mimetype = request.accept_mimetypes.best_match( - ["application/json", "application/octet-stream"], - "application/json" + ["application/octet-stream"] ) try: - if preferred_mimetype == "application/json": - return make_response( - ( - current_app.data.data_frame( - request.get_json()["filter"], axis=Axis.VAR - ) - ), - HTTPStatus.OK, - {"Content-Type": "application/json"}, - ) - elif preferred_mimetype == "application/octet-stream": + if preferred_mimetype == "application/octet-stream": + filter_json = request.get_json() + filter = filter_json["filter"] if filter_json else None return make_response( current_app.data.data_frame_to_fbs_matrix( - request.get_json()["filter"], axis=Axis.VAR + filter, axis=Axis.VAR ), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) @@ -648,73 +127,11 @@ class DataVarAPI(Resource): return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except FilterError as e: return make_response(e.message, HTTPStatus.BAD_REQUEST) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) except ValueError as e: return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) class DiffExpObsAPI(Resource): - @swagger.doc( - { - "summary": "Generate differential expression (DE) statistics for two specified subsets of data, " - "as indicated by the two provided observation complex filters", - "tags": ["diffexp"], - # TODO sort out params - # "parameters": [ - # # { - # # "in": "body", - # # "name": "mode", - # # "type": "string", - # # "required": True, - # # "description": "topN or varFilter" - # # }, - # { - # "in": "query", - # "name": "count", - # "type": "int32", - # "description": "TopN mode: how many vars to return" - # }, - # { - # "in": "body", - # "name": "varFilter", - # "schema": FilterModel, - # "description": "varFilter: Complex filter, only var for which vars to return" - # }, - # { - # "in": "body", - # "name": "set1", - # "schema": FilterModel, - # "required": True, - # "description": "Complex filter, only obs - observations in set1" - # }, - # { - # "in": "body", - # "name": "set2", - # "schema": FilterModel, - # "description": "Complex filter, only obs - observations in set2. - # If not included, inverse of set1." - # }, - # ], - "responses": { - "200": { - "description": "Statistics are encoded as an array of arrays, with fields ordered as: " - "varIndex, logfoldchange, pVal, pValAdj", - "examples": { - "application/json": [ - [328, -2.569_489, 2.655_706e-63, 3.642_036e-57], - [1250, -2.569_489, 2.655_706e-63, 3.642_036e-57], - ] - }, - }, - "400": {"description": "malformed filter"}, - "403": {"description": "non-interactive request"}, - "501": {"description": "diffexp is not implemented"}, - }, - } - ) def post(self): args = request.get_json() # confirm mode is present and legal @@ -783,40 +200,12 @@ class DiffExpObsAPI(Resource): class LayoutObsAPI(Resource): - @swagger.doc( - { - "summary": "Get the default layout for all observations.", - "tags": ["layout"], - "parameters": [], - "responses": { - "200": { - "description": "layout", - "examples": { - "application/json": { - "layout": { - "ndims": 2, - "coordinates": [ - [0, 0.284_483, 0.983_744], - [1, 0.038_844, 0.739_444], - ], - } - } - }, - }, - "400": {"description": "Data preparation error"}, - }, - } - ) def get(self): preferred_mimetype = request.accept_mimetypes.best_match( - ["application/json", "application/octet-stream"], - "application/json" + ["application/octet-stream"] ) try: - if preferred_mimetype == "application/json": - return make_response(current_app.data.layout({}), HTTPStatus.OK, {"Content-Type": "application/json"}) - - elif preferred_mimetype == "application/octet-stream": + if preferred_mimetype == "application/octet-stream": return make_response(current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) @@ -824,69 +213,19 @@ class LayoutObsAPI(Resource): return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except PrepareError as e: return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR) - except JSONEncodingValueError as e: - # JSON encoding failure, usually due to bad data - warnings.warn(JSON_NaN_to_num_warning_msg) - return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) except ValueError as e: return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) - # @swagger.doc({ - # "summary": "Observation layout for filtered subset.", - # "tags": ["layout"], - # "parameters": [ - # { - # "name": "filter", - # "description": "Complex Filter", - # "in": "body", - # "schema": FilterModel - # } - # ], - # "responses": { - # "200": { - # "description": "layout", - # "examples": { - # "application/json": { - # "layout": { - # "ndims": 2, - # "coordinates": [ - # [0, 0.284483, 0.983744], - # [1, 0.038844, 0.739444] - # ] - # } - # } - # } - # }, - # "400": { - # "description": "Malformed filter" - # }, - # "403": { - # "description": "Non-interactive request" - # }, - # } - # }) - # def put(self): - # try: - # filter = request.get_json()["filter"] - # interactive_limit = current_app.data.features["layout"]["obs"]["interactiveLimit"] - # layout = current_app.data.layout(filter, interactive_limit=interactive_limit) - # return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type}) - # except FilterError as e: - # return make_response(e.message, HTTPStatus.BAD_REQUEST) - # except InteractiveError: - # return make_response("Non-interactive request", HTTPStatus.FORBIDDEN) - def get_api_resources(): bp = Blueprint("api", __name__, url_prefix="/api/v0.2") - api = Api(bp, add_api_spec_resource=False) + api = Api(bp) # Initialization routes api.add_resource(SchemaAPI, "/schema") api.add_resource(ConfigAPI, "/config") # Data routes api.add_resource(AnnotationsObsAPI, "/annotations/obs") api.add_resource(AnnotationsVarAPI, "/annotations/var") - api.add_resource(DataObsAPI, "/data/obs") api.add_resource(DataVarAPI, "/data/var") # Computation routes api.add_resource(DiffExpObsAPI, "/diffexp/obs") diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index dde57e92..705f0ba8 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -1,16 +1,13 @@ import warnings import numpy as np -from pandas import DataFrame from pandas.core.dtypes.dtypes import CategoricalDtype import scanpy.api as sc -from scipy import sparse from server.app.driver.driver import CXGDriver from server.app.util.constants import Axis, DEFAULT_TOP_N from server.app.util.errors import ( FilterError, - InteractiveError, JSONEncodingValueError, PrepareError, ScanpyFileError, @@ -197,23 +194,6 @@ class ScanpyEngine(CXGDriver): f"to solve this problem. " ) - def filter_dataframe(self, filter): - """ - Filter cells from data and return a subset of the data. They can operate on both obs and var dimension with - indexing and filtering by annotation value. Filters are combined with the and operator. - See REST specs for info on filter format: - # TODO update this link to swagger when it's done - https://docs.google.com/document/d/1Fxjp1SKtCk7l8QP9-7KAjGXL0eldi_qEnNT0NmlGzXI/edit#heading=h.8qc9q57amldx - - :param filter: dictionary with filter params - :return: View into scanpy object with cells/genes filtered - """ - if not filter: - return self.data - obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False) - data = self._slice(self.data, obs_selector, var_selector) - return data - @staticmethod def _annotation_filter_to_mask(filter, d_axis, count): mask = np.ones((count,), dtype=bool) @@ -277,72 +257,6 @@ class ScanpyEngine(CXGDriver): ) return obs_selector, var_selector - @staticmethod - def _slice(data, obs_selector=None, vars_selector=None): - """ - Slice date using any selector that the AnnData object - supprots for slicing. If selector is None, will not slice - on that axis. - - This method exists to optimize filtering/slicing sparse data that has - access patterns which impact slicing performance. - - https://docs.scipy.org/doc/scipy/reference/sparse.html - """ - prefer_row_access = ( - sparse.isspmatrix_csr(data._X) - or sparse.isspmatrix_lil(data._X) - or sparse.isspmatrix_bsr(data._X) - ) - if prefer_row_access: - # Row-major slicing - if obs_selector is not None: - data = data[obs_selector, :] - if vars_selector is not None: - data = data[:, vars_selector] - else: - # Col-major slicing - if vars_selector is not None: - data = data[:, vars_selector] - if obs_selector is not None: - data = data[obs_selector, :] - - return data - - def annotation(self, filter, axis, fields=None): - """ - Gets annotation value for each observation - :param filter: filter: dictionary with filter params - :param axis: string obs or var - :param fields: list of keys for annotation to return, returns all annotation values if not set. - :return: dict: names - list of fields in order, data - list of lists or metadata - [observation ids, val1, val2...] - """ - try: - obs_selector, var_selector = self._filter_to_mask(filter) - except (KeyError, IndexError) as e: - raise FilterError(f"Error parsing filter: {e}") from e - if axis == Axis.OBS: - obs = self.data.obs[obs_selector] - if not fields: - fields = obs.columns.tolist() - result = { - "names": fields, - "data": DataFrame(obs[fields]).to_records(index=True).tolist(), - } - else: - var = self.data.var[var_selector] - if not fields: - fields = var.columns.tolist() - result = { - "names": fields, - "data": DataFrame(var[fields]).to_records(index=True).tolist(), - } - try: - return jsonify_scanpy(result) - except ValueError: - raise JSONEncodingValueError("Error encoding annotations to JSON") - def annotation_to_fbs_matrix(self, axis, fields=None): if axis == Axis.OBS: df = self.data.obs @@ -352,44 +266,6 @@ class ScanpyEngine(CXGDriver): df = df[fields] return encode_matrix_fbs(df, col_idx=df.columns) - def data_frame(self, filter, axis): - """ - Retrieves data for each variable for observations in data frame - :param filter: filter: dictionary with filter params - :param axis: string obs or var - :return: { - "var": list of variable ids, - "obs": [cellid, var1 expression, var2 expression, ...], - } - """ - try: - obs_selector, var_selector = self._filter_to_mask(filter) - except (KeyError, IndexError) as e: - raise FilterError(f"Error parsing filter: {e}") from e - _X = self.data._X[obs_selector, var_selector] - if sparse.issparse(_X): - _X = _X.toarray() - var_index_sliced = self.data.var.index[var_selector] - obs_index_sliced = self.data.obs.index[obs_selector] - if axis == Axis.OBS: - result = { - "var": var_index_sliced.tolist(), - "obs": DataFrame(_X, index=obs_index_sliced) - .to_records(index=True) - .tolist(), - } - else: - result = { - "obs": obs_index_sliced.tolist(), - "var": DataFrame(_X.T, index=var_index_sliced) - .to_records(index=True) - .tolist(), - } - try: - return jsonify_scanpy(result) - except ValueError: - raise JSONEncodingValueError("Error encoding dataframe to JSON") - def data_frame_to_fbs_matrix(self, filter, axis): """ Retrieves data 'X' and returns in a flatbuffer Matrix. @@ -405,7 +281,7 @@ class ScanpyEngine(CXGDriver): raise ValueError("Only VAR dimension access is supported") try: obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False) - except (KeyError, IndexError) as e: + except (KeyError, IndexError, TypeError) as e: raise FilterError(f"Error parsing filter: {e}") from e if obs_selector is not None: raise FilterError("filtering on obs unsupported") @@ -440,50 +316,6 @@ class ScanpyEngine(CXGDriver): "Error encoding differential expression to JSON" ) - def layout(self, filter, interactive_limit=None): - """ - Computes a n-d layout for cells through dimensionality reduction. - :param filter: filter: dictionary with filter params - :param interactive_limit: -- don't compute if total # genes in dataframes are larger than this - :return: [cellid, x, y, ...] - """ - try: - df = self.filter_dataframe(filter) - except (KeyError, IndexError) as e: - raise FilterError(f"Error parsing filter: {e}") from e - if interactive_limit and len(df.obs.index) > interactive_limit: - raise InteractiveError("Size data is too large for interactive computation") - # TODO Filtering cells is fine, but filtering genes does nothing because the neighbors are - # calculated using the original vars (geneset) and this doesn’t get updated when you use less. - # Need to recalculate neighbors (long) if user requests new layout filtered by var - # TODO for MVP we are pushing computation of layout to preprocessing and not allowing re-layout - # this will probably change after user feedback - # getattr(sc.tl, self.layout_method)(df, random_state=123) - try: - df_layout = df.obsm[f"X_{self.layout_method}"] - except ValueError as e: - raise PrepareError( - f"Layout has not been calculated using {self.layout_method}, " - f"please prepare your datafile and relaunch cellxgene" - ) from e - normalized_layout = DataFrame( - (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()), - index=df.obs.index, - ) - try: - return jsonify_scanpy( - { - "layout": { - "ndims": normalized_layout.shape[1], - "coordinates": normalized_layout.to_records( - index=True - ).tolist(), - } - } - ) - except ValueError: - raise JSONEncodingValueError("Error encoding layout to JSON") - def layout_to_fbs_matrix(self): """ Return the default 2-D layout for cells as a FBS Matrix. diff --git a/server/app/util/filter.py b/server/app/util/filter.py deleted file mode 100644 index ad1935be..00000000 --- a/server/app/util/filter.py +++ /dev/null @@ -1,86 +0,0 @@ -import json -from collections import defaultdict - -from numpy import float32, int32 - -from server.app.util.constants import Axis - - -class QueryStringError(Exception): - def __init__(self, key, message): - self.key = key - self.message = message - - -def _convert_variable(datatype, variable): - """ - Convert variable to number (float/int) - Used for dataset metadata and for query string - :param datatype: type to convert to - :param variable (string or None): value of variable - :return: converted variable - :raises: AssertionError - """ - assert datatype in ["boolean", "categorical", "float32", "int32", "string"] - if variable is None: - return variable - if datatype == "int32": - variable = int32(variable) - elif datatype == "float32": - variable = float32(variable) - elif datatype == "boolean": - variable = json.loads(variable) - assert isinstance(variable, bool) - return variable - - -def parse_filter(query_filter, schema): - """ - The filter comes in as arguments from a GET request - For categorical metadata keys filter based on axis:key=value - For continuous metadata keys filter by axis:key=min,max - Either value can be replaced by a * To have only a minimum - value axis:key=min,* To have only a maximum value axis:key=*,max - - They combine via AND so a cell's metadata would have to match every filter - - The results is a matrix with the cells the pass the filter and at this point all the genes - :param query_filter: flask's request.args - :param schema: dictionary schema - :raises QueryStringError - :return: - """ - query = defaultdict(lambda: defaultdict(list)) - - for key in query_filter: - axis, annotation = key.split(":", 1) - try: - Axis(axis) - except ValueError: - raise QueryStringError(key, f"Error: key {key} not in metadata schema") - ann_filter = {"name": annotation} - for ann in schema[axis]: - if ann["name"] == annotation: - dtype = ann["type"] - break - else: - raise QueryStringError(key, f"Error: {annotation} not a valid annotation name") - if dtype in ["string", "categorical", "boolean"]: - ann_filter["values"] = [_convert_variable(dtype, i) for i in query_filter.getlist(key)] - else: - value = query_filter.get(key) - try: - min_, max_ = value.split(",") - except ValueError: - raise QueryStringError(key, f"Error: min,max format required for range for {annotation}, got {value}") - if min_ == "*": - min_ = None - if max_ == "*": - max_ = None - try: - ann_filter["min"] = _convert_variable(dtype, min_) - ann_filter["max"] = _convert_variable(dtype, max_) - except ValueError: - raise QueryStringError(key, f"Error: expected type {query[key]['type']} for key {key}, got {value}") - query[axis]["annotation_value"].append(ann_filter) - return query diff --git a/server/app/util/models.py b/server/app/util/models.py deleted file mode 100644 index d5ebb9d9..00000000 --- a/server/app/util/models.py +++ /dev/null @@ -1,34 +0,0 @@ -from flask_restful_swagger_2 import Schema - - -class AnnotationModel(Schema): - type = "object" - description = "Filter by annotation key: value" - properties = { - "name": {"type": "string"}, - # TODO update to OpenAPI v3.0 when a library is available that supports it - # Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types - # Overloading the type key with a list seems to work ok and makes it to the page - "values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}}, - "min": {"type": ["int32", "float32"]}, - "max": {"type": ["int32", "float32"]}, - } - required = ["name"] - - -class IndexModel(Schema): - type = "object" - description = "Filter by index of observation/variable ex. [0, 5, 15]" - properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}} - - -class AxisModel(Schema): - type = "object" - description = "Axis of data -- obs or var" - properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()} - - -class FilterModel(Schema): - type = "object" - description = "Complex filter" - properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}} diff --git a/server/app/util/utils.py b/server/app/util/utils.py index fc274f87..f9705d1d 100644 --- a/server/app/util/utils.py +++ b/server/app/util/utils.py @@ -1,10 +1,6 @@ import json -from argparse import ArgumentTypeError - from numpy import float32, integer -from server.app.util.errors import MimeTypeError - class Float32JSONEncoder(json.JSONEncoder): def __init__(self, *args, **kwargs): @@ -30,31 +26,5 @@ def custom_format_warning(msg, *args, **kwargs): return f"[cellxgene] Warning: {msg} \n" -def get_mime_type( - default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None -): - mime_type = default - if query_param: - if query_param in acceptable_types: - mime_type = query_param - else: - raise MimeTypeError(f"Unsupported mime type {query_param} specified in query parameter 'accept-type'") - elif len(header): - mime_type = header.best_match(acceptable_types) - if not mime_type: - raise MimeTypeError(f"Unsupported mime type(s) {header} in HTTP Accept header") - return mime_type - - -def whole_number(value): - try: - value = int(value) - except ValueError as e: - raise ArgumentTypeError(f"{value} is not type int") from e - if value < 0: - raise ArgumentTypeError(f"{value} is not >= 0") - return value - - def jsonify_scanpy(data): return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False) diff --git a/server/app/web/templates/swagger.html b/server/app/web/templates/swagger.html deleted file mode 100644 index 115c7eb8..00000000 --- a/server/app/web/templates/swagger.html +++ /dev/null @@ -1,97 +0,0 @@ - - - - - cellxgene REST API - Swagger definition - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - diff --git a/server/app/web/webapp.py b/server/app/web/webapp.py index 04e1d19b..c2b51911 100644 --- a/server/app/web/webapp.py +++ b/server/app/web/webapp.py @@ -11,13 +11,6 @@ def index(): return render_template("index.html", datasetTitle=dataset_title) -# renders swagger documentation -@bp.route("/swagger") -def swag(): - return render_template("swagger.html") - - -# renders swagger documentation @bp.route("/favicon.png") def favicon(): return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png") diff --git a/server/requirements.txt b/server/requirements.txt index 7c09cd6c..00913583 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -5,7 +5,6 @@ Flask-Caching>=1.4.0 Flask-Compress>=1.4.0 Flask-Cors>=3.0.6 Flask-RESTful>=0.3.6 -flask-restful-swagger-2>=0.35 flatbuffers>=1.10.0 matplotlib>=2.2 numpy>=1.14.5 diff --git a/server/test/test_api.py b/server/test/test_api.py index f79a8fa7..eb7eb821 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -57,16 +57,6 @@ class EndPoints(unittest.TestCase): self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") self.assertEqual(len(result_data["config"]["features"]), 4) - def test_get_layout(self): - endpoint = "layout/obs" - url = f"{URL_BASE}{endpoint}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["layout"]["ndims"], 2) - self.assertEqual(len(result_data["layout"]["coordinates"]), 2638) - def test_get_layout_fbs(self): endpoint = "layout/obs" url = f"{URL_BASE}{endpoint}" @@ -82,53 +72,11 @@ class EndPoints(unittest.TestCase): self.assertIsNone(df['row_idx']) self.assertEqual(len(df['columns']), df['n_cols']) - # def test_put_layout(self): - # endpoint = "layout/obs" - # url = f"{URL_BASE}{endpoint}" - # obs_filter = { - # "filter": { - # "obs": { - # "annotation_value": [ - # {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, - # {"name": "n_counts", "min": 3000}, - # ], - # "index": [1, 99, [1000, 2000]] - # } - # } - # } - # result = self.session.put(url, json=obs_filter) - # self.assertEqual(result.status_code, HTTPStatus.OK) - # result_data = result.json() - # self.assertEqual(len(result_data["layout"]["coordinates"]), 15) - def test_bad_filter(self): - endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"] - for endpoint in endpoints: - url = f"{URL_BASE}{endpoint}" - result = self.session.put(url, json=BAD_FILTER) - self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) - - def test_get_annotations_obs(self): - endpoint = "annotations/obs" + endpoint = "data/var" url = f"{URL_BASE}{endpoint}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"]) - self.assertEqual(len(result_data["data"]), 2638) - self.assertEqual(len(result_data["data"][0]), 6) - - def test_get_annotations_obs_keys(self): - endpoint = "annotations/obs" - query = "annotation-name=n_genes&annotation-name=percent_mito" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.headers["Content-Type"], "application/json") - self.assertEqual(result.status_code, HTTPStatus.OK) - result_data = result.json() - self.assertEqual(result_data["names"], ["n_genes", "percent_mito"]) - self.assertEqual(len(result_data["data"][0]), 3) + result = self.session.put(url, json=BAD_FILTER) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) def test_get_annotations_obs_fbs(self): endpoint = "annotations/obs" @@ -146,6 +94,23 @@ class EndPoints(unittest.TestCase): self.assertEqual(len(df['columns']), df['n_cols']) self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain']) + def test_get_annotations_obs_keys_fbs(self): + endpoint = "annotations/obs" + query = "annotation-name=n_genes&annotation-name=percent_mito" + url = f"{URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df['n_rows'], 2638) + self.assertEqual(df['n_cols'], 2) + self.assertIsNotNone(df['columns']) + self.assertIsNotNone(df['col_idx']) + self.assertIsNone(df['row_idx']) + self.assertEqual(len(df['columns']), df['n_cols']) + self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito']) + def test_get_annotations_obs_error(self): endpoint = "annotations/obs" query = "annotation-name=notakey" @@ -153,50 +118,6 @@ class EndPoints(unittest.TestCase): result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) - def test_put_annotations_obs(self): - endpoint = "annotations/obs" - url = f"{URL_BASE}{endpoint}" - obs_filter = { - "filter": { - "obs": { - "annotation_value": [ - {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, - {"name": "n_counts", "min": 3000}, - ], - "index": [1, 99, [1000, 2000]], - } - } - } - result = self.session.put(url, json=obs_filter) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"]) - self.assertEqual(len(result_data["data"]), 15) - - def test_filter_put_annotations_obs(self): - endpoint = "annotations/obs" - query = "annotation-name=n_genes&annotation-name=percent_mito" - url = f"{URL_BASE}{endpoint}?{query}" - obs_filter = { - "filter": { - "obs": { - "annotation_value": [ - {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, - {"name": "n_counts", "min": 3000}, - ], - "index": [1, 99, [1000, 2000]], - } - } - } - result = self.session.put(url, json=obs_filter) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["names"], ["n_genes", "percent_mito"]) - self.assertEqual(len(result_data["data"][0]), 3) - self.assertEqual(len(result_data["data"]), 15) - def test_diff_exp(self): endpoint = "diffexp/obs" url = f"{URL_BASE}{endpoint}" @@ -227,28 +148,6 @@ class EndPoints(unittest.TestCase): result_data = result.json() self.assertEqual(len(result_data), 10) - def test_get_annotations_var(self): - endpoint = "annotations/var" - url = f"{URL_BASE}{endpoint}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["names"], ["name", "n_cells"]) - self.assertEqual(len(result_data["data"]), 1838) - self.assertEqual(len(result_data["data"][0]), 3) - - def test_get_annotations_var_keys(self): - endpoint = "annotations/var" - query = "annotation-name=n_cells" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["names"], ["n_cells"]) - self.assertEqual(len(result_data["data"][0]), 2) - def test_get_annotations_var_fbs(self): endpoint = "annotations/var" url = f"{URL_BASE}{endpoint}" @@ -265,6 +164,23 @@ class EndPoints(unittest.TestCase): self.assertEqual(len(df['columns']), df['n_cols']) self.assertListEqual(df['col_idx'], ['name', 'n_cells']) + def test_get_annotations_var_keys_fbs(self): + endpoint = "annotations/var" + query = "annotation-name=n_cells" + url = f"{URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df['n_rows'], 1838) + self.assertEqual(df['n_cols'], 1) + self.assertIsNotNone(df['columns']) + self.assertIsNotNone(df['col_idx']) + self.assertIsNone(df['row_idx']) + self.assertEqual(len(df['columns']), df['n_cols']) + self.assertListEqual(df['col_idx'], ['n_cells']) + def test_get_annotations_var_error(self): endpoint = "annotations/var" query = "annotation-name=notakey" @@ -272,95 +188,36 @@ class EndPoints(unittest.TestCase): result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) - def test_put_annotations_var(self): - endpoint = "annotations/var" - url = f"{URL_BASE}{endpoint}" - var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}} - result = self.session.put(url, json=var_filter) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["names"], ["name", "n_cells"]) - self.assertEqual(len(result_data["data"]), 2) - - def test_filter_put_annotations_var(self): - endpoint = "annotations/var" - query = "annotation-name=n_cells" - url = f"{URL_BASE}{endpoint}?{query}" - var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}} - result = self.session.put(url, json=var_filter) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(result_data["names"], ["n_cells"]) - self.assertEqual(len(result_data["data"][0]), 2) - self.assertEqual(len(result_data["data"]), 2) - - def test_get_data(self): - for axis in ["obs", "var"]: - endpoint = f"data/{axis}" - query = "accept-type=application/json" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(len(result_data["obs"]), 2638) - def test_data_mimetype_error(self): - for axis in ["obs", "var"]: - endpoint = f"data/{axis}" - query = "accept-type=xxx" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE) - url = f"{URL_BASE}{endpoint}" - header = {"Accept": "sdkljfa;dsjalkj"} - result = self.session.get(url, headers=header) - self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE) + endpoint = f"data/var" + header = {"Accept": "xxx"} + url = f"{URL_BASE}{endpoint}" + result = self.session.put(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE) - def test_json_default(self): - for axis in ["obs", "var"]: - endpoint = f"data/{axis}" - url = f"{URL_BASE}{endpoint}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - - def test_data_filter(self): - for axis in ["obs", "var"]: - endpoint = f"data/{axis}" - query = "accept-type=application/json&obs:louvain=NK cells&obs:louvain=CD8 T cells&obs:n_counts=3000,*" - url = f"{URL_BASE}{endpoint}?{query}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(len(result_data["obs"]), 38) - - def test_data_json_put(self): - for axis in ["obs", "var"]: - endpoint = f"data/{axis}" - url = f"{URL_BASE}{endpoint}" - header = {"Accept": "application/json"} - obs_filter = { - "filter": { - "obs": { - "annotation_value": [ - {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, - {"name": "n_counts", "min": 3000}, - ], - "index": [1, 99, [1000, 2000]], - } - } - } - result = self.session.put(url, headers=header, json=obs_filter) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - self.assertEqual(len(result_data["obs"]), 15) + def test_fbs_default(self): + endpoint = f"data/var" + url = f"{URL_BASE}{endpoint}" + result = self.session.put(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") def test_data_put_fbs(self): + endpoint = f"data/var" + url = f"{URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.put(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df['n_rows'], 2638) + self.assertEqual(df['n_cols'], 1838) + self.assertIsNotNone(df['columns']) + self.assertListEqual(df['col_idx'].tolist(), []) + self.assertIsNone(df['row_idx']) + self.assertEqual(len(df['columns']), df['n_cols']) + + def test_data_put_filter_fbs(self): endpoint = f"data/var" url = f"{URL_BASE}{endpoint}" header = {"Accept": "application/octet-stream"} @@ -384,94 +241,16 @@ class EndPoints(unittest.TestCase): self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4]) def test_data_put_single_var(self): - for axis in ["obs", "var"]: - endpoint = f"data/{axis}" - url = f"{URL_BASE}{endpoint}" - header = {"Accept": "application/json"} - var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}} - result = self.session.put(url, headers=header, json=var_filter) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data = result.json() - if axis == "obs": - self.assertEqual(len(result_data["obs"][0]), 2) - self.assertEqual(len(result_data["var"]), 1) - elif axis == "var": - self.assertEqual(len(result_data["obs"]), 2638) - self.assertEqual(len(result_data["var"][0]), 2639) - - def test_cache(self): - endpoint = "annotations/var" + endpoint = f"data/var" url = f"{URL_BASE}{endpoint}" - f1 = { - "filter": { - "var": { - "annotation_value": [ - { - "name": "name", - "values": [ - "HLA-DRB1", - "HLA-DQA1", - "HLA-DQB1", - "HLA-DPA1", - "HLA-DPB1", - "MS4A1", - "IL32", - "CCL5", - "CD79B", - "CD79A", - ], - } - ] - } - } - } - result = self.session.put(url, json=f1) + header = {"Accept": "application/octet-stream"} + var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}} + result = self.session.put(url, headers=header, json=var_filter) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data1 = result.json() - f2 = { - "filter": { - "var": { - "annotation_value": [ - { - "name": "name", - "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"], - } - ] - } - } - } - result = self.session.put(url, json=f2) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data2 = result.json() - self.assertNotEqual(result_data1, result_data2) - - def test_cache_nofilter(self): - endpoint = "annotations/var" - url = f"{URL_BASE}{endpoint}" - f1 = {"filter": {}} - result = self.session.put(url, json=f1) - self.assertEqual(result.status_code, HTTPStatus.OK) - result_data1 = result.json() - f2 = { - "filter": { - "var": { - "annotation_value": [ - { - "name": "name", - "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"], - } - ] - } - } - } - result = self.session.put(url, json=f2) - self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/json") - result_data2 = result.json() - self.assertNotEqual(result_data1, result_data2) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) def test_static(self): endpoint = "static" diff --git a/server/test/test_filter.py b/server/test/test_filter.py deleted file mode 100644 index d5c01450..00000000 --- a/server/test/test_filter.py +++ /dev/null @@ -1,82 +0,0 @@ -import json -from os import path -import unittest - -from numpy import float32, int32 -from werkzeug.datastructures import ImmutableMultiDict - -from server.app.util.filter import _convert_variable, parse_filter, QueryStringError - - -class UtilTest(unittest.TestCase): - """Test Case for endpoints""" - - def setUp(self): - with open(path.join(path.dirname(__file__), "schema.json")) as fh: - schema = json.load(fh) - self.schema = schema["annotations"] - - def test_convert(self): - five = _convert_variable("int32", "5") - self.assertEqual(five, int32(5)) - - def test_convert_zero(self): - zero = _convert_variable("int32", "0") - self.assertEqual(zero, 0) - - def test_convert_float(self): - str_to_convert = "4.38719237129" - val = _convert_variable("float32", str_to_convert) - self.assertAlmostEqual(val, float32(str_to_convert)) - - def test_convert_bool(self): - str_to_convert = "false" - val = _convert_variable("boolean", str_to_convert) - self.assertFalse(val) - str_to_convert = "true" - val = _convert_variable("boolean", str_to_convert) - self.assertTrue(val) - str_to_convert = "0" - with self.assertRaises(AssertionError): - val = _convert_variable("boolean", str_to_convert) - - def test_empty_convert(self): - empty = _convert_variable("int32", None) - self.assertIsNone(empty) - - def test_bad_convert(self): - with self.assertRaises(ValueError): - _convert_variable("int32", "5.5") - - def test_bad_datatype(self): - with self.assertRaises(AssertionError): - _convert_variable("jkasdslkja", 1) - - def test_complex_filter(self): - filter_dict = ImmutableMultiDict( - [("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")] - ) - filter_ = parse_filter(filter_dict, self.schema) - self.assertIn("obs", filter_) - self.assertEqual( - filter_["obs"]["annotation_value"], - [ - {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, - {"name": "n_counts", "max": None, "min": 3000.0}, - ], - ) - - def test_bad_filter(self): - bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")]) - with self.assertRaises(QueryStringError): - parse_filter(bad_annotation_type, self.schema) - bad_axis = ImmutableMultiDict([("xyz:n_genes", "100,1000")]) - with self.assertRaises(QueryStringError): - parse_filter(bad_axis, self.schema) - - def test_boolean_filter(self): - schema = {"obs": [{"name": "bool_filter", "type": "boolean"}]} - filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")]) - filter_ = parse_filter(filter_dict, schema) - self.assertIn("obs", filter_) - self.assertEqual(filter_["obs"]["annotation_value"], [{"name": "bool_filter", "values": [False]}]) diff --git a/server/test/test_nan_rest.py b/server/test/test_nan_rest.py index fe627f2f..ccceab03 100644 --- a/server/test/test_nan_rest.py +++ b/server/test/test_nan_rest.py @@ -2,6 +2,9 @@ from http import HTTPStatus from subprocess import Popen import unittest import time +import math + +import decode_fbs import requests @@ -43,9 +46,29 @@ class WithNaNs(unittest.TestCase): result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) - def test_errors(self): - endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"] - for endpoint in endpoints: - url = f"{URL_BASE}{endpoint}" - result = self.session.get(url) - self.assertEqual(result.status_code, HTTPStatus.INTERNAL_SERVER_ERROR) + def test_data(self): + endpoint = "data/var" + url = f"{URL_BASE}{endpoint}" + result = self.session.put(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][3][3])) + + def test_annotation_obs(self): + endpoint = "annotations/obs" + url = f"{URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][2][0])) + + def test_annotation_var(self): + endpoint = "annotations/var" + url = f"{URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][2][0])) diff --git a/server/test/test_nan_scanpy_engine.py b/server/test/test_nan_scanpy_engine.py index 45927022..09f45c5a 100644 --- a/server/test/test_nan_scanpy_engine.py +++ b/server/test/test_nan_scanpy_engine.py @@ -1,4 +1,3 @@ -import json import pytest import unittest import warnings @@ -7,7 +6,7 @@ import math import decode_fbs from server.app.scanpy_engine.scanpy_engine import ScanpyEngine -from server.app.util.errors import JSONEncodingValueError +from server.app.util.errors import FilterError class NaNTest(unittest.TestCase): @@ -42,10 +41,15 @@ class NaNTest(unittest.TestCase): self.assertEqual(data_frame_var["n_cols"], 100) self.assertTrue(math.isnan(data_frame_var["columns"][3][3])) - with pytest.raises(JSONEncodingValueError): - json.loads(self.data.data_frame(None, "obs")) - with pytest.raises(JSONEncodingValueError): - json.loads(self.data.data_frame(None, "var")) + with pytest.raises(FilterError): + self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") + with pytest.raises(FilterError): + filter_ = { + "filter": { + "obs": {"index": [1, 99, [200, 300]]} + } + } + self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_dataframe_obs_not_implemented(self): with self.assertRaises(ValueError) as cm: @@ -65,8 +69,3 @@ class NaNTest(unittest.TestCase): self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) - - with pytest.raises(JSONEncodingValueError): - json.loads(self.data.annotation(None, "obs")) - with pytest.raises(JSONEncodingValueError): - json.loads(self.data.annotation(None, "var")) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 133c91a4..561f1fe0 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -3,11 +3,13 @@ from os import path import pytest import time import unittest +import decode_fbs import numpy as np from pandas import Series from server.app.scanpy_engine.scanpy_engine import ScanpyEngine +from server.app.util.errors import FilterError class UtilTest(unittest.TestCase): @@ -45,55 +47,29 @@ class UtilTest(unittest.TestCase): def test_filter_idx(self): filter_ = { "filter": { - "var": {"index": [1, 99, [200, 300]]}, - "obs": {"index": [1, 99, [1000, 2000]]}, + "var": {"index": [1, 99, [200, 300]]} } } - data = self.data.filter_dataframe(filter_["filter"]) - self.assertEqual(data.shape, (1002, 102)) - - def test_filter_annotation(self): - filter_ = { - "filter": { - "obs": { - "annotation_value": [ - {"name": "louvain", "values": ["NK cells", "CD8 T cells"]} - ] - } - } - } - data = self.data.filter_dataframe(filter_["filter"]) - self.assertEqual(data.shape, (470, 1838)) - filter_ = { - "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} - } - data = self.data.filter_dataframe(filter_["filter"]) - self.assertEqual(data.shape, (497, 1838)) - - def test_filter_annotation_no_uns(self): - filter_ = { - "filter": { - "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]} - } - } - data = self.data.filter_dataframe(filter_["filter"]) - self.assertEqual(data.shape[1], 1) + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 102) def test_filter_complex(self): filter_ = { "filter": { - "var": {"index": [1, 99, [200, 300]]}, - "obs": { + "var": { "annotation_value": [ - {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}, - {"name": "n_counts", "min": 3000}, + {"name": "n_cells", "min": 10} ], - "index": [1, 99, [1000, 2000]], - }, + "index": [1, 99, [200, 300]] + } } } - data = self.data.filter_dataframe(filter_["filter"]) - self.assertEqual(data.shape, (15, 102)) + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 91) def test_obs_and_var_names(self): self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0) @@ -119,60 +95,42 @@ class UtilTest(unittest.TestCase): ) def test_layout(self): - layout = json.loads(self.data.layout(None)) - self.assertEqual(layout["layout"]["ndims"], 2) - self.assertEqual(len(layout["layout"]["coordinates"]), 2638) - self.assertEqual(layout["layout"]["coordinates"][0][0], 0) - for idx, val in enumerate(layout["layout"]["coordinates"]): - self.assertLessEqual(val[1], 1) - self.assertLessEqual(val[2], 1) + fbs = self.data.layout_to_fbs_matrix() + layout = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(layout["n_cols"], 2) + self.assertEqual(layout["n_rows"], 2638) + + X = layout["columns"][0] + self.assertTrue((X >= 0).all() and (X <= 1).all()) + Y = layout["columns"][1] + self.assertTrue((Y >= 0).all() and (Y <= 1).all()) def test_annotations(self): - annotations = json.loads(self.data.annotation(None, "obs")) + fbs = self.data.annotation_to_fbs_matrix("obs") + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 2638) + self.assertEqual(annotations["n_cols"], 5) self.assertEqual( - annotations["names"], + annotations["col_idx"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"], ) - self.assertEqual(len(annotations["data"]), 2638) - annotations = json.loads(self.data.annotation(None, "var")) - self.assertEqual(annotations["names"], ["name", "n_cells"]) - self.assertEqual(len(annotations["data"]), 1838) + + fbs = self.data.annotation_to_fbs_matrix("var") + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations['n_rows'], 1838) + self.assertEqual(annotations['n_cols'], 2) + self.assertEqual(annotations["col_idx"], ["name", "n_cells"]) def test_annotation_fields(self): - annotations = json.loads( - self.data.annotation(None, "obs", ["n_genes", "n_counts"]) - ) - self.assertEqual(annotations["names"], ["n_genes", "n_counts"]) - self.assertEqual(len(annotations["data"]), 2638) - annotations = json.loads(self.data.annotation(None, "var", ["name"])) - self.assertEqual(annotations["names"], ["name"]) - self.assertEqual(len(annotations["data"]), 1838) + fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 2638) + self.assertEqual(annotations['n_cols'], 2) - def test_filtered_annotation(self): - filter_ = { - "filter": { - "obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}, - "var": { - "annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}] - }, - } - } - annotations = json.loads(self.data.annotation(filter_["filter"], "obs")) - self.assertEqual( - annotations["names"], - ["name", "n_genes", "percent_mito", "n_counts", "louvain"], - ) - self.assertEqual(len(annotations["data"]), 497) - annotations = json.loads(self.data.annotation(filter_["filter"], "var")) - self.assertEqual(annotations["names"], ["name", "n_cells"]) - self.assertEqual(len(annotations["data"]), 2) - - def test_filtered_layout(self): - filter_ = { - "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} - } - layout = json.loads(self.data.layout(filter_["filter"])) - self.assertEqual(len(layout["layout"]["coordinates"]), 497) + fbs = self.data.annotation_to_fbs_matrix("var", ["name"]) + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations['n_rows'], 1838) + self.assertEqual(annotations['n_cols'], 1) def test_diffexp_topN(self): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} @@ -183,42 +141,51 @@ class UtilTest(unittest.TestCase): self.assertEqual(len(result), 20) def test_data_frame(self): - data_frame_obs = json.loads(self.data.data_frame(None, "obs")) - self.assertEqual(len(data_frame_obs["var"]), 1838) - self.assertEqual(len(data_frame_obs["obs"]), 2638) - data_frame_var = json.loads(self.data.data_frame(None, "var")) - self.assertEqual(len(data_frame_var["var"]), 1838) - self.assertEqual(len(data_frame_var["obs"]), 2638) + fbs = self.data.data_frame_to_fbs_matrix(None, "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 1838) + + with self.assertRaises(ValueError): + self.data.data_frame_to_fbs_matrix(None, "obs") def test_filtered_data_frame(self): + filter_ = { + "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}} + } + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 1040) + filter_ = { "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} } - data_frame_obs = json.loads(self.data.data_frame(filter_["filter"], "obs")) - self.assertEqual(len(data_frame_obs["var"]), 1838) - self.assertEqual(len(data_frame_obs["obs"]), 497) - self.assertIsInstance(data_frame_obs["obs"][0], (list, tuple)) - self.assertEqual(type(data_frame_obs["var"][0]), int) - data_frame_var = json.loads(self.data.data_frame(filter_["filter"], "var")) - self.assertEqual(len(data_frame_var["var"]), 1838) - self.assertEqual(len(data_frame_var["obs"]), 497) - self.assertIsInstance(data_frame_var["var"][0], (list, tuple)) - self.assertEqual(type(data_frame_var["obs"][0]), int) + with self.assertRaises(FilterError): + self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") - def test_data_single_gene(self): - for axis in ["obs", "var"]: - filter_ = { - "filter": { - "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]} - } + def test_data_named_gene(self): + filter_ = { + "filter": { + "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]} } - data_frame_var = json.loads(self.data.data_frame(filter_["filter"], axis)) - if axis == "obs": - self.assertEqual(type(data_frame_var["var"][0]), int) - self.assertIsInstance(data_frame_var["obs"][0], (list, tuple)) - elif axis == "var": - self.assertEqual(type(data_frame_var["obs"][0]), int) - self.assertIsInstance(data_frame_var["var"][0], (list, tuple)) + } + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 1) + self.assertEqual(data["col_idx"], [4]) + + filter_ = { + "filter": { + "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]} + } + } + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 3) + self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all()) if __name__ == "__main__": unittest.main()