diff --git a/client/__tests__/util/stateManager/sampleResponses.js b/client/__tests__/util/stateManager/sampleResponses.js index 8a3e6f9a..2b66bac0 100644 --- a/client/__tests__/util/stateManager/sampleResponses.js +++ b/client/__tests__/util/stateManager/sampleResponses.js @@ -34,28 +34,34 @@ const aSchemaResponse = { type: "float32" }, annotations: { - obs: [ - { name: "name", type: "string" }, - { name: "field1", type: "int32" }, - { name: "field2", type: "float32" }, - { name: "field3", type: "boolean" }, - { - name: "field4", - type: "categorical", - categories: field4Categories - } - ], - var: [ - { name: "name", type: "string" }, - { name: "fieldA", type: "int32" }, - { name: "fieldB", type: "float32" }, - { name: "fieldC", type: "boolean" }, - { - name: "fieldD", - type: "categorical", - categories: fieldDCategories - } - ] + obs: { + index: "name", + columns: [ + { name: "name", type: "string" }, + { name: "field1", type: "int32" }, + { name: "field2", type: "float32" }, + { name: "field3", type: "boolean" }, + { + name: "field4", + type: "categorical", + categories: field4Categories + } + ] + }, + var: { + index: "name", + columns: [ + { name: "name", type: "string" }, + { name: "fieldA", type: "int32" }, + { name: "fieldB", type: "float32" }, + { name: "fieldC", type: "boolean" }, + { + name: "fieldD", + type: "categorical", + categories: fieldDCategories + } + ] + } }, layout: { obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }], diff --git a/client/__tests__/util/stateManager/universe.test.js b/client/__tests__/util/stateManager/universe.test.js index 6e4dc306..3bac0b69 100644 --- a/client/__tests__/util/stateManager/universe.test.js +++ b/client/__tests__/util/stateManager/universe.test.js @@ -53,7 +53,7 @@ describe("createUniverseFromResponse", () => { expect(universe.obsAnnotations.dims).toEqual([ nObs, - REST.schema.schema.annotations.obs.length + REST.schema.schema.annotations.obs.columns.length ]); expect(universe.obsLayout.dims).toEqual([nObs, 2]); expect(universe.obsLayout.colIndex.keys()).toEqual( @@ -61,7 +61,7 @@ describe("createUniverseFromResponse", () => { ); expect(universe.varAnnotations.dims).toEqual([ nVar, - REST.schema.schema.annotations.var.length + REST.schema.schema.annotations.var.columns.length ]); expect(universe.varData.isEmpty()).toBeTruthy(); }); diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index c3cae4a3..6ba3424b 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -155,14 +155,18 @@ describe("createObsDimensionMap", () => { const { crossfilter } = defaultBigBang(); const annotationNames = _.map( - REST.schema.schema.annotations.obs, + REST.schema.schema.annotations.obs.columns, c => c.name ); - const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name"); + const obsIndexColName = REST.schema.schema.annotations.obs.index; + const schemaByObsName = _.keyBy( + REST.schema.schema.annotations.obs.columns, + "name" + ); expect(crossfilter).toBeDefined(); annotationNames.forEach(name => { const dim = crossfilter.dimensions[obsAnnoDimensionName(name)]; - if (name === "name") { + if (name === obsIndexColName) { expect(dim).toBeUndefined(); } else { const { type } = schemaByObsName[name]; diff --git a/client/src/actions/index.js b/client/src/actions/index.js index e4e0562e..d8c9c5ca 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -21,24 +21,32 @@ const doInitialDataLoad = () => dispatch({ type: "initial data load start" }); try { - const requestJson = _(["config", "schema"]) + /* + Step 1 - config & schema, all JSON + */ + const requestJson = ["config", "schema"] .map(r => `${globals.API.prefix}${globals.API.version}${r}`) - .map(url => doJsonRequest(url)) - .value(); - const requestBinary = _([ - "annotations/obs", - "annotations/var?annotation-name=name", - "layout/obs" - ]) - .map(r => `${globals.API.prefix}${globals.API.version}${r}`) - .map(url => doBinaryRequest(url)) - .value(); - - const results = await Promise.all(_.concat(requestJson, requestBinary)); - + .map(url => doJsonRequest(url)); + const stepOneResults = await Promise.all(requestJson); /* set config defaults */ - const config = { ...globals.configDefaults, ...results[0].config }; - const [, schema, obsAnno, varAnno, obsLayout] = [...results]; + const config = { ...globals.configDefaults, ...stepOneResults[0].config }; + const schema = stepOneResults[1]; + + /* + Step 2 - dataframes, all binary. NOTE: uses results of step 1. + */ + /* only load names for var annotations, if possible*/ + const varIndexName = schema?.schema?.annotations?.var?.index; + const varAnnotationsQuery = varIndexName + ? `?annotation-name=${varIndexName}` + : ""; + const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`; + const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"] + .map(r => `${globals.API.prefix}${globals.API.version}${r}`) + .map(url => doBinaryRequest(url)); + const stepTwoResults = await Promise.all(requestBinary); + const [obsAnno, varAnno, obsLayout] = [...stepTwoResults]; + const universe = Universe.createUniverseFromResponse( config, schema, @@ -91,6 +99,10 @@ needs expression data. Transparently utilizes cached data if it is already present. */ async function _doRequestExpressionData(dispatch, getState, genes) { + const state = getState(); + const { universe } = state; + const varIndexName = universe.schema.annotations.var.index; + /* helper for this function only */ const fetchData = async geneNames => { const res = await fetch( @@ -100,7 +112,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) { body: JSON.stringify({ filter: { var: { - annotation_value: [{ name: "name", values: geneNames }] + annotation_value: [{ name: varIndexName, values: geneNames }] } } }), @@ -123,8 +135,6 @@ async function _doRequestExpressionData(dispatch, getState, genes) { return Universe.convertDataFBStoObject(universe, data); }; - const state = getState(); - const { universe } = state; /* preload data already in cache */ let expressionData = _.transform( genes, @@ -241,6 +251,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async ( */ const state = getState(); const { universe } = state; + const varIndexName = universe.schema.annotations.var.index; // Legal values are null, Array or TypedArray. Null is initial state. if (!set1) set1 = []; @@ -277,7 +288,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async ( const data = await res.json(); // result is [ [varIdx, ...], ... ] const topNGenes = _.map(data, r => - universe.varAnnotations.at(r[0], "name") + universe.varAnnotations.at(r[0], varIndexName) ); /* diff --git a/client/src/components/continuous/continuous.js b/client/src/components/continuous/continuous.js index 6c56376b..959db97f 100644 --- a/client/src/components/continuous/continuous.js +++ b/client/src/components/continuous/continuous.js @@ -66,7 +66,8 @@ class Continuous extends React.Component { ? _.map(obsAnnotations.colIndex.keys(), key => { const isColorField = key.includes("color") || key.includes("Color"); - if (key === "name" || isColorField) return null; + if (key === schema.annotations.obs.index || isColorField) + return null; const summary = obsAnnotations.col(key).summarize(); const nonFiniteExtent = diff --git a/client/src/components/geneExpression/index.js b/client/src/components/geneExpression/index.js index 023d848d..799b7074 100644 --- a/client/src/components/geneExpression/index.js +++ b/client/src/components/geneExpression/index.js @@ -85,7 +85,8 @@ class GeneExpression extends React.Component { */ const { world } = this.props; const { varAnnotations } = world; - const geneNames = varAnnotations.col("name").asArray(); + const varIndexName = world.schema.annotations.var.index; + const geneNames = varAnnotations.col(varIndexName).asArray(); if (geneNames.length > 0) { const placeholder = []; let len = geneNames.length; @@ -107,6 +108,7 @@ class GeneExpression extends React.Component { handleClick(g) { const { world, dispatch, userDefinedGenes } = this.props; + const varIndexName = world.schema.annotations.var.index; const gene = g.target; if (userDefinedGenes.indexOf(gene) !== -1) { postUserErrorToast("That gene already exists"); @@ -114,7 +116,9 @@ class GeneExpression extends React.Component { postUserErrorToast( "That's too many genes, you can have at most 15 user defined genes" ); - } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) { + } else if ( + world.varAnnotations.col(varIndexName).indexOf(gene) === undefined + ) { postUserErrorToast("That doesn't appear to be a valid gene name."); } else { dispatch({ type: "single user defined gene start" }); @@ -127,6 +131,7 @@ class GeneExpression extends React.Component { handleBulkAddClick() { const { world, dispatch, userDefinedGenes } = this.props; + const varIndexName = world.schema.annotations.var.index; const { bulkAdd } = this.state; /* @@ -145,7 +150,9 @@ class GeneExpression extends React.Component { if (userDefinedGenes.indexOf(gene) !== -1) { return keepAroundErrorToast("That gene already exists"); } - if (world.varAnnotations.col("name").indexOf(gene) === undefined) { + if ( + world.varAnnotations.col(varIndexName).indexOf(gene) === undefined + ) { return keepAroundErrorToast( `${gene} doesn't appear to be a valid gene name.` ); @@ -168,7 +175,7 @@ class GeneExpression extends React.Component { userDefinedGenesLoading, differential } = this.props; - + const varIndexName = world?.schema?.annotations?.var?.index; const { tab, bulkAdd } = this.state; return ( @@ -243,7 +250,7 @@ class GeneExpression extends React.Component { itemRenderer={renderGene.bind(this)} items={ world && world.varAnnotations - ? world.varAnnotations.col("name").asArray() + ? world.varAnnotations.col(varIndexName).asArray() : ["No genes"] } popoverProps={{ minimal: true }} @@ -322,7 +329,7 @@ class GeneExpression extends React.Component { {differential.diffExp ? _.map(differential.diffExp, (value, index) => { - const name = world.varAnnotations.at(value[0], "name"); + const name = world.varAnnotations.at(value[0], varIndexName); const values = world.varData.col(name); if (!values) { return null; diff --git a/client/src/reducers/controls.js b/client/src/reducers/controls.js index 89096749..35d3cfb6 100644 --- a/client/src/reducers/controls.js +++ b/client/src/reducers/controls.js @@ -93,9 +93,10 @@ const Controls = ( } case "request differential expression success": { const { world } = prevSharedState; + const varIndexName = world.schema.annotations.var.index; const _diffexpGenes = []; action.data.forEach(d => { - _diffexpGenes.push(world.varAnnotations.at(d[0], "name")); + _diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName)); }); return { ...state, diff --git a/client/src/reducers/crossfilter.js b/client/src/reducers/crossfilter.js index 4c8ea570..06f8e30e 100644 --- a/client/src/reducers/crossfilter.js +++ b/client/src/reducers/crossfilter.js @@ -90,8 +90,9 @@ const CrossfilterReducer = ( case "request differential expression success": { const { world } = prevSharedState; + const varIndexName = world.schema.annotations.var.index; const genes = _.map(action.data, d => - world.varAnnotations.at(d[0], "name") + world.varAnnotations.at(d[0], varIndexName) ); const crossfilter = _.reduce( genes, @@ -109,10 +110,11 @@ const CrossfilterReducer = ( case "clear differential expression": { const { world } = prevSharedState; + const varIndexName = world.schema.annotations.var.index; const crossfilter = _.reduce( action.diffExp, (xfltr, values) => { - const name = world.varAnnotations.at(values[0], "name"); + const name = world.varAnnotations.at(values[0], varIndexName); return xfltr.delDimension(diffexpDimensionName(name)); }, state diff --git a/client/src/util/stateManager/controlsHelpers.js b/client/src/util/stateManager/controlsHelpers.js index 27ecef46..3f2e08ea 100644 --- a/client/src/util/stateManager/controlsHelpers.js +++ b/client/src/util/stateManager/controlsHelpers.js @@ -56,13 +56,14 @@ function topNCategories(summary) { export function createCategoricalSelection(maxCategoryItems, world) { const res = {}; + const obsIndexName = world.schema.annotations.obs.index; _.forEach(world.obsAnnotations.colIndex.keys(), key => { const summary = world.obsAnnotations.col(key).summarize(); if (summary.categories) { const isColorField = key.includes("color") || key.includes("Color"); const isSelectableCategory = !isColorField && - key !== "name" && + key !== obsIndexName && summary.categories.length < maxCategoryItems; if (isSelectableCategory) { const [categoryValues, categoryValueCounts] = topNCategories(summary); diff --git a/client/src/util/stateManager/universe.js b/client/src/util/stateManager/universe.js index bf64d10c..35c586cf 100644 --- a/client/src/util/stateManager/universe.js +++ b/client/src/util/stateManager/universe.js @@ -124,7 +124,7 @@ function reconcileSchemaCategoriesWithSummary(universe) { cases, add a 'categories' field to the schema so it is accessible. */ - universe.schema.annotations.obs.forEach(s => { + universe.schema.annotations.obs.columns.forEach(s => { if ( s.type === "string" || s.type === "boolean" || @@ -179,10 +179,10 @@ export function createUniverseFromResponse( /* Index schema for ease of use */ universe.schema.annotations.obsByName = fromEntries( - universe.schema.annotations.obs.map(v => [v.name, v]) + universe.schema.annotations.obs.columns.map(v => [v.name, v]) ); universe.schema.annotations.varByName = fromEntries( - universe.schema.annotations.var.map(v => [v.name, v]) + universe.schema.annotations.var.columns.map(v => [v.name, v]) ); universe.schema.layout.obsByName = fromEntries( universe.schema.layout.obs.map(v => [v.name, v]) @@ -213,8 +213,9 @@ export function convertDataFBStoObject(universe, arrayBuffer) { throw new Error("Unexpected non-floating point response from server."); } + const varIndexName = universe.schema.annotations.var.index; for (let c = 0; c < colIdx.length; c += 1) { - const varName = universe.varAnnotations.at(colIdx[c], "name"); + const varName = universe.varAnnotations.at(colIdx[c], varIndexName); result[varName] = columns[c]; } return result; diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js index 6852e745..1cb03f3c 100644 --- a/client/src/util/stateManager/world.js +++ b/client/src/util/stateManager/world.js @@ -263,10 +263,14 @@ function deduceDimensionType(attributes, fieldName) { export function createObsDimensions(crossfilter, world, XYdimNames) { /* create and return a crossfilter with a dimension for every obs annotation - for which we have a supported type, *except* 'name' + for which we have a supported type, *except* for the index column, indicated + by schema.annotations.obs.index. */ const { schema, obsLayout, obsAnnotations } = world; - const annoList = schema.annotations.obs.filter(anno => anno.name !== "name"); + const indexName = schema.annotations.obs.index; + const annoList = schema.annotations.obs.columns.filter( + anno => anno.name !== indexName + ); crossfilter = annoList.reduce((xfltr, anno) => { const dimType = deduceDimensionType(anno, anno.name); const colData = obsAnnotations.col(anno.name).asArray(); diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 8bb88b6b..f1439264 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -50,41 +50,61 @@ class ScanpyEngine(CXGDriver): "diffexp_lfc_cutoff": 0.01, } - def _alias_annotation_names(self, axis, name): - """ - Do all user-specified annotation aliasing. + @staticmethod + def _create_unique_column_name(df, col_name_prefix): + """ given the columns of a dataframe, and a name prefix, return a column name which + does not exist in the dataframe, AND which is prefixed by `prefix` - As a *critical* side-effect, ensure the indices are simple number ranges - (accomplished by calling pandas.DataFrame.reset_index()) + The approach is to append a numeric suffix, starting at zero and increasing by + one, until an unused name is found (eg, prefix_0, prefix_1, ...). """ - if name == "name": - # a noop, so skip it - return + suffix = 0 + while f"{col_name_prefix}{suffix}" in df: + suffix += 1 + return f"{col_name_prefix}{suffix}" - ax_name = str(axis) - df_axis = getattr(self.data, ax_name) - if name is None: - # reset index to simple range; alias "name" to point at the - # previously specified index. - df_axis.reset_index(inplace=True) - df_axis.rename(inplace=True, columns={"index": "name"}) - elif name in df_axis.columns: - if name not in df_axis.columns: + def _alias_annotation_names(self): + """ + The front-end relies on the existance of a unique, human-readable + index for obs & var (eg, var is typically gene name, obs the cell name). + The user can specify these via the --obs-names and --var-names config. + If they are not specified, use the existing index to create them, giving + the resulting column a unique name (eg, "name"). + + In both cases, enforce that the result is unique, and communicate the + index column name to the front-end via the obs_names and var_names config + (which is incorporated into the schema). + """ + for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")): + name = self.config[config_name] + df_axis = getattr(self.data, str(ax_name)) + if name is None: + # Default: create unique names from index + if not df_axis.index.is_unique: + raise KeyError( + f"Values in {ax_name}.index must be unique. " + "Please prepare data to contain unique index values, or specify an " + "alternative with --{ax_name}-name." + ) + name = self._create_unique_column_name(df_axis.columns, "name_") + self.config[config_name] = name + # reset index to simple range; alias name to point at the + # previously specified index. + df_axis.rename_axis(name, inplace=True) + df_axis.reset_index(inplace=True) + elif name in df_axis.columns: + # User has specified alternative column for unique names, and it exists + if not df_axis[name].is_unique: + raise KeyError( + f"Values in {ax_name}.{name} must be unique. " + "Please prepare data to contain unique values." + ) + df_axis.reset_index(drop=True, inplace=True) + else: + # user specified a non-existent column name raise KeyError( f"Annotation name {name}, specified in --{ax_name}-name does not exist." ) - if not df_axis[name].is_unique: - raise KeyError( - f"Values in -{ax_name}-name must be unique. " - "Please prepare data to contain unique values." - ) - # reset index to simple range; alias user-specified annotation to "name" - df_axis.reset_index(drop=True, inplace=True) - df_axis.rename(inplace=True, columns={name: "name"}) - else: - raise KeyError( - f"Annotation name {name}, specified in --{ax_name}_name does not exist." - ) @staticmethod def _can_cast_to_float32(ann): @@ -114,7 +134,16 @@ class ScanpyEngine(CXGDriver): "nVar": self.gene_count, "type": str(self.data.X.dtype), }, - "annotations": {"obs": [], "var": []}, + "annotations": { + "obs": { + "index": self.config["obs_names"], + "columns": [] + }, + "var": { + "index": self.config["var_names"], + "columns": [] + } + }, "layout": {"obs": []} } for ax in Axis: @@ -139,7 +168,7 @@ class ScanpyEngine(CXGDriver): raise TypeError( f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene." ) - self.schema["annotations"][ax].append(ann_schema) + self.schema["annotations"][ax]["columns"].append(ann_schema) for layout in self.config['layout']: layout_schema = { @@ -173,8 +202,11 @@ class ScanpyEngine(CXGDriver): @requires_data def _validate_and_initialize(self): - self._alias_annotation_names(Axis.OBS, self.config["obs_names"]) - self._alias_annotation_names(Axis.VAR, self.config["var_names"]) + # var and obs column names must be unique + if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique: + raise KeyError(f"All annotation column names must be unique.") + + self._alias_annotation_names() self._validate_data_types() self.cell_count = self.data.shape[0] self.gene_count = self.data.shape[1] diff --git a/server/test/schema.json b/server/test/schema.json index e6afb72c..828920e3 100644 --- a/server/test/schema.json +++ b/server/test/schema.json @@ -5,48 +5,54 @@ "type": "float32" }, "annotations": { - "obs": [ - { - "name": "name", - "type": "string" - }, - { - "name": "n_genes", - "type": "int32" - }, - { - "name": "percent_mito", - "type": "float32" - }, - { - "name": "n_counts", - "type": "float32" - }, - { - "name": "louvain", - "type": "categorical", - "categories": [ - "CD4 T cells", - "CD14+ Monocytes", - "B cells", - "CD8 T cells", - "NK cells", - "FCGR3A+ Monocytes", - "Dendritic cells", - "Megakaryocytes" - ] - } - ], - "var": [ - { - "name": "name", - "type": "string" - }, - { - "name": "n_cells", - "type": "int32" - } - ] + "obs": { + "index": "name_0", + "columns": [ + { + "name": "name_0", + "type": "string" + }, + { + "name": "n_genes", + "type": "int32" + }, + { + "name": "percent_mito", + "type": "float32" + }, + { + "name": "n_counts", + "type": "float32" + }, + { + "name": "louvain", + "type": "categorical", + "categories": [ + "CD4 T cells", + "CD14+ Monocytes", + "B cells", + "CD8 T cells", + "NK cells", + "FCGR3A+ Monocytes", + "Dendritic cells", + "Megakaryocytes" + ] + } + ] + }, + "var": { + "index": "name_0", + "columns": [ + { + "name": "name_0", + "type": "string" + }, + { + "name": "n_cells", + "type": "int32" + } + ] + } }, "layout": { "obs": [ diff --git a/server/test/test_api.py b/server/test/test_api.py index dd796481..24791879 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -23,7 +23,8 @@ class EndPoints(unittest.TestCase): session = requests.Session() for i in range(90): try: - session.get(f"{URL_BASE}schema") + result = session.get(f"{URL_BASE}schema") + cls.schema = result.json() except requests.exceptions.ConnectionError: time.sleep(1) @@ -45,7 +46,8 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638) - self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5) + self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2) + self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5) def test_config(self): endpoint = "config" @@ -95,7 +97,8 @@ class EndPoints(unittest.TestCase): self.assertIsNotNone(df['col_idx']) self.assertIsNone(df['row_idx']) self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain']) + obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] + self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain']) def test_get_annotations_obs_keys_fbs(self): endpoint = "annotations/obs" @@ -165,7 +168,8 @@ class EndPoints(unittest.TestCase): self.assertIsNotNone(df['col_idx']) self.assertIsNone(df['row_idx']) self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'], ['name', 'n_cells']) + var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells']) def test_get_annotations_var_keys_fbs(self): endpoint = "annotations/var" @@ -247,7 +251,8 @@ class EndPoints(unittest.TestCase): endpoint = f"data/var" url = f"{URL_BASE}{endpoint}" header = {"Accept": "application/octet-stream"} - var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}} + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}} result = self.session.put(url, headers=header, json=var_filter) self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") diff --git a/server/test/test_nan_scanpy_engine.py b/server/test/test_nan_scanpy_engine.py index 656b1f33..ffe62cf6 100644 --- a/server/test/test_nan_scanpy_engine.py +++ b/server/test/test_nan_scanpy_engine.py @@ -58,14 +58,16 @@ class NaNTest(unittest.TestCase): def test_annotation(self): annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) + obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] self.assertEqual( annotations["col_idx"], - ["name", "n_genes", "percent_mito", "n_counts", "louvain"] + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] ) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var")) - self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"]) + var_index_col_name = self.data.schema["annotations"]["var"]["index"] + self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index e6f453fa..3f917d4c 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -31,9 +31,11 @@ class EngineTest(unittest.TestCase): self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_mandatory_annotations(self): - self.assertIn("name", self.data.data.obs) + obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] + self.assertIn(obs_index_col_name, self.data.data.obs) self.assertEqual(list(self.data.data.obs.index), list(range(2638))) - self.assertIn("name", self.data.data.var) + var_index_col_name = self.data.schema["annotations"]["var"]["index"] + self.assertIn(var_index_col_name, self.data.data.var) self.assertEqual(list(self.data.data.var.index), list(range(1838))) @pytest.mark.filterwarnings("ignore:Scanpy data matrix") @@ -70,12 +72,14 @@ class EngineTest(unittest.TestCase): self.assertEqual(data["n_cols"], 91) def test_obs_and_var_names(self): - self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0) - self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0) + self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0) + self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0) def test_schema(self): with open(path.join(path.dirname(__file__), "schema.json")) as fh: schema = json.load(fh) + print(schema) + print(self.data.schema) self.assertEqual(self.data.schema, schema) def test_schema_produces_error(self): @@ -108,16 +112,18 @@ class EngineTest(unittest.TestCase): annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations["n_cols"], 5) + obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] self.assertEqual( annotations["col_idx"], - ["name", "n_genes", "percent_mito", "n_counts", "louvain"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], ) fbs = self.data.annotation_to_fbs_matrix("var") annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations['n_cols'], 2) - self.assertEqual(annotations["col_idx"], ["name", "n_cells"]) + var_index_col_name = self.data.schema["annotations"]["var"]["index"] + self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) def test_annotation_fields(self): fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) @@ -125,7 +131,8 @@ class EngineTest(unittest.TestCase): self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations['n_cols'], 2) - fbs = self.data.annotation_to_fbs_matrix("var", ["name"]) + var_index_col_name = self.data.schema["annotations"]["var"]["index"] + fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations['n_cols'], 1) @@ -163,9 +170,10 @@ class EngineTest(unittest.TestCase): self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_data_named_gene(self): + var_index_col_name = self.data.schema["annotations"]["var"]["index"] filter_ = { "filter": { - "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]} + "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]} } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") @@ -176,7 +184,7 @@ class EngineTest(unittest.TestCase): filter_ = { "filter": { - "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]} + "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]} } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")