do not hard-wire column names in annotations (#785)

* enforce column name uniqueness for obs and var * parameterize the column name containing obs and var user-readable names * use the new annotation index value from schema * update f/e unit tests * PR review suggestions * lint
2026-09-15 20:57:56 +08:00 · 2019-05-24 21:00:54 -07:00
parent a8c2e408d1
commit 3dc45d6330
16 changed files with 246 additions and 155 deletions
--- a/client/tests/util/stateManager/sampleResponses.js
+++ b/client/tests/util/stateManager/sampleResponses.js
@@ -34,28 +34,34 @@ const aSchemaResponse = {
      type: "float32"
    },
    annotations: {
-      obs: [
-        { name: "name", type: "string" },
-        { name: "field1", type: "int32" },
-        { name: "field2", type: "float32" },
-        { name: "field3", type: "boolean" },
-        {
-          name: "field4",
-          type: "categorical",
-          categories: field4Categories
-        }
-      ],
-      var: [
-        { name: "name", type: "string" },
-        { name: "fieldA", type: "int32" },
-        { name: "fieldB", type: "float32" },
-        { name: "fieldC", type: "boolean" },
-        {
-          name: "fieldD",
-          type: "categorical",
-          categories: fieldDCategories
-        }
-      ]
+      obs: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "field1", type: "int32" },
+          { name: "field2", type: "float32" },
+          { name: "field3", type: "boolean" },
+          {
+            name: "field4",
+            type: "categorical",
+            categories: field4Categories
+          }
+        ]
+      },
+      var: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "fieldA", type: "int32" },
+          { name: "fieldB", type: "float32" },
+          { name: "fieldC", type: "boolean" },
+          {
+            name: "fieldD",
+            type: "categorical",
+            categories: fieldDCategories
+          }
+        ]
+      }
    },
    layout: {
      obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
--- a/client/tests/util/stateManager/universe.test.js
+++ b/client/tests/util/stateManager/universe.test.js
@@ -53,7 +53,7 @@ describe("createUniverseFromResponse", () => {

    expect(universe.obsAnnotations.dims).toEqual([
      nObs,
-      REST.schema.schema.annotations.obs.length
+      REST.schema.schema.annotations.obs.columns.length
    ]);
    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
    expect(universe.obsLayout.colIndex.keys()).toEqual(
@@ -61,7 +61,7 @@ describe("createUniverseFromResponse", () => {
    );
    expect(universe.varAnnotations.dims).toEqual([
      nVar,
-      REST.schema.schema.annotations.var.length
+      REST.schema.schema.annotations.var.columns.length
    ]);
    expect(universe.varData.isEmpty()).toBeTruthy();
  });
--- a/client/tests/util/stateManager/world.test.js
+++ b/client/tests/util/stateManager/world.test.js
@@ -155,14 +155,18 @@ describe("createObsDimensionMap", () => {

    const { crossfilter } = defaultBigBang();
    const annotationNames = _.map(
-      REST.schema.schema.annotations.obs,
+      REST.schema.schema.annotations.obs.columns,
      c => c.name
    );
-    const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
+    const obsIndexColName = REST.schema.schema.annotations.obs.index;
+    const schemaByObsName = _.keyBy(
+      REST.schema.schema.annotations.obs.columns,
+      "name"
+    );
    expect(crossfilter).toBeDefined();
    annotationNames.forEach(name => {
      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === "name") {
+      if (name === obsIndexColName) {
        expect(dim).toBeUndefined();
      } else {
        const { type } = schemaByObsName[name];
--- a/client/src/actions/index.js
+++ b/client/src/actions/index.js
@@ -21,24 +21,32 @@ const doInitialDataLoad = () =>
    dispatch({ type: "initial data load start" });

    try {
-      const requestJson = _(["config", "schema"])
+      /*
+      Step 1 - config & schema, all JSON
+      */
+      const requestJson = ["config", "schema"]
        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url))
-        .value();
-      const requestBinary = _([
-        "annotations/obs",
-        "annotations/var?annotation-name=name",
-        "layout/obs"
-      ])
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url))
-        .value();
-
-      const results = await Promise.all(_.concat(requestJson, requestBinary));
-
+        .map(url => doJsonRequest(url));
+      const stepOneResults = await Promise.all(requestJson);
      /* set config defaults */
-      const config = { ...globals.configDefaults, ...results[0].config };
-      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
+      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
+      const schema = stepOneResults[1];
+
+      /*
+      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
+      */
+      /* only load names for var annotations, if possible*/
+      const varIndexName = schema?.schema?.annotations?.var?.index;
+      const varAnnotationsQuery = varIndexName
+        ? `?annotation-name=${varIndexName}`
+        : "";
+      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
+      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url));
+      const stepTwoResults = await Promise.all(requestBinary);
+      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
+
      const universe = Universe.createUniverseFromResponse(
        config,
        schema,
@@ -91,6 +99,10 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  const state = getState();
+  const { universe } = state;
+  const varIndexName = universe.schema.annotations.var.index;
+
  /* helper for this function only */
  const fetchData = async geneNames => {
    const res = await fetch(
@@ -100,7 +112,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
        body: JSON.stringify({
          filter: {
            var: {
-              annotation_value: [{ name: "name", values: geneNames }]
+              annotation_value: [{ name: varIndexName, values: geneNames }]
            }
          }
        }),
@@ -123,8 +135,6 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
    return Universe.convertDataFBStoObject(universe, data);
  };

-  const state = getState();
-  const { universe } = state;
  /* preload data already in cache */
  let expressionData = _.transform(
    genes,
@@ -241,6 +251,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
    */
    const state = getState();
    const { universe } = state;
+    const varIndexName = universe.schema.annotations.var.index;

    // Legal values are null, Array or TypedArray.  Null is initial state.
    if (!set1) set1 = [];
@@ -277,7 +288,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
    const data = await res.json();
    // result is [ [varIdx, ...], ... ]
    const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], "name")
+      universe.varAnnotations.at(r[0], varIndexName)
    );

    /*
--- a/client/src/components/continuous/continuous.js
+++ b/client/src/components/continuous/continuous.js
@@ -66,7 +66,8 @@ class Continuous extends React.Component {
          ? _.map(obsAnnotations.colIndex.keys(), key => {
              const isColorField =
                key.includes("color") || key.includes("Color");
-              if (key === "name" || isColorField) return null;
+              if (key === schema.annotations.obs.index || isColorField)
+                return null;

              const summary = obsAnnotations.col(key).summarize();
              const nonFiniteExtent =
--- a/client/src/components/geneExpression/index.js
+++ b/client/src/components/geneExpression/index.js
@@ -85,7 +85,8 @@ class GeneExpression extends React.Component {
    */
    const { world } = this.props;
    const { varAnnotations } = world;
-    const geneNames = varAnnotations.col("name").asArray();
+    const varIndexName = world.schema.annotations.var.index;
+    const geneNames = varAnnotations.col(varIndexName).asArray();
    if (geneNames.length > 0) {
      const placeholder = [];
      let len = geneNames.length;
@@ -107,6 +108,7 @@ class GeneExpression extends React.Component {

  handleClick(g) {
    const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
    const gene = g.target;
    if (userDefinedGenes.indexOf(gene) !== -1) {
      postUserErrorToast("That gene already exists");
@@ -114,7 +116,9 @@ class GeneExpression extends React.Component {
      postUserErrorToast(
        "That's too many genes, you can have at most 15 user defined genes"
      );
-    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+    } else if (
+      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+    ) {
      postUserErrorToast("That doesn't appear to be a valid gene name.");
    } else {
      dispatch({ type: "single user defined gene start" });
@@ -127,6 +131,7 @@ class GeneExpression extends React.Component {

  handleBulkAddClick() {
    const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
    const { bulkAdd } = this.state;

    /*
@@ -145,7 +150,9 @@ class GeneExpression extends React.Component {
          if (userDefinedGenes.indexOf(gene) !== -1) {
            return keepAroundErrorToast("That gene already exists");
          }
-          if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+          if (
+            world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+          ) {
            return keepAroundErrorToast(
              `${gene} doesn't appear to be a valid gene name.`
            );
@@ -168,7 +175,7 @@ class GeneExpression extends React.Component {
      userDefinedGenesLoading,
      differential
    } = this.props;
-
+    const varIndexName = world?.schema?.annotations?.var?.index;
    const { tab, bulkAdd } = this.state;

    return (
@@ -243,7 +250,7 @@ class GeneExpression extends React.Component {
                itemRenderer={renderGene.bind(this)}
                items={
                  world && world.varAnnotations
-                    ? world.varAnnotations.col("name").asArray()
+                    ? world.varAnnotations.col(varIndexName).asArray()
                    : ["No genes"]
                }
                popoverProps={{ minimal: true }}
@@ -322,7 +329,7 @@ class GeneExpression extends React.Component {
          <ExpressionButtons />
          {differential.diffExp
            ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], "name");
+                const name = world.varAnnotations.at(value[0], varIndexName);
                const values = world.varData.col(name);
                if (!values) {
                  return null;
--- a/client/src/reducers/controls.js
+++ b/client/src/reducers/controls.js
@@ -93,9 +93,10 @@ const Controls = (
    }
    case "request differential expression success": {
      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
      const _diffexpGenes = [];
      action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
+        _diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
      });
      return {
        ...state,
--- a/client/src/reducers/crossfilter.js
+++ b/client/src/reducers/crossfilter.js
@@ -90,8 +90,9 @@ const CrossfilterReducer = (

    case "request differential expression success": {
      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
      const genes = _.map(action.data, d =>
-        world.varAnnotations.at(d[0], "name")
+        world.varAnnotations.at(d[0], varIndexName)
      );
      const crossfilter = _.reduce(
        genes,
@@ -109,10 +110,11 @@ const CrossfilterReducer = (

    case "clear differential expression": {
      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
      const crossfilter = _.reduce(
        action.diffExp,
        (xfltr, values) => {
-          const name = world.varAnnotations.at(values[0], "name");
+          const name = world.varAnnotations.at(values[0], varIndexName);
          return xfltr.delDimension(diffexpDimensionName(name));
        },
        state
--- a/client/src/util/stateManager/controlsHelpers.js
+++ b/client/src/util/stateManager/controlsHelpers.js
@@ -56,13 +56,14 @@ function topNCategories(summary) {

 export function createCategoricalSelection(maxCategoryItems, world) {
  const res = {};
+  const obsIndexName = world.schema.annotations.obs.index;
  _.forEach(world.obsAnnotations.colIndex.keys(), key => {
    const summary = world.obsAnnotations.col(key).summarize();
    if (summary.categories) {
      const isColorField = key.includes("color") || key.includes("Color");
      const isSelectableCategory =
        !isColorField &&
-        key !== "name" &&
+        key !== obsIndexName &&
        summary.categories.length < maxCategoryItems;
      if (isSelectableCategory) {
        const [categoryValues, categoryValueCounts] = topNCategories(summary);
--- a/client/src/util/stateManager/universe.js
+++ b/client/src/util/stateManager/universe.js
@@ -124,7 +124,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
  cases, add a 'categories' field to the schema so it is accessible.
  */

-  universe.schema.annotations.obs.forEach(s => {
+  universe.schema.annotations.obs.columns.forEach(s => {
    if (
      s.type === "string" ||
      s.type === "boolean" ||
@@ -179,10 +179,10 @@ export function createUniverseFromResponse(

  /* Index schema for ease of use */
  universe.schema.annotations.obsByName = fromEntries(
-    universe.schema.annotations.obs.map(v => [v.name, v])
+    universe.schema.annotations.obs.columns.map(v => [v.name, v])
  );
  universe.schema.annotations.varByName = fromEntries(
-    universe.schema.annotations.var.map(v => [v.name, v])
+    universe.schema.annotations.var.columns.map(v => [v.name, v])
  );
  universe.schema.layout.obsByName = fromEntries(
    universe.schema.layout.obs.map(v => [v.name, v])
@@ -213,8 +213,9 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
    throw new Error("Unexpected non-floating point response from server.");
  }

+  const varIndexName = universe.schema.annotations.var.index;
  for (let c = 0; c < colIdx.length; c += 1) {
-    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    const varName = universe.varAnnotations.at(colIdx[c], varIndexName);
    result[varName] = columns[c];
  }
  return result;
--- a/client/src/util/stateManager/world.js
+++ b/client/src/util/stateManager/world.js
@@ -263,10 +263,14 @@ function deduceDimensionType(attributes, fieldName) {
 export function createObsDimensions(crossfilter, world, XYdimNames) {
  /*
  create and return a crossfilter with a dimension for every obs annotation
-  for which we have a supported type, *except* 'name'
+  for which we have a supported type, *except* for the index column, indicated
+  by schema.annotations.obs.index.
  */
  const { schema, obsLayout, obsAnnotations } = world;
-  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
+  const indexName = schema.annotations.obs.index;
+  const annoList = schema.annotations.obs.columns.filter(
+    anno => anno.name !== indexName
+  );
  crossfilter = annoList.reduce((xfltr, anno) => {
    const dimType = deduceDimensionType(anno, anno.name);
    const colData = obsAnnotations.col(anno.name).asArray();
--- a/server/app/scanpy_engine/scanpy_engine.py
+++ b/server/app/scanpy_engine/scanpy_engine.py
@@ -50,41 +50,61 @@ class ScanpyEngine(CXGDriver):
            "diffexp_lfc_cutoff": 0.01,
        }

-    def _alias_annotation_names(self, axis, name):
-        """
-        Do all user-specified annotation aliasing.
+    @staticmethod
+    def _create_unique_column_name(df, col_name_prefix):
+        """ given the columns of a dataframe, and a name prefix, return a column name which
+            does not exist in the dataframe, AND which is prefixed by `prefix`

-        As a *critical* side-effect, ensure the indices are simple number ranges
-        (accomplished by calling pandas.DataFrame.reset_index())
+            The approach is to append a numeric suffix, starting at zero and increasing by
+            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
        """
-        if name == "name":
-            # a noop, so skip it
-            return
+        suffix = 0
+        while f"{col_name_prefix}{suffix}" in df:
+            suffix += 1
+        return f"{col_name_prefix}{suffix}"

-        ax_name = str(axis)
-        df_axis = getattr(self.data, ax_name)
-        if name is None:
-            # reset index to simple range; alias "name" to point at the
-            # previously specified index.
-            df_axis.reset_index(inplace=True)
-            df_axis.rename(inplace=True, columns={"index": "name"})
-        elif name in df_axis.columns:
-            if name not in df_axis.columns:
+    def _alias_annotation_names(self):
+        """
+        The front-end relies on the existance of a unique, human-readable
+        index for obs & var (eg, var is typically gene name, obs the cell name).
+        The user can specify these via the --obs-names and --var-names config.
+        If they are not specified, use the existing index to create them, giving
+        the resulting column a unique name (eg, "name").
+
+        In both cases, enforce that the result is unique, and communicate the
+        index column name to the front-end via the obs_names and var_names config
+        (which is incorporated into the schema).
+        """
+        for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
+            name = self.config[config_name]
+            df_axis = getattr(self.data, str(ax_name))
+            if name is None:
+                # Default: create unique names from index
+                if not df_axis.index.is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.index must be unique. "
+                        "Please prepare data to contain unique index values, or specify an "
+                        "alternative with --{ax_name}-name."
+                    )
+                name = self._create_unique_column_name(df_axis.columns, "name_")
+                self.config[config_name] = name
+                # reset index to simple range; alias name to point at the
+                # previously specified index.
+                df_axis.rename_axis(name, inplace=True)
+                df_axis.reset_index(inplace=True)
+            elif name in df_axis.columns:
+                # User has specified alternative column for unique names, and it exists
+                if not df_axis[name].is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.{name} must be unique. "
+                        "Please prepare data to contain unique values."
+                    )
+                df_axis.reset_index(drop=True, inplace=True)
+            else:
+                # user specified a non-existent column name
                raise KeyError(
                    f"Annotation name {name}, specified in --{ax_name}-name does not exist."
                )
-            if not df_axis[name].is_unique:
-                raise KeyError(
-                    f"Values in -{ax_name}-name must be unique. "
-                    "Please prepare data to contain unique values."
-                )
-            # reset index to simple range; alias user-specified annotation to "name"
-            df_axis.reset_index(drop=True, inplace=True)
-            df_axis.rename(inplace=True, columns={name: "name"})
-        else:
-            raise KeyError(
-                f"Annotation name {name}, specified in --{ax_name}_name does not exist."
-            )

    @staticmethod
    def _can_cast_to_float32(ann):
@@ -114,7 +134,16 @@ class ScanpyEngine(CXGDriver):
                "nVar": self.gene_count,
                "type": str(self.data.X.dtype),
            },
-            "annotations": {"obs": [], "var": []},
+            "annotations": {
+                "obs": {
+                    "index": self.config["obs_names"],
+                    "columns": []
+                },
+                "var": {
+                    "index": self.config["var_names"],
+                    "columns": []
+                }
+            },
            "layout": {"obs": []}
        }
        for ax in Axis:
@@ -139,7 +168,7 @@ class ScanpyEngine(CXGDriver):
                    raise TypeError(
                        f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
                    )
-                self.schema["annotations"][ax].append(ann_schema)
+                self.schema["annotations"][ax]["columns"].append(ann_schema)

        for layout in self.config['layout']:
            layout_schema = {
@@ -173,8 +202,11 @@ class ScanpyEngine(CXGDriver):

    @requires_data
    def _validate_and_initialize(self):
-        self._alias_annotation_names(Axis.OBS, self.config["obs_names"])
-        self._alias_annotation_names(Axis.VAR, self.config["var_names"])
+        # var and obs column names must be unique
+        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
+            raise KeyError(f"All annotation column names must be unique.")
+
+        self._alias_annotation_names()
        self._validate_data_types()
        self.cell_count = self.data.shape[0]
        self.gene_count = self.data.shape[1]
--- a/server/test/schema.json
+++ b/server/test/schema.json
@@ -5,48 +5,54 @@
    "type": "float32"
  },
  "annotations": {
-    "obs": [
-      {
-        "name": "name",
-        "type": "string"
-      },
-      {
-        "name": "n_genes",
-        "type": "int32"
-      },
-      {
-        "name": "percent_mito",
-        "type": "float32"
-      },
-      {
-        "name": "n_counts",
-        "type": "float32"
-      },
-      {
-        "name": "louvain",
-        "type": "categorical",
-        "categories": [
-          "CD4 T cells",
-          "CD14+ Monocytes",
-          "B cells",
-          "CD8 T cells",
-          "NK cells",
-          "FCGR3A+ Monocytes",
-          "Dendritic cells",
-          "Megakaryocytes"
-        ]
-      }
-    ],
-    "var": [
-      {
-        "name": "name",
-        "type": "string"
-      },
-      {
-        "name": "n_cells",
-        "type": "int32"
-      }
-    ]
+    "obs": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_genes",
+          "type": "int32"
+        },
+        {
+          "name": "percent_mito",
+          "type": "float32"
+        },
+        {
+          "name": "n_counts",
+          "type": "float32"
+        },
+        {
+          "name": "louvain",
+          "type": "categorical",
+          "categories": [
+            "CD4 T cells",
+            "CD14+ Monocytes",
+            "B cells",
+            "CD8 T cells",
+            "NK cells",
+            "FCGR3A+ Monocytes",
+            "Dendritic cells",
+            "Megakaryocytes"
+          ]
+        }
+      ]
+    },
+    "var": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_cells",
+          "type": "int32"
+        }
+      ]
+    }
  },
  "layout": {
    "obs": [
--- a/server/test/test_api.py
+++ b/server/test/test_api.py
@@ -23,7 +23,8 @@ class EndPoints(unittest.TestCase):
        session = requests.Session()
        for i in range(90):
            try:
-                session.get(f"{URL_BASE}schema")
+                result = session.get(f"{URL_BASE}schema")
+                cls.schema = result.json()
            except requests.exceptions.ConnectionError:
                time.sleep(1)

@@ -45,7 +46,8 @@ class EndPoints(unittest.TestCase):
        self.assertEqual(result.headers["Content-Type"], "application/json")
        result_data = result.json()
        self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
-        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)

    def test_config(self):
        endpoint = "config"
@@ -95,7 +97,8 @@ class EndPoints(unittest.TestCase):
        self.assertIsNotNone(df['col_idx'])
        self.assertIsNone(df['row_idx'])
        self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])

    def test_get_annotations_obs_keys_fbs(self):
        endpoint = "annotations/obs"
@@ -165,7 +168,8 @@ class EndPoints(unittest.TestCase):
        self.assertIsNotNone(df['col_idx'])
        self.assertIsNone(df['row_idx'])
        self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])

    def test_get_annotations_var_keys_fbs(self):
        endpoint = "annotations/var"
@@ -247,7 +251,8 @@ class EndPoints(unittest.TestCase):
        endpoint = f"data/var"
        url = f"{URL_BASE}{endpoint}"
        header = {"Accept": "application/octet-stream"}
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
        result = self.session.put(url, headers=header, json=var_filter)
        self.assertEqual(result.status_code, HTTPStatus.OK)
        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
--- a/server/test/test_nan_scanpy_engine.py
+++ b/server/test/test_nan_scanpy_engine.py
@@ -58,14 +58,16 @@ class NaNTest(unittest.TestCase):

    def test_annotation(self):
        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
        self.assertEqual(
            annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"]
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
        )
        self.assertEqual(annotations["n_rows"], 100)
        self.assertTrue(math.isnan(annotations["columns"][2][0]))

        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
        self.assertEqual(annotations["n_rows"], 100)
        self.assertTrue(math.isnan(annotations["columns"][2][0]))
--- a/server/test/test_scanpy_engine.py
+++ b/server/test/test_scanpy_engine.py
@@ -31,9 +31,11 @@ class EngineTest(unittest.TestCase):
        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)

    def test_mandatory_annotations(self):
-        self.assertIn("name", self.data.data.obs)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
        self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
-        self.assertIn("name", self.data.data.var)
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
        self.assertEqual(list(self.data.data.var.index), list(range(1838)))

    @pytest.mark.filterwarnings("ignore:Scanpy data matrix")
@@ -70,12 +72,14 @@ class EngineTest(unittest.TestCase):
        self.assertEqual(data["n_cols"], 91)

    def test_obs_and_var_names(self):
-        self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
-        self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)

    def test_schema(self):
        with open(path.join(path.dirname(__file__), "schema.json")) as fh:
            schema = json.load(fh)
+            print(schema)
+            print(self.data.schema)
            self.assertEqual(self.data.schema, schema)

    def test_schema_produces_error(self):
@@ -108,16 +112,18 @@ class EngineTest(unittest.TestCase):
        annotations = decode_fbs.decode_matrix_FBS(fbs)
        self.assertEqual(annotations["n_rows"], 2638)
        self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
        self.assertEqual(
            annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
        )

        fbs = self.data.annotation_to_fbs_matrix("var")
        annotations = decode_fbs.decode_matrix_FBS(fbs)
        self.assertEqual(annotations['n_rows'], 1838)
        self.assertEqual(annotations['n_cols'], 2)
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])

    def test_annotation_fields(self):
        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -125,7 +131,8 @@ class EngineTest(unittest.TestCase):
        self.assertEqual(annotations["n_rows"], 2638)
        self.assertEqual(annotations['n_cols'], 2)

-        fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
        annotations = decode_fbs.decode_matrix_FBS(fbs)
        self.assertEqual(annotations['n_rows'], 1838)
        self.assertEqual(annotations['n_cols'], 1)
@@ -163,9 +170,10 @@ class EngineTest(unittest.TestCase):
            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")

    def test_data_named_gene(self):
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
        filter_ = {
            "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
            }
        }
        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -176,7 +184,7 @@ class EngineTest(unittest.TestCase):

        filter_ = {
            "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
            }
        }
        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")