Add obs index to label file (#928)

* remove debug logging

* load and save obs/row index in label file

* lint

* update tests

client/src/reducers/annotations.js

@@ -13,7 +13,6 @@ const Annotations = (
   switch (action.type) {
     /* CATEGORY */
     case "annotation: activate add new label mode":
-      console.log(action.type, action);
       return {
         ...state,
         isAddingNewLabel: true,
@@ -26,28 +25,24 @@ const Annotations = (
         categoryAddingNewLabel: null
       };
     case "annotation: add new label to category":
-      console.log(action.type, action);
       return {
         ...state,
         isAddingNewLabel: false,
         categoryAddingNewLabel: null
       };
     case "annotation: activate category edit mode":
-      console.log(action.type, action);
       return {
         ...state,
         isEditingCategoryName: true,
         categoryEditable: action.data
       };
     case "annotation: disable category edit mode":
-      console.log(action.type, action);
       return {
         ...state,
         isEditingCategoryName: false,
         categoryEditable: null
       };
     case "annotation: category edited":
-      console.log(action.type, action);
       return {
         ...state,
         isEditingCategoryName: true,
@@ -56,7 +51,6 @@ const Annotations = (
 
     /* LABEL */
     case "annotation: activate edit label mode":
-      console.log(action.type, action);
       return {
         ...state,
         isEditingLabelName: true,
@@ -66,19 +60,16 @@ const Annotations = (
         }
       };
     case "annotation: cancel edit label mode":
-      console.log(action.type, action);
       return {
         ...state,
         isEditingLabelName: false,
         labelEditable: { category: null, label: null }
       };
     case "annotation: label edited":
-      console.log(action.type, action);
       return {
         ...state,
         isEditingLabelName: false,
         labelEditable: { category: null, label: null }
-        /* Bruce to persist new label name */
       };
     default:
       return state;

server/app/scanpy_engine/labels.py

@@ -8,7 +8,7 @@ import pandas as pd
 
 def read_labels(fname):
     if exists(fname) and getsize(fname) > 0:
-        return pd.read_csv(fname, dtype='category')
+        return pd.read_csv(fname, dtype='category', index_col=0)
     else:
         return pd.DataFrame()
 
@@ -16,7 +16,7 @@ def read_labels(fname):
 def write_labels(fname, df):
     rotate_fname(fname)
     if not df.empty:
-        df.to_csv(fname, index=False)
+        df.to_csv(fname)
     else:
         open(fname, 'a').close()
 

server/app/scanpy_engine/scanpy_engine.py

@@ -22,15 +22,6 @@ from server.app.scanpy_engine.diffexp import diffexp_ttest
 from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
 from server.app.scanpy_engine.labels import read_labels, write_labels
 
-"""
-Sort order for methods
-1. Initialize
-2. Helper
-3. Filter
-4. Data & Metadata
-5. Computation
-"""
-
 
 class ScanpyEngine(CXGDriver):
     def __init__(self, data=None, args={}):
@@ -81,6 +72,8 @@ class ScanpyEngine(CXGDriver):
         index column name to the front-end via the obs_names and var_names config
         (which is incorporated into the schema).
         """
+        self.original_obs_index = self.data.obs.index
+
         for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
             name = self.config[config_name]
             df_axis = getattr(self.data, str(ax_name))
@@ -255,7 +248,7 @@ class ScanpyEngine(CXGDriver):
         self.cell_count = self.data.shape[0]
         self.gene_count = self.data.shape[1]
         self._default_and_validate_layouts()
-        self._validate_label_file()
+        self._validate_label_data()
         self._create_schema()
 
     @requires_data
@@ -340,7 +333,7 @@ class ScanpyEngine(CXGDriver):
                         )
 
     @requires_data
-    def _validate_label_file(self):
+    def _validate_label_data(self):
         """
         labels is None if disabled, empty if enabled by no data
         """
@@ -350,6 +343,17 @@ class ScanpyEngine(CXGDriver):
         # all lables must have a name, which must be unique and not used in obs column names
         if not self.labels.columns.is_unique:
             raise KeyError(f"All column names specified in {self.config['label_file']} must be unique.")
+
+        # the label index must be unique, and must have same values the anndata obs index
+        if not self.labels.index.is_unique:
+            raise KeyError(f"All row index values specified in the label file "
+                           f"`{self.config['label_file']}` must be unique.")
+
+        if not self.labels.index.equals(self.original_obs_index):
+            raise KeyError("Label file row index does not match H5AD file index. "
+                           "Please ensure that column zero (0) in the label file contain the same "
+                           "index values as the H5AD file.")
+
         duplicate_columns = list(set(self.labels.columns) & set(self.data.obs.columns))
         if len(duplicate_columns) > 0:
             raise KeyError(f"Labels file may not contain column names which overlap "
@@ -427,7 +431,7 @@ class ScanpyEngine(CXGDriver):
     def annotation_to_fbs_matrix(self, axis, fields=None):
         if axis == Axis.OBS:
             if self.labels is not None and not self.labels.empty:
-                df = pandas.concat([self.data.obs, self.labels], axis=1, join_axes=[self.data.obs.index], copy=False)
+                df = self.data.obs.join(self.labels, self.config['obs_names'])
             else:
                 df = self.data.obs
         else:
@@ -446,6 +450,8 @@ class ScanpyEngine(CXGDriver):
             raise ValueError("Only OBS dimension access is supported")
 
         new_label_df = decode_matrix_fbs(fbs)
+        new_label_df.index = self.original_obs_index
+        self._validate_label_data()  # paranoia
 
         # if any of the new column labels overlap with our existing labels, raise error
         duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))

server/test/test_fbs.py

@@ -25,7 +25,6 @@ class FbsTests(unittest.TestCase):
 
     def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
         d = decode_fbs.decode_matrix_FBS(fbs)
-        print(d)
         self.assertEqual(d["n_rows"], dims[0])
         self.assertEqual(d["n_cols"], dims[1])
         self.assertIsNone(d["row_idx"])

server/test/test_scanpy_engine.py

@@ -248,9 +248,10 @@ class WritableAnnotationTest(unittest.TestCase):
         res = self.data.annotation_put_fbs("obs", fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
         self.assertTrue(path.exists(self.label_file))
-        df = pd.read_csv(self.label_file)
+        df = pd.read_csv(self.label_file, index_col=0)
         self.assertEqual(df.shape, (n_rows, 2))
         self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
+        self.assertTrue(self.data.original_obs_index.equals(df.index))
         self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
         self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
 
@@ -262,7 +263,7 @@ class WritableAnnotationTest(unittest.TestCase):
         res = self.data.annotation_put_fbs("obs", fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
         self.assertTrue(path.exists(self.label_file))
-        df = pd.read_csv(self.label_file)
+        df = pd.read_csv(self.label_file, index_col=0)
         self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
         self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
         self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))