From 109c9e70ec215f7d4c697aa7a9fb1ac79aee61b9 Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Thu, 19 Sep 2019 09:16:01 -0700 Subject: [PATCH] Add obs index to label file (#928) * remove debug logging * load and save obs/row index in label file * lint * update tests --- client/src/reducers/annotations.js | 9 ------- server/app/scanpy_engine/labels.py | 4 +-- server/app/scanpy_engine/scanpy_engine.py | 30 ++++++++++++++--------- server/test/test_fbs.py | 1 - server/test/test_scanpy_engine.py | 5 ++-- 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/client/src/reducers/annotations.js b/client/src/reducers/annotations.js index 9bbcea97..f1224114 100644 --- a/client/src/reducers/annotations.js +++ b/client/src/reducers/annotations.js @@ -13,7 +13,6 @@ const Annotations = ( switch (action.type) { /* CATEGORY */ case "annotation: activate add new label mode": - console.log(action.type, action); return { ...state, isAddingNewLabel: true, @@ -26,28 +25,24 @@ const Annotations = ( categoryAddingNewLabel: null }; case "annotation: add new label to category": - console.log(action.type, action); return { ...state, isAddingNewLabel: false, categoryAddingNewLabel: null }; case "annotation: activate category edit mode": - console.log(action.type, action); return { ...state, isEditingCategoryName: true, categoryEditable: action.data }; case "annotation: disable category edit mode": - console.log(action.type, action); return { ...state, isEditingCategoryName: false, categoryEditable: null }; case "annotation: category edited": - console.log(action.type, action); return { ...state, isEditingCategoryName: true, @@ -56,7 +51,6 @@ const Annotations = ( /* LABEL */ case "annotation: activate edit label mode": - console.log(action.type, action); return { ...state, isEditingLabelName: true, @@ -66,19 +60,16 @@ const Annotations = ( } }; case "annotation: cancel edit label mode": - console.log(action.type, action); return { ...state, isEditingLabelName: false, labelEditable: { category: null, label: null } }; case "annotation: label edited": - console.log(action.type, action); return { ...state, isEditingLabelName: false, labelEditable: { category: null, label: null } - /* Bruce to persist new label name */ }; default: return state; diff --git a/server/app/scanpy_engine/labels.py b/server/app/scanpy_engine/labels.py index bb62b880..32a74dc3 100644 --- a/server/app/scanpy_engine/labels.py +++ b/server/app/scanpy_engine/labels.py @@ -8,7 +8,7 @@ import pandas as pd def read_labels(fname): if exists(fname) and getsize(fname) > 0: - return pd.read_csv(fname, dtype='category') + return pd.read_csv(fname, dtype='category', index_col=0) else: return pd.DataFrame() @@ -16,7 +16,7 @@ def read_labels(fname): def write_labels(fname, df): rotate_fname(fname) if not df.empty: - df.to_csv(fname, index=False) + df.to_csv(fname) else: open(fname, 'a').close() diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 975e1adc..4e9ed9d2 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -22,15 +22,6 @@ from server.app.scanpy_engine.diffexp import diffexp_ttest from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs from server.app.scanpy_engine.labels import read_labels, write_labels -""" -Sort order for methods -1. Initialize -2. Helper -3. Filter -4. Data & Metadata -5. Computation -""" - class ScanpyEngine(CXGDriver): def __init__(self, data=None, args={}): @@ -81,6 +72,8 @@ class ScanpyEngine(CXGDriver): index column name to the front-end via the obs_names and var_names config (which is incorporated into the schema). """ + self.original_obs_index = self.data.obs.index + for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")): name = self.config[config_name] df_axis = getattr(self.data, str(ax_name)) @@ -255,7 +248,7 @@ class ScanpyEngine(CXGDriver): self.cell_count = self.data.shape[0] self.gene_count = self.data.shape[1] self._default_and_validate_layouts() - self._validate_label_file() + self._validate_label_data() self._create_schema() @requires_data @@ -340,7 +333,7 @@ class ScanpyEngine(CXGDriver): ) @requires_data - def _validate_label_file(self): + def _validate_label_data(self): """ labels is None if disabled, empty if enabled by no data """ @@ -350,6 +343,17 @@ class ScanpyEngine(CXGDriver): # all lables must have a name, which must be unique and not used in obs column names if not self.labels.columns.is_unique: raise KeyError(f"All column names specified in {self.config['label_file']} must be unique.") + + # the label index must be unique, and must have same values the anndata obs index + if not self.labels.index.is_unique: + raise KeyError(f"All row index values specified in the label file " + f"`{self.config['label_file']}` must be unique.") + + if not self.labels.index.equals(self.original_obs_index): + raise KeyError("Label file row index does not match H5AD file index. " + "Please ensure that column zero (0) in the label file contain the same " + "index values as the H5AD file.") + duplicate_columns = list(set(self.labels.columns) & set(self.data.obs.columns)) if len(duplicate_columns) > 0: raise KeyError(f"Labels file may not contain column names which overlap " @@ -427,7 +431,7 @@ class ScanpyEngine(CXGDriver): def annotation_to_fbs_matrix(self, axis, fields=None): if axis == Axis.OBS: if self.labels is not None and not self.labels.empty: - df = pandas.concat([self.data.obs, self.labels], axis=1, join_axes=[self.data.obs.index], copy=False) + df = self.data.obs.join(self.labels, self.config['obs_names']) else: df = self.data.obs else: @@ -446,6 +450,8 @@ class ScanpyEngine(CXGDriver): raise ValueError("Only OBS dimension access is supported") new_label_df = decode_matrix_fbs(fbs) + new_label_df.index = self.original_obs_index + self._validate_label_data() # paranoia # if any of the new column labels overlap with our existing labels, raise error duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns)) diff --git a/server/test/test_fbs.py b/server/test/test_fbs.py index 0346a24b..da32903a 100644 --- a/server/test/test_fbs.py +++ b/server/test/test_fbs.py @@ -25,7 +25,6 @@ class FbsTests(unittest.TestCase): def fbs_checks(self, fbs, dims, expected_types, expected_column_idx): d = decode_fbs.decode_matrix_FBS(fbs) - print(d) self.assertEqual(d["n_rows"], dims[0]) self.assertEqual(d["n_cols"], dims[1]) self.assertIsNone(d["row_idx"]) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 5e2513a2..190f800f 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -248,9 +248,10 @@ class WritableAnnotationTest(unittest.TestCase): res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.label_file)) - df = pd.read_csv(self.label_file) + df = pd.read_csv(self.label_file, index_col=0) self.assertEqual(df.shape, (n_rows, 2)) self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) + self.assertTrue(self.data.original_obs_index.equals(df.index)) self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) @@ -262,7 +263,7 @@ class WritableAnnotationTest(unittest.TestCase): res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.label_file)) - df = pd.read_csv(self.label_file) + df = pd.read_csv(self.label_file, index_col=0) self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))