diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index c8052323..0b970ab0 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -11,18 +11,20 @@ Sort order for methods class CXGDriver(metaclass=ABCMeta): - def __init__(self, data=None, args={}): + def __init__(self, data_locator=None, args={}): self.config = self._get_default_config() self.config.update(args) - if data: - self._load_data(data) + if data_locator: + self._load_data(data_locator) + self.data_locator = data_locator else: self.data = None - def update(self, data=None, args={}): + def update(self, data_locator=None, args={}): self.config.update(args) - if data: - self._load_data(data) + if data_locator: + self._load_data(data_locator) + self.data_locator = data_locator @staticmethod def _get_default_config(): diff --git a/server/app/scanpy_engine/labels.py b/server/app/scanpy_engine/labels.py index 32a74dc3..f06aeb40 100644 --- a/server/app/scanpy_engine/labels.py +++ b/server/app/scanpy_engine/labels.py @@ -8,15 +8,18 @@ import pandas as pd def read_labels(fname): if exists(fname) and getsize(fname) > 0: - return pd.read_csv(fname, dtype='category', index_col=0) + return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#') else: return pd.DataFrame() -def write_labels(fname, df): +def write_labels(fname, df, header=None): rotate_fname(fname) if not df.empty: - df.to_csv(fname) + f = open(fname, 'a', newline="") + if header is not None: + f.write(header) + df.to_csv(f) else: open(fname, 'a').close() diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index addcbf69..0392a9cb 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -1,6 +1,7 @@ import warnings import copy import threading +from datetime import datetime import numpy as np import pandas @@ -8,6 +9,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype import anndata from scipy import sparse +from server import __version__ as cellxgene_version from server.app.driver.driver import CXGDriver from server.app.util.constants import Axis, DEFAULT_TOP_N, MAX_LAYOUTS from server.app.util.errors import ( @@ -32,15 +34,15 @@ def has_method(o, name): class ScanpyEngine(CXGDriver): - def __init__(self, data=None, args={}): - super().__init__(data, args) + def __init__(self, data_locator=None, args={}): + super().__init__(data_locator, args) # lock used to protect label file write ops self.label_lock = threading.Lock() if self.data: self._validate_and_initialize() - def update(self, data=None, args={}): - super().__init__(data, args) + def update(self, data_locator=None, args={}): + super().__init__(data_locator, args) if self.data: self._validate_and_initialize() @@ -484,7 +486,13 @@ class ScanpyEngine(CXGDriver): # so treat this as a critical section. with self.label_lock: self.labels = new_label_df - write_labels(fname, self.labels) + lastmod = self.data_locator.lastmodtime() + lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") + header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \ + f"using cellxgene version {cellxgene_version}\n" \ + f"# Input data file was {self.data_locator.uri_or_path}, " \ + f"which was last modified on {lastmodstr}\n" + write_labels(fname, self.labels, header) return jsonify_scanpy({"status": "OK"}) diff --git a/server/app/util/data_locator.py b/server/app/util/data_locator.py index b2402047..e4d1af5d 100644 --- a/server/app/util/data_locator.py +++ b/server/app/util/data_locator.py @@ -1,6 +1,7 @@ import os import tempfile import fsspec +from datetime import datetime class DataLocator(): @@ -48,6 +49,14 @@ class DataLocator(): def size(self): return self.fs.size(self.cname) + def lastmodtime(self): + """ return datetime object representing last modification time, or None if unavailable """ + info = self.fs.info(self.cname) + if self.islocal() and info is not None: + return datetime.fromtimestamp(info['mtime']) + else: + return getattr(info, 'LastModified', None) + def isfile(self): return self.fs.isfile(self.cname) diff --git a/server/test/test_scanpy_engine_data_load.py b/server/test/test_scanpy_engine_data_load.py index cba82dc1..3379a5dc 100644 --- a/server/test/test_scanpy_engine_data_load.py +++ b/server/test/test_scanpy_engine_data_load.py @@ -37,7 +37,7 @@ class DataLoadEngineTest(unittest.TestCase): self.data._create_schema() def test_delayed_load_data(self): - self.data.update(data=self.data_file) + self.data.update(data_locator=self.data_file) self.data._create_schema() self.assertEqual(self.data.cell_count, 2638) self.assertEqual(self.data.gene_count, 1838) @@ -45,7 +45,7 @@ class DataLoadEngineTest(unittest.TestCase): self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_diffexp_topN(self): - self.data.update(data=self.data_file) + self.data.update(data_locator=self.data_file) f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) diff --git a/server/test/test_writable_annotation.py b/server/test/test_writable_annotation.py index bf260053..4e817566 100644 --- a/server/test/test_writable_annotation.py +++ b/server/test/test_writable_annotation.py @@ -60,7 +60,7 @@ class WritableAnnotationTest(unittest.TestCase): res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.label_file)) - df = pd.read_csv(self.label_file, index_col=0) + df = pd.read_csv(self.label_file, index_col=0, header=0, comment='#') self.assertEqual(df.shape, (n_rows, 2)) self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) self.assertTrue(self.data.original_obs_index.equals(df.index)) @@ -75,7 +75,7 @@ class WritableAnnotationTest(unittest.TestCase): res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.label_file)) - df = pd.read_csv(self.label_file, index_col=0) + df = pd.read_csv(self.label_file, index_col=0, header=0, comment='#') self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))