Add provenance header to labels CSV file (#1041)

* add last mod time lookup to data locator

* store data locator in Driver

* save metadata header in labels csv

* lint

* fix tests

* change datetime format to second precision

server/app/driver/driver.py

@@ -11,18 +11,20 @@ Sort order for methods
 
 
 class CXGDriver(metaclass=ABCMeta):
-    def __init__(self, data=None, args={}):
+    def __init__(self, data_locator=None, args={}):
         self.config = self._get_default_config()
         self.config.update(args)
-        if data:
-            self._load_data(data)
+        if data_locator:
+            self._load_data(data_locator)
+            self.data_locator = data_locator
         else:
             self.data = None
 
-    def update(self, data=None, args={}):
+    def update(self, data_locator=None, args={}):
         self.config.update(args)
-        if data:
-            self._load_data(data)
+        if data_locator:
+            self._load_data(data_locator)
+            self.data_locator = data_locator
 
     @staticmethod
     def _get_default_config():

server/app/scanpy_engine/labels.py

@@ -8,15 +8,18 @@ import pandas as pd
 
 def read_labels(fname):
     if exists(fname) and getsize(fname) > 0:
-        return pd.read_csv(fname, dtype='category', index_col=0)
+        return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#')
     else:
         return pd.DataFrame()
 
 
-def write_labels(fname, df):
+def write_labels(fname, df, header=None):
     rotate_fname(fname)
     if not df.empty:
-        df.to_csv(fname)
+        f = open(fname, 'a', newline="")
+        if header is not None:
+            f.write(header)
+        df.to_csv(f)
     else:
         open(fname, 'a').close()
 

server/app/scanpy_engine/scanpy_engine.py

@@ -1,6 +1,7 @@
 import warnings
 import copy
 import threading
+from datetime import datetime
 
 import numpy as np
 import pandas
@@ -8,6 +9,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 import anndata
 from scipy import sparse
 
+from server import __version__ as cellxgene_version
 from server.app.driver.driver import CXGDriver
 from server.app.util.constants import Axis, DEFAULT_TOP_N, MAX_LAYOUTS
 from server.app.util.errors import (
@@ -32,15 +34,15 @@ def has_method(o, name):
 
 
 class ScanpyEngine(CXGDriver):
-    def __init__(self, data=None, args={}):
-        super().__init__(data, args)
+    def __init__(self, data_locator=None, args={}):
+        super().__init__(data_locator, args)
         # lock used to protect label file write ops
         self.label_lock = threading.Lock()
         if self.data:
             self._validate_and_initialize()
 
-    def update(self, data=None, args={}):
-        super().__init__(data, args)
+    def update(self, data_locator=None, args={}):
+        super().__init__(data_locator, args)
         if self.data:
             self._validate_and_initialize()
 
@@ -484,7 +486,13 @@ class ScanpyEngine(CXGDriver):
         # so treat this as a critical section.
         with self.label_lock:
             self.labels = new_label_df
-            write_labels(fname, self.labels)
+            lastmod = self.data_locator.lastmodtime()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \
+                     f"using cellxgene version {cellxgene_version}\n" \
+                     f"# Input data file was {self.data_locator.uri_or_path}, " \
+                     f"which was last modified on {lastmodstr}\n"
+            write_labels(fname, self.labels, header)
 
         return jsonify_scanpy({"status": "OK"})
 

server/app/util/data_locator.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 import fsspec
+from datetime import datetime
 
 
 class DataLocator():
@@ -48,6 +49,14 @@ class DataLocator():
     def size(self):
         return self.fs.size(self.cname)
 
+    def lastmodtime(self):
+        """ return datetime object representing last modification time, or None if unavailable """
+        info = self.fs.info(self.cname)
+        if self.islocal() and info is not None:
+            return datetime.fromtimestamp(info['mtime'])
+        else:
+            return getattr(info, 'LastModified', None)
+
     def isfile(self):
         return self.fs.isfile(self.cname)
 

server/test/test_scanpy_engine_data_load.py

@@ -37,7 +37,7 @@ class DataLoadEngineTest(unittest.TestCase):
             self.data._create_schema()
 
     def test_delayed_load_data(self):
-        self.data.update(data=self.data_file)
+        self.data.update(data_locator=self.data_file)
         self.data._create_schema()
         self.assertEqual(self.data.cell_count, 2638)
         self.assertEqual(self.data.gene_count, 1838)
@@ -45,7 +45,7 @@ class DataLoadEngineTest(unittest.TestCase):
         self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
 
     def test_diffexp_topN(self):
-        self.data.update(data=self.data_file)
+        self.data.update(data_locator=self.data_file)
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))

server/test/test_writable_annotation.py

@@ -60,7 +60,7 @@ class WritableAnnotationTest(unittest.TestCase):
         res = self.data.annotation_put_fbs("obs", fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
         self.assertTrue(path.exists(self.label_file))
-        df = pd.read_csv(self.label_file, index_col=0)
+        df = pd.read_csv(self.label_file, index_col=0, header=0, comment='#')
         self.assertEqual(df.shape, (n_rows, 2))
         self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
         self.assertTrue(self.data.original_obs_index.equals(df.index))
@@ -75,7 +75,7 @@ class WritableAnnotationTest(unittest.TestCase):
         res = self.data.annotation_put_fbs("obs", fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
         self.assertTrue(path.exists(self.label_file))
-        df = pd.read_csv(self.label_file, index_col=0)
+        df = pd.read_csv(self.label_file, index_col=0, header=0, comment='#')
         self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
         self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
         self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))