added support for gateway ttl and manual pruning

cellxgene_gateway/cache_entry.py

@@ -44,17 +44,17 @@ class CacheEntry:
     @classmethod
     def for_dataset(cls, dataset, file_path, port):
         return cls(
-            "",
+            None,
             dataset,
             file_path,
             port,
             current_time_stamp(),
             current_time_stamp(),
             "loading",
-            "",
+            None,
-            "",
+            None,
-            "",
+            None,
-            "",
+            None,
         )
 
     def set_loaded(self, pid):
@@ -66,6 +66,12 @@ class CacheEntry:
         self.stderr = stderr
         self.http_status = http_status
         self.status = "error"
+    
+    def append_output(self, output):
+        if self.all_output == None:
+            self.all_output = output
+        else:
+            self.all_output += output
 
     def serve_content(self, path):
         dataset = self.dataset

cellxgene_gateway/env.py

@@ -15,6 +15,7 @@ gateway_host = os.environ.get("GATEWAY_HOST")
 gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
 ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
+ttl = os.environ.get("GATEWAY_TTL")
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,

cellxgene_gateway/gateway.py

@@ -187,9 +187,21 @@ def do_GET_status():
         "cache_status.html", entry_list=cache.entry_list
     )
 
+pruner = PruneProcessCache(cache)
+
+@app.route("/prune/<path:path>", methods=["DELETE"])
+def do_prune(path):
+    dataset = get_dataset(path)
+    match = cache.check_entry(dataset)
+    if match is None:
+        return "not found"
+    else:
+        pruner.prune(match)
+        return "success"
+
 def main():
     env.validate()
-    background_thread = Thread(target=PruneProcessCache(cache))
+    background_thread = Thread(target=pruner)
     background_thread.start()
 
     app.run(host="0.0.0.0", port=5005, debug=False)

cellxgene_gateway/prune_process_cache.py

@@ -12,12 +12,22 @@ import time
 import psutil
 
 from cellxgene_gateway.util import current_time_stamp
+from cellxgene_gateway.env import ttl
 
 
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
 
+    def prune(self, process):
+        self.cache.entry_list.remove(process)
+        pid = process.pid
+        if pid != None:
+            p = psutil.Process(pid)
+            p.terminate()
+            p = psutil.Process(pid + 2)
+            p.terminate()
+
     def __call__(self):
         while True:
             time.sleep(60)
@@ -26,14 +36,13 @@ class PruneProcessCache:
 
             processes_to_delete = []
             for p in self.cache.entry_list:
-                if timestamp - p.timestamp > 3600:
+                if timestamp - p.timestamp > (3600 if ttl is None else ttl):
                     processes_to_delete.append(p)
                     processes_to_delete
 
             for process in processes_to_delete:
-                self.cache.entry_list.remove(process)
+                try:
-                pid = process.pid
+                    self.prune(process)
-                p = psutil.Process(pid)
+                except Exception as detail:
-                p.terminate()
+                    print('failed to prune process:', detail)
-                p = psutil.Process(pid + 2)
+
-                p.terminate()

cellxgene_gateway/subprocess_backend.py

@@ -64,7 +64,7 @@ class SubprocessBackend:
 
                 raise ProcessException.from_pid_object(cache_entry)
             else:
-                cache_entry.all_output += output
+                cache_entry.append_output(output)
 
         cache_entry.set_loaded(process.pid)