Minimum working iterator version

Checkpointing. This is a minimum working version using the iterator. I basically hacked consuming the iterator since I want to switch directions, but this is a good place to come back to.

backend/app/__init__.py

@@ -16,8 +16,9 @@ CORS(app)
 
 #Config
 CONFIG_FILE = os.environ.get("CXG_CONFIG_FILE", default="scanpy-test.cfg")
-CXG_DIR = os.environ.get("CXG_DIRECTORY", default="/Users/charlotteweaver/Documents/Git/cxg-v2/")
+CXG_DIR = os.environ.get("CXG_DIRECTORY", default="/Users/charlotteweaver/Documents/Git/cxg-v2/data/")
 SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
+ENGINE = os.environ.get("CXG_ENGINE", default="scanpy")
 # TODO remove the 2 when this is prod
 CXG_API_BASE = os.environ.get("CXG_API_BASE2", default="http://0.0.0.0:5005/api/")
 
@@ -28,9 +29,8 @@ app.config.from_pyfile(os.path.join(CXG_DIR, "config", CONFIG_FILE), silent=True
 app.config.update(
     SECRET_KEY=SECRET_KEY,
     CXG_API_BASE=CXG_API_BASE,
-)
-app.config.update(
-    DATA=os.path.join(CXG_DIR, app.config["DATA_DIR"]),
+    ENGINE=ENGINE,
+    DATA=CXG_DIR
 )
 
 app.config['PROFILE'] = True
@@ -40,7 +40,7 @@ app.config['PROFILE'] = True
 data = None
 if app.config["ENGINE"] == "scanpy":
     from .scanpy_engine.scanpy_engine import ScanpyEngine
-    data = ScanpyEngine(app.config["DATA"])
+    data = ScanpyEngine(app.config["DATA"], schema="data_schema.json")
 
 REACTIVE_LIMIT = 1_000_000
 

backend/app/driver/driver.py

@@ -1,12 +1,13 @@
 from abc import ABCMeta, abstractmethod
 
+
 class CXGDriver(metaclass=ABCMeta):
 
 	def __init__(self, data, schema=None, graph_method=None, diffexp_method=None):
 		self.data = self._load_data(data)
 
-	@abstractmethod
 	@staticmethod
+	@abstractmethod
 	def _load_data(data):
 		pass
 

backend/app/rest_api/rest.py

@@ -1,10 +1,10 @@
 from flask import (
-    Blueprint, render_template, request, url_for
+    Blueprint, request
 )
 from flask_restful_swagger_2 import Api, swagger, Resource
-from ..util.utils import make_payload
-from ..util.filter import parse_filter, QueryStringError
 
+from ..util.utils import make_payload
+from ..util.filter import parse_filter
 
 
 class InitializeAPI(Resource):
@@ -88,15 +88,17 @@ class InitializeAPI(Resource):
         }
     })
     def get(self):
-        from cxg import data, REACTIVE_LIMIT
+        from app import data, REACTIVE_LIMIT
         return make_payload({
             "schema": data.schema,
-            "ranges": data.metadata_ranges(data.ADATA, data.schema),
-            "cellcount": data.cell_count(),
+            "cellcount": data.cell_count,
             "reactivelimit": REACTIVE_LIMIT,
-            "genes": data.all_genes(),
+            "genes": data.genes(),
+            "ranges": data.metadata_ranges(),
+
         })
 
+
 class CellsAPI(Resource):
     @swagger.doc({
         'summary': 'filter based on metadata fields to get a subset cells, expression data, and metadata',
@@ -189,7 +191,7 @@ class CellsAPI(Resource):
         }
     })
     def get(self):
-        from cxg import data
+        from app import data
         payload = {
             "cellids": [],
             "metadata": [],
@@ -199,14 +201,16 @@ class CellsAPI(Resource):
         }
         # get query params
         filter = parse_filter(request.args, data.schema)
-        filtered_data = data.filter_cells(filter)
-        payload["metadata"], payload["cellids"] = data.metadata(filtered_data)
-        payload["ranges"] = data.metadata_ranges(filtered_data, data.schema)
+        filtered_data = list(data.filter_cells(filter))
+        payload["metadata"] = list(data.metadata(filtered_data))
+        payload["ranges"] = list(data.metadata_ranges(filtered_data))
+        payload["cellids"] = filtered_data
         payload["cellcount"] = len(payload["cellids"])
-        payload["graph"] = data.create_graph(filtered_data)
+        payload["graph"] = list(data.create_graph(filtered_data))
         return make_payload(payload)
 
 
+
 def get_api_resources():
     bp = Blueprint('api', __name__, url_prefix='/api/v2.0')
     api = Api(bp, add_api_spec_resource=False)

backend/app/scanpy_engine/scanpy_engine.py

@@ -1,15 +1,16 @@
 import scanpy.api as sc
 import numpy as np
+import os
 
 from ..util.schema_parse import parse_schema
-from ..driver import CXGDriver
+from ..driver.driver import CXGDriver
 
 
 class ScanpyEngine(CXGDriver):
 
-	def __init__(self, data, schema=None, graph_method=None, diffexp_method=None):
+	def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"):
 		self.data = self._load_data(data)
-		self.schema = self._load_or_infer_schema(schema)
+		self.schema = self._load_or_infer_schema(data, schema)
 		self._set_cell_ids()
 		self.cell_count = self.data.shape[0]
 		# TODO Do I need this?
@@ -20,14 +21,15 @@ class ScanpyEngine(CXGDriver):
 
 	@staticmethod
 	def _load_data(data):
-		return sc.read(data)
+		return sc.read(os.path.join(data, "data.h5ad"))
 
-	def _load_or_infer_schema(schema):
+	@staticmethod
+	def _load_or_infer_schema(data, schema):
 		data_schema = None
 		if not schema:
 			pass
 		else:
-			data_schema = parse_schema(schema)
+			data_schema = parse_schema(os.path.join(data,schema))
 		return data_schema
 
 	def _set_cell_ids(self):
@@ -38,8 +40,12 @@ class ScanpyEngine(CXGDriver):
 	def cells(self):
 		return list(self.data.obs.index)
 
-	def cellids(self):
-		return list(self.data.obs.index)
+	def cellids(self, cells_iterator=None):
+		if cells_iterator:
+			data = self.data.obs.iloc[[i for i in cells_iterator], :]
+		else:
+			data = self.data.obs
+		return list(data.index)
 
 	def genes(self):
 		return self.data.var.index.tolist()
@@ -50,7 +56,7 @@ class ScanpyEngine(CXGDriver):
 		:param filter:
 		:return: iterator through cell ids
 		"""
-		cell_idx = np.ones((self.cell_count(),), dtype=bool)
+		cell_idx = np.ones((self.cell_count,), dtype=bool)
 		# TODO does this need to be a generator too?
 		for key, value in filter.items():
 			if value["variable_type"] == "categorical":
@@ -66,9 +72,31 @@ class ScanpyEngine(CXGDriver):
 					key_idx = np.array((getattr(self.data.obs, key) <= min_).data)
 					cell_idx = np.logical_and(cell_idx, key_idx)
 		# If this is slow, could vectorize with logical array and then loop through that
-		for idx in self.cell_count:
+		for idx in range(self.cell_count):
 			if cell_idx[idx]:
-				yield self.data.obs['cxg_cell_id'][idx]
+				yield self.data.obs.index[idx]
+
+
+	def metadata_ranges(self, cells_iterator=None):
+		metadata_ranges = {}
+		if cells_iterator:
+			data = self.data.obs.iloc[[i for i in cells_iterator], :]
+		else:
+			data = self.data.obs
+		for field in self.schema:
+			if self.schema[field]["variabletype"] == "categorical":
+				group_by = field
+				if group_by == "CellName":
+					group_by = 'cell_name'
+				metadata_ranges[field] = {"options": data.groupby(group_by).size().to_dict()}
+			else:
+				metadata_ranges[field] = {
+					"range": {
+						"min": data[field].min(),
+						"max": data[field].max()
+					}
+				}
+		return metadata_ranges
 
 	# Should this return the order of metadata fields as the first value?
 	def metadata(self, cells_iterator, fields=None):
@@ -83,7 +111,7 @@ class ScanpyEngine(CXGDriver):
 		if not fields:
 			fields = self.data.obs.columns.tolist()
 		for cell_id in cells_iterator:
-			yield [cell_id] + self.data.obs.loc[cell_id, [fields]].tolist()
+			yield [cell_id] + self.data.obs.loc[cell_id, fields].tolist()
 
 
 	def create_graph(self, cells_iterator):

backend/app/util/utils.py

@@ -1,6 +1,6 @@
 import json
 from numpy import float32, integer
-from flask import make_response, jsonify
+from flask import make_response, jsonify, Response
 
 class Float32JSONEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -31,4 +31,13 @@ def make_payload(data, errormessage="", errorcode=200):
         }
     }), errorcode)
 
+def make_streaming_response(data_generator, errorcode=200, content_type="application/json"):
+    # TODO headers
+    return Response(data_generator, status=errorcode, content_type=content_type)
+
+
+
+
+
+
 

backend/run.py

@@ -1,3 +1,3 @@
-from cxg import app
+from app import app
 
 app.run(host='0.0.0.0', debug=True, port=5005)