From 1621d20bc4e799b3a2388ddd367becd243f595f0 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Fri, 22 Jun 2018 10:01:52 -0700 Subject: [PATCH] Minimum working iterator version Checkpointing. This is a minimum working version using the iterator. I basically hacked consuming the iterator since I want to switch directions, but this is a good place to come back to. --- backend/app/__init__.py | 10 ++--- backend/app/driver/driver.py | 3 +- backend/app/rest_api/rest.py | 28 +++++++----- backend/app/scanpy_engine/scanpy_engine.py | 52 +++++++++++++++++----- backend/app/util/utils.py | 11 ++++- backend/run.py | 2 +- 6 files changed, 74 insertions(+), 32 deletions(-) diff --git a/backend/app/__init__.py b/backend/app/__init__.py index 2bc0d452..166bc39b 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -16,8 +16,9 @@ CORS(app) #Config CONFIG_FILE = os.environ.get("CXG_CONFIG_FILE", default="scanpy-test.cfg") -CXG_DIR = os.environ.get("CXG_DIRECTORY", default="/Users/charlotteweaver/Documents/Git/cxg-v2/") +CXG_DIR = os.environ.get("CXG_DIRECTORY", default="/Users/charlotteweaver/Documents/Git/cxg-v2/data/") SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine") +ENGINE = os.environ.get("CXG_ENGINE", default="scanpy") # TODO remove the 2 when this is prod CXG_API_BASE = os.environ.get("CXG_API_BASE2", default="http://0.0.0.0:5005/api/") @@ -28,9 +29,8 @@ app.config.from_pyfile(os.path.join(CXG_DIR, "config", CONFIG_FILE), silent=True app.config.update( SECRET_KEY=SECRET_KEY, CXG_API_BASE=CXG_API_BASE, -) -app.config.update( - DATA=os.path.join(CXG_DIR, app.config["DATA_DIR"]), + ENGINE=ENGINE, + DATA=CXG_DIR ) app.config['PROFILE'] = True @@ -40,7 +40,7 @@ app.config['PROFILE'] = True data = None if app.config["ENGINE"] == "scanpy": from .scanpy_engine.scanpy_engine import ScanpyEngine - data = ScanpyEngine(app.config["DATA"]) + data = ScanpyEngine(app.config["DATA"], schema="data_schema.json") REACTIVE_LIMIT = 1_000_000 diff --git a/backend/app/driver/driver.py b/backend/app/driver/driver.py index 07fc1187..085ae8ba 100644 --- a/backend/app/driver/driver.py +++ b/backend/app/driver/driver.py @@ -1,12 +1,13 @@ from abc import ABCMeta, abstractmethod + class CXGDriver(metaclass=ABCMeta): def __init__(self, data, schema=None, graph_method=None, diffexp_method=None): self.data = self._load_data(data) - @abstractmethod @staticmethod + @abstractmethod def _load_data(data): pass diff --git a/backend/app/rest_api/rest.py b/backend/app/rest_api/rest.py index e33b7aac..6603f4b7 100644 --- a/backend/app/rest_api/rest.py +++ b/backend/app/rest_api/rest.py @@ -1,10 +1,10 @@ from flask import ( - Blueprint, render_template, request, url_for + Blueprint, request ) from flask_restful_swagger_2 import Api, swagger, Resource -from ..util.utils import make_payload -from ..util.filter import parse_filter, QueryStringError +from ..util.utils import make_payload +from ..util.filter import parse_filter class InitializeAPI(Resource): @@ -88,15 +88,17 @@ class InitializeAPI(Resource): } }) def get(self): - from cxg import data, REACTIVE_LIMIT + from app import data, REACTIVE_LIMIT return make_payload({ "schema": data.schema, - "ranges": data.metadata_ranges(data.ADATA, data.schema), - "cellcount": data.cell_count(), + "cellcount": data.cell_count, "reactivelimit": REACTIVE_LIMIT, - "genes": data.all_genes(), + "genes": data.genes(), + "ranges": data.metadata_ranges(), + }) + class CellsAPI(Resource): @swagger.doc({ 'summary': 'filter based on metadata fields to get a subset cells, expression data, and metadata', @@ -189,7 +191,7 @@ class CellsAPI(Resource): } }) def get(self): - from cxg import data + from app import data payload = { "cellids": [], "metadata": [], @@ -199,14 +201,16 @@ class CellsAPI(Resource): } # get query params filter = parse_filter(request.args, data.schema) - filtered_data = data.filter_cells(filter) - payload["metadata"], payload["cellids"] = data.metadata(filtered_data) - payload["ranges"] = data.metadata_ranges(filtered_data, data.schema) + filtered_data = list(data.filter_cells(filter)) + payload["metadata"] = list(data.metadata(filtered_data)) + payload["ranges"] = list(data.metadata_ranges(filtered_data)) + payload["cellids"] = filtered_data payload["cellcount"] = len(payload["cellids"]) - payload["graph"] = data.create_graph(filtered_data) + payload["graph"] = list(data.create_graph(filtered_data)) return make_payload(payload) + def get_api_resources(): bp = Blueprint('api', __name__, url_prefix='/api/v2.0') api = Api(bp, add_api_spec_resource=False) diff --git a/backend/app/scanpy_engine/scanpy_engine.py b/backend/app/scanpy_engine/scanpy_engine.py index f42bad54..e70fa53c 100644 --- a/backend/app/scanpy_engine/scanpy_engine.py +++ b/backend/app/scanpy_engine/scanpy_engine.py @@ -1,15 +1,16 @@ import scanpy.api as sc import numpy as np +import os from ..util.schema_parse import parse_schema -from ..driver import CXGDriver +from ..driver.driver import CXGDriver class ScanpyEngine(CXGDriver): - def __init__(self, data, schema=None, graph_method=None, diffexp_method=None): + def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"): self.data = self._load_data(data) - self.schema = self._load_or_infer_schema(schema) + self.schema = self._load_or_infer_schema(data, schema) self._set_cell_ids() self.cell_count = self.data.shape[0] # TODO Do I need this? @@ -20,14 +21,15 @@ class ScanpyEngine(CXGDriver): @staticmethod def _load_data(data): - return sc.read(data) + return sc.read(os.path.join(data, "data.h5ad")) - def _load_or_infer_schema(schema): + @staticmethod + def _load_or_infer_schema(data, schema): data_schema = None if not schema: pass else: - data_schema = parse_schema(schema) + data_schema = parse_schema(os.path.join(data,schema)) return data_schema def _set_cell_ids(self): @@ -38,8 +40,12 @@ class ScanpyEngine(CXGDriver): def cells(self): return list(self.data.obs.index) - def cellids(self): - return list(self.data.obs.index) + def cellids(self, cells_iterator=None): + if cells_iterator: + data = self.data.obs.iloc[[i for i in cells_iterator], :] + else: + data = self.data.obs + return list(data.index) def genes(self): return self.data.var.index.tolist() @@ -50,7 +56,7 @@ class ScanpyEngine(CXGDriver): :param filter: :return: iterator through cell ids """ - cell_idx = np.ones((self.cell_count(),), dtype=bool) + cell_idx = np.ones((self.cell_count,), dtype=bool) # TODO does this need to be a generator too? for key, value in filter.items(): if value["variable_type"] == "categorical": @@ -66,9 +72,31 @@ class ScanpyEngine(CXGDriver): key_idx = np.array((getattr(self.data.obs, key) <= min_).data) cell_idx = np.logical_and(cell_idx, key_idx) # If this is slow, could vectorize with logical array and then loop through that - for idx in self.cell_count: + for idx in range(self.cell_count): if cell_idx[idx]: - yield self.data.obs['cxg_cell_id'][idx] + yield self.data.obs.index[idx] + + + def metadata_ranges(self, cells_iterator=None): + metadata_ranges = {} + if cells_iterator: + data = self.data.obs.iloc[[i for i in cells_iterator], :] + else: + data = self.data.obs + for field in self.schema: + if self.schema[field]["variabletype"] == "categorical": + group_by = field + if group_by == "CellName": + group_by = 'cell_name' + metadata_ranges[field] = {"options": data.groupby(group_by).size().to_dict()} + else: + metadata_ranges[field] = { + "range": { + "min": data[field].min(), + "max": data[field].max() + } + } + return metadata_ranges # Should this return the order of metadata fields as the first value? def metadata(self, cells_iterator, fields=None): @@ -83,7 +111,7 @@ class ScanpyEngine(CXGDriver): if not fields: fields = self.data.obs.columns.tolist() for cell_id in cells_iterator: - yield [cell_id] + self.data.obs.loc[cell_id, [fields]].tolist() + yield [cell_id] + self.data.obs.loc[cell_id, fields].tolist() def create_graph(self, cells_iterator): diff --git a/backend/app/util/utils.py b/backend/app/util/utils.py index 26fe85eb..4f0fff77 100644 --- a/backend/app/util/utils.py +++ b/backend/app/util/utils.py @@ -1,6 +1,6 @@ import json from numpy import float32, integer -from flask import make_response, jsonify +from flask import make_response, jsonify, Response class Float32JSONEncoder(json.JSONEncoder): def default(self, obj): @@ -31,4 +31,13 @@ def make_payload(data, errormessage="", errorcode=200): } }), errorcode) +def make_streaming_response(data_generator, errorcode=200, content_type="application/json"): + # TODO headers + return Response(data_generator, status=errorcode, content_type=content_type) + + + + + + diff --git a/backend/run.py b/backend/run.py index b7608b65..6a8dee0c 100644 --- a/backend/run.py +++ b/backend/run.py @@ -1,3 +1,3 @@ -from cxg import app +from app import app app.run(host='0.0.0.0', debug=True, port=5005)