fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-03 20:38:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Correct tabs to spaces

Browse Source
This commit is contained in:
Charlotte Weaver
2018-06-26 11:22:34 -07:00
parent c7f9dc0bb9
commit 59dcfe2bc4
2 changed files with 216 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/driver/driver.py
+61 -61
View File
@@ -2,77 +2,77 @@ from abc import ABCMeta, abstractmethod
class CXGDriver(metaclass=ABCMeta): class CXGDriver(metaclass=ABCMeta):
def __init__(self, data, schema=None, graph_method=None, diffexp_method=None): def __init__(self, data, schema=None, graph_method=None, diffexp_method=None):
self.data = self._load_data(data) self.data = self._load_data(data)
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def _load_data(data): def _load_data(data):
pass pass
@abstractmethod @abstractmethod
def _load_or_infer_schema(data): def _load_or_infer_schema(data):
pass pass
@abstractmethod @abstractmethod
def _set_cell_ids(self): def _set_cell_ids(self):
pass pass
@abstractmethod @abstractmethod
def cells(self): def cells(self):
pass pass
@abstractmethod @abstractmethod
def cellids(self): def cellids(self):
pass pass
@abstractmethod @abstractmethod
def genes(self): def genes(self):
pass pass
@abstractmethod @abstractmethod
def filter_cells(self, filter): def filter_cells(self, filter):
""" """
Filter cells from data and return a subset of the data Filter cells from data and return a subset of the data
:param filter: :param filter:
:return: filtered dataframe :return: filtered dataframe
""" """
pass pass
# Should this return the order of metadata fields as the first value? # Should this return the order of metadata fields as the first value?
@abstractmethod @abstractmethod
def metadata(self, df, fields=None): def metadata(self, df, fields=None):
""" """
Generator for metadata. Gets the metadata values cell by cell and returns all value Generator for metadata. Gets the metadata values cell by cell and returns all value
or only certain values if names is not None or only certain values if names is not None
:param df: from filter_cells, dataframe :param df: from filter_cells, dataframe
:param fields: list of keys for metadata to return, returns all metadata values if not set. :param fields: list of keys for metadata to return, returns all metadata values if not set.
:return: Iterator for cellid + list of cells metadata values ex. [cell-id, val1, val2, val3] :return: Iterator for cellid + list of cells metadata values ex. [cell-id, val1, val2, val3]
""" """
pass pass
@abstractmethod @abstractmethod
def create_graph(self, df): def create_graph(self, df):
""" """
Computes a n-d layout for cells through dimensionality reduction. Computes a n-d layout for cells through dimensionality reduction.
:param df: from filter_cells, dataframe :param df: from filter_cells, dataframe
:return: Iterator for [cellid-1, pos1, pos2], [cellid-2, pos1, pos2] :return: Iterator for [cellid-1, pos1, pos2], [cellid-2, pos1, pos2]
""" """
pass pass
@abstractmethod @abstractmethod
def diffexp(self, df1, df2): def diffexp(self, df1, df2):
""" """
Computes the top differentially expressed genes between two clusters Computes the top differentially expressed genes between two clusters
:param df1: First set of cells :param df1: First set of cells
:param df2: Second set of cells :param df2: Second set of cells
:return: Up in the air: I recommend [gene name, mean_expression_cells1, :return: Up in the air: I recommend [gene name, mean_expression_cells1,
mean_expression_cells2, average_difference, statistic_value] mean_expression_cells2, average_difference, statistic_value]
""" """
pass pass
@abstractmethod @abstractmethod
def expression(self, df): def expression(self, df):
pass pass
backend/app/scanpy_engine/scanpy_engine.py
+155 -156
View File
@@ -10,175 +10,174 @@ from ..driver.driver import CXGDriver
class ScanpyEngine(CXGDriver): class ScanpyEngine(CXGDriver):
def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"): def __init__(self, data, schema=None, graph_method="umap", diffexp_method="ttest"):
self.data = self._load_data(data) self.data = self._load_data(data)
self.schema = self._load_or_infer_schema(data, schema) self.schema = self._load_or_infer_schema(data, schema)
self._set_cell_ids() self._set_cell_ids()
self.cell_count = self.data.shape[0] self.cell_count = self.data.shape[0]
self.gene_count = self.data.shape[1] self.gene_count = self.data.shape[1]
self.graph_method = graph_method self.graph_method = graph_method
self.diffexp_method = diffexp_method self.diffexp_method = diffexp_method
@staticmethod @staticmethod
def _load_data(data): def _load_data(data):
return sc.read(os.path.join(data, "data.h5ad")) return sc.read(os.path.join(data, "data.h5ad"))
@staticmethod @staticmethod
def _load_or_infer_schema(data, schema): def _load_or_infer_schema(data, schema):
data_schema = None data_schema = None
if not schema: if not schema:
pass pass
else: else:
data_schema = parse_schema(os.path.join(data, schema)) data_schema = parse_schema(os.path.join(data, schema))
return data_schema return data_schema
def _set_cell_ids(self): def _set_cell_ids(self):
self.data.obs['cxg_cell_id'] = list(range(self.data.obs.shape[0])) self.data.obs['cxg_cell_id'] = list(range(self.data.obs.shape[0]))
self.data.obs["cell_name"] = list(self.data.obs.index) self.data.obs["cell_name"] = list(self.data.obs.index)
self.data.obs.set_index('cxg_cell_id', inplace=True) self.data.obs.set_index('cxg_cell_id', inplace=True)
def cells(self): def cells(self):
return list(self.data.obs.index) return list(self.data.obs.index)
def cellids(self, df=None): def cellids(self, df=None):
if df: if df:
return list(df.obs.index) return list(df.obs.index)
else: else:
return list(self.data.obs.index) return list(self.data.obs.index)
def genes(self): def genes(self):
return self.data.var.index.tolist() return self.data.var.index.tolist()
def filter_cells(self, filter): def filter_cells(self, filter):
""" """
Filter cells from data and return a subset of the data Filter cells from data and return a subset of the data
:param filter: :param filter:
:return: iterator through cell ids """
""" cell_idx = np.ones((self.cell_count,), dtype=bool)
cell_idx = np.ones((self.cell_count,), dtype=bool) for key, value in filter.items():
for key, value in filter.items(): if value["variable_type"] == "categorical":
if value["variable_type"] == "categorical": key_idx = np.in1d(getattr(self.data.obs, key), value["query"])
key_idx = np.in1d(getattr(self.data.obs, key), value["query"]) cell_idx = np.logical_and(cell_idx, key_idx)
cell_idx = np.logical_and(cell_idx, key_idx) else:
else: min_ = value["query"]["min"]
min_ = value["query"]["min"] max_ = value["query"]["max"]
max_ = value["query"]["max"] if min_:
if min_: key_idx = np.array((getattr(self.data.obs, key) >= min_).data)
key_idx = np.array((getattr(self.data.obs, key) >= min_).data) cell_idx = np.logical_and(cell_idx, key_idx)
cell_idx = np.logical_and(cell_idx, key_idx) if max_:
if max_: key_idx = np.array((getattr(self.data.obs, key) <= min_).data)
key_idx = np.array((getattr(self.data.obs, key) <= min_).data) cell_idx = np.logical_and(cell_idx, key_idx)
cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :]
return self.data[cell_idx, :]
def metadata_ranges(self, df=None): def metadata_ranges(self, df=None):
metadata_ranges = {} metadata_ranges = {}
if not df: if not df:
df = self.data df = self.data
for field in self.schema: for field in self.schema:
if self.schema[field]["variabletype"] == "categorical": if self.schema[field]["variabletype"] == "categorical":
group_by = field group_by = field
if group_by == "CellName": if group_by == "CellName":
group_by = 'cell_name' group_by = 'cell_name'
metadata_ranges[field] = {"options": df.obs.groupby(group_by).size().to_dict()} metadata_ranges[field] = {"options": df.obs.groupby(group_by).size().to_dict()}
else: else:
metadata_ranges[field] = { metadata_ranges[field] = {
"range": { "range": {
"min": df.obs[field].min(), "min": df.obs[field].min(),
"max": df.obs[field].max() "max": df.obs[field].max()
} }
} }
return metadata_ranges return metadata_ranges
def metadata(self, df, fields=None): def metadata(self, df, fields=None):
""" """
Generator for metadata. Gets the metadata values cell by cell and returns all value Generator for metadata. Gets the metadata values cell by cell and returns all value
or only certain values if names is not None or only certain values if names is not None
""" """
metadata = df.obs.to_dict(orient="records") metadata = df.obs.to_dict(orient="records")
for idx in range(len(metadata)): for idx in range(len(metadata)):
metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None) metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None)
return metadata return metadata
def create_graph(self, df): def create_graph(self, df):
""" """
Computes a n-d layout for cells through dimensionality reduction. Computes a n-d layout for cells through dimensionality reduction.
""" """
getattr(sc.tl, self.graph_method)(df) getattr(sc.tl, self.graph_method)(df)
graph = df.obsm["X_{graph_method}".format(graph_method=self.graph_method)] graph = df.obsm["X_{graph_method}".format(graph_method=self.graph_method)]
normalized_graph = (graph - graph.min()) / (graph.max() - graph.min()) normalized_graph = (graph - graph.min()) / (graph.max() - graph.min())
return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist() return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist()
def diffexp(self, cell_list_1, cell_list_2, pval, num_genes): def diffexp(self, cell_list_1, cell_list_2, pval, num_genes):
cells_idx_1 = np.in1d(self.data.obs["cell_name"], cell_list_1) cells_idx_1 = np.in1d(self.data.obs["cell_name"], cell_list_1)
cells_idx_2 = np.in1d(self.data.obs["cell_name"], cell_list_2) cells_idx_2 = np.in1d(self.data.obs["cell_name"], cell_list_2)
expression_1 = self.data.X[cells_idx_1, :] expression_1 = self.data.X[cells_idx_1, :]
expression_2 = self.data.X[cells_idx_2, :] expression_2 = self.data.X[cells_idx_2, :]
diff_exp = stats.ttest_ind(expression_1, expression_2) diff_exp = stats.ttest_ind(expression_1, expression_2)
set1 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic > 0) set1 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic > 0)
set2 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic < 0) set2 = np.logical_and(diff_exp.pvalue < pval, diff_exp.statistic < 0)
stat1 = diff_exp.statistic[set1] stat1 = diff_exp.statistic[set1]
stat2 = diff_exp.statistic[set2] stat2 = diff_exp.statistic[set2]
sort_set1 = np.argsort(stat1)[::-1] sort_set1 = np.argsort(stat1)[::-1]
sort_set2 = np.argsort(stat2) sort_set2 = np.argsort(stat2)
pval1 = diff_exp.pvalue[set1][sort_set1] pval1 = diff_exp.pvalue[set1][sort_set1]
pval2 = diff_exp.pvalue[set2][sort_set2] pval2 = diff_exp.pvalue[set2][sort_set2]
mean_ex1_set1 = np.mean(expression_1[:, set1], axis=0)[sort_set1] mean_ex1_set1 = np.mean(expression_1[:, set1], axis=0)[sort_set1]
mean_ex2_set1 = np.mean(expression_2[:, set1], axis=0)[sort_set1] mean_ex2_set1 = np.mean(expression_2[:, set1], axis=0)[sort_set1]
mean_ex1_set2 = np.mean(expression_1[:, set2], axis=0)[sort_set2] mean_ex1_set2 = np.mean(expression_1[:, set2], axis=0)[sort_set2]
mean_ex2_set2 = np.mean(expression_2[:, set2], axis=0)[sort_set2] mean_ex2_set2 = np.mean(expression_2[:, set2], axis=0)[sort_set2]
mean_diff1 = mean_ex1_set1 - mean_ex2_set1 mean_diff1 = mean_ex1_set1 - mean_ex2_set1
mean_diff2 = mean_ex1_set2 - mean_ex2_set2 mean_diff2 = mean_ex1_set2 - mean_ex2_set2
genes_cellset_1 = self.data.var_names[set1][sort_set1] genes_cellset_1 = self.data.var_names[set1][sort_set1]
genes_cellset_2 = self.data.var_names[set2][sort_set2] genes_cellset_2 = self.data.var_names[set2][sort_set2]
return { return {
"celllist1": { "celllist1": {
"topgenes": genes_cellset_1.tolist()[:num_genes], "topgenes": genes_cellset_1.tolist()[:num_genes],
"mean_expression_cellset1": mean_ex1_set1.tolist()[:num_genes], "mean_expression_cellset1": mean_ex1_set1.tolist()[:num_genes],
"mean_expression_cellset2": mean_ex2_set1.tolist()[:num_genes], "mean_expression_cellset2": mean_ex2_set1.tolist()[:num_genes],
"pval": pval1.tolist()[:num_genes], "pval": pval1.tolist()[:num_genes],
"ave_diff": mean_diff1.tolist()[:num_genes] "ave_diff": mean_diff1.tolist()[:num_genes]
}, },
"celllist2": { "celllist2": {
"topgenes": genes_cellset_2.tolist()[:num_genes], "topgenes": genes_cellset_2.tolist()[:num_genes],
"mean_expression_cellset1": mean_ex1_set2.tolist()[:num_genes], "mean_expression_cellset1": mean_ex1_set2.tolist()[:num_genes],
"mean_expression_cellset2": mean_ex2_set2.tolist()[:num_genes], "mean_expression_cellset2": mean_ex2_set2.tolist()[:num_genes],
"pval": pval2.tolist()[:num_genes], "pval": pval2.tolist()[:num_genes],
"ave_diff": mean_diff2.tolist()[:num_genes] "ave_diff": mean_diff2.tolist()[:num_genes]
}, },
} }
def expression(self, cells=None, genes=None): def expression(self, cells=None, genes=None):
""" """
:param df: :param df:
:return: :return:
""" """
if cells: if cells:
cells_idx = np.in1d(self.data.obs["cell_name"], cells) cells_idx = np.in1d(self.data.obs["cell_name"], cells)
else: else:
cells_idx = np.ones((self.cell_count,), dtype=bool) cells_idx = np.ones((self.cell_count,), dtype=bool)
if genes: if genes:
genes_idx = np.in1d(self.data.var_names, genes) genes_idx = np.in1d(self.data.var_names, genes)
else: else:
genes_idx = np.ones((self.gene_count,), dtype=bool) genes_idx = np.ones((self.gene_count,), dtype=bool)
index = np.ix_(cells_idx, genes_idx) index = np.ix_(cells_idx, genes_idx)
expression = self.data.X[index] expression = self.data.X[index]
if not genes: if not genes:
genes = self.data.var.index.tolist() genes = self.data.var.index.tolist()
if not cells: if not cells:
cells = self.data.obs["cell_name"].tolist() cells = self.data.obs["cell_name"].tolist()
cell_data = [] cell_data = []
for idx, cell in enumerate(cells): for idx, cell in enumerate(cells):
cell_data.append({ cell_data.append({
"cellname": cell, "cellname": cell,
"e": list(expression[idx]), "e": list(expression[idx]),
}) })
return { return {
"genes": genes, "genes": genes,
"cells": cell_data, "cells": cell_data,
"nonzero_gene_count": int(np.sum(expression.any(axis=0))) "nonzero_gene_count": int(np.sum(expression.any(axis=0)))
} }