fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-28 02:08:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

hosted gene sets routes, plus a few bug fixes (#2155)

Browse Source 
* first cut at hosted gs routes

* lint

* update tests to match csv parser changes

* update tests to new API

* update gene set name validation rules to match requirements

* add path mapping from dataset to geneset

* add test cases for geneset GET route

* fix test assertion

* remove debugging code

* update gene set uri mapping function

* fix error message

* allow extra user-specified headers in gene set csv file

* clarify comment
This commit is contained in:
Bruce Martin
2021-04-27 13:58:58 -07:00
committed by GitHub
parent ebeb1c8818
commit f2e9aecebe
19 changed files with 702 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/fixtures/pbmc3k-genesets.csv
Vendored
+5 -5
View File
@@ -1,10 +1,10 @@
# Test fixture
gene_set_name, gene_set_description, gene_symbol, gene_description
gene_set_name,gene_set_description,gene_symbol,gene_description
first gene set name,,F5, a gene_description
first gene set name,a description, NO_SUCH_GENE, non-existent gene
first gene set name,a description, F5, duplicate gene
first gene set name, a description, SUMO3,
first gene set name,, SRM,
first gene set name,a description,NO_SUCH_GENE, non-existent gene
first gene set name,a description,F5, duplicate gene
first gene set name, a description,SUMO3,
first gene set name,,SRM,
second gene set,,RER1
second gene set,,SIK1
third gene set,,NO_SUCH_GENE
1 # Test fixture
2 gene_set_name, gene_set_description, gene_symbol, gene_description gene_set_name,gene_set_description,gene_symbol,gene_description
3 first gene set name,,F5, a gene_description
4 first gene set name,a description, NO_SUCH_GENE, non-existent gene first gene set name,a description,NO_SUCH_GENE, non-existent gene
5 first gene set name,a description, F5, duplicate gene first gene set name,a description,F5, duplicate gene
6 first gene set name, a description, SUMO3, first gene set name, a description,SUMO3,
7 first gene set name,, SRM, first gene set name,,SRM,
8 second gene set,,RER1
9 second gene set,,SIK1
10 third gene set,,NO_SUCH_GENE
backend/test/test_czi_hosted/unit/__init__.py
+19 -3
View File
@@ -48,7 +48,14 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),)
annotations = AnnotationsHostedTileDB(
{
"user-annotations": True,
"genesets-save": False,
},
tmp_dir,
DbUtils("postgresql://postgres:test_pw@localhost:5432"),
)
return data, tmp_dir, annotations
@@ -70,12 +77,21 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
single_dataset__datapath=data_locator.path,
)
config.update_default_dataset_config(
embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
embeddings__names=["umap"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsLocalFile(None, annotations_file)
annotations = AnnotationsLocalFile(
{
"user-annotations": True,
"genesets-save": False,
},
None,
annotations_file,
)
return data, tmp_dir, annotations
backend/test/test_czi_hosted/unit/common/test_api.py
+177 -1
View File
@@ -3,6 +3,7 @@ import time
import unittest
import zlib
from http import HTTPStatus
import hashlib
import pandas as pd
import requests
@@ -315,6 +316,95 @@ class EndPoints(object):
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def test_genesets_config(self):
result = self.session.get(f"{self.URL_BASE}config")
config_data = result.json()
params = config_data["config"]["parameters"]
annotations_genesets = params["annotations_genesets"]
annotations_genesets_readonly = params["annotations_genesets_readonly"]
annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
self.assertTrue(annotations_genesets)
self.assertTrue(annotations_genesets_readonly)
self.assertEqual(annotations_genesets_summary_methods, ["mean"])
def test_get_genesets(self):
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertIsNotNone(result_data["genesets"])
def test_get_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "summarize/var"
# single column
filter = f"var:{index_col_name}=F5"
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
# multi-column
col_names = ["F5", "BEB3", "SIK1"]
filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
def test_post_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "summarize/var"
headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"}
# single column
filter = f"var:{index_col_name}=F5"
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
result = self.session.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
# multi-column
col_names = ["F5", "BEB3", "SIK1"]
filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
result = self.session.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
def _setupClass(child_class, command_line):
child_class.ps, child_class.server = start_test_server(command_line)
child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
@@ -333,7 +423,8 @@ class EndPointsAnnotations(EndPoints):
def test_get_user_annotations_existing_obs_keys_fbs(self):
self._test_get_user_annotations_obs_keys_fbs(
"cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
"cluster-test",
{"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
)
def test_put_user_annotations_obs_fbs(self):
@@ -416,6 +507,91 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
def tearDownClass(cls):
stop_test_server(cls.ps)
def test_get_genesets_json(self):
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertIsNotNone(result_data["genesets"])
self.assertIsNotNone(result_data["tid"])
self.assertEqual(
result_data,
{
"genesets": [
{
"genes": [
{"gene_description": " a gene_description", "gene_symbol": "F5"},
{"gene_description": "", "gene_symbol": "SUMO3"},
{"gene_description": "", "gene_symbol": "SRM"},
],
"geneset_description": "a description",
"geneset_name": "first gene set name",
},
{
"genes": [
{"gene_description": "", "gene_symbol": "RER1"},
{"gene_description": "", "gene_symbol": "SIK1"},
],
"geneset_description": "",
"geneset_name": "second gene set",
},
{"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
{"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
{"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
{
"genes": [
{"gene_description": "", "gene_symbol": "ACD"},
{"gene_description": "", "gene_symbol": "AATF"},
{"gene_description": "", "gene_symbol": "F5"},
{"gene_description": "", "gene_symbol": "PIGU"},
],
"geneset_description": "",
"geneset_name": "summary test",
},
],
"tid": 0,
},
)
def test_get_genesets_csv(self):
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "text/csv"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "text/csv")
self.assertEqual(
result.text,
"""gene_set_name,gene_set_description,gene_symbol,gene_description\r
first gene set name,a description,F5, a gene_description\r
first gene set name,a description,SUMO3,\r
first gene set name,a description,SRM,\r
second gene set,,RER1,\r
second gene set,,SIK1,\r
third gene set,,,\r
fourth_gene_set,fourth description,,\r
fifth_dataset,,,\r
summary test,,ACD,\r
summary test,,AATF,\r
summary test,,F5,\r
summary test,,PIGU,\r
""",
)
def test_put_genesets(self):
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
test1 = {"tid": 3, "genesets": []}
result = self.session.put(url, json=test1)
self.assertEqual(result.status_code, HTTPStatus.METHOD_NOT_ALLOWED)
class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
"""Test Case for endpoints"""
backend/test/test_server/unit/common/test_api.py
+2 -2
View File
@@ -560,7 +560,7 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
"genesets": [
{
"genes": [
{"gene_description": "a gene_description", "gene_symbol": "F5"},
{"gene_description": " a gene_description", "gene_symbol": "F5"},
{"gene_description": "", "gene_symbol": "SUMO3"},
{"gene_description": "", "gene_symbol": "SRM"},
],
@@ -602,7 +602,7 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
self.assertEqual(
result.text,
"""gene_set_name,gene_set_description,gene_symbol,gene_description\r
first gene set name,a description,F5,a gene_description\r
first gene set name,a description,F5, a gene_description\r
first gene set name,a description,SUMO3,\r
first gene set name,a description,SRM,\r
second gene set,,RER1,\r