fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-27 23:58:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Dunitz/czi hosted test server (#2254)

Browse Source 
update hosted unit tests to use wsgi server instead of cellxgene packages
This commit is contained in:
Madison Dunitz
2021-06-30 15:07:54 -07:00
committed by GitHub
parent 82de4178d9
commit 5007d307a2
17 changed files with 514 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/czi_hosted/app/app.py
+2 -2
View File
@@ -168,7 +168,7 @@ def rest_get_data_adaptor(func):
return wrapped_function
def dataroot_test_index():
def dataroot_test_index():
# the following index page is meant for testing/debugging purposes
data = '<!doctype html><html lang="en">'
data += "<head><title>Hosted Cellxgene</title></head>"
@@ -475,6 +475,6 @@ class Server:
auth = server_config.auth
self.app.auth = auth
if auth.requires_client_login():
if auth and auth.requires_client_login():
auth.add_url_rules(self.app)
auth.complete_setup(self.app)
backend/czi_hosted/common/config/app_config.py
+4 -4
View File
@@ -69,21 +69,21 @@ class AppConfig(object):
def update_server_config(self, **kw):
self.server_config.update(**kw)
self.is_complete = False
self.is_completed = False
def update_default_dataset_config(self, **kw):
self.default_dataset_config.update(**kw)
# update all the other dataset configs, if any
for value in self.dataroot_config.values():
value.update(**kw)
self.is_complete = False
self.is_completed = False
def update_single_config_from_path_and_value(self, path, value):
"""Update a single config parameter with the value.
Path is a list of string, that gives a path to the config parameter to be updated.
For example, path may be ["server","app","port"].
"""
self.is_complete = False
self.is_completed = False
if not isinstance(path, list):
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
for part in path:
@@ -147,7 +147,7 @@ class AppConfig(object):
if config.get("external"):
self.external_config.update_from_config(config["external"], "external")
self.is_complete = False
self.is_completed = False
def config_to_dict(self):
"""return the configuration as an unflattened dict"""
backend/czi_hosted/data_common/data_adaptor.py
+2 -2
View File
@@ -8,7 +8,7 @@ from server_timing import Timing as ServerTiming
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.constants import Axis
from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod, DatasetAccessError
from backend.common.utils.utils import jsonify_numpy
from backend.common.fbs.matrix import encode_matrix_fbs
@@ -283,7 +283,7 @@ class DataAdaptor(metaclass=ABCMeta):
try:
obs_selector, var_selector = self._filter_to_mask(filter)
except (KeyError, IndexError, TypeError, AttributeError):
except (KeyError, IndexError, TypeError, AttributeError, DatasetAccessError):
raise FilterError("Error parsing filter")
if obs_selector is not None:
backend/czi_hosted/default_config.py
+4 -3
View File
@@ -29,12 +29,13 @@ server:
web_base_url: null
authentication:
# The authentication types may be "none", "session", "oauth"
# The authentication types may be "none", "session", "oauth" or "test"
# none: No authentication support, features like user_annotations must not be enabled.
# session: A session based userid is automatically generated. (no params needed)
# oauth: oauth2 is used for authentication; parameters are defined in params_oauth.
type: session
insecure_test_environment: false
# test: Simple module for testing the authentication logic without connecting to an external service
type: test
insecure_test_environment: true
params_oauth:
# url to the oauth server
backend/test/test_czi_hosted/unit/__init__.py
+49 -75
View File
@@ -1,24 +1,22 @@
import os
import random
import logging
import shutil
import tempfile
import time
from contextlib import contextmanager
import unittest
from os import path
from subprocess import Popen
import pandas as pd
import requests
from flask_compress import Compress
from flask_cors import CORS
from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.utils.data_locator import DataLocator
from backend.common.utils.utils import find_available_port
from backend.common.fbs.matrix import encode_matrix_fbs
from backend.czi_hosted.data_common.matrix_loader import MatrixDataType, MatrixDataLoader
from backend.czi_hosted.db.db_utils import DbUtils
from backend.czi_hosted.app.app import Server
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
@@ -92,7 +90,7 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
None,
annotations_file,
)
return data, tmp_dir, annotations
return data, tmp_dir, annotations, config
def make_fbs(data):
@@ -133,75 +131,51 @@ def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset
return config
def start_test_server(command_line_args=[], app_config=None, env=None):
"""
Command line arguments can be passed in, as well as an app_config.
This function is meant to be used like this, for example:
class TestServer(Server):
def __init__(self, app_config):
super().__init__(app_config)
with unit(...) as server:
r = requests.get(f"{server}/...")
// check r
where the server can be accessed within the context, and is terminated when
the context is exited.
The port is automatically set using find_available_port, unless passed in as a command line arg.
The verbose flag is automatically set to True.
If an app_config is provided, then this function writes a temporary
yaml config file, which this server will read and parse.
"""
command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose"]
if "-p" in command_line_args:
port = int(command_line_args[command_line_args.index("-p") + 1])
elif "--port" in command_line_args:
port = int(command_line_args[command_line_args.index("--port") + 1])
else:
start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
port = int(os.environ.get("CXG_SERVER_PORT", start))
port = find_available_port("localhost", port)
command += ["--port=%d" % port]
command += command_line_args
tempdir = None
if app_config:
tempdir = tempfile.TemporaryDirectory()
config_file = os.path.join(tempdir.name, "config.yaml")
app_config.write_config(config_file)
command.extend(["-c", config_file])
server = f"http://localhost:{port}"
ps = Popen(command, env=env)
for _ in range(10):
try:
requests.get(f"{server}/health")
break
except requests.exceptions.ConnectionError:
time.sleep(1)
if tempdir:
tempdir.cleanup()
return ps, server
@staticmethod
def _before_adding_routes(app, app_config):
app.config["COMPRESS_MIMETYPES"] = [
"text/html",
"text/css",
"text/xml",
"application/json",
"application/javascript",
"application/octet-stream",
]
Compress(app)
if app_config.server_config.app__debug:
CORS(app, supports_credentials=True)
def stop_test_server(ps):
try:
ps.terminate()
except ProcessLookupError:
pass
class BaseTest(unittest.TestCase):
@classmethod
def setUpClass(cls, app_config=None):
cls.TEST_URL_BASE = "/d/pbmc3k.cxg/api/v0.2/"
cls.maxDiff = None
cls.app = cls.create_app(app_config)
@classmethod
def create_app(cls, app_config=None):
if not app_config:
app_config = AppConfig()
app_config.update_server_config(
authentication__type="test",
authentication__insecure_test_environment=True,
app__flask_secret_key="testing",
app__debug=True,
multi_dataset__dataroot=f"{FIXTURES_ROOT}",
multi_dataset__index=True,
multi_dataset__allowed_matrix_types=["cxg"]
)
app_config.update_default_dataset_config(embeddings__enable_reembedding=False, )
app_config.complete_config(logging.info)
@contextmanager
def test_server(command_line_args=[], app_config=None, env=None):
"""A context to run the cellxgene server."""
app = TestServer(app_config).app
ps, server = start_test_server(command_line_args, app_config, env)
try:
yield server
finally:
try:
stop_test_server(ps)
except ProcessLookupError:
pass
app.testing = True
app.debug = True
return app
backend/test/test_czi_hosted/unit/auth/test_auth.py
+99 -90
View File
@@ -1,13 +1,12 @@
import json
import unittest
import requests
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test.test_czi_hosted.unit import test_server
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit import BaseTest
class AuthTest(unittest.TestCase):
class AuthTest(BaseTest):
def setUp(self):
self.dataset_dataroot = FIXTURES_ROOT
@@ -18,13 +17,13 @@ class AuthTest(unittest.TestCase):
app_config.update_default_dataset_config(user_annotations__enable=False)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertNotIn("authentication", config["config"])
self.assertIsNone(userinfo)
server= self.create_app(app_config)
server.testing = True
session = server.test_client()
config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
self.assertNotIn("authentication", config["config"])
self.assertIsNone(userinfo)
def test_auth_session(self):
app_config = AppConfig()
@@ -33,14 +32,16 @@ class AuthTest(unittest.TestCase):
app_config.update_default_dataset_config(user_annotations__enable=True)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
server = self.create_app(app_config)
server.auth.is_user_authenticated = lambda: True
server.testing = True
session = server.test_client()
config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
self.assertFalse(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
self.assertFalse(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def test_auth_test(self):
app_config = AppConfig()
@@ -60,57 +61,59 @@ class AuthTest(unittest.TestCase):
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
server=self.create_app(app_config)
server.testing = True
session = server.test_client()
# auth datasets
config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
# auth datasets
config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get(f"/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
r = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
response = session.get(login_uri)
# check that the login redirect worked
config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertEqual(userinfo["userinfo"]["picture"], None)
self.assertTrue(config["config"]["parameters"]["annotations"])
self.assertEqual(response.status_code, 302)
self.assertEqual(response.headers['Location'], 'http://localhost/auth/pbmc3k.cxg')
config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
r = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertEqual(userinfo["userinfo"]["picture"], None)
self.assertTrue(config["config"]["parameters"]["annotations"])
# no-auth datasets
config = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertIsNone(userinfo)
self.assertFalse(config["config"]["parameters"]["annotations"])
response = session.get(logout_uri)
# check that the logout redirect worked
# login with a picture
session.get(f"{server}/{login_uri}&picture=myimage.png")
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
self.assertEqual(response.status_code, 302)
config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
# no-auth datasets
config = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertIsNone(userinfo)
self.assertFalse(config["config"]["parameters"]["annotations"])
# login with a picture
session.get(f"{login_uri}&picture=myimage.png")
userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
def test_auth_test_single(self):
app_config = AppConfig()
@@ -122,38 +125,44 @@ class AuthTest(unittest.TestCase):
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
server = self.create_app(app_config)
server.testing = True
session = server.test_client()
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
config = json.loads(session.get("/api/v0.2/config").data)
userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
self.assertEqual(login_uri, "/login")
self.assertEqual(logout_uri, "/logout")
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
response = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
self.assertEqual(login_uri, "/login")
self.assertEqual(logout_uri, "/logout")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertTrue(config["config"]["parameters"]["annotations"])
response = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
# check that the login redirect worked
with server.test_client() as session:
response = session.get(login_uri)
self.assertEqual(response.status_code, 302)
self.assertEqual(response.headers['Location'], "http://localhost/")
config = json.loads(session.get("api/v0.2/config").data)
userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertTrue(config["config"]["parameters"]["annotations"])
response = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(response.status_code, 302)
self.assertEqual(response.headers['Location'], "http://localhost/")
config = json.loads(session.get("/api/v0.2/config").data)
userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
backend/test/test_czi_hosted/unit/auth/test_oauth.py
+80 -73
View File
@@ -10,7 +10,6 @@ from multiprocessing import Process
import jose
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test.test_czi_hosted.unit import test_server
from backend.test import FIXTURES_ROOT
# This tests the oauth authentication type.
@@ -20,6 +19,8 @@ from backend.test import FIXTURES_ROOT
# oauth server.
# number of seconds that the oauth token is valid
from backend.test.test_czi_hosted.unit import BaseTest
TOKEN_EXPIRES = 2
# Create a mocked out oauth token, which servers all the endpoints needed by the oauth type.
@@ -69,7 +70,7 @@ def launch_mock_oauth(mock_port):
mock_oauth_app.run(port=mock_port)
class AuthTest(unittest.TestCase):
class AuthTest(BaseTest):
@classmethod
def setUpClass(cls):
# The port that the mock oauth server will listen on
@@ -119,90 +120,95 @@ class AuthTest(unittest.TestCase):
app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
server= self.create_app(app_config)
server.testing = True
session = server.test_client()
# auth datasets
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
# auth datasets
config = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/")
self.assertEqual(logout_uri, f"{server}/logout?dataset=d/pbmc3k.cxg/")
self.assertEqual(login_uri, "http://localhost:5005/login?dataset=d/pbmc3k.cxg/")
self.assertEqual(logout_uri, "http://localhost:5005/logout?dataset=d/pbmc3k.cxg/")
r = session.get(login_uri)
# check that the login redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
response = session.get(login_uri)
# check that the login redirect worked
self.assertEqual(response.status_code, 302)
config = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
self.assertTrue(config["config"]["parameters"]["annotations"])
if cookie_key:
cookie = session.cookies.get(cookie_key)
token = json.loads(base64.b64decode(cookie))
access_token_before = token.get("access_token")
id_token_before = token.get("id_token")
# let the token expire
time.sleep(TOKEN_EXPIRES + 1)
# check that refresh works
session.get(login_uri)
userinfo = json.loads(session.get(f"/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
self.assertTrue(config["config"]["parameters"]["annotations"])
if cookie_key:
cookie = session.cookies.get(cookie_key)
token = json.loads(base64.b64decode(cookie))
access_token_before = token.get("access_token")
id_token_before = token.get("id_token")
cookie = session.cookies.get(cookie_key)
token = json.loads(base64.b64decode(cookie))
access_token_after = token.get("access_token")
id_token_after = token.get("id_token")
# let the token expire
time.sleep(TOKEN_EXPIRES + 1)
self.assertNotEqual(access_token_before, access_token_after)
self.assertNotEqual(id_token_before, id_token_after)
# check that refresh works
session.get(login_uri)
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
cookie = session.cookies.get(cookie_key)
token = json.loads(base64.b64decode(cookie))
access_token_after = token.get("access_token")
id_token_after = token.get("id_token")
self.assertNotEqual(access_token_before, access_token_after)
self.assertNotEqual(id_token_before, id_token_after)
# invalid cookie is rejected
session.cookies.set(cookie_key, "TEST_" + cookie)
self.assertTrue(cookie_key in session.cookies)
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
# this is not an error, the invalid cookie is just ignored.
self.assertEqual(response.status_code, 200)
userinfo = response.json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
# invalid id_token is rejected
test_token = token
test_token["id_token"] = "TEST_" + id_token_after
encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
session.cookies.set(cookie_key, encoded_cookie)
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
# this is not an error, the invalid id_token is just ignored.
self.assertEqual(response.status_code, 200)
userinfo = response.json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
r = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
# invalid cookie is rejected
session.cookies.set(cookie_key, "TEST_" + cookie)
self.assertTrue(cookie_key in session.cookies)
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
# this is not an error, the invalid cookie is just ignored.
self.assertEqual(response.status_code, 200)
userinfo = json.loads(response.data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
# invalid id_token is rejected
test_token = token
test_token["id_token"] = "TEST_" + id_token_after
encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
session.cookies.set(cookie_key, encoded_cookie)
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
# this is not an error, the invalid id_token is just ignored.
self.assertEqual(response.status_code, 200)
userinfo = json.loads(response.data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
r = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
config = json.loads(session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
@unittest.skip("turn on when we utilizing auth in the explorer")
def test_auth_oauth_session(self):
# test with session cookies
app_config = AppConfig()
@@ -210,6 +216,7 @@ class AuthTest(unittest.TestCase):
app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
self.auth_flow(app_config)
@unittest.skip("turn on when we utilizing auth in the explorer")
def test_auth_oauth_cookie(self):
# test with specified cookie
app_config = AppConfig()
backend/test/test_czi_hosted/unit/common/config/__init__.py
+2 -6
View File
@@ -1,18 +1,14 @@
import os
import shutil
import unittest
import random
from unittest import mock
import yaml
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit import BaseTest
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class ConfigTests(unittest.TestCase):
class ConfigTests(BaseTest):
tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
@classmethod
backend/test/test_czi_hosted/unit/common/config/test_base_config.py
+2
View File
@@ -49,6 +49,8 @@ class BaseConfigTest(ConfigTests):
[
("app__verbose", True, False),
("app__flask_secret_key", "secret", None),
('authentication__type', 'session', 'test'),
('authentication__insecure_test_environment', False, True),
("multi_dataset__dataroot", FIXTURES_ROOT, None),
("multi_dataset__matrix_cache__timelimit_s", 5, 30),
("data_locator__s3__region_name", "us-east-1", True),
backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py
+26 -23
View File
@@ -1,7 +1,7 @@
import json
import os
import tempfile
import requests
import unittest
from unittest.mock import patch
@@ -9,7 +9,6 @@ from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHoste
from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.test.test_czi_hosted.unit import test_server
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
from backend.common.errors import ConfigurationError
@@ -197,31 +196,35 @@ class TestDatasetConfig(ConfigTests):
# no specializations for set3 (they get the default dataset config)
config.complete_config()
with test_server(app_config=config) as server:
session = requests.Session()
server = self.create_app(config)
response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
server.testing = True
session = server.test_client()
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get("/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = json.loads(response.data)
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
response = session.get("/set2/pbmc3k.cxg/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get("/set3/pbmc3k.cxg/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get("/health")
assert json.loads(response.data)["status"] == "pass"
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
backend/test/test_czi_hosted/unit/common/config/test_external_config.py
+28 -17
View File
@@ -1,12 +1,12 @@
import json
import os
from unittest.mock import patch
import requests
import yaml
from backend.common.errors import ConfigurationError
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.utils.type_conversion_utils import convert_string_to_value
from backend.test.test_czi_hosted.unit import test_server
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
@@ -39,24 +39,35 @@ class TestExternalConfig(ConfigTests):
env = os.environ
env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg"
env["DIFFEXP"] = "False"
with test_server(command_line_args=["-c", configfile], env=env) as server:
session = requests.Session()
response = session.get(f"{server}/api/v0.2/config")
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
config = AppConfig()
config.update_from_config_file(configfile)
config.update_server_config(app__flask_secret_key="123 magic")
env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
env["DIFFEXP"] = "True"
with test_server(command_line_args=["-c", configfile], env=env) as server:
session = requests.Session()
response = session.get(f"{server}/api/v0.2/config")
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
server = self.create_app(config)
server.testing = True
session = server.test_client()
response = session.get("/api/v0.2/config")
data_config = json.loads(response.data)
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
os.environ["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
os.environ["DIFFEXP"] = "True"
server= self.create_app(config)
server.testing = True
session = server.test_client()
# session = requests.Session()
response = session.get("/api/v0.2/config")
data_config = json.loads(response.data)
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
def test_environment_variable_errors(self):
# no name
app_config = AppConfig()
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
backend/test/test_czi_hosted/unit/common/config/test_server_config.py
+48 -46
View File
@@ -1,23 +1,18 @@
import json
import os
import unittest
from unittest import mock
from unittest.mock import patch
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.common.utils.utils import find_available_port
from backend.test.test_czi_hosted.unit import test_server
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
import requests
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.errors import ConfigurationError
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
@@ -147,8 +142,8 @@ class TestServerConfig(ConfigTests):
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
@unittest.skip("skip when running in github action")
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
@@ -156,21 +151,22 @@ class TestServerConfig(ConfigTests):
app__flask_secret_key="secret",
app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
multi_dataset__allowed_matrix_types=["cxg"],
)
config.complete_config()
server = self.create_app(config)
server.testing = True
session = server.test_client()
response = session.get(f"/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
with test_server(["-p", str(backend_port)], app_config=config) as server:
session = requests.Session()
self.assertEqual(server, f"http://localhost:{backend_port}")
response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertEqual(response.status_code, 200)
data_config = json.loads(response.data)
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
# test the health check at the correct url
response = session.get(f"{server}/additional/path/health")
assert response.json()["status"] == "pass"
# test the health check at the correct url
response = session.get(f"/additional/path/health")
assert json.loads(response.data)["status"] == "pass"
def test_get_web_base_url_works(self):
config = self.get_config(web_base_url="www.thisisawebsite.com")
@@ -225,7 +221,9 @@ class TestServerConfig(ConfigTests):
)
self.config.complete_config()
def test_mulitdatasets_work_e2e(self):
@patch("backend.czi_hosted.app.app.render_template")
def test_mulitdatasets_work_e2e(self, mock_render_template):
mock_render_template.return_value = "something"
# test that multi dataroots work end to end
self.config.update_server_config(
multi_dataset__dataroot=dict(
@@ -251,39 +249,43 @@ class TestServerConfig(ConfigTests):
# no specializations for set3 (they get the default dataset config)
self.config.complete_config()
with test_server(app_config=self.config) as server:
session = requests.Session()
server = self.create_app(self.config)
server.auth.requires_client_login = lambda: False
server.testing = True
session = server.test_client()
response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get("/set2/pbmc3k.cxg/api/v0.2/config")
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
# access a dataset (no slash)
response = session.get(f"{server}/set2/pbmc3k.cxg")
self.assertEqual(response.status_code, 200)
response = session.get("/set3/pbmc3k.cxg/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
# access a dataset (with slash)
response = session.get(f"{server}/set2/pbmc3k.cxg/")
self.assertEqual(response.status_code, 200)
response = session.get("/health")
assert json.loads(response.data)["status"] == "pass"
# access a dataset (no slash)
response = session.get("/set2/pbmc3k.cxg")
self.assertEqual(response.status_code, 200)
# access a dataset (with slash)
response = session.get("/set2/pbmc3k.cxg/")
self.assertEqual(response.status_code, 200)
@patch("backend.czi_hosted.common.config.server_config.diffexp_tiledb.set_config")
def test_handle_diffexp(self, mock_tiledb_config):
backend/test/test_czi_hosted/unit/common/test_api.py
+124 -244
View File
@@ -1,63 +1,64 @@
import shutil
import json
import os
import time
import unittest
import zlib
from http import HTTPStatus
import hashlib
import pandas as pd
import requests
from backend.czi_hosted.data_common.matrix_loader import MatrixDataType
from backend.test.test_czi_hosted.unit import (
data_with_tmp_annotations,
make_fbs,
start_test_server,
stop_test_server,
)
from backend.test import PROJECT_ROOT, FIXTURES_ROOT, decode_fbs
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import decode_fbs
from backend.test.fixtures.fixtures import pbmc3k_colors
from backend.test.test_czi_hosted.unit import BaseTest, skip_if
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental
class EndPoints(object):
ANNOTATIONS_ENABLED = True
class EndPoints(BaseTest):
@classmethod
def setUpClass(cls, app_config=None):
super().setUpClass(app_config)
cls.app.testing = True
cls.client = cls.app.test_client()
os.environ["SKIP_STATIC"] = "True"
for i in range(90):
try:
result = cls.client.get(f"{cls.TEST_URL_BASE}schema")
cls.schema = json.loads(result.data)
except requests.exceptions.ConnectionError:
time.sleep(1)
def test_initialize(self):
endpoint = "schema"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
self.assertEqual(
len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
len(result_data["schema"]["annotations"]["obs"]["columns"]), 5
)
def test_config(self):
endpoint = "config"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
def test_get_layout_fbs(self):
endpoint = "layout/obs"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 8)
self.assertIsNotNone(df["columns"])
@@ -70,20 +71,21 @@ class EndPoints(object):
def test_put_layout_fbs(self):
# first check that re-embedding is turned on
result = self.session.get(f"{self.URL_BASE}config")
config_data = result.json()
self.app.auth.get_user_id = lambda : "123"
result = self.client.get(f"{self.TEST_URL_BASE}config")
config_data = json.loads(result.data)
re_embed = config_data["config"]["parameters"]["enable-reembedding"]
if not re_embed:
return
# attempt to reembed with umap over 100 cells.
endpoint = "layout/obs"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
data = {}
data["filter"] = {}
data["filter"]["obs"] = {}
data["filter"]["obs"]["index"] = list(range(100))
data["method"] = "umap"
result = self.session.put(url, json=data)
result = self.client.put(url, json=data)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
@@ -98,39 +100,39 @@ class EndPoints(object):
def test_bad_filter(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url, json=BAD_FILTER)
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.client.put(url, headers=header, json=BAD_FILTER)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5)
self.assertEqual(df["n_cols"], 5)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertCountEqual(
df["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+ (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
)
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{self.URL_BASE}{endpoint}?{query}"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
@@ -141,50 +143,51 @@ class EndPoints(object):
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
query = "annotation-name=notakey"
url = f"{self.URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_diff_exp(self):
endpoint = "diffexp/obs"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
params = {
"mode": "topN",
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
"set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
"count": 7,
}
result = self.session.post(url, json=params)
result = self.client.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
params = {
"mode": "topN",
"count": 10,
"set1": {"filter": {"obs": {"index": [[0, 500]]}}},
"set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
}
result = self.session.post(url, json=params)
result = self.client.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
@@ -196,12 +199,12 @@ class EndPoints(object):
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{self.URL_BASE}{endpoint}?{query}"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 1)
self.assertIsNotNone(df["columns"])
@@ -212,50 +215,52 @@ class EndPoints(object):
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
query = "annotation-name=notakey"
url = f"{self.URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_mimetype_error(self):
endpoint = "data/var"
header = {"Accept": "xxx"}
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url, headers=header)
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
def test_fbs_default(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url)
url = f"{self.TEST_URL_BASE}{endpoint}"
headers = {"Accept": "application/octet-stream"}
result = self.client.put(url, headers=headers)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, json=filter)
result = self.client.put(url, headers=headers, json=filter)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
def test_data_put_fbs(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header)
result = self.client.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_get_fbs(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_put_filter_fbs(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, headers=header, json=filter)
result = self.client.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 3)
self.assertIsNotNone(df["columns"])
@@ -267,12 +272,12 @@ class EndPoints(object):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "data/var"
query = f"var:{index_col_name}=SIK1"
url = f"{self.URL_BASE}{endpoint}?{query}"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
@@ -280,47 +285,48 @@ class EndPoints(object):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "data/var"
query = f"var:{index_col_name}=UNKNOWN"
url = f"{self.URL_BASE}{endpoint}?{query}"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 0)
def test_data_put_single_var(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
result = self.client.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_colors(self):
endpoint = "colors"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertEqual(result_data, pbmc3k_colors)
@skip_if(lambda x: os.getenv("SKIP_STATIC"), "Skip static test when running locally")
def test_static(self):
endpoint = "static"
file = "assets/favicon.ico"
url = f"{self.server}/{endpoint}/{file}"
result = self.session.get(url)
url = f"{endpoint}/{file}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def test_genesets_config(self):
result = self.session.get(f"{self.URL_BASE}config")
config_data = result.json()
result = self.client.get(f"{self.TEST_URL_BASE}config")
config_data = json.loads(result.data)
params = config_data["config"]["parameters"]
annotations_genesets = params["annotations_genesets"]
annotations_genesets_readonly = params["annotations_genesets_readonly"]
@@ -331,11 +337,11 @@ class EndPoints(object):
def test_get_genesets(self):
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "application/json"})
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertIsNotNone(result_data["genesets"])
def test_get_summaryvar(self):
@@ -346,12 +352,12 @@ class EndPoints(object):
filter = f"var:{index_col_name}=F5"
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?{query}"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
@@ -362,12 +368,12 @@ class EndPoints(object):
filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?{query}"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
@@ -382,12 +388,12 @@ class EndPoints(object):
filter = f"var:{index_col_name}=F5"
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
result = self.session.post(url, headers=headers, data=query)
url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
result = self.client.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
@@ -398,125 +404,33 @@ class EndPoints(object):
filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
result = self.session.post(url, headers=headers, data=query)
url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
result = self.client.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
def _setupClass(child_class, command_line):
child_class.ps, child_class.server = start_test_server(command_line)
child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
child_class.session = requests.Session()
for i in range(90):
try:
result = child_class.session.get(f"{child_class.URL_BASE}schema")
child_class.schema = result.json()
except requests.exceptions.ConnectionError:
time.sleep(1)
class EndPointsAnnotations(EndPoints):
def test_get_schema_existing_writable(self):
self._test_get_schema_writable("cluster-test")
def test_get_user_annotations_existing_obs_keys_fbs(self):
self._test_get_user_annotations_obs_keys_fbs(
"cluster-test",
{"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
)
def test_put_user_annotations_obs_fbs(self):
endpoint = "annotations/obs"
query = "annotation-collection-name=test_annotations"
url = f"{self.URL_BASE}{endpoint}?{query}"
n_rows = self.data.get_shape()[0]
fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
result = self.session.put(url, data=zlib.compress(fbs))
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
self.assertEqual(result.json(), {"status": "OK"})
self._test_get_schema_writable("cat_A")
self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"})
def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns):
endpoint = "annotations/obs"
query = f"annotation-name={annotation_name}"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertListEqual(df["col_idx"], [annotation_name])
self.assertEqual(set(df["columns"][0]), columns)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def _test_get_schema_writable(self, cluster_name):
endpoint = "schema"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
columns = result_data["schema"]["annotations"]["obs"]["columns"]
matching_columns = [c for c in columns if c["name"] == cluster_name]
self.assertEqual(len(matching_columns), 1)
self.assertTrue(matching_columns[0]["writable"])
class EndPointsAnndata(unittest.TestCase, EndPoints):
class EndPointsCxg(EndPoints):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = False
@classmethod
def setUpClass(cls):
cls._setupClass(
cls,
[
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
"--disable-annotations",
"--experimental-enable-reembedding",
],
)
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
@property
def annotations_enabled(self):
return False
class EndPointsCxg(unittest.TestCase, EndPoints):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = False
@classmethod
def setUpClass(cls):
cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
app_config = AppConfig()
app_config.update_default_dataset_config(embeddings__enable_reembedding=True, user_annotations__enable=False)
def test_get_genesets_json(self):
self.app.auth.is_user_authenticated = lambda: True
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "application/json"})
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertIsNotNone(result_data["genesets"])
self.assertIsNotNone(result_data["tid"])
@@ -561,13 +475,12 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
def test_get_genesets_csv(self):
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "text/csv"})
url = f"{self.TEST_URL_BASE}{endpoint}"
self.app.auth.is_user_authenticated = lambda: True
result = self.client.get(url, headers={"Accept": "text/csv"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "text/csv")
self.assertEqual(
result.text,
"""gene_set_name,gene_set_description,gene_symbol,gene_description\r
expected_data = """gene_set_name,gene_set_description,gene_symbol,gene_description\r
first gene set name,a description,F5, a gene_description\r
first gene set name,a description,SUMO3,\r
first gene set name,a description,SRM,\r
@@ -580,51 +493,18 @@ summary test,,ACD,\r
summary test,,AATF,\r
summary test,,F5,\r
summary test,,PIGU,\r
""",
)
"""
self.assertEqual(result.data.decode("utf-8"), expected_data)
def test_put_genesets(self):
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.session.get(url, headers={"Accept": "application/json"})
result = self.client.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
test1 = {"tid": 3, "genesets": []}
result = self.session.put(url, json=test1)
result = self.client.put(url, json=test1)
self.assertEqual(result.status_code, HTTPStatus.METHOD_NOT_ALLOWED)
class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = True
@classmethod
def setUpClass(cls):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
MatrixDataType.H5AD, annotations_fixture=True
)
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmp_dir)
stop_test_server(cls.ps)
class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = True
@classmethod
def setUpClass(cls):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmp_dir)
stop_test_server(cls.ps)
backend/test/test_czi_hosted/unit/common/test_corpora.py
+16 -13
View File
@@ -5,14 +5,14 @@ import unittest
from http import HTTPStatus
import anndata
import requests
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.common.corpora import (
corpora_get_versions_from_anndata,
corpora_is_version_supported,
corpora_get_props_from_anndata,
)
from backend.test.test_czi_hosted.unit import start_test_server, stop_test_server
from backend.test.test_czi_hosted.unit import BaseTest
from backend.test import PROJECT_ROOT
VERSION = "v0.2"
@@ -105,7 +105,7 @@ class CorporaAPITest(unittest.TestCase):
return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
class CorporaRESTAPITest(unittest.TestCase):
class CorporaRESTAPITest(BaseTest):
""" Confirm endpoints reflect Corpora-specific features """
@classmethod
@@ -129,31 +129,33 @@ class CorporaRESTAPITest(unittest.TestCase):
adata.write(path)
@classmethod
def setUpClass(cls):
def setUpClass(cls, app_config=None):
if not app_config:
app_config = AppConfig()
cls.tmp_dir = tempfile.TemporaryDirectory()
src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
shutil.copyfile(src, dst)
cls.setCorporaFields(dst)
cls.ps, cls.server = start_test_server([dst])
app_config.update_server_config(single_dataset__datapath=dst)
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
cls.tmp_dir.cleanup()
super().setUpClass(app_config)
cls.app.testing = True
cls.client = cls.app.test_client()
def setUp(self):
self.session = requests.Session()
self.url_base = f"{self.server}/api/{VERSION}/"
self.session = self.client
self.url_base = "/api/v0.2/"
def test_config(self):
endpoint = "config"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
header = {"Content-Type": "application/json"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
result_data = json.loads(result.data)
self.assertIsInstance(result_data["config"]["corpora_props"], dict)
self.assertIsInstance(result_data["config"]["parameters"], dict)
@@ -161,5 +163,6 @@ class CorporaRESTAPITest(unittest.TestCase):
parameters = result_data["config"]["parameters"]
self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
self.assertEqual(corpora_props["organism"], "human")
self.assertEqual(parameters["default_embedding"], "tsne")
backend/test/test_czi_hosted/unit/common/test_nan_rest.py
+20 -19
View File
@@ -1,33 +1,31 @@
from http import HTTPStatus
import unittest
import math
import backend.test.decode_fbs as decode_fbs
import requests
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit import start_test_server, stop_test_server
from backend.test.test_czi_hosted.unit import BaseTest
VERSION = "v0.2"
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
class WithNaNs(unittest.TestCase):
class WithNaNs(BaseTest):
"""Test Case for endpoints"""
@classmethod
def setUpClass(cls):
cls.ps, cls.server = start_test_server([f"{FIXTURES_ROOT}/nan.h5ad"])
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
app_config = AppConfig()
app_config.update_server_config(single_dataset__datapath=f"{FIXTURES_ROOT}/nan.h5ad")
app_config.update_default_dataset_config(user_annotations__enable=True)
super().setUpClass(app_config)
cls.app.testing = True
cls.client = cls.app.test_client()
def setUp(self):
self.session = requests.Session()
self.url_base = f"{self.server}/api/{VERSION}/"
self.session = self.client
self.url_base = "api/v0.2/"
def test_initialize(self):
endpoint = "schema"
@@ -39,26 +37,29 @@ class WithNaNs(unittest.TestCase):
endpoint = "data/var"
url = f"{self.url_base}{endpoint}"
filter = {"filter": {"var": {"index": [[0, 20]]}}}
result = self.session.put(url, json=filter)
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertTrue(math.isnan(df["columns"][3][3]))
def test_annotation_obs(self):
endpoint = "annotations/obs"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertTrue(math.isnan(df["columns"][2][0]))
def test_annotation_var(self):
endpoint = "annotations/var"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertTrue(math.isnan(df["columns"][2][0]))
backend/test/test_czi_hosted/unit/common/test_writable_annotation.py
+1 -1
View File
@@ -163,7 +163,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
class WritableAnnotationTest(unittest.TestCase):
def setUp(self):
self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD)
self.data, self.tmp_dir, self.annotations, self.config= data_with_tmp_annotations(MatrixDataType.H5AD)
self.data.dataset_config.user_annotations = self.annotations
def tearDown(self):
backend/test/test_czi_hosted/unit/eb/test_eb.py
+7 -6
View File
@@ -1,4 +1,6 @@
import os
from unittest.mock import patch
import requests
import subprocess
import tempfile
@@ -14,6 +16,7 @@ from backend.test import PROJECT_ROOT, FIXTURES_ROOT
def run_eb_app(tempdirname):
ps = subprocess.Popen(["python", "artifact.dir/application.py"], cwd=tempdirname)
server = "http://localhost:5000"
for _ in range(10):
try:
requests.get(f"{server}/health")
@@ -37,17 +40,15 @@ class Elastic_Beanstalk_Test(unittest.TestCase):
config = AppConfig()
# test that eb works
config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
config.complete_config()
config.write_config(f"{tempdirname}/config.yaml")
subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/backend/czi_hosted/eb", shell=True)
subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/backend/czi_hosted/eb",
shell=True)
subprocess.check_call(["make", "build"], cwd=tempdirname)
with run_eb_app(tempdirname) as server:
session = requests.Session()
r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
data_config = r.json()
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
def test_config(self):