From a63bf9d5a35d5e74494c8f64ea2e019ea65eb765 Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Wed, 16 Sep 2020 14:46:59 -0700
Subject: [PATCH 01/73] Change psycopg to be binary (#1842)
---
.github/workflows/compatibility_tests.yml | 1 +
server/requirements-dev.txt | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml
index bd3c41e8..f0e55f38 100644
--- a/.github/workflows/compatibility_tests.yml
+++ b/.github/workflows/compatibility_tests.yml
@@ -25,6 +25,7 @@ jobs:
cellxgene-main-with-python-and-anndata-versions:
name: python versions x anndata versions
runs-on: ubuntu-latest
+ continue-on-error: true
strategy:
matrix:
python-version: [3.6, 3.7, 3.8]
diff --git a/server/requirements-dev.txt b/server/requirements-dev.txt
index 0fb57ce8..0882128a 100644
--- a/server/requirements-dev.txt
+++ b/server/requirements-dev.txt
@@ -3,7 +3,7 @@ black
bumpversion>=0.5
codecov>=2.0.15
parameterized>=0.7.0
-psycopg2==2.7.7
+psycopg2-binary>=2.8.5
pytest>=3.6.3
python-jose>=3.2.0
scanpy>=1.4.6
From 25c272ae8e33edab808f369c2bc65d0cc7a08342 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Wed, 16 Sep 2020 17:37:52 -0700
Subject: [PATCH 02/73] minor fix to auth redirect (#1845)
The previous version added and extra "/" to the url after login:
e.g: https://cellxgene.dev.single-cell.czi.technology/d/pbmc3k.cxg//
---
server/auth/auth_oauth.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py
index 2b4ddf45..a5107934 100644
--- a/server/auth/auth_oauth.py
+++ b/server/auth/auth_oauth.py
@@ -158,7 +158,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
def login(self):
callbackurl = f"{self.api_base_url}/oauth2/callback"
return_path = request.args.get("dataset", "")
- return_to = f"{self.web_base_url}/{return_path}/"
+ return_to = f"{self.web_base_url}/{return_path}"
# save the return path in the session cookie, accessed in the callback function
session["oauth_callback_redirect"] = return_to
response = self.client.authorize_redirect(redirect_uri=callbackurl)
From 14fbe0aa77a50d8a9a78ed2608cb2e19d9810bb9 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Thu, 17 Sep 2020 17:14:08 -0700
Subject: [PATCH 03/73] Fix the /health endpoint (#1847)
* Fix the /health endpoint
#1846
Keep both the old and new locations until the deployments are upgraded.
---
server/app/app.py | 38 ++++++++++++++++++----
server/test/unit/common/test_app_config.py | 14 ++++++--
2 files changed, 44 insertions(+), 8 deletions(-)
diff --git a/server/app/app.py b/server/app/app.py
index d926f515..7e02b4e1 100644
--- a/server/app/app.py
+++ b/server/app/app.py
@@ -99,6 +99,10 @@ def dataset_index(url_dataroot=None, dataset=None):
)
+# TODO: This route will be deprecated, but needs to be left for a short time until all the
+# deployments are upgraded to the new location for the health check (or else the upgrade will
+# fail). Once the upgrade is complete, the deployments can move to the new health check URL
+# and this route will be removed.
@webbp.route("/health", methods=["GET"])
@cache_control_always(no_store=True)
def health():
@@ -224,6 +228,13 @@ def dataroot_index():
return redirect(config.server_config.multi_dataset__index)
+class HealthAPI(Resource):
+ @cache_control(no_store=True)
+ def get(self):
+ config = current_app.app_config
+ return health_check(config)
+
+
class DatasetResource(Resource):
"""Base class for all Resources that act on datasets."""
@@ -312,8 +323,18 @@ class LayoutObsAPI(DatasetResource):
return common_rest.layout_obs_put(request, data_adaptor)
-def get_api_resources(bp_api, url_dataroot=None):
- api = Api(bp_api)
+def get_api_base_resources(bp_base):
+ """Add resources that are accessed from the api_base_url"""
+ api = Api(bp_base)
+
+ # Diagnostics routes
+ api.add_resource(HealthAPI, "/health")
+ return api
+
+
+def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
+ """Add resources that refer to a dataset"""
+ api = Api(bp_dataroot)
def add_resource(resource, url):
"""convenience function to make the outer function less verbose"""
@@ -385,18 +406,23 @@ class Server:
parse = urlparse(api_base_url)
api_path = parse.path
+ bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
+ base_resources = get_api_base_resources(bp_base)
+ self.app.register_blueprint(base_resources.blueprint)
+
if app_config.is_multi_dataset():
# NOTE: These routes only allow the dataset to be in the directory
# of the dataroot, and not a subdirectory. We may want to change
# the route format at some point
for dataroot_dict in server_config.multi_dataset__dataroot.values():
url_dataroot = dataroot_dict["base_url"]
- bp_api = Blueprint(
+ bp_dataroot = Blueprint(
f"api_dataset_{url_dataroot}", __name__,
url_prefix=f"{api_path}/{url_dataroot}/" + api_version
)
- resources = get_api_resources(bp_api, url_dataroot)
- self.app.register_blueprint(resources.blueprint)
+ dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
+ self.app.register_blueprint(dataroot_resources.blueprint)
+
self.app.add_url_rule(
f"/{url_dataroot}//",
f"dataset_index_{url_dataroot}",
@@ -412,7 +438,7 @@ class Server:
else:
bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
- resources = get_api_resources(bp_api)
+ resources = get_api_dataroot_resources(bp_api)
self.app.register_blueprint(resources.blueprint)
self.app.add_url_rule(
"/static/",
diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py
index f8a20093..6b86d771 100644
--- a/server/test/unit/common/test_app_config.py
+++ b/server/test/unit/common/test_app_config.py
@@ -142,7 +142,7 @@ class AppConfigTest(unittest.TestCase):
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
- app__api_base_url=f"http://localhost:{backend_port}/additional/path/before/dataroot",
+ app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset"
)
@@ -151,11 +151,21 @@ class AppConfigTest(unittest.TestCase):
with test_server(["-p", str(backend_port)], app_config=config) as server:
session = requests.Session()
self.assertEqual(server, f"http://localhost:{backend_port}")
- response = session.get(f"{server}/additional/path/before/dataroot/d/pbmc3k.h5ad/api/v0.2/config")
+ response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+ # test the health check at the correct url
+ response = session.get(f"{server}/additional/path/health")
+ assert response.json()["status"] == "pass"
+
+ # also check that the old URL still works.
+ # NOTE: this old URL location will soon be deprecated, and when that happens
+ # this check can be removed.
+ response = session.get(f"{server}/health")
+ assert response.json()["status"] == "pass"
+
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
From 210042814fb560713bff50d5dfe68b2f55ed7499 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Fri, 18 Sep 2020 13:16:28 -0700
Subject: [PATCH 04/73] Info Drawer format adjustments (#1853)
This PR tweaks the look and feel of the info drawer in response to QA from @signechambers1
---
.../e2e/__snapshots__/e2e.test.js.snap | 4 +-
.../__snapshots__/e2eAnnotations.test.js.snap | 16 +++---
.../src/components/infoDrawer/infoFormat.js | 55 +++++++++++++++----
.../leftSidebar/topLeftLogoAndTitle.js | 2 -
client/src/components/menubar/infoMenu.js | 2 +-
client/src/components/util/truncate.js | 7 ++-
6 files changed, 61 insertions(+), 25 deletions(-)
diff --git a/client/__tests__/e2e/__snapshots__/e2e.test.js.snap b/client/__tests__/e2e/__snapshots__/e2e.test.js.snap
index e7207fb7..d33b3262 100644
--- a/client/__tests__/e2e/__snapshots__/e2e.test.js.snap
+++ b/client/__tests__/e2e/__snapshots__/e2e.test.js.snap
@@ -1,5 +1,5 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
-exports[`did launch page launched 1`] = `"pbm c3k c3k "`;
+exports[`did launch page launched 1`] = `"pbm c3k c3k "`;
-exports[`metadata loads categories and values from dataset appear 1`] = `"
"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"
"`;
diff --git a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
index 67b743ce..01ed89a8 100644
--- a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
+++ b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
@@ -2,22 +2,22 @@
exports[`annotations stacked bar graph renders 1`] = `
Array [
- "",
- "",
+ "",
+ "",
]
`;
exports[`annotations stacked bar graph renders 2`] = `
Array [
- "",
- "",
+ "",
+ "",
]
`;
-exports[`annotations truncate midpoint whitespace 1`] = `"123 456 456 "`;
+exports[`annotations truncate midpoint whitespace 1`] = `"123 456 456 "`;
-exports[`annotations truncate midpoint whitespace 2`] = `"123 456 456 "`;
+exports[`annotations truncate midpoint whitespace 2`] = `"123 456 456 "`;
-exports[`annotations truncate single character 1`] = `"T "`;
+exports[`annotations truncate single character 1`] = `"T "`;
-exports[`annotations truncate single character 2`] = `"T "`;
+exports[`annotations truncate single character 2`] = `"T "`;
diff --git a/client/src/components/infoDrawer/infoFormat.js b/client/src/components/infoDrawer/infoFormat.js
index 8361b6de..eacaa406 100644
--- a/client/src/components/infoDrawer/infoFormat.js
+++ b/client/src/components/infoDrawer/infoFormat.js
@@ -1,6 +1,8 @@
import { H3, H1, UL, Classes } from "@blueprintjs/core";
import React from "react";
+import Truncate from "../util/truncate";
+
const renderContributors = (contributors, affiliations, skeleton) => {
// eslint-disable-next-line no-constant-condition -- Temp removed contributor section to avoid publishing PII
if (!contributors || contributors.length === 0 || true) return null;
@@ -79,23 +81,56 @@ const renderOrganism = (organism, skeleton) => {
);
};
+const ONTOLOGY_KEY = "ontology_term_id";
+const CAT_WIDTH = "30%";
+const VAL_WIDTH = "35%";
// Render list of metadata attributes found in categorical field
-// Ignores categories with empty or null values
const renderSingleValueCategories = (singleValueCategories, skeleton) => {
if (singleValueCategories.size === 0) return null;
return (
<>
Dataset Metadata
- {Array.from(singleValueCategories).map((pair) => {
- if (!pair[1] || pair[1] === "") return null;
- return (
- {`${pair[0]}: ${pair[1]}`}
- );
- })}
+ {Array.from(singleValueCategories).reduce((elems, pair) => {
+ const [category, value] = pair;
+ // If the value is empty skip it
+ if (!value) return elems;
+
+ // If this category is a ontology term, let's add its value to the previous node
+ if (String(category).includes(ONTOLOGY_KEY)) {
+ const prevElem = elems.pop();
+ // Props aren't extensible so we must clone and alter the component to append the new child
+ elems.push(
+ React.cloneElement(
+ prevElem,
+ prevElem.props,
+ // Concat returns a new array
+ prevElem.props.children.concat([
+
+ {value}
+ ,
+ ])
+ )
+ );
+ } else {
+ // Create the list item
+ elems.push(
+
+
+ {`${category}:`}
+
+
+ {value}
+
+
+ );
+ }
+ return elems;
+ }, [])}
>
);
diff --git a/client/src/components/leftSidebar/topLeftLogoAndTitle.js b/client/src/components/leftSidebar/topLeftLogoAndTitle.js
index f2346593..6224c1a1 100644
--- a/client/src/components/leftSidebar/topLeftLogoAndTitle.js
+++ b/client/src/components/leftSidebar/topLeftLogoAndTitle.js
@@ -2,7 +2,6 @@
import React from "react";
import { connect } from "react-redux";
import { Button } from "@blueprintjs/core";
-import { IconNames } from "@blueprintjs/icons";
import * as globals from "../../globals";
import Logo from "../framework/logo";
@@ -60,7 +59,6 @@ class LeftSideBar extends React.Component {
{
handleClick(dispatch)}
- icon={IconNames.BOOK}
+ icon={IconNames.INFO_SIGN}
text="Dataset Overview"
/>
diff --git a/client/src/components/util/truncate.js b/client/src/components/util/truncate.js
index da06a58f..33e64dcc 100644
--- a/client/src/components/util/truncate.js
+++ b/client/src/components/util/truncate.js
@@ -7,6 +7,8 @@ const SPLIT_STYLE = {
display: "flex",
overflow: "hidden",
justifyContent: "flex-start",
+ width: "100%", // There are probably additional styles that we don't want to stack
+ padding: 0,
};
const FIRST_HALF_STYLE = {
@@ -40,7 +42,7 @@ export default (props) => {
) {
throw Error("Only pass a single child with text to Truncate");
}
- const originalString = children.props.children;
+ const originalString = String(children.props.children);
let firstString;
let secondString;
@@ -58,7 +60,7 @@ export default (props) => {
}
}
- const inheritedColor = children.props.style.color;
+ const inheritedColor = children.props.style?.color;
const splitStyle = { ...children.props.style, ...SPLIT_STYLE };
const secondHalfContentStyle = {
@@ -93,6 +95,7 @@ export default (props) => {
preventOverflow: { enabled: false },
hide: { enabled: false },
}}
+ targetProps={{ style: children.props.style }}
>
{newChildren}
From a817a94eec2555dfeb5e6246bef648390ef0d590 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Fri, 18 Sep 2020 19:05:14 -0700
Subject: [PATCH 05/73] Bug reading the config file. (#1857)
The config file had a bug where it expected both a "server" and "dataset" section.
If one didn't exist, then it would raise an exception.
It should use the default server config or the defaul dataset config in those cases.
Added a test case that would have caught this.
---
server/common/app_config.py | 6 ++--
server/test/unit/common/test_app_config.py | 41 ++++++++++++++++++++++
2 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/server/common/app_config.py b/server/common/app_config.py
index e0a49e46..3102bae0 100644
--- a/server/common/app_config.py
+++ b/server/common/app_config.py
@@ -95,8 +95,10 @@ class AppConfig(object):
with open(config_file) as fyaml:
config = yaml.load(fyaml, Loader=yaml.FullLoader)
- self.server_config.update_from_config(config["server"], "server")
- self.default_dataset_config.update_from_config(config["dataset"], "dataset")
+ if config.get("server"):
+ self.server_config.update_from_config(config["server"], "server")
+ if config.get("dataset"):
+ self.default_dataset_config.update_from_config(config["dataset"], "dataset")
per_dataset_config = config.get("per_dataset_config", {})
for key, dataroot_config in per_dataset_config.items():
diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py
index 6b86d771..c3dd1436 100644
--- a/server/test/unit/common/test_app_config.py
+++ b/server/test/unit/common/test_app_config.py
@@ -206,3 +206,44 @@ class AppConfigTest(unittest.TestCase):
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
+
+ def test_configfile_no_dataset_section(self):
+ # test a config file without a dataset section
+
+ with tempfile.TemporaryDirectory() as tempdir:
+ configfile = os.path.join(tempdir, "config.yaml")
+ with open(configfile, "w") as fconfig:
+ config = """
+ server:
+ multi_dataset:
+ dataroot: test_dataroot
+
+ """
+ fconfig.write(config)
+
+ app_config = AppConfig()
+ app_config.update_from_config_file(configfile)
+ server_changes = app_config.server_config.changes_from_default()
+ dataset_changes = app_config.default_dataset_config.changes_from_default()
+ self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
+ self.assertEqual(dataset_changes, [])
+
+ def test_configfile_no_server_section(self):
+ # test a config file without a dataset section
+
+ with tempfile.TemporaryDirectory() as tempdir:
+ configfile = os.path.join(tempdir, "config.yaml")
+ with open(configfile, "w") as fconfig:
+ config = """
+ dataset:
+ user_annotations:
+ enable: false
+ """
+ fconfig.write(config)
+
+ app_config = AppConfig()
+ app_config.update_from_config_file(configfile)
+ server_changes = app_config.server_config.changes_from_default()
+ dataset_changes = app_config.default_dataset_config.changes_from_default()
+ self.assertEqual(server_changes, [])
+ self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
From 3e2d7174fd0a31e16ed4f2264012cd34fa07ed58 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Wed, 23 Sep 2020 11:46:56 -0700
Subject: [PATCH 06/73] Add user email to the userinfo response (#1862)
We are planning to display the user's email address in the front end.
#1830
---
server/common/app_config.py | 3 ++-
server/test/unit/auth/test_oauth.py | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/server/common/app_config.py b/server/common/app_config.py
index 3102bae0..080e1892 100644
--- a/server/common/app_config.py
+++ b/server/common/app_config.py
@@ -308,7 +308,8 @@ class AppConfig(object):
userinfo["userinfo"] = {
"is_authenticated": auth.is_user_authenticated(),
"username": auth.get_user_name(),
- "user_id": auth.get_user_id()
+ "user_id": auth.get_user_id(),
+ "email": auth.get_user_email()
}
return userinfo
else:
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index 42734a7b..d5be070b 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -122,6 +122,7 @@ class AuthTest(unittest.TestCase):
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
+ self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
self.assertTrue(config["config"]["parameters"]["annotations"])
if cookie_key:
From 374bb112792c5f568a4dc20f565926fa9487ebb0 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Mon, 28 Sep 2020 10:34:47 -0700
Subject: [PATCH 07/73] Handle case where new drag starts while existing lasso
is not finished (#1864)
* handle case where new drag starts while existing lasso is not finished
* flip variable
---
client/src/components/graph/setupLasso.js | 29 +++++++++++++++++++----
1 file changed, 24 insertions(+), 5 deletions(-)
diff --git a/client/src/components/graph/setupLasso.js b/client/src/components/graph/setupLasso.js
index 526d47a5..6610e04c 100644
--- a/client/src/components/graph/setupLasso.js
+++ b/client/src/components/graph/setupLasso.js
@@ -10,6 +10,7 @@ const Lasso = () => {
let lassoPolygon;
let lassoPath;
let closePath;
+ let lassoInProgress;
const polygonToPath = (polygon) =>
`M${polygon.map((d) => d.join(",")).join("L")}`;
@@ -25,8 +26,18 @@ const Lasso = () => {
lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
if (lassoPath) {
+ // If the existing path is in progress
+ if (lassoInProgress) {
+ // cancel the existing lasso
+ handleCancel();
+ // Don't continue with current drag start
+ return;
+ }
+
lassoPath.remove();
}
+ // We're starting a new drag
+ lassoInProgress = true;
lassoPath = g
.append("path")
@@ -67,25 +78,33 @@ const Lasso = () => {
}
};
+ const handleCancel = () => {
+ lassoPath.remove();
+ closePath = closePath?.remove();
+ lassoPath = null;
+ lassoPolygon = null;
+ closePath = null;
+ dispatch.call("cancel");
+ };
+
const handleDragEnd = () => {
// remove the close path
closePath.remove();
closePath = null;
- // succesfully closed
+ // successfully closed
if (
distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
closeDistance
) {
+ lassoInProgress = false;
+
lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
dispatch.call("end", lasso, lassoPolygon);
// otherwise cancel
} else {
- lassoPath.remove();
- lassoPath = null;
- lassoPolygon = null;
- dispatch.call("cancel");
+ handleCancel();
}
};
From 21dfdb91a934ee3181bcaca7ddad899a2045d04e Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Mon, 28 Sep 2020 13:17:14 -0700
Subject: [PATCH 08/73] skip user annos when building dataset metadata (#1881)
---
client/src/components/infoDrawer/infoDrawer.js | 2 ++
1 file changed, 2 insertions(+)
diff --git a/client/src/components/infoDrawer/infoDrawer.js b/client/src/components/infoDrawer/infoDrawer.js
index 49189ff2..57bfd77b 100644
--- a/client/src/components/infoDrawer/infoDrawer.js
+++ b/client/src/components/infoDrawer/infoDrawer.js
@@ -38,6 +38,8 @@ class InfoDrawer extends PureComponent {
const singleValueCategories = (
await Promise.all(nonUserAnnoCategories)
).reduce((acc, categoryData, i) => {
+ // Actually check to see if it is null(user anno)
+ if (!categoryData) return acc;
const catName = allCategoryNames[i];
const column = categoryData.icol(0);
From 863ca8be03e86d8fb1dd068c98c2516d62a6ca4a Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Mon, 28 Sep 2020 16:44:56 -0700
Subject: [PATCH 09/73] Fix license years and add CZI (#1882)
---
LICENSE.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/LICENSE.txt b/LICENSE.txt
index e0bb8c7d..a34ab341 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,6 +1,6 @@
The MIT License (MIT)
-Copyright (c) 2013
+Copyright (c) 2017-2020 Chan Zuckerberg Initiative
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
From 1145f61c78b39e12e12d42177c9825026c0a1396 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Tue, 29 Sep 2020 13:42:24 -0700
Subject: [PATCH 10/73] auth: logging out should keep the user on the same
page (#1877)
previous behavior is that logout would redirect to the index page.
---
server/auth/auth_oauth.py | 25 +++++++++++++++++++++++--
server/test/unit/auth/test_oauth.py | 4 ++--
2 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py
index a5107934..a2b6077f 100644
--- a/server/auth/auth_oauth.py
+++ b/server/auth/auth_oauth.py
@@ -114,6 +114,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
parse = urlparse(self.api_base_url)
app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
+ app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"])
app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
def complete_setup(self, flask_app):
@@ -166,12 +167,29 @@ class AuthTypeOAuth(AuthTypeClientBase):
return response
def logout(self):
+ """
+ We would like for the user to remain on the same dataset after logout. oauth requires that
+ the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of
+ indirection is used. We first redirect to a single path "logout_redirect", and logout_redirect
+ will redirect the user's browser back to the current page.
+ """
self.remove_tokens()
- params = {"returnTo": self.web_base_url, "client_id": self.client_id}
+ redirect_path = request.args.get("dataset", "")
+ redirect_to = f"{self.web_base_url}/{redirect_path}"
+ session["oauth_logout_redirect"] = redirect_to
+
+ return_to = f"{self.api_base_url}/logout_redirect"
+ params = {"returnTo": return_to, "client_id": self.client_id}
response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
self.update_response(response)
return response
+ def logout_redirect(self):
+ oauth_logout_redirect = session.pop("oauth_logout_redirect", "/")
+ response = redirect(oauth_logout_redirect)
+ self.update_response(response)
+ return response
+
def callback(self):
data = self.client.authorize_access_token()
tokens = Tokens(
@@ -253,7 +271,10 @@ class AuthTypeOAuth(AuthTypeClientBase):
def get_logout_url(self, data_adaptor):
"""Return the url for the logout route"""
- return f"{self.api_base_url}/logout"
+ if data_adaptor and current_app.app_config.is_multi_dataset():
+ return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/"
+ else:
+ return f"{self.api_base_url}/logout"
def check_jwt_payload(self, id_token):
try:
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index d5be070b..728fccfe 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -112,7 +112,7 @@ class AuthTest(unittest.TestCase):
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/")
- self.assertEqual(logout_uri, f"{server}/logout")
+ self.assertEqual(logout_uri, f"{server}/logout?dataset=d/pbmc3k.cxg/")
r = session.get(login_uri)
# check that the login redirect worked
@@ -151,7 +151,7 @@ class AuthTest(unittest.TestCase):
r = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
- self.assertEqual(r.url, f"{server}")
+ self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
From af3c6e1d8e853dc474b3ddf690b939d04ac7da5e Mon Sep 17 00:00:00 2001
From: Madison Dunitz
Date: Tue, 29 Sep 2020 16:42:46 -0500
Subject: [PATCH 11/73] config refactor (#1854)
* split out config
* add tests for base and app config, refactor client config out of app config
* refactor default config retrieval
* create config test class and helper functions
* move default_config into server to fix import issue
---
Makefile | 3 +-
server/__init__.py | 1 -
server/app/app.py | 27 +-
server/auth/__init__.py | 1 -
server/auth/auth.py | 2 +-
server/auth/auth_none.py | 1 -
server/auth/auth_oauth.py | 13 +-
server/cli/convert_to_cxg.py | 74 +-
server/cli/launch.py | 71 +-
server/cli/prepare.py | 26 +-
server/cli/upgrade.py | 3 +-
server/common/annotations/hosted_tiledb.py | 17 +-
server/common/app_config.py | 963 ------------------
server/common/aws_secret_utils.py | 61 --
server/common/config/__init__.py | 66 ++
server/common/config/app_config.py | 183 ++++
server/common/config/base_config.py | 113 ++
server/common/config/client_config.py | 125 +++
server/common/config/dataset_config.py | 234 +++++
server/common/config/server_config.py | 390 +++++++
server/common/errors.py | 8 +-
server/common/rest.py | 5 +-
server/common/utils/cxg_generation_utils.py | 2 +-
server/common/utils/matrix_utils.py | 11 +-
server/common/utils/type_conversion_utils.py | 20 +-
server/converters/h5ad_data_file.py | 16 +-
server/data_common/data_adaptor.py | 20 +-
server/db/cellxgene_orm.py | 7 +-
server/db/db_utils.py | 10 +-
server/{common => }/default_config.py | 1 -
server/eb/app.py | 13 +-
server/test/__init__.py | 13 +-
server/test/fixtures/database/__init__.py | 26 +-
.../test/fixtures/dataset_config_outline.py | 37 +
server/test/fixtures/server_config_outline.py | 62 ++
server/test/performance/run_diffexp.py | 2 +-
server/test/test_database/test_database.py | 37 +-
server/test/unit/auth/test_auth.py | 14 +-
server/test/unit/auth/test_oauth.py | 13 +-
server/test/unit/cli/test_launch.py | 28 +
server/test/unit/common/config/__init__.py | 246 +++++
.../unit/common/config/test_app_config.py | 140 +++
.../unit/common/config/test_base_config.py | 63 ++
.../unit/common/config/test_dataset_config.py | 260 +++++
.../unit/common/config/test_server_config.py | 335 ++++++
server/test/unit/common/test_api.py | 36 +-
server/test/unit/common/test_app_config.py | 249 -----
server/test/unit/common/test_corpora.py | 19 +-
.../unit/common/test_writable_annotation.py | 16 +-
.../common/utils/test_cxg_generation_utils.py | 52 +-
.../unit/common/utils/test_matrix_utils.py | 1 -
.../common/utils/test_sanitization_utils.py | 1 -
.../utils/test_type_conversion_utils.py | 42 +-
.../unit/converters/test_h5ad_data_file.py | 75 +-
.../test_anndata_adaptor_data_load.py | 2 +-
.../unit/data_common/test_matrix_loader.py | 4 +-
server/test/unit/eb/test_eb.py | 6 +-
57 files changed, 2667 insertions(+), 1599 deletions(-)
delete mode 100644 server/common/app_config.py
create mode 100644 server/common/config/__init__.py
create mode 100644 server/common/config/app_config.py
create mode 100644 server/common/config/base_config.py
create mode 100644 server/common/config/client_config.py
create mode 100644 server/common/config/dataset_config.py
create mode 100644 server/common/config/server_config.py
rename server/{common => }/default_config.py (99%)
create mode 100644 server/test/fixtures/dataset_config_outline.py
create mode 100644 server/test/fixtures/server_config_outline.py
create mode 100644 server/test/unit/cli/test_launch.py
create mode 100644 server/test/unit/common/config/__init__.py
create mode 100644 server/test/unit/common/config/test_app_config.py
create mode 100644 server/test/unit/common/config/test_base_config.py
create mode 100644 server/test/unit/common/config/test_dataset_config.py
create mode 100644 server/test/unit/common/config/test_server_config.py
delete mode 100644 server/test/unit/common/test_app_config.py
diff --git a/Makefile b/Makefile
index 5b588b77..56dccf59 100644
--- a/Makefile
+++ b/Makefile
@@ -83,7 +83,8 @@ lint: lint-server lint-client
.PHONY: lint-server
lint-server:
- flake8 server
+ flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821'
+
.PHONY: lint-client
lint-client:
diff --git a/server/__init__.py b/server/__init__.py
index 94238d9a..100b1874 100644
--- a/server/__init__.py
+++ b/server/__init__.py
@@ -1,6 +1,5 @@
import logging
import sys
-
from server.common.utils.utils import import_plugins
__version__ = "0.16.0"
diff --git a/server/app/app.py b/server/app/app.py
index 7e02b4e1..da167453 100644
--- a/server/app/app.py
+++ b/server/app/app.py
@@ -6,8 +6,17 @@ from urllib.parse import urlparse
import hashlib
import os
-from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \
- send_from_directory
+from flask import (
+ Flask,
+ redirect,
+ current_app,
+ make_response,
+ render_template,
+ abort,
+ Blueprint,
+ request,
+ send_from_directory,
+)
from flask_restful import Api, Resource
from server_timing import Timing as ServerTiming
@@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None):
cache_manager = current_app.matrix_data_cache_manager
with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
- args = {
- "SCRIPTS" : scripts,
- "INLINE_SCRIPTS" : inline_scripts
- }
+ args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
return render_template("index.html", **args)
except DatasetAccessError as e:
@@ -417,8 +423,9 @@ class Server:
for dataroot_dict in server_config.multi_dataset__dataroot.values():
url_dataroot = dataroot_dict["base_url"]
bp_dataroot = Blueprint(
- f"api_dataset_{url_dataroot}", __name__,
- url_prefix=f"{api_path}/{url_dataroot}/" + api_version
+ f"api_dataset_{url_dataroot}",
+ __name__,
+ url_prefix=f"{api_path}/{url_dataroot}/" + api_version,
)
dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
self.app.register_blueprint(dataroot_resources.blueprint)
@@ -433,7 +440,7 @@ class Server:
f"/{url_dataroot}//static/",
f"static_assets_{url_dataroot}",
view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
- methods=["GET"]
+ methods=["GET"],
)
else:
@@ -444,7 +451,7 @@ class Server:
"/static/",
"static_assets",
view_func=lambda filename: send_from_directory("../common/web/static", filename),
- methods=["GET"]
+ methods=["GET"],
)
self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
diff --git a/server/auth/__init__.py b/server/auth/__init__.py
index 1b33bebc..dfadadde 100644
--- a/server/auth/__init__.py
+++ b/server/auth/__init__.py
@@ -1,4 +1,3 @@
-
# import the built in auth types so they can be registered
import server.auth.auth_none # noqa: F401
diff --git a/server/auth/auth.py b/server/auth/auth.py
index bb86ea64..145616cc 100644
--- a/server/auth/auth.py
+++ b/server/auth/auth.py
@@ -76,7 +76,7 @@ class AuthTypeFactory:
@staticmethod
def register(name, auth_type):
- assert(issubclass(auth_type, AuthTypeBase))
+ assert issubclass(auth_type, AuthTypeBase)
AuthTypeFactory.auth_types[name] = auth_type
@staticmethod
diff --git a/server/auth/auth_none.py b/server/auth/auth_none.py
index c482e3c4..61f82400 100644
--- a/server/auth/auth_none.py
+++ b/server/auth/auth_none.py
@@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory
class AuthTypeNone(AuthTypeBase):
-
def __init__(self, app_config):
super().__init__()
diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py
index a2b6077f..d7162318 100644
--- a/server/auth/auth_oauth.py
+++ b/server/auth/auth_oauth.py
@@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase):
return
valid_keys = {
- "verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss",
- "verify_sub", "verify_jti", "verify_at_hash", "leeway"}
+ "verify_signature",
+ "verify_aud",
+ "verify_iat",
+ "verify_exp",
+ "verify_nbf",
+ "verify_iss",
+ "verify_sub",
+ "verify_jti",
+ "verify_at_hash",
+ "leeway",
+ }
keys = set(self.jwt_decode_options.keys())
unknown = keys - valid_keys
if unknown:
diff --git a/server/cli/convert_to_cxg.py b/server/cli/convert_to_cxg.py
index 4e456c08..0cd67e3c 100644
--- a/server/cli/convert_to_cxg.py
+++ b/server/cli/convert_to_cxg.py
@@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile
name="convert",
short_help="Converts an H5AD dataset to the CXG format.",
help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
- "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
- "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
- "usually with the generated CXG file.",
+ "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
+ "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
+ "usually with the generated CXG file.",
)
@click.argument(
- "input-file",
- nargs=1,
- type=click.Path(exists=True, dir_okay=False),
+ "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"-o",
"--output-directory",
help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
- "CXG extension.",
+ "CXG extension.",
)
@click.option(
"-b",
"--backed",
help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
- "but will use less memory.",
+ "but will use less memory.",
default=False,
show_default=True,
is_flag=True,
@@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile
"-t",
"--title",
help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
- "the dataset title will be the filename.",
+ "the dataset title will be the filename.",
)
@click.option(
"-a",
"--about",
help="A fully qualified URL that provides more information about the dataset and will be included as "
- "metadata about the CXG file.",
+ "metadata about the CXG file.",
)
@click.option(
"-s",
"--sparse-threshold",
help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
- "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
- "convert to dense array.",
+ "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
+ "convert to dense array.",
default=0.0,
show_default=True,
)
-@click.option("--obs-names",
- help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
- "the one designated by the dataframe generated-index.")
-@click.option("--var-names",
- help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
- "the one designated by the dataframe generated-index.")
+@click.option(
+ "--obs-names",
+ help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
+ "the one designated by the dataframe generated-index.",
+)
+@click.option(
+ "--var-names",
+ help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+ "the one designated by the dataframe generated-index.",
+)
@click.option(
"--disable-custom-colors",
help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
@@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile
@click.option(
"--disable-corpora-schema",
help="When set, conversion process will neither extract nor store Corpora schema information. See "
- "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
- "information.",
+ "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
+ "information.",
default=False,
show_default=True,
is_flag=True,
@@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def convert_to_cxg(
- input_file,
- output_directory,
- backed,
- title,
- about,
- sparse_threshold,
- obs_names,
- var_names,
- disable_custom_colors,
- disable_corpora_schema,
- overwrite,
+ input_file,
+ output_directory,
+ backed,
+ title,
+ about,
+ sparse_threshold,
+ obs_names,
+ var_names,
+ disable_custom_colors,
+ disable_corpora_schema,
+ overwrite,
):
"""
Convert a dataset file into CXG.
"""
- h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names,
- use_corpora_schema=not disable_corpora_schema)
+ h5ad_data_file = H5ADDataFile(
+ input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
+ )
# Get the directory that will hold all the CXG files
cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
- h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold,
- convert_anndata_colors_to_cxg_colors=not disable_custom_colors)
+ h5ad_data_file.to_cxg(
+ cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
+ )
def get_output_directory(input_filename, output_directory, should_overwrite):
diff --git a/server/cli/launch.py b/server/cli/launch.py
index d46161d4..a05fd6a1 100644
--- a/server/cli/launch.py
+++ b/server/cli/launch.py
@@ -3,15 +3,14 @@ import functools
import logging
import sys
import webbrowser
-from os import devnull
-
+import os
import click
from flask_compress import Compress
from flask_cors import CORS
+from server.default_config import default_config
from server.app.app import Server
-from server.common.app_config import AppConfig
-from server.common.default_config import default_config
+from server.common.config.app_config import AppConfig
from server.common.errors import DatasetAccessError, ConfigurationError
from server.common.utils.utils import sort_options
@@ -33,7 +32,7 @@ def annotation_args(func):
multiple=False,
metavar="",
help="CSV file to initialize editing of existing annotations; will be altered in-place. "
- "Incompatible with --annotations-dir.",
+ "Incompatible with --annotations-dir.",
)
@click.option(
"--annotations-dir",
@@ -42,7 +41,7 @@ def annotation_args(func):
multiple=False,
metavar="",
help="Directory of where to save output annotations; filename will be specified in the application. "
- "Incompatible with --annotations-file.",
+ "Incompatible with --annotations-file.",
)
@click.option(
"--experimental-annotations-ontology",
@@ -170,7 +169,7 @@ def server_args(func):
default=DEFAULT_CONFIG.server_config.app__debug,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
- "or when you want more information about an error condition.",
+ "or when you want more information about an error condition.",
)
@click.option(
"--verbose",
@@ -203,7 +202,7 @@ def server_args(func):
multiple=True,
metavar="",
help="Additional script files to include in HTML page. If not specified, "
- "no additional script files will be included.",
+ "no additional script files will be included.",
show_default=False,
)
@functools.wraps(func)
@@ -223,7 +222,7 @@ def launch_args(func):
default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
metavar="",
help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
- " to folder containing H5AD and/or CXG datasets.",
+ " to folder containing H5AD and/or CXG datasets.",
hidden=True,
) # TODO, unhide when dataroot is supported)
@click.argument("datapath", required=False, metavar="")
@@ -307,32 +306,32 @@ class CliLaunchServer(Server):
)
@launch_args
def launch(
- datapath,
- dataroot,
- verbose,
- debug,
- open_browser,
- port,
- host,
- embedding,
- obs_names,
- var_names,
- max_category_items,
- disable_custom_colors,
- diffexp_lfc_cutoff,
- title,
- scripts,
- about,
- disable_annotations,
- annotations_file,
- annotations_dir,
- backed,
- disable_diffexp,
- experimental_annotations_ontology,
- experimental_annotations_ontology_obo,
- experimental_enable_reembedding,
- config_file,
- dump_default_config,
+ datapath,
+ dataroot,
+ verbose,
+ debug,
+ open_browser,
+ port,
+ host,
+ embedding,
+ obs_names,
+ var_names,
+ max_category_items,
+ disable_custom_colors,
+ diffexp_lfc_cutoff,
+ title,
+ scripts,
+ about,
+ disable_annotations,
+ annotations_file,
+ annotations_dir,
+ backed,
+ disable_diffexp,
+ experimental_annotations_ontology,
+ experimental_annotations_ontology_obo,
+ experimental_enable_reembedding,
+ config_file,
+ dump_default_config,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -443,7 +442,7 @@ def launch(
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not server_config.app__verbose:
- f = open(devnull, "w")
+ f = open(os.devnull, "w")
sys.stdout = f
try:
diff --git a/server/cli/prepare.py b/server/cli/prepare.py
index df535db0..67735172 100644
--- a/server/cli/prepare.py
+++ b/server/cli/prepare.py
@@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options
default=False,
is_flag=True,
help="Do not run quality control metrics. By default cellxgene runs them "
- "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+ "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
)
@click.option(
"--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def prepare(
- data,
- embedding,
- recipe,
- output,
- plotting,
- sparse,
- overwrite,
- set_obs_names,
- set_var_names,
- skip_qc,
- make_obs_names_unique,
- make_var_names_unique,
+ data,
+ embedding,
+ recipe,
+ output,
+ plotting,
+ sparse,
+ overwrite,
+ set_obs_names,
+ set_var_names,
+ skip_qc,
+ make_obs_names_unique,
+ make_var_names_unique,
):
"""
Preprocess data for use with cellxgene.
diff --git a/server/cli/upgrade.py b/server/cli/upgrade.py
index 953f92ba..222d7e81 100644
--- a/server/cli/upgrade.py
+++ b/server/cli/upgrade.py
@@ -10,7 +10,8 @@ from .. import __version__
SEMVER_FORMAT = re.compile(
r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*["
r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+("
- r"?:\.[0-9a-zA-Z-]+)*))?$")
+ r"?:\.[0-9a-zA-Z-]+)*))?$"
+)
def log_upgrade_check():
diff --git a/server/common/annotations/hosted_tiledb.py b/server/common/annotations/hosted_tiledb.py
index f3df75ed..54f7dac2 100644
--- a/server/common/annotations/hosted_tiledb.py
+++ b/server/common/annotations/hosted_tiledb.py
@@ -31,7 +31,8 @@ class AnnotationsHostedTileDB(Annotations):
unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
if unsanitary_original_category_names:
raise AnnotationCategoryNameError(
- f"{unsanitary_original_category_names} are not valid category names, please resubmit")
+ f"{unsanitary_original_category_names} are not valid category names, please resubmit"
+ )
def is_safe_collection_name(self, name):
"""
@@ -68,11 +69,11 @@ class AnnotationsHostedTileDB(Annotations):
index_dims = None
schema_hints = json.loads(schema_hints)
- if '__pandas_attribute_repr' in tileDBArray.meta:
+ if "__pandas_attribute_repr" in tileDBArray.meta:
# backwards compatibility... unsure if necessary at this point
- repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
- if '__pandas_index_dims' in tileDBArray.meta:
- index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
+ repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
+ if "__pandas_index_dims" in tileDBArray.meta:
+ index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
data = tileDBArray[:]
indexes = list()
@@ -80,12 +81,12 @@ class AnnotationsHostedTileDB(Annotations):
for col_name, col_val in data.items():
# If the column values are byte literals, decode them
if isinstance(col_val[0], bytes):
- col_val = [value.decode('utf-8') for value in col_val]
+ col_val = [value.decode("utf-8") for value in col_val]
if schema_hints and col_name in schema_hints:
type = schema_hints.get(col_name).get("type")
if type and type == "categorical":
- new_col = pd.Series(col_val, dtype='category')
+ new_col = pd.Series(col_val, dtype="category")
data[col_name] = new_col
elif repr_meta and col_name in repr_meta:
new_col = pd.Series(col_val, dtype=repr_meta[col_name])
@@ -127,7 +128,7 @@ class AnnotationsHostedTileDB(Annotations):
tiledb_uri=uri,
user_id=user_id,
dataset_id=str(dataset_id),
- schema_hints=json.dumps(dataframe_schema_type_hints)
+ schema_hints=json.dumps(dataframe_schema_type_hints),
)
if not df.empty:
self.check_category_names(df)
diff --git a/server/common/app_config.py b/server/common/app_config.py
deleted file mode 100644
index 080e1892..00000000
--- a/server/common/app_config.py
+++ /dev/null
@@ -1,963 +0,0 @@
-import copy
-import os
-import sys
-import warnings
-from os.path import splitext, basename, isdir
-from urllib.parse import urlparse, quote_plus
-
-import yaml
-from flatten_dict import flatten, unflatten
-
-import server.compute.diffexp_cxg as diffexp_tiledb
-import server.compute.scanpy
-from server import display_version as cellxgene_display_version
-from server.auth.auth import AuthTypeFactory
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.data_locator import discover_s3_region_name
-from server.common.default_config import get_default_config
-from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
-from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
-from server.db.db_utils import DbUtils
-
-DEFAULT_SERVER_PORT = 5005
-# anything bigger than this will generate a special message
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
-
-
-class AppFeature(object):
- def __init__(self, path, available=False, method="POST", extra={}):
- self.path = path
- self.available = available
- self.method = method
- self.extra = extra
- for k, v in extra.items():
- setattr(self, k, v)
-
- def todict(self):
- d = dict(available=self.available, method=self.method, path=self.path)
- d.update(self.extra)
- return d
-
-
-class AppConfig(object):
- """AppConfig stores all the configuration for cellxgene. The configuration is divided into two main parts:
- server attributes, and dataset attributes. The server_config contains attributes that refer to the server process
- as a whole. The default_dataset_config referes to attributes that are associated with the features and
- presentations of a dataset. The dataset config attributes can be overridden depending on the url by which the
- dataset was accessed. These are stored in dataroot_config.
- AppConfig has methods to initialize, modify, and access the configuration.
- """
-
- def __init__(self):
-
- # the default configuration (see default_config.py)
- self.default_config = get_default_config()
- # the server configuration
- self.server_config = ServerConfig(self, self.default_config["server"])
- # the dataset config, unless overridden by an entry in dataroot_config
- self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
- # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
- # attribute of the server_config.
- self.dataroot_config = {}
-
- # Set to true when config_completed is called
- self.is_completed = False
-
- def get_dataset_config(self, dataroot_key):
- if self.server_config.single_dataset__datapath:
- return self.default_dataset_config
- else:
- return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
-
- def check_config(self):
- """Verify all the attributes have been checked"""
- if not self.is_completed:
- raise ConfigurationError("The configuration has not been completed")
- self.server_config.check_config()
- self.default_dataset_config.check_config()
- for dataset_config in self.dataroot_config.values():
- dataset_config.check_config()
-
- def update_server_config(self, **kw):
- self.server_config.update(**kw)
- self.is_complete = False
-
- def update_default_dataset_config(self, **kw):
- self.default_dataset_config.update(**kw)
- # update all the other dataset configs, if any
- for value in self.dataroot_config.values():
- value.update(**kw)
- self.is_complete = False
-
- def update_from_config_file(self, config_file):
- with open(config_file) as fyaml:
- config = yaml.load(fyaml, Loader=yaml.FullLoader)
-
- if config.get("server"):
- self.server_config.update_from_config(config["server"], "server")
- if config.get("dataset"):
- self.default_dataset_config.update_from_config(config["dataset"], "dataset")
-
- per_dataset_config = config.get("per_dataset_config", {})
- for key, dataroot_config in per_dataset_config.items():
- # first create and initialize the dataroot with the default config
- self.add_dataroot_config(key, **config["dataset"])
- # then apply the per dataset configuration
- self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
-
- self.is_complete = False
-
- def write_config(self, config_file):
- """output the config to a yaml file"""
- server = self.server_config.create_mapping(self.server_config.default_config)
- dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
- config = dict(server={}, dataset={})
- for attrname in server.keys():
- config["server__" + attrname] = getattr(self.server_config, attrname)
- for attrname in dataset.keys():
- config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
- if self.dataroot_config:
- config["per_dataset_config"] = {}
- for dataroot_tag, dataroot_config in self.dataroot_config.items():
- dataset = dataroot_config.create_mapping(dataroot_config.default_config)
- for attrname in dataset.keys():
- config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
-
- config = unflatten(config, splitter=lambda key: key.split("__"))
- yaml.dump(config, open(config_file, "w"))
-
- def changes_from_default(self):
- """Return all the attribute that are different from the default"""
- diff_server = self.server_config.changes_from_default()
- diff_dataset = self.default_dataset_config.changes_from_default()
- diff = dict(server=diff_server, dataset=diff_dataset)
- return diff
-
- def add_dataroot_config(self, dataroot_tag, **kw):
- """Create a new dataset config object based on the default dataset config, and kw parameters"""
- if dataroot_tag in self.dataroot_config:
- raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
- if type(self.server_config.multi_dataset__dataroot) != dict:
- raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
- if dataroot_tag not in self.server_config.multi_dataset__dataroot:
- raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
-
- self.is_completed = False
- self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
- flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
- config = {key: value[1] for key, value in flat_config.items()}
- self.dataroot_config[dataroot_tag].update(**config)
- self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
-
- def complete_config(self, messagefn=None):
- """The configure options are checked, and any additional setup based on the config
- parameters is done"""
-
- if messagefn is None:
- def noop(message):
- pass
-
- messagefn = noop
-
- # TODO: to give better error messages we can add a mapping between where each config
- # attribute originated (e.g. command line argument or config file), then in the error
- # messages we can give correct context for attributes with bad value.
- context = dict(messagefn=messagefn)
-
- self.server_config.complete_config(context)
- self.default_dataset_config.complete_config(context)
- for dataroot_config in self.dataroot_config.values():
- dataroot_config.complete_config(context)
-
- self.is_completed = True
- self.check_config()
-
- def get_matrix_data_cache_manager(self):
- return self.server_config.matrix_data_cache_manager
-
- def is_multi_dataset(self):
- return self.server_config.multi_dataset__dataroot is not None
-
- def get_title(self, data_adaptor):
- return (
- self.server_config.single_dataset__title
- if self.server_config.single_dataset__title
- else data_adaptor.get_title()
- )
-
- def get_about(self, data_adaptor):
- return (
- self.server_config.single_dataset__about
- if self.server_config.single_dataset__about
- else data_adaptor.get_about()
- )
-
- def get_client_config(self, data_adaptor):
- """
- Return the configuration as required by the /config REST route
- """
-
- server_config = self.server_config
- dataset_config = data_adaptor.dataset_config
- annotation = dataset_config.user_annotations
- auth = server_config.auth
-
- # FIXME The current set of config is not consistently presented:
- # we have camalCase, hyphen-text, and underscore_text
-
- # make sure the configuration has been checked.
- self.check_config()
-
- # features
- features = [f.todict() for f in data_adaptor.get_features(annotation)]
-
- # display_names
- title = self.get_title(data_adaptor)
- about = self.get_about(data_adaptor)
-
- display_names = dict(engine=data_adaptor.get_name(), dataset=title)
-
- # library_versions
- library_versions = {}
- library_versions.update(data_adaptor.get_library_versions())
- library_versions["cellxgene"] = cellxgene_display_version
-
- # links
- links = {"about-dataset": about}
-
- # parameters
- parameters = {
- "layout": dataset_config.embeddings__names,
- "max-category-items": dataset_config.presentation__max_categories,
- "obs_names": server_config.single_dataset__obs_names,
- "var_names": server_config.single_dataset__var_names,
- "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
- "backed": server_config.adaptor__anndata_adaptor__backed,
- "disable-diffexp": not dataset_config.diffexp__enable,
- "enable-reembedding": dataset_config.embeddings__enable_reembedding,
- "annotations": False,
- "annotations_file": None,
- "annotations_dir": None,
- "annotations_cell_ontology_enabled": False,
- "annotations_cell_ontology_obopath": None,
- "annotations_cell_ontology_terms": None,
- "custom_colors": dataset_config.presentation__custom_colors,
- "diffexp-may-be-slow": False,
- "about_legal_tos": dataset_config.app__about_legal_tos,
- "about_legal_privacy": dataset_config.app__about_legal_privacy,
- }
-
- # corpora dataset_props
- # TODO/Note: putting info from the dataset into the /config is not ideal.
- # However, it is definitely not part of /schema, and we do not have a top-level
- # route for data properties. Consider creating one at some point.
- corpora_props = data_adaptor.get_corpora_props()
- if corpora_props and "default_embedding" in corpora_props:
- default_embedding = corpora_props["default_embedding"]
- if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
- default_embedding = default_embedding[2:] # drop X_ prefix
- if default_embedding in data_adaptor.get_embedding_names():
- parameters["default_embedding"] = default_embedding
-
- data_adaptor.update_parameters(parameters)
- if annotation:
- annotation.update_parameters(parameters, data_adaptor)
-
- # gather it all together
- c = {}
- config = c["config"] = {}
- config["features"] = features
- config["displayNames"] = display_names
- config["library_versions"] = library_versions
- config["links"] = links
- config["parameters"] = parameters
- config["corpora_props"] = corpora_props
- config["limits"] = {
- "column_request_max": server_config.limits__column_request_max,
- "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
- }
-
- if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
- config["authentication"] = {
- "requires_client_login": auth.requires_client_login(),
- }
- if auth.requires_client_login():
- config["authentication"].update({
- "login": auth.get_login_url(data_adaptor),
- "logout": auth.get_logout_url(data_adaptor),
- })
-
- return c
-
- def get_client_userinfo(self, data_adaptor):
- """
- Return the userinfo as required by the /userinfo REST route
- """
-
- server_config = self.server_config
- dataset_config = data_adaptor.dataset_config
- auth = server_config.auth
-
- # make sure the configuration has been checked.
- self.check_config()
-
- if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
- userinfo = {}
- userinfo["userinfo"] = {
- "is_authenticated": auth.is_user_authenticated(),
- "username": auth.get_user_name(),
- "user_id": auth.get_user_id(),
- "email": auth.get_user_email()
- }
- return userinfo
- else:
- return None
-
-
-class BaseConfig(object):
- """This class handles the mechanics of updating and checking attributes.
- Derived classes are expected to store the actual attributes"""
-
- def __init__(self, app_config, default_config, dictval_cases={}):
- # reference back to the app_config
- self.app_config = app_config
- # the complete set of attribute and their default values (unflattened)
- self.default_config = default_config
- # attributes where the value may be a dict (and therefore are not flattened)
- self.dictval_cases = dictval_cases
- # used to make sure every attribute value is checked
- self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()}
-
- def create_mapping(self, config):
- """Create a mapping from attribute names to (location in the config tree, value)"""
- dc = copy.deepcopy(config)
- mapping = {}
-
- # special cases where the value could be a dict.
- # If its value is not None, the entry is added to the mapping, and not included
- # in the flattening below.
- for dictval_case in self.dictval_cases:
- cur = dc
- for part in dictval_case[:-1]:
- cur = cur.get(part, {})
- val = cur.get(dictval_case[-1])
- if val is not None:
- key = "__".join(dictval_case)
- mapping[key] = (dictval_case, val)
- del cur[dictval_case[-1]]
-
- flat_config = flatten(dc)
- for key, value in flat_config.items():
- # name of the attribute
- attr = "__".join(key)
- mapping[attr] = (key, value)
-
- return mapping
-
- def check_attr(self, attrname, vtype):
- val = getattr(self, attrname)
- if type(vtype) in (list, tuple):
- if type(val) not in vtype:
- tnames = ",".join([x.__name__ for x in vtype])
- raise ConfigurationError(
- f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
- )
- else:
- if type(val) != vtype:
- raise ConfigurationError(
- f"Invalid type for attribute: {attrname}, "
- f"expected type {vtype.__name__}, got {type(val).__name__}"
- )
-
- self.attr_checked[attrname] = True
-
- def check_config(self):
- mapping = self.create_mapping(self.default_config)
- for key in mapping.keys():
- if not self.attr_checked[key]:
- raise ConfigurationError(f"The attr '{key}' has not been checked")
-
- def update(self, **kw):
- for key, value in kw.items():
- if not hasattr(self, key):
- raise ConfigurationError(f"unknown config parameter {key}.")
- try:
- if type(value) == tuple:
- # convert tuple values to list values
- value = list(value)
- setattr(self, key, value)
- except KeyError:
- raise ConfigurationError(f"Unable to set config parameter {key}.")
-
- self.attr_checked[key] = False
-
- def update_from_config(self, config, prefix):
- mapping = self.create_mapping(config)
- for attr, (key, value) in mapping.items():
- if not hasattr(self, attr):
- raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
- try:
- setattr(self, attr, value)
- except KeyError:
- raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}")
-
- self.attr_checked[attr] = False
-
- def changes_from_default(self):
- """Return all the attribute that are different from the default"""
- mapping = self.create_mapping(self.default_config)
- diff = []
- for attrname, (key, defval) in mapping.items():
- curval = getattr(self, attrname)
- if curval != defval:
- diff.append((attrname, curval, defval))
- return diff
-
-
-class ServerConfig(BaseConfig):
- """Manages the config attribute associated with the server."""
-
- def __init__(self, app_config, default_config):
- dictval_cases = [
- ("app", "csp_directives"),
- ("authentication", "params_oauth", "cookie"),
- ("authentication", "params_oauth", "jwt_decode_options"),
- ("adaptor", "cxg_adaptor", "tiledb_ctx"),
- ("multi_dataset", "dataroot"),
- ]
- super().__init__(app_config, default_config, dictval_cases)
-
- dc = default_config
- try:
- self.app__verbose = dc["app"]["verbose"]
- self.app__debug = dc["app"]["debug"]
- self.app__host = dc["app"]["host"]
- self.app__port = dc["app"]["port"]
- self.app__open_browser = dc["app"]["open_browser"]
- self.app__force_https = dc["app"]["force_https"]
- self.app__flask_secret_key = dc["app"]["flask_secret_key"]
- self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
- self.app__server_timing_headers = dc["app"]["server_timing_headers"]
- self.app__csp_directives = dc["app"]["csp_directives"]
- self.app__api_base_url = dc["app"]["api_base_url"]
- self.app__web_base_url = dc["app"]["web_base_url"]
-
- self.authentication__type = dc["authentication"]["type"]
- self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][
- "oauth_api_base_url"
- ]
- self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
- self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
- self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][
- "jwt_decode_options"]
- self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
- self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
-
- self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
- self.multi_dataset__index = dc["multi_dataset"]["index"]
- self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
- self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
- self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"]
-
- self.single_dataset__datapath = dc["single_dataset"]["datapath"]
- self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
- self.single_dataset__var_names = dc["single_dataset"]["var_names"]
- self.single_dataset__about = dc["single_dataset"]["about"]
- self.single_dataset__title = dc["single_dataset"]["title"]
-
- self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"]
- self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"]
- self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"]
-
- self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"]
-
- self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
- self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
-
- self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"]
- self.limits__column_request_max = dc["limits"]["column_request_max"]
-
- except KeyError as e:
- raise ConfigurationError(f"Unexpected config: {str(e)}")
-
- # The matrix data cache manager is created during the complete_config and stored here.
- self.matrix_data_cache_manager = None
-
- # The authentication object
- self.auth = None
-
- def complete_config(self, context):
- self.handle_app(context)
- self.handle_data_source(context)
- self.handle_authentication(context)
- self.handle_data_locator(context)
- self.handle_adaptor(context) # may depend on data_locator
- self.handle_single_dataset(context) # may depend on adaptor
- self.handle_multi_dataset(context) # may depend on adaptor
- self.handle_diffexp(context)
- self.handle_limits(context)
-
- self.check_config()
-
- def handle_app(self, context):
- self.check_attr("app__verbose", bool)
- self.check_attr("app__debug", bool)
- self.check_attr("app__host", str)
- self.check_attr("app__port", (type(None), int))
- self.check_attr("app__open_browser", bool)
- self.check_attr("app__force_https", bool)
- self.check_attr("app__flask_secret_key", (type(None), str))
- self.check_attr("app__generate_cache_control_headers", bool)
- self.check_attr("app__server_timing_headers", bool)
- self.check_attr("app__csp_directives", (type(None), dict))
- self.check_attr("app__api_base_url", (type(None), str))
- self.check_attr("app__web_base_url", (type(None), str))
-
- if self.app__port:
- try:
- if not is_port_available(self.app__host, self.app__port):
- raise ConfigurationError(
- f"The port selected {self.app__port} is in use, please configure an open port."
- )
- except OverflowError:
- raise ConfigurationError(f"Invalid port: {self.app__port}")
- else:
- try:
- default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
- except ValueError:
- raise ConfigurationError(
- "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
- )
- try:
- self.app__port = find_available_port(self.app__host, default_server_port)
- except OverflowError:
- raise ConfigurationError(f"Invalid port: {default_server_port}")
-
- if self.app__debug:
- context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
- self.app__verbose = True
- self.app__open_browser = False
- else:
- warnings.formatwarning = custom_format_warning
-
- if not self.app__verbose:
- sys.tracebacklimit = 0
-
- # secret key:
- # first, from CXG_SECRET_KEY environment variable
- # second, from config file
- self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
-
- # CSP Directives are a dict of string: list(string) or string: string
- if self.app__csp_directives is not None:
- for k, v in self.app__csp_directives.items():
- if not isinstance(k, str):
- raise ConfigurationError("CSP directive names must be a string.")
- if isinstance(v, list):
- for policy in v:
- if not isinstance(policy, str):
- raise ConfigurationError("CSP directive value must be a string or list of strings.")
- elif not isinstance(v, str):
- raise ConfigurationError("CSP directive value must be a string or list of strings.")
-
- if self.app__web_base_url is None:
- self.app__web_base_url = self.app__api_base_url
-
- def handle_authentication(self, context):
- self.check_attr("authentication__type", (type(None), str))
-
- # oauth
- ptypes = str if self.authentication__type == "oauth" else (type(None), str)
- self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes)
- self.check_attr("authentication__params_oauth__client_id", ptypes)
- self.check_attr("authentication__params_oauth__client_secret", ptypes)
- self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict))
- self.check_attr("authentication__params_oauth__session_cookie", bool)
-
- if self.authentication__params_oauth__session_cookie:
- self.check_attr("authentication__params_oauth__cookie", (type(None), dict))
- else:
- self.check_attr("authentication__params_oauth__cookie", dict)
- # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
- # second, from config file
- self.authentication__params__oauth__client_secret = os.environ.get(
- "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
-
- self.auth = AuthTypeFactory.create(self.authentication__type, self)
- if self.auth is None:
- raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
-
- def handle_data_locator(self, context):
- self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
- if self.data_locator__s3__region_name is True:
- path = self.single_dataset__datapath or self.multi_dataset__dataroot
- if type(path) == dict:
- # if multi_dataset__dataroot is a dict, then use the first key
- # that is in s3. NOTE: it is not supported to have dataroots
- # in different regions.
- paths = [val.get("dataroot") for val in path.values()]
- for path in paths:
- if path.startswith("s3://"):
- break
- if path.startswith("s3://"):
- region_name = discover_s3_region_name(path)
- if region_name is None:
- raise ConfigurationError(f"Unable to discover s3 region name from {path}")
- else:
- region_name = None
- self.data_locator__s3__region_name = region_name
-
- def handle_data_source(self, context):
- self.check_attr("single_dataset__datapath", (str, type(None)))
- self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
-
- if self.single_dataset__datapath is None:
- if self.multi_dataset__dataroot is None:
- # TODO: change the error message once dataroot is fully supported
- raise ConfigurationError("missing datapath")
- return
- else:
- if self.multi_dataset__dataroot is not None:
- raise ConfigurationError("must supply only one of datapath or dataroot")
-
- def handle_single_dataset(self, context):
- self.check_attr("single_dataset__datapath", (str, type(None)))
- self.check_attr("single_dataset__title", (str, type(None)))
- self.check_attr("single_dataset__about", (str, type(None)))
- self.check_attr("single_dataset__obs_names", (str, type(None)))
- self.check_attr("single_dataset__var_names", (str, type(None)))
-
- if self.single_dataset__datapath is None:
- return
-
- # create the matrix data cache manager:
- if self.matrix_data_cache_manager is None:
- self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
-
- # preload this data set
- matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
- try:
- matrix_data_loader.pre_load_validation()
- except DatasetAccessError as e:
- raise ConfigurationError(str(e))
-
- file_size = matrix_data_loader.file_size()
- file_basename = basename(self.single_dataset__datapath)
- if file_size > BIG_FILE_SIZE_THRESHOLD:
- context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
- else:
- context["messagefn"](f"Loading data from {file_basename}.")
-
- if self.single_dataset__about:
-
- def url_check(url):
- try:
- result = urlparse(url)
- if all([result.scheme, result.netloc]):
- return True
- else:
- return False
- except ValueError:
- return False
-
- if not url_check(self.single_dataset__about):
- raise ConfigurationError(
- "Must provide an absolute URL for --about. (Example format: http://example.com)"
- )
-
- def handle_multi_dataset(self, context):
- self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
- self.check_attr("multi_dataset__index", (type(None), bool, str))
- self.check_attr("multi_dataset__allowed_matrix_types", list)
- self.check_attr("multi_dataset__matrix_cache__max_datasets", int)
- self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float))
-
- if self.multi_dataset__dataroot is None:
- return
-
- if type(self.multi_dataset__dataroot) == str:
- default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
- self.multi_dataset__dataroot = dict(d=default_dict)
-
- for tag, dataroot_dict in self.multi_dataset__dataroot.items():
- if "base_url" not in dataroot_dict:
- raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
- if "dataroot" not in dataroot_dict:
- raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
-
- base_url = dataroot_dict["base_url"]
-
- # sanity check for well formed base urls
- bad = False
- if type(base_url) != str:
- bad = True
- elif os.path.normpath(base_url) != base_url:
- bad = True
- else:
- base_url_parts = base_url.split("/")
- if [quote_plus(part) for part in base_url_parts] != base_url_parts:
- bad = True
- if ".." in base_url_parts:
- bad = True
- if bad:
- raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
-
- # verify all the base_urls are unique
- base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
- if len(base_urls) > len(set(base_urls)):
- raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
-
- # error checking
- for mtype in self.multi_dataset__allowed_matrix_types:
- try:
- MatrixDataType(mtype)
- except ValueError:
- raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
-
- # create the matrix data cache manager:
- if self.matrix_data_cache_manager is None:
- self.matrix_data_cache_manager = MatrixDataCacheManager(
- max_cached=self.multi_dataset__matrix_cache__max_datasets,
- timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
- )
-
- def handle_diffexp(self, context):
- self.check_attr("diffexp__alg_cxg__max_workers", (str, int))
- self.check_attr("diffexp__alg_cxg__cpu_multiplier", int)
- self.check_attr("diffexp__alg_cxg__target_workunit", int)
-
- max_workers = self.diffexp__alg_cxg__max_workers
- cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
- cpu_count = os.cpu_count()
- max_workers = min(max_workers, cpu_multiplier * cpu_count)
- diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
-
- def handle_adaptor(self, context):
- # cxg
- self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
- regionkey = "vfs.s3.region"
- if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
- if type(self.data_locator__s3__region_name) == str:
- self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
-
- from server.data_cxg.cxg_adaptor import CxgAdaptor
-
- CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
-
- # anndata
- self.check_attr("adaptor__anndata_adaptor__backed", bool)
-
- def handle_limits(self, context):
- self.check_attr("limits__diffexp_cellcount_max", (type(None), int))
- self.check_attr("limits__column_request_max", (type(None), int))
-
- def exceeds_limit(self, limit_name, value):
- limit_value = getattr(self, "limits__" + limit_name, None)
- if limit_value is None: # disabled
- return False
- return value > limit_value
-
- def get_api_base_url(self):
- if self.app__api_base_url == "local":
- return f"http://{self.app__host}:{self.app__port}"
- if self.app__api_base_url and self.app__api_base_url.endswith("/"):
- return self.app__api_base_url[:-1]
- return self.app__api_base_url
-
- def get_web_base_url(self):
- if self.app__web_base_url == "local":
- return f"http://{self.app__host}:{self.app__port}"
- if self.app__web_base_url is None:
- return self.get_api_base_url()
- if self.app__web_base_url.endswith("/"):
- return self.app__web_base_url[:-1]
- return self.api__web_base_url
-
-
-class DatasetConfig(BaseConfig):
- """Manages the config attribute associated with a dataset."""
-
- def __init__(self, tag, app_config, default_config):
- super().__init__(app_config, default_config)
- self.tag = tag
- dc = default_config
- try:
- self.app__scripts = dc["app"]["scripts"]
- self.app__inline_scripts = dc["app"]["inline_scripts"]
- self.app__about_legal_tos = dc["app"]["about_legal_tos"]
- self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
- self.app__authentication_enable = dc["app"]["authentication_enable"]
-
- self.presentation__max_categories = dc["presentation"]["max_categories"]
- self.presentation__custom_colors = dc["presentation"]["custom_colors"]
-
- self.user_annotations__enable = dc["user_annotations"]["enable"]
- self.user_annotations__type = dc["user_annotations"]["type"]
- self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
- self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
- self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
- self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
- self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
- self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
- dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ] # noqa E501
-
- self.embeddings__names = dc["embeddings"]["names"]
- self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
-
- self.diffexp__enable = dc["diffexp"]["enable"]
- self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
- self.diffexp__top_n = dc["diffexp"]["top_n"]
-
- except KeyError as e:
- raise ConfigurationError(f"Unexpected config: {str(e)}")
-
- # The annotation object is created during complete_config and stored here.
- self.user_annotations = None
-
- def complete_config(self, context):
- self.handle_app(context)
- self.handle_presentation(context)
- self.handle_user_annotations(context)
- self.handle_embeddings(context)
- self.handle_diffexp(context)
-
- def handle_app(self, context):
- self.check_attr("app__scripts", list)
- self.check_attr("app__inline_scripts", list)
- self.check_attr("app__about_legal_tos", (type(None), str))
- self.check_attr("app__about_legal_privacy", (type(None), str))
- self.check_attr("app__authentication_enable", bool)
-
- # scripts can be string (filename) or dict (attributes). Convert string to dict.
- scripts = []
- for s in self.app__scripts:
- if isinstance(s, str):
- scripts.append({"src": s})
- elif isinstance(s, dict) and isinstance(s["src"], str):
- scripts.append(s)
- else:
- raise ConfigurationError("Scripts must be string or dict")
- self.app__scripts = scripts
-
- def handle_presentation(self, context):
- self.check_attr("presentation__max_categories", int)
- self.check_attr("presentation__custom_colors", bool)
-
- def handle_user_annotations(self, context):
- self.check_attr("user_annotations__enable", bool)
- self.check_attr("user_annotations__type", str)
- self.check_attr("user_annotations__local_file_csv__directory", (type(None), str))
- self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
- self.check_attr("user_annotations__ontology__enable", bool)
- self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
- self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
- self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
-
- if self.user_annotations__enable:
- server_config = self.app_config.server_config
- if not self.app__authentication_enable:
- raise ConfigurationError("user annotations requires authentication to be enabled")
- if not server_config.auth.is_valid_authentication_type():
- auth_type = server_config.authentication__type
- raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
-
- # TODO, replace this with a factory pattern once we have more than one way
- # to do annotations. currently only local_file_csv
- if self.user_annotations__type == "local_file_csv":
- dirname = self.user_annotations__local_file_csv__directory
- filename = self.user_annotations__local_file_csv__file
-
- if filename is not None and dirname is not None:
- raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
-
- if filename is not None:
- lf_name, lf_ext = splitext(filename)
- if lf_ext and lf_ext != ".csv":
- raise ConfigurationError(f"annotation file type must be .csv: {filename}")
-
- if dirname is not None and not isdir(dirname):
- try:
- os.mkdir(dirname)
- except OSError:
- raise ConfigurationError("Unable to create directory specified by --annotations-dir")
-
- self.user_annotations = AnnotationsLocalFile(dirname, filename)
-
- # if the user has specified a fixed label file, go ahead and validate it
- # so that we can remove errors early in the process.
- server_config = self.app_config.server_config
- if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
- with server_config.matrix_data_cache_manager.data_adaptor(
- self.tag, server_config.single_dataset__datapath, self.app_config
- ) as data_adaptor:
- data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
-
- if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
- try:
- self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
- except OntologyLoadFailure as e:
- raise ConfigurationError("Unable to load ontology terms\n" + str(e))
- elif self.user_annotations__type == "hosted_tiledb_array":
- self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
- self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
- self.user_annotations = AnnotationsHostedTileDB(
- directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
- db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
- )
- else:
- raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
- else:
- if self.user_annotations__type == "local_file_csv":
- dirname = self.user_annotations__local_file_csv__directory
- filename = self.user_annotations__local_file_csv__file
- if filename is not None:
- context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
- if dirname is not None:
- context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
-
- if self.user_annotations__ontology__enable:
- context["messagefn"](
- "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
- )
- if self.user_annotations__ontology__obo_location is not None:
- context["messagefn"](
- "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
- )
-
- def handle_embeddings(self, context):
- self.check_attr("embeddings__names", list)
- self.check_attr("embeddings__enable_reembedding", bool)
-
- server_config = self.app_config.server_config
- if self.embeddings__enable_reembedding:
- if server_config.single_dataset__datapath:
- matrix_data_loader = MatrixDataLoader(
- server_config.single_dataset__datapath, app_config=self.app_config
- )
- if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
- raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
- if server_config.adaptor__anndata_adaptor__backed:
- raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
- try:
- server.compute.scanpy.get_scanpy_module()
- except NotImplementedError:
- raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
-
- def handle_diffexp(self, context):
- self.check_attr("diffexp__enable", bool)
- self.check_attr("diffexp__lfc_cutoff", float)
- self.check_attr("diffexp__top_n", int)
-
- server_config = self.app_config.server_config
- if server_config.single_dataset__datapath:
- with server_config.matrix_data_cache_manager.data_adaptor(
- self.tag, server_config.single_dataset__datapath, self.app_config
- ) as data_adaptor:
- if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
- context["messagefn"](
- "CAUTION: due to the size of your dataset, "
- "running differential expression may take longer or fail."
- )
diff --git a/server/common/aws_secret_utils.py b/server/common/aws_secret_utils.py
index 47f7690b..070ed160 100644
--- a/server/common/aws_secret_utils.py
+++ b/server/common/aws_secret_utils.py
@@ -1,72 +1,11 @@
import logging
-import os
-import sys
import boto3
from flask import json
-from server.common.data_locator import discover_s3_region_name
from server.common.errors import SecretKeyRetrievalError
-def handle_config_from_secret(app_config):
- """Update configuration from the secret manager"""
- secret_name = os.getenv("CXG_AWS_SECRET_NAME")
- if not secret_name:
- return
-
- # need to find the secret manager region.
- # 1. from CXG_AWS_SECRET_REGION_NAME
- # 2. discover from dataroot location (if on s3)
- # 3. discover from config file location (if on s3)
- secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
- if secret_region_name is None:
- secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
- if not secret_region_name:
- from server.eb.app import config_file
- secret_region_name = discover_s3_region_name(config_file)
- if not secret_region_name:
- logging.error("Could not determine the AWS Secret Manager region")
- sys.exit(1)
-
- secrets = get_secret_key(secret_region_name, secret_name)
-
- if not secrets:
- return
-
- server_attrs = (
- ("flask_secret_key", "app__flask_secret_key"),
- ("oauth_client_secret", "authentication__params_oauth__client_secret"),
- )
- default_dataset_attrs = (
- ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
- )
-
- # update server configuration attributes
- for key, attr in server_attrs:
- cur_val = getattr(app_config.server_config, attr)
- if cur_val:
- continue
-
- # replace the attr with the secret if it is not set
- val = secrets.get(key)
- if val:
- logging.info(f"set {attr} from secret")
- app_config.update_server_config(**{attr : val})
-
- # update default dataset configuration attributes
- for key, attr in default_dataset_attrs:
- cur_val = getattr(app_config.default_dataset_config, attr)
- if cur_val:
- continue
-
- # replace the attr with the secret if it is not set
- val = secrets.get(key)
- if val:
- logging.info(f"set {attr} from secret")
- app_config.update_default_dataset_config(**{attr : val})
-
-
def get_secret_key(region_name, secret_name):
session = boto3.session.Session()
client = session.client(service_name="secretsmanager", region_name=region_name)
diff --git a/server/common/config/__init__.py b/server/common/config/__init__.py
new file mode 100644
index 00000000..fb2439e7
--- /dev/null
+++ b/server/common/config/__init__.py
@@ -0,0 +1,66 @@
+import logging
+import os
+import sys
+
+from server.common.aws_secret_utils import get_secret_key
+from server.common.data_locator import discover_s3_region_name
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
+
+
+def handle_config_from_secret(app_config):
+ """Update configuration from the secret manager"""
+ secret_name = os.getenv("CXG_AWS_SECRET_NAME")
+ if not secret_name:
+ return
+
+ # need to find the secret manager region.
+ # 1. from CXG_AWS_SECRET_REGION_NAME
+ # 2. discover from dataroot location (if on s3)
+ # 3. discover from config file location (if on s3)
+ secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
+ if secret_region_name is None:
+ secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
+ if not secret_region_name:
+ from server.eb.app import config_file
+
+ secret_region_name = discover_s3_region_name(config_file)
+ if not secret_region_name:
+ logging.error("Could not determine the AWS Secret Manager region")
+ sys.exit(1)
+
+ secrets = get_secret_key(secret_region_name, secret_name)
+
+ if not secrets:
+ return
+
+ server_attrs = (
+ ("flask_secret_key", "app__flask_secret_key"),
+ ("oauth_client_secret", "authentication__params_oauth__client_secret"),
+ )
+ default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
+
+ # update server configuration attributes
+ for key, attr in server_attrs:
+ cur_val = getattr(app_config.server_config, attr)
+ if cur_val:
+ continue
+
+ # replace the attr with the secret if it is not set
+ val = secrets.get(key)
+ if val:
+ logging.info(f"set {attr} from secret")
+ app_config.update_server_config(**{attr: val})
+
+ # update default dataset configuration attributes
+ for key, attr in default_dataset_attrs:
+ cur_val = getattr(app_config.default_dataset_config, attr)
+ if cur_val:
+ continue
+
+ # replace the attr with the secret if it is not set
+ val = secrets.get(key)
+ if val:
+ logging.info(f"set {attr} from secret")
+ app_config.update_default_dataset_config(**{attr: val})
diff --git a/server/common/config/app_config.py b/server/common/config/app_config.py
new file mode 100644
index 00000000..422dcb8e
--- /dev/null
+++ b/server/common/config/app_config.py
@@ -0,0 +1,183 @@
+import yaml
+from flatten_dict import unflatten
+
+from server.default_config import get_default_config
+from server.common.config.dataset_config import DatasetConfig
+from server.common.config.server_config import ServerConfig
+from server.common.errors import ConfigurationError
+
+
+class AppConfig(object):
+ """
+ AppConfig stores all the configuration for cellxgene.
+ AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+ The server_config contains attributes that refer to the server process as a whole.
+ The default_dataset_config refers to attributes that are associated with the features and
+ presentations of a dataset.
+ The dataset config attributes can be overridden depending on the url by which the
+ dataset was accessed. These are stored in dataroot_config.
+ AppConfig has methods to initialize, modify, and access the configuration.
+ """
+
+ def __init__(self):
+
+ # the default configuration (see default_config.py)
+ # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+ # defaults within the config class?
+ self.default_config = get_default_config()
+ # the server configuration
+ self.server_config = ServerConfig(self, self.default_config["server"])
+ # the dataset config, unless overridden by an entry in dataroot_config
+ self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+ # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
+ # attribute of the server_config. The default dataset config will apply to all datasets unless a different set
+ # of config vars was passed for a specific dataset under the multidataset config. For example:
+ """
+ per_dataset_config:
+ d1:
+ user_annotations:
+ enable: false
+ d2:
+ user_annotations:
+ enable: true
+ """
+ # dataroot config
+ self.dataroot_config = {}
+
+ # Set to true when config_completed is called
+ self.is_completed = False
+
+ def get_dataset_config(self, dataroot_key):
+ if self.server_config.single_dataset__datapath:
+ return self.default_dataset_config
+ else:
+ return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
+
+ def check_config(self):
+ """Verify all the attributes in the config have been type checked"""
+ if not self.is_completed:
+ raise ConfigurationError("The configuration has not been completed")
+ self.server_config.check_config()
+ self.default_dataset_config.check_config()
+ for dataset_config in self.dataroot_config.values():
+ dataset_config.check_config()
+
+ def update_server_config(self, **kw):
+ self.server_config.update(**kw)
+ self.is_complete = False
+
+ def update_default_dataset_config(self, **kw):
+ self.default_dataset_config.update(**kw)
+ # update all the other dataset configs, if any
+ for value in self.dataroot_config.values():
+ value.update(**kw)
+ self.is_complete = False
+
+ def update_from_config_file(self, config_file):
+ try:
+ with open(config_file) as yml_file:
+ config = yaml.safe_load(yml_file)
+ except yaml.YAMLError as e:
+ raise ConfigurationError(f"The specified config file contained an error: {e}")
+ except OSError as e:
+ raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+ if config.get("server"):
+ self.server_config.update_from_config(config["server"], "server")
+ if config.get("dataset"):
+ self.default_dataset_config.update_from_config(config["dataset"], "dataset")
+
+ per_dataset_config = config.get("per_dataset_config", {})
+ for key, dataroot_config in per_dataset_config.items():
+ # first create and initialize the dataroot with the default config
+ self.add_dataroot_config(key, **config["dataset"])
+ # then apply the per dataset configuration
+ self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
+
+ self.is_complete = False
+
+ def write_config(self, config_file):
+ """output the config to a yaml file"""
+ server = self.server_config.create_mapping(self.server_config.default_config)
+ dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+ config = dict(server={}, dataset={})
+ for attrname in server.keys():
+ config["server__" + attrname] = getattr(self.server_config, attrname)
+ for attrname in dataset.keys():
+ config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
+ if self.dataroot_config:
+ config["per_dataset_config"] = {}
+ for dataroot_tag, dataroot_config in self.dataroot_config.items():
+ dataset = dataroot_config.create_mapping(dataroot_config.default_config)
+ for attrname in dataset.keys():
+ config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+
+ config = unflatten(config, splitter=lambda key: key.split("__"))
+ yaml.dump(config, open(config_file, "w"))
+
+ def changes_from_default(self):
+ """Return all the attribute that are different from the default"""
+ diff_server = self.server_config.changes_from_default()
+ diff_dataset = self.default_dataset_config.changes_from_default()
+ diff = dict(server=diff_server, dataset=diff_dataset)
+ return diff
+
+ def add_dataroot_config(self, dataroot_tag, **kw):
+ """Create a new dataset config object based on the default dataset config, and kw parameters"""
+ if dataroot_tag in self.dataroot_config:
+ raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
+ if type(self.server_config.multi_dataset__dataroot) != dict:
+ raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
+ if dataroot_tag not in self.server_config.multi_dataset__dataroot:
+ raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
+
+ self.is_completed = False
+ self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
+ flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+ config = {key: value[1] for key, value in flat_config.items()}
+ self.dataroot_config[dataroot_tag].update(**config)
+ self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
+
+ def complete_config(self, messagefn=None):
+ """The configure options are checked, and any additional setup based on the config
+ parameters is done"""
+
+ if messagefn is None:
+
+ def noop(message):
+ pass
+
+ messagefn = noop
+
+ # TODO: to give better error messages we can add a mapping between where each config
+ # attribute originated (e.g. command line argument or config file), then in the error
+ # messages we can give correct context for attributes with bad value.
+ context = dict(messagefn=messagefn)
+
+ self.server_config.complete_config(context)
+ self.default_dataset_config.complete_config(context)
+ for dataroot_config in self.dataroot_config.values():
+ dataroot_config.complete_config(context)
+
+ self.is_completed = True
+ self.check_config()
+
+ def get_matrix_data_cache_manager(self):
+ return self.server_config.matrix_data_cache_manager
+
+ def is_multi_dataset(self):
+ return self.server_config.multi_dataset__dataroot is not None
+
+ def get_title(self, data_adaptor):
+ return (
+ self.server_config.single_dataset__title
+ if self.server_config.single_dataset__title
+ else data_adaptor.get_title()
+ )
+
+ def get_about(self, data_adaptor):
+ return (
+ self.server_config.single_dataset__about
+ if self.server_config.single_dataset__about
+ else data_adaptor.get_about()
+ )
diff --git a/server/common/config/base_config.py b/server/common/config/base_config.py
new file mode 100644
index 00000000..a0b46f1b
--- /dev/null
+++ b/server/common/config/base_config.py
@@ -0,0 +1,113 @@
+import copy
+
+from flatten_dict import flatten
+from server.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+ """
+ This class handles the mechanics of updating and checking attributes.
+ Derived classes are expected to store the actual attributes
+ Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+ """
+
+ def __init__(self, app_config, default_config, dictval_cases={}):
+ # reference back to the app_config
+ self.app_config = app_config
+ # the complete set of attributes and their default values (unflattened)
+ self.default_config = default_config
+ # attributes where the value may be a dict (and therefore are not flattened)
+ self.dictval_cases = dictval_cases
+ # used to make sure every attribute value is checked
+ self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+ def create_mapping(self, config):
+ """
+ Create a dictionary where the keys are the name of attributes (using double underscore convention)
+ For example: authentication__type
+
+ The values are a tuple,
+ - the first item of the tuple is a tuple of path elements (location in config 'tree')
+ - the second item is the value of the config parameter
+
+ For example: (('authentication', 'type'), 'session'))
+ """
+ config_copy = copy.deepcopy(config)
+ mapping = {}
+
+ # special cases where the value could be a dict.
+ # If its value is not None, the entry is added to the mapping, and not included
+ # in the flattening below.
+ for dictval_case in self.dictval_cases:
+ cur = config_copy
+ for part in dictval_case[:-1]:
+ cur = cur.get(part, {})
+ val = cur.get(dictval_case[-1])
+ if val is not None:
+ key = "__".join(dictval_case)
+ mapping[key] = (dictval_case, val)
+ del cur[dictval_case[-1]]
+
+ flat_config = flatten(config_copy)
+ for key, value in flat_config.items():
+ # name of the attribute
+ attr = "__".join(key)
+ mapping[attr] = (key, value)
+
+ return mapping
+
+ def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+ val = getattr(self, attrname)
+ if type(vtype) in (list, tuple):
+ if type(val) not in vtype:
+ tnames = ",".join([x.__name__ for x in vtype])
+ raise ConfigurationError(
+ f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+ )
+ else:
+ if type(val) != vtype:
+ raise ConfigurationError(
+ f"Invalid type for attribute: {attrname}, "
+ f"expected type {vtype.__name__}, got {type(val).__name__}"
+ )
+
+ self.attr_checked[attrname] = True
+
+ def check_config(self):
+ mapping = self.create_mapping(self.default_config)
+ for key in mapping.keys():
+ if not self.attr_checked[key]:
+ raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+ def update(self, **kw):
+ for key, value in kw.items():
+ if not hasattr(self, key):
+ raise ConfigurationError(f"unknown config parameter {key}.")
+ try:
+ if type(value) == tuple:
+ # convert tuple values to list values
+ value = list(value)
+ setattr(self, key, value)
+ except KeyError:
+ raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+ self.attr_checked[key] = False
+
+ def update_from_config(self, config, prefix):
+ mapping = self.create_mapping(config)
+ for attr, (key, value) in mapping.items():
+ if not hasattr(self, attr):
+ raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+ setattr(self, attr, value)
+
+ self.attr_checked[attr] = False
+
+ def changes_from_default(self):
+ """Return all the attribute that are different from the default"""
+ mapping = self.create_mapping(self.default_config)
+ diff = []
+ for attrname, (key, defval) in mapping.items():
+ curval = getattr(self, attrname)
+ if curval != defval:
+ diff.append((attrname, curval, defval))
+ return diff
diff --git a/server/common/config/client_config.py b/server/common/config/client_config.py
new file mode 100644
index 00000000..9fec9042
--- /dev/null
+++ b/server/common/config/client_config.py
@@ -0,0 +1,125 @@
+from server import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+ """
+ Return the configuration as required by the /config REST route
+ """
+
+ server_config = app_config.server_config
+ dataset_config = data_adaptor.dataset_config
+ annotation = dataset_config.user_annotations
+ auth = server_config.auth
+
+ # FIXME The current set of config is not consistently presented:
+ # we have camalCase, hyphen-text, and underscore_text
+
+ # make sure the configuration has been checked.
+ app_config.check_config()
+
+ # features
+ features = [f.todict() for f in data_adaptor.get_features(annotation)]
+
+ # display_names
+ title = app_config.get_title(data_adaptor)
+ about = app_config.get_about(data_adaptor)
+
+ display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+ # library_versions
+ library_versions = {}
+ library_versions.update(data_adaptor.get_library_versions())
+ library_versions["cellxgene"] = cellxgene_display_version
+
+ # links
+ links = {"about-dataset": about}
+
+ # parameters
+ parameters = {
+ "layout": dataset_config.embeddings__names,
+ "max-category-items": dataset_config.presentation__max_categories,
+ "obs_names": server_config.single_dataset__obs_names,
+ "var_names": server_config.single_dataset__var_names,
+ "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+ "backed": server_config.adaptor__anndata_adaptor__backed,
+ "disable-diffexp": not dataset_config.diffexp__enable,
+ "enable-reembedding": dataset_config.embeddings__enable_reembedding,
+ "annotations": False,
+ "annotations_file": None,
+ "annotations_dir": None,
+ "annotations_cell_ontology_enabled": False,
+ "annotations_cell_ontology_obopath": None,
+ "annotations_cell_ontology_terms": None,
+ "custom_colors": dataset_config.presentation__custom_colors,
+ "diffexp-may-be-slow": False,
+ "about_legal_tos": dataset_config.app__about_legal_tos,
+ "about_legal_privacy": dataset_config.app__about_legal_privacy,
+ }
+
+ # corpora dataset_props
+ # TODO/Note: putting info from the dataset into the /config is not ideal.
+ # However, it is definitely not part of /schema, and we do not have a top-level
+ # route for data properties. Consider creating one at some point.
+ corpora_props = data_adaptor.get_corpora_props()
+ if corpora_props and "default_embedding" in corpora_props:
+ default_embedding = corpora_props["default_embedding"]
+ if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+ default_embedding = default_embedding[2:] # drop X_ prefix
+ if default_embedding in data_adaptor.get_embedding_names():
+ parameters["default_embedding"] = default_embedding
+
+ data_adaptor.update_parameters(parameters)
+ if annotation:
+ annotation.update_parameters(parameters, data_adaptor)
+
+ # gather it all together
+ client_config = {}
+ config = client_config["config"] = {}
+ config["features"] = features
+ config["displayNames"] = display_names
+ config["library_versions"] = library_versions
+ config["links"] = links
+ config["parameters"] = parameters
+ config["corpora_props"] = corpora_props
+ config["limits"] = {
+ "column_request_max": server_config.limits__column_request_max,
+ "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+ }
+
+ if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+ config["authentication"] = {
+ "requires_client_login": auth.requires_client_login(),
+ }
+ if auth.requires_client_login():
+ config["authentication"].update(
+ {
+ # Todo why are these stored on the data_adaptor?
+ "login": auth.get_login_url(data_adaptor),
+ "logout": auth.get_logout_url(data_adaptor),
+ }
+ )
+
+ return client_config
+
+
+def get_client_userinfo(app_config, data_adaptor):
+ """
+ Return the userinfo as required by the /userinfo REST route
+ """
+
+ server_config = app_config.server_config
+ dataset_config = data_adaptor.dataset_config
+ auth = server_config.auth
+
+ # make sure the configuration has been checked.
+ app_config.check_config()
+
+ if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+ userinfo = {}
+ userinfo["userinfo"] = {
+ "is_authenticated": auth.is_user_authenticated(),
+ "username": auth.get_user_name(),
+ "user_id": auth.get_user_id(),
+ "email": auth.get_user_email(),
+ }
+ return userinfo
diff --git a/server/common/config/dataset_config.py b/server/common/config/dataset_config.py
new file mode 100644
index 00000000..8c8231ac
--- /dev/null
+++ b/server/common/config/dataset_config.py
@@ -0,0 +1,234 @@
+import os
+from os.path import splitext, isdir
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError, OntologyLoadFailure
+from server.compute.scanpy import get_scanpy_module
+from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from server.db.db_utils import DbUtils
+
+
+class DatasetConfig(BaseConfig):
+ """Manages the config attribute associated with a dataset."""
+
+ def __init__(self, tag, app_config, default_config):
+ super().__init__(app_config, default_config)
+ self.tag = tag
+ try:
+ self.app__scripts = default_config["app"]["scripts"]
+ self.app__inline_scripts = default_config["app"]["inline_scripts"]
+ self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
+ self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
+ self.app__authentication_enable = default_config["app"]["authentication_enable"]
+
+ self.presentation__max_categories = default_config["presentation"]["max_categories"]
+ self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+ self.user_annotations__enable = default_config["user_annotations"]["enable"]
+ self.user_annotations__type = default_config["user_annotations"]["type"]
+ self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+ "directory"
+ ] # noqa E501
+ self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+ self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
+ self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
+ "obo_location"
+ ] # noqa E501
+ self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
+ "hosted_tiledb_array"
+ ][
+ "db_uri"
+ ] # noqa E501
+ self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
+ "hosted_tiledb_array"
+ ][
+ "hosted_file_directory"
+ ] # noqa E501
+
+ self.embeddings__names = default_config["embeddings"]["names"]
+ self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
+
+ self.diffexp__enable = default_config["diffexp"]["enable"]
+ self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+ self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+ except KeyError as e:
+ raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+ # The annotation object is created during complete_config and stored here.
+ self.user_annotations = None
+
+ def complete_config(self, context):
+ self.handle_app()
+ self.handle_presentation()
+ self.handle_user_annotations(context)
+ self.handle_embeddings()
+ self.handle_diffexp(context)
+
+ def handle_app(self):
+ self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+ self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+ self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
+ self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
+ self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
+
+ # scripts can be string (filename) or dict (attributes). Convert string to dict.
+ scripts = []
+ for script in self.app__scripts:
+ try:
+ if isinstance(script, str):
+ scripts.append({"src": script})
+ elif isinstance(script, dict) and isinstance(script["src"], str):
+ scripts.append(script)
+ else:
+ raise Exception
+ except Exception as e:
+ raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+ self.app__scripts = scripts
+
+ def handle_presentation(self):
+ self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+ self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+ def handle_user_annotations(self, context):
+ self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+ self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+ self.validate_correct_type_of_configuration_attribute(
+ "user_annotations__local_file_csv__directory", (type(None), str)
+ ) # noqa E501
+ self.validate_correct_type_of_configuration_attribute(
+ "user_annotations__local_file_csv__file", (type(None), str)
+ ) # noqa E501
+ self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
+ self.validate_correct_type_of_configuration_attribute(
+ "user_annotations__ontology__obo_location", (type(None), str)
+ ) # noqa E501
+ self.validate_correct_type_of_configuration_attribute(
+ "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
+ ) # noqa E501
+ self.validate_correct_type_of_configuration_attribute(
+ "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
+ ) # noqa E501
+ if self.user_annotations__enable:
+ server_config = self.app_config.server_config
+ if not self.app__authentication_enable:
+ raise ConfigurationError("user annotations requires authentication to be enabled")
+ if not server_config.auth.is_valid_authentication_type():
+ auth_type = server_config.authentication__type
+ raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
+ if self.user_annotations__type == "local_file_csv":
+ self.handle_local_file_csv_annotations()
+ elif self.user_annotations__type == "hosted_tiledb_array":
+ self.handle_hosted_tiledb_annotations()
+ else:
+ raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
+ if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
+ try:
+ self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
+ except OntologyLoadFailure as e:
+ raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+ else:
+ self.check_annotation_config_vars_not_set(context)
+
+ def handle_local_file_csv_annotations(self):
+ dirname = self.user_annotations__local_file_csv__directory
+ filename = self.user_annotations__local_file_csv__file
+ if filename is not None and dirname is not None:
+ raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+
+ if filename is not None:
+ lf_name, lf_ext = splitext(filename)
+ if lf_ext and lf_ext != ".csv":
+ raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+ if dirname is not None and not isdir(dirname):
+ try:
+ os.mkdir(dirname)
+ except OSError:
+ raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+
+ self.user_annotations = AnnotationsLocalFile(dirname, filename)
+
+ # if the user has specified a fixed label file, go ahead and validate it
+ # so that we can remove errors early in the process.
+ server_config = self.app_config.server_config
+ if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+ with server_config.matrix_data_cache_manager.data_adaptor(
+ self.tag, server_config.single_dataset__datapath, self.app_config
+ ) as data_adaptor:
+ data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+
+ def handle_hosted_tiledb_annotations(self):
+ self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
+ self.validate_correct_type_of_configuration_attribute(
+ "user_annotations__hosted_tiledb_array__hosted_file_directory", str
+ ) # noqa E501
+ self.user_annotations = AnnotationsHostedTileDB(
+ directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
+ db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
+ )
+
+ def check_annotation_config_vars_not_set(self, context):
+ if self.user_annotations__type is not None:
+ dirname = self.user_annotations__local_file_csv__directory
+ filename = self.user_annotations__local_file_csv__file
+ db_uri = self.user_annotations__hosted_tiledb_array__db_uri
+ hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
+ if filename is not None:
+ context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+ if dirname is not None:
+ context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
+ if db_uri is not None:
+ context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
+ if hosted_file_dirname is not None:
+ context["messagefn"](
+ "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
+ )
+
+ if self.user_annotations__ontology__enable:
+ context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
+ if self.user_annotations__ontology__obo_location is not None:
+ context["messagefn"](
+ "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
+ )
+
+ def handle_embeddings(self):
+ self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+ self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
+
+ server_config = self.app_config.server_config
+ if self.embeddings__enable_reembedding:
+ if server_config.single_dataset__datapath:
+ matrix_data_loader = MatrixDataLoader(
+ server_config.single_dataset__datapath, app_config=self.app_config
+ )
+ if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
+ raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
+ if server_config.adaptor__anndata_adaptor__backed:
+ raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
+
+ try:
+ get_scanpy_module()
+ except NotImplementedError:
+ # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
+ raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
+
+ def handle_diffexp(self, context):
+ self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+ self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+ self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+ server_config = self.app_config.server_config
+ if server_config.single_dataset__datapath:
+ with server_config.matrix_data_cache_manager.data_adaptor(
+ self.tag, server_config.single_dataset__datapath, self.app_config
+ ) as data_adaptor:
+ if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+ context["messagefn"](
+ "CAUTION: due to the size of your dataset, "
+ "running differential expression may take longer or fail."
+ )
diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py
new file mode 100644
index 00000000..508b3d4f
--- /dev/null
+++ b/server/common/config/server_config.py
@@ -0,0 +1,390 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse, quote_plus
+
+from server.auth.auth import AuthTypeFactory
+from server.common.config.base_config import BaseConfig
+from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from server.common.errors import ConfigurationError, DatasetAccessError
+from server.common.data_locator import discover_s3_region_name
+from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from server.compute import diffexp_cxg as diffexp_tiledb
+from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+
+
+class ServerConfig(BaseConfig):
+ """Manages the config attribute associated with the server."""
+
+ def __init__(self, app_config, default_config):
+ dictval_cases = [
+ ("app", "csp_directives"),
+ ("authentication", "params_oauth", "cookie"),
+ ("authentication", "params_oauth", "jwt_decode_options"),
+ ("adaptor", "cxg_adaptor", "tiledb_ctx"),
+ ("multi_dataset", "dataroot"),
+ ]
+ super().__init__(app_config, default_config, dictval_cases)
+
+ try:
+ self.app__verbose = default_config["app"]["verbose"]
+ self.app__debug = default_config["app"]["debug"]
+ self.app__host = default_config["app"]["host"]
+ self.app__port = default_config["app"]["port"]
+ self.app__open_browser = default_config["app"]["open_browser"]
+ self.app__force_https = default_config["app"]["force_https"]
+ self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+ self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
+ self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
+ self.app__csp_directives = default_config["app"]["csp_directives"]
+ self.app__api_base_url = default_config["app"]["api_base_url"]
+ self.app__web_base_url = default_config["app"]["web_base_url"]
+
+ self.authentication__type = default_config["authentication"]["type"]
+ self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
+ "oauth_api_base_url"
+ ] # noqa E501
+ self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
+ self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
+ "client_secret"
+ ] # noqa E501
+ self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
+ "jwt_decode_options"
+ ] # noqa E501
+ self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
+ "session_cookie"
+ ] # noqa E501
+ self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
+
+ self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
+ self.multi_dataset__index = default_config["multi_dataset"]["index"]
+ self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
+ self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
+ "max_datasets"
+ ] # noqa E501
+ self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
+ "timelimit_s"
+ ] # noqa E501
+
+ self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+ self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+ self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+ self.single_dataset__about = default_config["single_dataset"]["about"]
+ self.single_dataset__title = default_config["single_dataset"]["title"]
+
+ self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
+ self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
+ self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
+
+ self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+ self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
+ self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+ self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+ self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+ except KeyError as e:
+ raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+ # The matrix data cache manager is created during the complete_config and stored here.
+ self.matrix_data_cache_manager = None
+
+ # The authentication object
+ self.auth = None
+
+ def complete_config(self, context):
+ self.handle_app(context)
+ self.handle_data_source()
+ self.handle_authentication()
+ self.handle_data_locator()
+ self.handle_adaptor() # may depend on data_locator
+ self.handle_single_dataset(context) # may depend on adaptor
+ self.handle_multi_dataset() # may depend on adaptor
+ self.handle_diffexp()
+ self.handle_limits()
+
+ self.check_config()
+
+ def handle_app(self, context):
+ self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+ self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+ self.validate_correct_type_of_configuration_attribute("app__host", str)
+ self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+ self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+ self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+ self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str))
+ self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
+ self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
+ self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
+ self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
+ self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
+
+ if self.app__port:
+ try:
+ if not is_port_available(self.app__host, self.app__port):
+ raise ConfigurationError(
+ f"The port selected {self.app__port} is in use, please configure an open port."
+ )
+ except OverflowError:
+ raise ConfigurationError(f"Invalid port: {self.app__port}")
+ else:
+ try:
+ default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+ except ValueError:
+ raise ConfigurationError(
+ "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+ )
+ try:
+ self.app__port = find_available_port(self.app__host, default_server_port)
+ except OverflowError:
+ raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+ if self.app__debug:
+ context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+ self.app__verbose = True
+ self.app__open_browser = False
+ else:
+ warnings.formatwarning = custom_format_warning
+
+ if not self.app__verbose:
+ sys.tracebacklimit = 0
+
+ # secret key:
+ # first, from CXG_SECRET_KEY environment variable
+ # second, from config file
+ self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
+
+ # CSP Directives are a dict of string: list(string) or string: string
+ if self.app__csp_directives is not None:
+ for k, v in self.app__csp_directives.items():
+ if not isinstance(k, str):
+ raise ConfigurationError("CSP directive names must be a string.")
+ if isinstance(v, list):
+ for policy in v:
+ if not isinstance(policy, str):
+ raise ConfigurationError("CSP directive value must be a string or list of strings.")
+ elif not isinstance(v, str):
+ raise ConfigurationError("CSP directive value must be a string or list of strings.")
+
+ if self.app__web_base_url is None:
+ self.app__web_base_url = self.app__api_base_url
+
+ def handle_authentication(self):
+ self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+
+ # oauth
+ ptypes = str if self.authentication__type == "oauth" else (type(None), str)
+ self.validate_correct_type_of_configuration_attribute(
+ "authentication__params_oauth__oauth_api_base_url", ptypes
+ ) # noqa E501
+ self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
+ self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
+ self.validate_correct_type_of_configuration_attribute(
+ "authentication__params_oauth__jwt_decode_options", (type(None), dict)
+ ) # noqa E501
+ self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
+
+ if self.authentication__params_oauth__session_cookie:
+ self.validate_correct_type_of_configuration_attribute(
+ "authentication__params_oauth__cookie", (type(None), dict)
+ ) # noqa E501
+ else:
+ self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
+ # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
+ # second, from config file
+ self.authentication__params_oauth__client_secret = os.environ.get(
+ "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
+ )
+
+ self.auth = AuthTypeFactory.create(self.authentication__type, self)
+ if self.auth is None:
+ raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+
+ def handle_data_locator(self):
+ self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+ if self.data_locator__s3__region_name is True:
+ path = self.single_dataset__datapath or self.multi_dataset__dataroot
+
+ if type(path) == dict:
+ # if multi_dataset__dataroot is a dict, then use the first key
+ # that is in s3. NOTE: it is not supported to have dataroots
+ # in different regions.
+ paths = [val.get("dataroot") for val in path.values()]
+ for path in paths:
+ if path.startswith("s3://"):
+ break
+ if path.startswith("s3://"):
+ region_name = discover_s3_region_name(path)
+ if region_name is None:
+ raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+ else:
+ region_name = None
+ self.data_locator__s3__region_name = region_name
+
+ def handle_data_source(self):
+ self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+ self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+
+ if self.single_dataset__datapath and self.multi_dataset__dataroot:
+ raise ConfigurationError(
+ "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
+ )
+ if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
+ raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
+
+ def handle_single_dataset(self, context):
+ self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+ self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+ self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+ self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+ self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+ if self.single_dataset__datapath is None:
+ return
+
+ # create the matrix data cache manager:
+ if self.matrix_data_cache_manager is None:
+ self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
+
+ # preload this data set
+ matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+ try:
+ matrix_data_loader.pre_load_validation()
+ except DatasetAccessError as e:
+ raise ConfigurationError(str(e))
+
+ file_size = matrix_data_loader.file_size()
+ file_basename = basename(self.single_dataset__datapath)
+ if file_size > BIG_FILE_SIZE_THRESHOLD:
+ context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+ else:
+ context["messagefn"](f"Loading data from {file_basename}.")
+
+ if self.single_dataset__about:
+
+ def url_check(url):
+ try:
+ result = urlparse(url)
+ if all([result.scheme, result.netloc]):
+ return True
+ else:
+ return False
+ except ValueError:
+ return False
+
+ if not url_check(self.single_dataset__about):
+ raise ConfigurationError(
+ "Must provide an absolute URL for --about. (Example format: http://example.com)"
+ )
+
+ def handle_multi_dataset(self):
+ self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+ self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
+ self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
+ self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
+ self.validate_correct_type_of_configuration_attribute(
+ "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
+ ) # noqa E501
+
+ if self.multi_dataset__dataroot is None:
+ return
+
+ if type(self.multi_dataset__dataroot) == str:
+ default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
+ self.multi_dataset__dataroot = dict(d=default_dict)
+
+ for tag, dataroot_dict in self.multi_dataset__dataroot.items():
+ if "base_url" not in dataroot_dict:
+ raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
+ if "dataroot" not in dataroot_dict:
+ raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
+
+ base_url = dataroot_dict["base_url"]
+
+ # sanity check for well formed base urls
+ bad = False
+ if type(base_url) != str:
+ bad = True
+ elif os.path.normpath(base_url) != base_url:
+ bad = True
+ else:
+ base_url_parts = base_url.split("/")
+ if [quote_plus(part) for part in base_url_parts] != base_url_parts:
+ bad = True
+ if ".." in base_url_parts:
+ bad = True
+ if bad:
+ raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
+
+ # verify all the base_urls are unique
+ base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
+ if len(base_urls) > len(set(base_urls)):
+ raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
+
+ # error checking
+ for mtype in self.multi_dataset__allowed_matrix_types:
+ try:
+ MatrixDataType(mtype)
+ except ValueError:
+ raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
+
+ # create the matrix data cache manager:
+ if self.matrix_data_cache_manager is None:
+ self.matrix_data_cache_manager = MatrixDataCacheManager(
+ max_cached=self.multi_dataset__matrix_cache__max_datasets,
+ timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
+ )
+
+ def handle_diffexp(self):
+ self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
+ self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
+ self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
+
+ max_workers = self.diffexp__alg_cxg__max_workers
+ cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
+ cpu_count = os.cpu_count()
+ max_workers = min(max_workers, cpu_multiplier * cpu_count)
+ diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
+
+ def handle_adaptor(self):
+ # cxg
+ self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
+ regionkey = "vfs.s3.region"
+ if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
+ if type(self.data_locator__s3__region_name) == str:
+ self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
+
+ from server.data_cxg.cxg_adaptor import CxgAdaptor
+
+ CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
+
+ # anndata
+ self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+ def handle_limits(self):
+ self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+ self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+ def exceeds_limit(self, limit_name, value):
+ limit_value = getattr(self, "limits__" + limit_name, None)
+ if limit_value is None: # disabled
+ return False
+ return value > limit_value
+
+ def get_api_base_url(self):
+ if self.app__api_base_url == "local":
+ return f"http://{self.app__host}:{self.app__port}"
+ if self.app__api_base_url and self.app__api_base_url.endswith("/"):
+ return self.app__api_base_url[:-1]
+ return self.app__api_base_url
+
+ def get_web_base_url(self):
+ if self.app__web_base_url == "local":
+ return f"http://{self.app__host}:{self.app__port}"
+ if self.app__web_base_url is None:
+ return self.get_api_base_url()
+ if self.app__web_base_url.endswith("/"):
+ return self.app__web_base_url[:-1]
+ return self.app__web_base_url
diff --git a/server/common/errors.py b/server/common/errors.py
index 5e8281e6..ca339bee 100644
--- a/server/common/errors.py
+++ b/server/common/errors.py
@@ -42,14 +42,14 @@ define_request_exception(
define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
define_request_exception(
- "AuthenticationError",
- "Raised when there is an authentication error",
- default_status_code=HTTPStatus.UNAUTHORIZED)
+ "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
+)
define_request_exception(
"AnnotationCategoryNameError",
"Raised when an annotation category name cant be saved",
- default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+ default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+)
define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
define_exception("ConfigurationError", "Raised when checking configuration errors")
diff --git a/server/common/rest.py b/server/common/rest.py
index be099f94..9baeca4c 100644
--- a/server/common/rest.py
+++ b/server/common/rest.py
@@ -6,6 +6,7 @@ from http import HTTPStatus
from flask import make_response, jsonify, current_app, abort
from werkzeug.urls import url_unquote
+from server.common.config.client_config import get_client_config, get_client_userinfo
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.common.errors import (
FilterError,
@@ -117,12 +118,12 @@ def schema_get(data_adaptor):
def config_get(app_config, data_adaptor):
- config = app_config.get_client_config(data_adaptor)
+ config = get_client_config(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
def userinfo_get(app_config, data_adaptor):
- config = app_config.get_client_userinfo(data_adaptor)
+ config = get_client_userinfo(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
diff --git a/server/common/utils/cxg_generation_utils.py b/server/common/utils/cxg_generation_utils.py
index f29f4bd0..d3ec2b40 100644
--- a/server/common/utils/cxg_generation_utils.py
+++ b/server/common/utils/cxg_generation_utils.py
@@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
def convert_matrix_to_cxg_array(
- matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
+ matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
):
"""
Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
diff --git a/server/common/utils/matrix_utils.py b/server/common/utils/matrix_utils.py
index 3eeddc10..60dfb19b 100644
--- a/server/common/utils/matrix_utils.py
+++ b/server/common/utils/matrix_utils.py
@@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
number_of_non_zero_elements += np.count_nonzero(matrix_subset)
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
if end_row_index != total_number_of_rows:
- percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
- end_row_index * total_number_of_columns)
+ percentage_of_non_zero_elements = (
+ 100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
+ )
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
- f"{percentage_of_non_zero_elements:6.2f}")
+ f"{percentage_of_non_zero_elements:6.2f}"
+ )
else:
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (exact): "
- f"{percentage_of_non_zero_elements:6.2f}")
+ f"{percentage_of_non_zero_elements:6.2f}"
+ )
return False
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py
index ac6b3fe4..4bc6bf88 100644
--- a/server/common/utils/type_conversion_utils.py
+++ b/server/common/utils/type_conversion_utils.py
@@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
schema_type_hints_by_column_name = {}
for column_name, column_values in dataframe.items():
- dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \
- get_dtype_and_schema_of_array(column_values)
+ (
+ dtypes_by_column_name[column_name],
+ schema_type_hints_by_column_name[column_name],
+ ) = get_dtype_and_schema_of_array(column_values)
return dtypes_by_column_name, schema_type_hints_by_column_name
@@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series):
def get_dtype_and_schema_of_array(array: pd.Series):
- return (get_dtype_from_dtype(array.dtype, array_values=array),
- get_schema_type_hint_from_dtype(array.dtype, array_values=array))
+ return (
+ get_dtype_from_dtype(array.dtype, array_values=array),
+ get_schema_type_hint_from_dtype(array.dtype, array_values=array),
+ )
def get_dtype_from_dtype(dtype, array_values=None):
@@ -133,9 +137,11 @@ def can_cast_to_int32(dtype, array_values=None):
if np.can_cast(dtype, np.int32):
return True
ii32 = np.iinfo(np.int32)
- if not ordered_array_values.empty and (
- ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \
- ordered_array_values.empty:
+ if (
+ not ordered_array_values.empty
+ and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+ or ordered_array_values.empty
+ ):
return True
return False
diff --git a/server/converters/h5ad_data_file.py b/server/converters/h5ad_data_file.py
index c8223822..c79dd785 100644
--- a/server/converters/h5ad_data_file.py
+++ b/server/converters/h5ad_data_file.py
@@ -24,14 +24,14 @@ class H5ADDataFile:
another format (currently just CXG is supported). """
def __init__(
- self,
- input_filename,
- backed=False,
- dataset_title=None,
- dataset_about=None,
- obs_index_column_name=None,
- vars_index_column_name=None,
- use_corpora_schema=True,
+ self,
+ input_filename,
+ backed=False,
+ dataset_title=None,
+ dataset_about=None,
+ obs_index_column_name=None,
+ vars_index_column_name=None,
+ use_corpora_schema=True,
):
self.input_filename = input_filename
self.backed = backed
diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py
index 20dafdd0..3945ea7a 100644
--- a/server/data_common/data_adaptor.py
+++ b/server/data_common/data_adaptor.py
@@ -5,7 +5,7 @@ import numpy as np
import pandas as pd
from server_timing import Timing as ServerTiming
-from server.common.app_config import AppFeature, AppConfig
+from server.common.config.app_config import AppConfig
from server.common.constants import Axis
from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
from server.common.utils.utils import jsonify_numpy
@@ -173,7 +173,7 @@ class DataAdaptor(metaclass=ABCMeta):
mask = np.zeros((count,), dtype=np.bool)
for i in filter:
if type(i) == list:
- mask[i[0]: i[1]] = True
+ mask[i[0] : i[1]] = True
else:
mask[i] = True
return mask
@@ -314,7 +314,7 @@ class DataAdaptor(metaclass=ABCMeta):
top_n = self.dataset_config.diffexp__top_n
if self.server_config.exceeds_limit(
- "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+ "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
):
raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
@@ -388,3 +388,17 @@ class DataAdaptor(metaclass=ABCMeta):
except RuntimeError:
lastmod = None
return lastmod
+
+
+class AppFeature(object):
+ def __init__(self, path, available=False, method="POST", extra={}):
+ self.path = path
+ self.available = available
+ self.method = method
+ self.extra = extra
+ [setattr(self, key, value) for key, value in extra.items()]
+
+ def todict(self):
+ d = dict(available=self.available, method=self.method, path=self.path)
+ d.update(self.extra)
+ return d
diff --git a/server/db/cellxgene_orm.py b/server/db/cellxgene_orm.py
index f2209b63..2860f6b1 100644
--- a/server/db/cellxgene_orm.py
+++ b/server/db/cellxgene_orm.py
@@ -1,11 +1,6 @@
import uuid
-from sqlalchemy import (
- Column,
- DateTime,
- ForeignKey,
- String,
- func, JSON)
+from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
diff --git a/server/db/db_utils.py b/server/db/db_utils.py
index 1664303f..b3030bf2 100644
--- a/server/db/db_utils.py
+++ b/server/db/db_utils.py
@@ -42,9 +42,9 @@ class DbUtils:
def get_or_create_dataset(self, dataset_name):
try:
- dataset_id = self.query(
- table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
- )[0].id
+ dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[
+ 0
+ ].id
except IndexError:
dataset_id = uuid.uuid4()
dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
@@ -54,9 +54,7 @@ class DbUtils:
def get_or_create_user(self, user_id):
try:
- user_id = self.query(
- table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id]
- )[0].id
+ user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id
except IndexError:
user = CellxGeneUser(id=user_id)
self.session.add(user)
diff --git a/server/common/default_config.py b/server/default_config.py
similarity index 99%
rename from server/common/default_config.py
rename to server/default_config.py
index 16e0c27d..88e96fe5 100644
--- a/server/common/default_config.py
+++ b/server/default_config.py
@@ -204,7 +204,6 @@ dataset:
enable: true
lfc_cutoff: 0.01
top_n: 10
-
"""
diff --git a/server/eb/app.py b/server/eb/app.py
index 7a5e834c..0844f599 100644
--- a/server/eb/app.py
+++ b/server/eb/app.py
@@ -9,7 +9,7 @@ from flask import json
import logging
from flask_talisman import Talisman
from flask_cors import CORS
-from server.common.aws_secret_utils import handle_config_from_secret
+from server.common.config import handle_config_from_secret
from server.common.errors import SecretKeyRetrievalError
@@ -26,7 +26,7 @@ SERVERDIR = os.path.dirname(os.path.realpath(__file__))
sys.path.append(SERVERDIR)
try:
- from server.common.app_config import AppConfig
+ from server.common.config.app_config import AppConfig
from server.app.app import Server
from server.common.data_locator import DataLocator, discover_s3_region_name
except Exception:
@@ -61,8 +61,7 @@ class WSGIServer(Server):
csp = {
"default-src": ["'self'"],
"connect-src": ["'self'"] + extra_connect_src,
- "script-src": ["'self'", "'unsafe-eval'"]
- + obsolete_browser_script_hash + script_hashes,
+ "script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes,
"style-src": ["'self'", "'unsafe-inline'"],
"img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"],
"object-src": ["'none'"],
@@ -104,7 +103,7 @@ class WSGIServer(Server):
if len(script_hashes) == 0:
logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy")
- return (script_hashes)
+ return script_hashes
@staticmethod
def compute_inline_csp_hashes(app, app_config):
@@ -173,9 +172,7 @@ try:
sys.exit(1)
# features are unsupported in the current hosted server
- app_config.update_default_dataset_config(
- embeddings__enable_reembedding=False,
- )
+ app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
app_config.complete_config(logging.info)
diff --git a/server/test/__init__.py b/server/test/__init__.py
index 8ed6e3e7..bbb1de0b 100644
--- a/server/test/__init__.py
+++ b/server/test/__init__.py
@@ -13,7 +13,8 @@ import requests
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
+from server.common.config.app_config import AppConfig
+from server.common.config import DEFAULT_SERVER_PORT
from server.common.data_locator import DataLocator
from server.common.utils.utils import find_available_port
from server.data_common.fbs.matrix import encode_matrix_fbs
@@ -33,8 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
- multi_dataset__dataroot=data_locator.path,
- authentication__type="test",
+ multi_dataset__dataroot=data_locator.path, authentication__type="test",
)
config.update_default_dataset_config(
embeddings__names=["umap"],
@@ -42,16 +42,13 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
diffexp__lfc_cutoff=0.01,
user_annotations__type="hosted_tiledb_array",
user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
- user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir
+ user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
- annotations = AnnotationsHostedTileDB(
- tmp_dir,
- DbUtils("postgresql://postgres:test_pw@localhost:5432"),
- )
+ annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),)
return data, tmp_dir, annotations
diff --git a/server/test/fixtures/database/__init__.py b/server/test/fixtures/database/__init__.py
index 5a7fa984..0088f6d4 100644
--- a/server/test/fixtures/database/__init__.py
+++ b/server/test/fixtures/database/__init__.py
@@ -29,34 +29,26 @@ class TestDatabase:
def _create_test_user(self):
user = CellxGeneUser(id="test_user_id")
- user2 = CellxGeneUser(id='1234')
+ user2 = CellxGeneUser(id="1234")
self.db.session.add(user)
self.db.session.add(user2)
self.db.session.commit()
def _create_test_dataset(self):
- dataset = CellxGeneDataset(
- name="test_dataset",
- )
+ dataset = CellxGeneDataset(name="test_dataset",)
self.db.session.add(dataset)
self.db.session.commit()
def _create_test_annotation(self):
- dataset = self.db.query([CellxGeneDataset],
- [CellxGeneDataset.name == "test_dataset"],
- )[0]
- annotation = Annotation(
- tiledb_uri="tiledb_uri",
- user_id="test_user_id",
- dataset_id=str(dataset.id)
- )
+ dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0]
+ annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id))
self.db.session.add(annotation)
self.db.session.commit()
@staticmethod
def get_random_string():
letters = string.ascii_lowercase
- return ''.join(random.choice(letters) for i in range(12))
+ return "".join(random.choice(letters) for i in range(12))
def _create_test_users(self, user_count: int = 10):
users = []
@@ -80,10 +72,8 @@ class TestDatabase:
for i in range(annotation_count):
dataset = self.order_by_random(CellxGeneDataset)
user = self.order_by_random(CellxGeneUser)
- annotations.append(Annotation(
- tiledb_uri=self.get_random_string(),
- user_id=user.id,
- dataset_id=str(dataset.id)
- ))
+ annotations.append(
+ Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id))
+ )
self.db.session.add_all(annotations)
self.db.session.commit()
diff --git a/server/test/fixtures/dataset_config_outline.py b/server/test/fixtures/dataset_config_outline.py
new file mode 100644
index 00000000..a3d99d78
--- /dev/null
+++ b/server/test/fixtures/dataset_config_outline.py
@@ -0,0 +1,37 @@
+f"""
+dataset:
+ app:
+ scripts: {scripts} #list of strs (filenames) or dicts containing keys
+ inline_scripts: {inline_scripts} #list of strs (filenames)
+
+ about_legal_tos: {about_legal_tos}
+ about_legal_privacy: {about_legal_privacy}
+
+ authentication_enable: {authentication_enable}
+
+ presentation:
+ max_categories: {max_categories}
+ custom_colors: {custom_colors}
+
+ user_annotations:
+ enable: {enable_users_annotations}
+ type: {annotation_type}
+ hosted_tiledb_array:
+ db_uri: {db_uri}
+ hosted_file_directory: {hosted_file_directory}
+ local_file_csv:
+ directory: {local_file_csv_directory}
+ file: {local_file_csv_file}
+ ontology:
+ enable: {ontology_enabled}
+ obo_location: {obo_location}
+
+ embeddings:
+ names: {embedding_names}
+ enable_reembedding: {enable_reembedding}
+
+ diffexp:
+ enable: {enable_difexp}
+ lfc_cutoff: {lfc_cutoff}
+ top_n: {top_n}
+"""
diff --git a/server/test/fixtures/server_config_outline.py b/server/test/fixtures/server_config_outline.py
new file mode 100644
index 00000000..4bbb4675
--- /dev/null
+++ b/server/test/fixtures/server_config_outline.py
@@ -0,0 +1,62 @@
+f"""server:
+ app:
+ verbose: {verbose}
+ debug: {debug}
+ host: {host}
+ port: {port}
+ open_browser: {open_browser}
+ force_https: {force_https}
+ flask_secret_key: {flask_secret_key}
+ generate_cache_control_headers: {generate_cache_control_headers}
+ server_timing_headers: {server_timing_headers}
+ csp_directives: {csp_directives}
+ api_base_url: {api_base_url}
+ web_base_url: {web_base_url}
+ authentication:
+ type: {auth_type}
+ params_oauth:
+ oauth_api_base_url: {oauth_api_base_url}
+ client_id: {client_id}
+ client_secret: {client_secret}
+ jwt_decode_options: {jwt_decode_options}
+ session_cookie: {session_cookie}
+ cookie: {cookie}
+
+ multi_dataset:
+ dataroot: {dataroot}
+ index: {index}
+ allowed_matrix_types: {allowed_matrix_types}
+ matrix_cache:
+ max_datasets: {max_cached_datasets}
+ timelimit_s: {timelimit_s}
+
+ single_dataset:
+ datapath: {dataset_datapath}
+ obs_names: {obs_names}
+ var_names: {var_names}
+ about: {about}
+ title: {title}
+
+ diffexp:
+ alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count)
+ max_workers: {diffexp_max_workers}
+ cpu_multiplier: {cpu_multiplier}
+ target_workunit: {target_workunit} # The target number of matrix elements that are evaluated in one thread.
+
+ data_locator:
+ s3:
+ region_name: {data_locater_region_name}
+
+ adaptor:
+ cxg_adaptor:
+ tiledb_ctx:
+ sm.tile_cache_size: {cxg_tile_cache_size}
+ sm.num_reader_threads: {cxg_num_reader_threads}
+
+ anndata_adaptor:
+ backed: {anndata_backed}
+
+ limits:
+ column_request_max: {column_request_max}
+ diffexp_cellcount_max: {diffexp_cellcount_max}
+"""
diff --git a/server/test/performance/run_diffexp.py b/server/test/performance/run_diffexp.py
index bbbb3094..5cd1fbef 100644
--- a/server/test/performance/run_diffexp.py
+++ b/server/test/performance/run_diffexp.py
@@ -7,7 +7,7 @@ import numpy as np
import server.compute.diffexp_cxg as diffexp_cxg
import server.compute.diffexp_generic as diffexp_generic
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
from server.data_common.matrix_loader import MatrixDataLoader
from server.data_cxg.cxg_adaptor import CxgAdaptor
diff --git a/server/test/test_database/test_database.py b/server/test/test_database/test_database.py
index 4ecb4f27..91798fdb 100644
--- a/server/test/test_database/test_database.py
+++ b/server/test/test_database/test_database.py
@@ -16,45 +16,48 @@ class DatabaseTest(unittest.TestCase):
del cls.db
def test_user_creation(self):
- one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id')
- self.assertEqual(one_user.id, 'test_user_id')
+ one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id")
+ self.assertEqual(one_user.id, "test_user_id")
user_count = self.db.session.query(CellxGeneUser).count()
self.assertGreater(user_count, 10)
def test_dataset_creation(self):
- one_dataset = self.db.query(table_args=[CellxGeneDataset],
- filter_args=[CellxGeneDataset.name == 'test_dataset'])
- self.assertEqual(one_dataset[0].name, 'test_dataset')
+ one_dataset = self.db.query(
+ table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"]
+ )
+ self.assertEqual(one_dataset[0].name, "test_dataset")
dataset_count = self.db.session.query(CellxGeneDataset).count()
self.assertGreater(dataset_count, 10)
def test_annotation_creation(self):
- one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0]
- self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri')
+ one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0]
+ self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri")
annotation_count = self.db.session.query(Annotation).count()
self.assertGreater(annotation_count, 10)
def test_get_most_recent_annotation_for_user_dataset(self):
- dataset_id = str(self.db.query(table_args=[CellxGeneDataset],
- filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id)
+ dataset_id = str(
+ self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id
+ )
# have to commit separately because created_at time written on the db server
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0'))
+ self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0"))
self.db.session.commit()
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1'))
+ self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1"))
self.db.session.commit()
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2'))
+ self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2"))
self.db.session.commit()
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3'))
+ self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3"))
self.db.session.commit()
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4'))
+ self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4"))
self.db.session.commit()
- most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id,
- Annotation.user_id == 'test_user_id'])
+ most_recent_annotation = self.db.query_for_most_recent(
+ Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"]
+ )
- self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4')
+ self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4")
diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py
index b07e38d3..f7f6f358 100644
--- a/server/test/unit/auth/test_auth.py
+++ b/server/test/unit/auth/test_auth.py
@@ -2,7 +2,7 @@ import unittest
import requests
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
from server.test import FIXTURES_ROOT, test_server
@@ -12,9 +12,7 @@ class AuthTest(unittest.TestCase):
def test_auth_none(self):
c = AppConfig()
- c.update_server_config(
- authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot
- )
+ c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
c.update_default_dataset_config(user_annotations__enable=False)
c.complete_config()
@@ -28,9 +26,7 @@ class AuthTest(unittest.TestCase):
def test_auth_session(self):
c = AppConfig()
- c.update_server_config(
- authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot
- )
+ c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
c.update_default_dataset_config(user_annotations__enable=True)
c.complete_config()
@@ -107,8 +103,8 @@ class AuthTest(unittest.TestCase):
def test_auth_test_single(self):
c = AppConfig()
c.update_server_config(
- authentication__type="test",
- single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")
+ authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
+ )
c.complete_config()
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index 728fccfe..8b250ed6 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -9,7 +9,7 @@ from flask import Flask, jsonify, make_response, request, redirect
from multiprocessing import Process
import jose
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
from server.test import FIXTURES_ROOT, test_server
# This tests the oauth authentication type.
@@ -46,7 +46,7 @@ def token():
"scope": "openid profile email",
"expires_in": TOKEN_EXPIRES,
"token_type": "Bearer",
- "expires_at": expires_at
+ "expires_at": expires_at,
}
return make_response(jsonify(r))
@@ -89,9 +89,8 @@ class AuthTest(unittest.TestCase):
authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}",
authentication__params_oauth__client_id="mock_client_id",
authentication__params_oauth__client_secret="mock_client_secret",
- authentication__params_oauth__jwt_decode_options={
- "verify_signature": False, "verify_iss": False
- })
+ authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
+ )
app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
app_config.complete_config()
@@ -161,9 +160,7 @@ class AuthTest(unittest.TestCase):
def test_auth_oauth_session(self):
# test with session cookies
app_config = AppConfig()
- app_config.update_server_config(
- authentication__params_oauth__session_cookie=True,
- )
+ app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
self.auth_flow(app_config)
def test_auth_oauth_cookie(self):
diff --git a/server/test/unit/cli/test_launch.py b/server/test/unit/cli/test_launch.py
new file mode 100644
index 00000000..ac388cc0
--- /dev/null
+++ b/server/test/unit/cli/test_launch.py
@@ -0,0 +1,28 @@
+import filecmp
+import os
+import shutil
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.test import FIXTURES_ROOT
+
+
+class CLIPLaunchTests(unittest.TestCase):
+ tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ os.mkdir(cls.tmp_dir)
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ shutil.rmtree(cls.tmp_dir)
+
+
+def test_dump_default_config(self):
+ os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+ with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+ expected_config.write(yaml.dump(default_config))
+ filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")
diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py
new file mode 100644
index 00000000..fa730df2
--- /dev/null
+++ b/server/test/unit/common/config/__init__.py
@@ -0,0 +1,246 @@
+import os
+import shutil
+import unittest
+import random
+from unittest import mock
+
+from server.test import FIXTURES_ROOT
+
+
+def mockenv(**envvars):
+ return mock.patch.dict(os.environ, envvars)
+
+
+class ConfigTests(unittest.TestCase):
+ tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ shutil.rmtree(cls.tmp_fixtures_directory)
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ os.makedirs(cls.tmp_fixtures_directory)
+
+ def custom_server_config(
+ self,
+ verbose="false",
+ debug="false",
+ host="localhost",
+ port="null",
+ open_browser="false",
+ force_https="false",
+ flask_secret_key="null",
+ generate_cache_control_headers="false",
+ server_timing_headers="false",
+ csp_directives="null",
+ api_base_url="null",
+ web_base_url="null",
+ auth_type="session",
+ oauth_api_base_url="null",
+ client_id="null",
+ client_secret="null",
+ jwt_decode_options="null",
+ session_cookie="true",
+ cookie="null",
+ dataroot="null",
+ index="false",
+ allowed_matrix_types=[],
+ max_cached_datasets=5,
+ timelimit_s=5,
+ dataset_datapath="null",
+ obs_names="null",
+ var_names="null",
+ about="null",
+ title="null",
+ diffexp_max_workers=64,
+ cpu_multiplier=4,
+ target_workunit="16_000_000",
+ data_locater_region_name="us-east-1",
+ cxg_tile_cache_size=8589934592,
+ cxg_num_reader_threads=32,
+ anndata_backed="false",
+ column_request_max=32,
+ diffexp_cellcount_max="null",
+ config_file_name="server_config.yaml",
+ ):
+ configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+ server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
+ with open(server_config_outline_path, "r") as config_skeleton:
+ config = config_skeleton.read()
+ server_config = eval(config)
+ with open(configfile, "w") as server_config_file:
+ server_config_file.write(server_config)
+ return configfile
+
+ def custom_app_config(
+ self,
+ verbose="false",
+ debug="false",
+ host="localhost",
+ port="null",
+ open_browser="false",
+ force_https="false",
+ flask_secret_key="null",
+ generate_cache_control_headers="false",
+ server_timing_headers="false",
+ csp_directives="null",
+ api_base_url="null",
+ web_base_url="null",
+ auth_type="session",
+ oauth_api_base_url="null",
+ client_id="null",
+ client_secret="null",
+ jwt_decode_options="null",
+ session_cookie="true",
+ cookie="null",
+ dataroot="null",
+ index="false",
+ allowed_matrix_types=[],
+ max_cached_datasets=5,
+ timelimit_s=5,
+ dataset_datapath="null",
+ obs_names="null",
+ var_names="null",
+ about="null",
+ title="null",
+ diffexp_max_workers=64,
+ cpu_multiplier=4,
+ target_workunit="16_000_000",
+ data_locater_region_name="us-east-1",
+ cxg_tile_cache_size=8589934592,
+ cxg_num_reader_threads=32,
+ anndata_backed="false",
+ column_request_max=32,
+ diffexp_cellcount_max="null",
+ scripts=[],
+ inline_scripts=[],
+ about_legal_tos="null",
+ about_legal_privacy="null",
+ authentication_enable="true",
+ max_categories=1000,
+ custom_colors="true",
+ enable_users_annotations="true",
+ annotation_type="local_file_csv",
+ db_uri="null",
+ hosted_file_directory="null",
+ local_file_csv_directory="null",
+ local_file_csv_file="null",
+ ontology_enabled="false",
+ obo_location="null",
+ embedding_names=[],
+ enable_reembedding="false",
+ enable_difexp="true",
+ lfc_cutoff=0.01,
+ top_n=10,
+ config_file_name="app_config.yml",
+ ):
+ random_num = random.randrange(999999)
+ configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+ server_config = self.custom_server_config(
+ verbose=verbose,
+ debug=debug,
+ host=host,
+ port=port,
+ open_browser=open_browser,
+ force_https=force_https,
+ flask_secret_key=flask_secret_key,
+ generate_cache_control_headers=generate_cache_control_headers,
+ server_timing_headers=server_timing_headers,
+ csp_directives=csp_directives,
+ api_base_url=api_base_url,
+ web_base_url=web_base_url,
+ auth_type=auth_type,
+ oauth_api_base_url=oauth_api_base_url,
+ client_id=client_id,
+ client_secret=client_secret,
+ jwt_decode_options=jwt_decode_options,
+ session_cookie=session_cookie,
+ cookie=cookie,
+ dataroot=dataroot,
+ index=index,
+ allowed_matrix_types=allowed_matrix_types,
+ max_cached_datasets=max_cached_datasets,
+ timelimit_s=timelimit_s,
+ dataset_datapath=dataset_datapath,
+ obs_names=obs_names,
+ var_names=var_names,
+ about=about,
+ title=title,
+ diffexp_max_workers=diffexp_max_workers,
+ cpu_multiplier=cpu_multiplier,
+ target_workunit=target_workunit,
+ data_locater_region_name=data_locater_region_name,
+ cxg_tile_cache_size=cxg_tile_cache_size,
+ cxg_num_reader_threads=cxg_num_reader_threads,
+ anndata_backed=anndata_backed,
+ column_request_max=column_request_max,
+ diffexp_cellcount_max=diffexp_cellcount_max,
+ config_file_name=f"temp_server_config_{random_num}.yml",
+ )
+ dataset_config = self.custom_dataset_config(
+ scripts=scripts,
+ inline_scripts=inline_scripts,
+ about_legal_tos=about_legal_tos,
+ about_legal_privacy=about_legal_privacy,
+ authentication_enable=authentication_enable,
+ max_categories=max_categories,
+ custom_colors=custom_colors,
+ enable_users_annotations=enable_users_annotations,
+ annotation_type=annotation_type,
+ db_uri=db_uri,
+ hosted_file_directory=hosted_file_directory,
+ local_file_csv_directory=local_file_csv_directory,
+ local_file_csv_file=local_file_csv_file,
+ ontology_enabled=ontology_enabled,
+ obo_location=obo_location,
+ embedding_names=embedding_names,
+ enable_reembedding=enable_reembedding,
+ enable_difexp=enable_difexp,
+ lfc_cutoff=lfc_cutoff,
+ top_n=top_n,
+ config_file_name=f"temp_dataset_config_{random_num}.yml",
+ )
+ with open(server_config) as server_config:
+ with open(dataset_config) as dataset_config:
+ with open(configfile, "w") as app_config_file:
+ for line in server_config:
+ app_config_file.write(line)
+ for line in dataset_config:
+ app_config_file.write(line)
+
+ return configfile
+
+ def custom_dataset_config(
+ self,
+ scripts=[],
+ inline_scripts=[],
+ about_legal_tos="null",
+ about_legal_privacy="null",
+ authentication_enable="true",
+ max_categories=1000,
+ custom_colors="true",
+ enable_users_annotations="true",
+ annotation_type="local_file_csv",
+ db_uri="null",
+ hosted_file_directory="null",
+ local_file_csv_directory="null",
+ local_file_csv_file="null",
+ ontology_enabled="false",
+ obo_location="null",
+ embedding_names=[],
+ enable_reembedding="false",
+ enable_difexp="true",
+ lfc_cutoff=0.01,
+ top_n=10,
+ config_file_name="dataset_config.yml",
+ ):
+ configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+ dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
+ with open(dataset_config_outline_path, "r") as config_skeleton:
+ config = config_skeleton.read()
+ dataset_config = eval(config)
+ with open(configfile, "w") as dataset_config_file:
+ dataset_config_file.write(dataset_config)
+
+ return configfile
diff --git a/server/test/unit/common/config/test_app_config.py b/server/test/unit/common/config/test_app_config.py
new file mode 100644
index 00000000..726ff86f
--- /dev/null
+++ b/server/test/unit/common/config/test_app_config.py
@@ -0,0 +1,140 @@
+import os
+import tempfile
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.common.config.app_config import AppConfig
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+from server.test import FIXTURES_ROOT
+
+
+class AppConfigTest(ConfigTests):
+ def setUp(self):
+ self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+ self.config = AppConfig()
+ self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+ self.server_config = self.config.server_config
+ self.config.complete_config()
+
+ message_list = []
+
+ def noop(message):
+ message_list.append(message)
+
+ messagefn = noop
+ self.context = dict(messagefn=messagefn, messages=message_list)
+
+ def get_config(self, **kwargs):
+ file_name = self.custom_app_config(
+ dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ return config
+
+ def test_get_default_config_correctly_reads_default_config_file(self):
+ app_default_config = AppConfig().default_config
+
+ expected_config = yaml.load(default_config, Loader=yaml.Loader)
+
+ server_config = app_default_config['server']
+ dataset_config = app_default_config['dataset']
+
+ expected_server_config = expected_config['server']
+ expected_dataset_config = expected_config['dataset']
+
+ self.assertDictEqual(app_default_config, expected_config)
+ self.assertDictEqual(server_config, expected_server_config)
+ self.assertDictEqual(dataset_config, expected_dataset_config)
+
+ def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
+ datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
+ file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+
+ self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
+
+ def test_update_server_config_updates_server_config_and_config_status(self):
+ config = self.get_config()
+ config.complete_config()
+ config.check_config()
+ config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+ with self.assertRaises(ConfigurationError):
+ config.server_config.check_config()
+
+ def test_write_config_outputs_yaml_with_all_config_vars(self):
+ config = self.get_config()
+ config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
+ with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
+ default_config_yml = yaml.safe_load(default_config)
+
+ with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
+ output_config_yml = yaml.safe_load(output_config)
+ self.maxDiff = None
+ self.assertEqual(default_config_yml, output_config_yml)
+
+ def test_update_app_config(self):
+ config = AppConfig()
+ config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
+ vars = config.server_config.changes_from_default()
+ self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
+
+ config = AppConfig()
+ config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
+ vars = config.server_config.changes_from_default()
+ self.assertCountEqual(vars, [])
+
+ config = AppConfig()
+ config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
+ vars = config.default_dataset_config.changes_from_default()
+ self.assertCountEqual(vars, [])
+
+ config = AppConfig()
+ config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+ vars = config.default_dataset_config.changes_from_default()
+ self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+ def test_configfile_no_dataset_section(self):
+ # test a config file without a dataset section
+
+ with tempfile.TemporaryDirectory() as tempdir:
+ configfile = os.path.join(tempdir, "config.yaml")
+ with open(configfile, "w") as fconfig:
+ config = """
+ server:
+ multi_dataset:
+ dataroot: test_dataroot
+
+ """
+ fconfig.write(config)
+
+ app_config = AppConfig()
+ app_config.update_from_config_file(configfile)
+ server_changes = app_config.server_config.changes_from_default()
+ dataset_changes = app_config.default_dataset_config.changes_from_default()
+ self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
+ self.assertEqual(dataset_changes, [])
+
+ def test_configfile_no_server_section(self):
+ # test a config file without a dataset section
+
+ with tempfile.TemporaryDirectory() as tempdir:
+ configfile = os.path.join(tempdir, "config.yaml")
+ with open(configfile, "w") as fconfig:
+ config = """
+ dataset:
+ user_annotations:
+ enable: false
+ """
+ fconfig.write(config)
+
+ app_config = AppConfig()
+ app_config.update_from_config_file(configfile)
+ server_changes = app_config.server_config.changes_from_default()
+ dataset_changes = app_config.default_dataset_config.changes_from_default()
+ self.assertEqual(server_changes, [])
+ self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
diff --git a/server/test/unit/common/config/test_base_config.py b/server/test/unit/common/config/test_base_config.py
new file mode 100644
index 00000000..d41082cf
--- /dev/null
+++ b/server/test/unit/common/config/test_base_config.py
@@ -0,0 +1,63 @@
+import unittest
+
+from server.common.config.app_config import AppConfig
+from server.test import FIXTURES_ROOT
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+
+
+class BaseConfigTest(ConfigTests):
+ def setUp(self):
+ self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+ self.config = AppConfig()
+ self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+ self.server_config = self.config.server_config
+ self.config.complete_config()
+
+ message_list = []
+
+ def noop(message):
+ message_list.append(message)
+
+ messagefn = noop
+ self.context = dict(messagefn=messagefn, messages=message_list)
+
+ def get_config(self, **kwargs):
+ file_name = self.custom_app_config(
+ dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ return config
+
+ def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
+ config = AppConfig()
+ mapping = config.default_dataset_config.create_mapping(config.default_config)
+ self.assertIsNotNone(mapping["server__app__verbose"])
+ self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
+ self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
+ self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
+
+ def test_changes_from_default_returns_list_of_nondefault_config_values(self):
+ config = self.get_config(verbose="true", lfc_cutoff=0.05)
+ server_changes = config.server_config.changes_from_default()
+ dataset_changes = config.default_dataset_config.changes_from_default()
+
+ self.assertEqual(
+ server_changes,
+ [
+ ("app__verbose", True, False),
+ ("multi_dataset__dataroot", FIXTURES_ROOT, None),
+ ("multi_dataset__matrix_cache__timelimit_s", 5, 30),
+ ("data_locator__s3__region_name", "us-east-1", True),
+ ],
+ )
+ self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
+
+ def test_check_config_throws_error_if_attr_has_not_been_checked(self):
+ config = self.get_config(verbose="true")
+ config.complete_config()
+ config.check_config()
+ config.update_server_config(app__verbose=False)
+ with self.assertRaises(ConfigurationError):
+ config.check_config()
diff --git a/server/test/unit/common/config/test_dataset_config.py b/server/test/unit/common/config/test_dataset_config.py
new file mode 100644
index 00000000..a32d1f87
--- /dev/null
+++ b/server/test/unit/common/config/test_dataset_config.py
@@ -0,0 +1,260 @@
+import os
+import tempfile
+
+import requests
+import unittest
+from unittest.mock import patch
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.app_config import AppConfig
+from server.common.config.base_config import BaseConfig
+from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT
+
+from server.common.errors import ConfigurationError
+from server.test.unit.common.config import ConfigTests
+
+
+class TestDatasetConfig(ConfigTests):
+ def setUp(self):
+ self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+ self.config = AppConfig()
+ self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+ self.dataset_config = self.config.default_dataset_config
+ self.config.complete_config()
+ message_list = []
+
+ def noop(message):
+ message_list.append(message)
+
+ messagefn = noop
+ self.context = dict(messagefn=messagefn, messages=message_list)
+
+ def get_config(self, **kwargs):
+ file_name = self.custom_app_config(
+ dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ return config
+
+ def test_init_datatset_config_sets_vars_from_default_config(self):
+ config = AppConfig()
+ self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
+ self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
+ self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
+ self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
+
+ @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+ def test_complete_config_checks_all_attr(self, mock_check_attrs):
+ mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+ self.dataset_config.complete_config(self.context)
+ self.assertEqual(mock_check_attrs.call_count, 21)
+
+ def test_app_sets_script_vars(self):
+ config = self.get_config(scripts=["path/to/script"])
+ config.default_dataset_config.handle_app()
+
+ self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
+
+ config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
+ config.default_dataset_config.handle_app()
+ self.assertEqual(
+ config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
+ )
+
+ config = self.get_config(scripts=["path/to/script", "different/script/path"])
+ config.default_dataset_config.handle_app()
+ # TODO @madison -- is this the desired functionality?
+ self.assertEqual(
+ config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
+ )
+
+ config = self.get_config(scripts=[{"more": "different/script/path"}])
+ with self.assertRaises(ConfigurationError):
+ config.default_dataset_config.handle_app()
+
+ def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
+ config = self.get_config(enable_users_annotations="true", authentication_enable="false")
+ config.server_config.complete_config(self.context)
+ with self.assertRaises(ConfigurationError):
+ config.default_dataset_config.handle_user_annotations(self.context)
+
+ config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
+ with self.assertRaises(ConfigurationError):
+ config.server_config.complete_config(self.context)
+
+ def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
+ config = self.get_config(
+ enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
+ )
+ config.default_dataset_config.handle_user_annotations(self.context)
+
+ self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
+
+ @patch("server.common.config.dataset_config.DbUtils")
+ def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
+ mock_db_utils.return_value = "123"
+ config = self.get_config(
+ enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+ )
+ config.server_config.complete_config(self.context)
+ config.default_dataset_config.handle_user_annotations(self.context)
+ self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+
+ config = self.get_config(
+ enable_users_annotations="true",
+ authentication_enable="true",
+ annotation_type="hosted_tiledb_array",
+ db_uri="gotta/set/this",
+ hosted_file_directory="and/this",
+ )
+ config.server_config.complete_config(self.context)
+ config.default_dataset_config.handle_user_annotations(self.context)
+ self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
+
+ config = self.get_config(
+ enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
+ )
+ config.server_config.complete_config(self.context)
+ with self.assertRaises(ConfigurationError):
+ config.default_dataset_config.handle_user_annotations(self.context)
+
+ def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
+ config = self.get_config(
+ enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+ )
+ config.server_config.complete_config(self.context)
+ config.default_dataset_config.handle_local_file_csv_annotations()
+ self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+ cwd = os.getcwd()
+ self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
+
+ def test_handle_embeddings__checks_data_file_types(self):
+ file_name = self.custom_app_config(
+ embedding_names=["name1", "name2"],
+ enable_reembedding="true",
+ dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+ anndata_backed="true",
+ config_file_name=self.config_file_name,
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ config.server_config.complete_config(self.context)
+ with self.assertRaises(ConfigurationError):
+ config.default_dataset_config.handle_embeddings()
+
+ def test_handle_diffexp__raises_warning_for_large_datasets(self):
+ config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
+ config.server_config.complete_config(self.context)
+ config.default_dataset_config.handle_diffexp(self.context)
+ self.assertEqual(len(self.context["messages"]), 0)
+
+ def test_multi_dataset(self):
+ config = AppConfig()
+ # test for illegal url_dataroots
+ for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+ config.update_server_config(
+ multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ )
+ with self.assertRaises(ConfigurationError):
+ config.complete_config()
+
+ # test for legal url_dataroots
+ for legal in ("d", "this.is-okay_", "a/b"):
+ config.update_server_config(
+ multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ )
+ config.complete_config()
+
+ # test that multi dataroots work end to end
+ config.update_server_config(
+ multi_dataset__dataroot=dict(
+ s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+ s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+ s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+ )
+ )
+
+ # Change this default to test if the dataroot overrides below work.
+ config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+ # specialize the configs for set1
+ config.add_dataroot_config(
+ "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+ )
+
+ # specialize the configs for set2
+ config.add_dataroot_config(
+ "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+ )
+
+ # no specializations for set3 (they get the default dataset config)
+ config.complete_config()
+
+ with test_server(app_config=config) as server:
+ session = requests.Session()
+
+ response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+ data_config = response.json()
+ assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+ assert data_config["config"]["parameters"]["annotations"] is False
+ assert data_config["config"]["parameters"]["disable-diffexp"] is False
+ assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+ response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+ data_config = response.json()
+ assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+ assert data_config["config"]["parameters"]["annotations"] is True
+ assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+ response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+ data_config = response.json()
+ assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+ assert data_config["config"]["parameters"]["annotations"] is True
+ assert data_config["config"]["parameters"]["disable-diffexp"] is False
+ assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+ response = session.get(f"{server}/health")
+ assert response.json()["status"] == "pass"
+
+ def test_configfile_with_specialization(self):
+ # test that per_dataset_config config load the default config, then the specialized config
+
+ with tempfile.TemporaryDirectory() as tempdir:
+ configfile = os.path.join(tempdir, "config.yaml")
+ with open(configfile, "w") as fconfig:
+ config = """
+ server:
+ multi_dataset:
+ dataroot:
+ test:
+ base_url: test
+ dataroot: fake_dataroot
+
+ dataset:
+ user_annotations:
+ enable: false
+ type: hosted_tiledb_array
+ hosted_tiledb_array:
+ db_uri: fake_db_uri
+ hosted_file_directory: fake_dir
+
+ per_dataset_config:
+ test:
+ user_annotations:
+ enable: true
+ """
+ fconfig.write(config)
+
+ app_config = AppConfig()
+ app_config.update_from_config_file(configfile)
+
+ test_config = app_config.dataroot_config["test"]
+
+ # test config from default
+ self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
+ self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
+
+ # test config from specialization
+ self.assertTrue(test_config.user_annotations__enable)
diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py
new file mode 100644
index 00000000..78ee9570
--- /dev/null
+++ b/server/test/unit/common/config/test_server_config.py
@@ -0,0 +1,335 @@
+import os
+import unittest
+from unittest import mock
+from unittest.mock import patch
+
+from server.common.config.base_config import BaseConfig
+from server.common.utils.utils import find_available_port
+from server.test import PROJECT_ROOT, FIXTURES_ROOT
+
+import requests
+
+from server.common.config.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.test import test_server
+from server.test.unit.common.config import ConfigTests
+
+
+def mockenv(**envvars):
+ return mock.patch.dict(os.environ, envvars)
+
+
+class TestServerConfig(ConfigTests):
+ def setUp(self):
+ self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+ self.config = AppConfig()
+ self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+ self.server_config = self.config.server_config
+ self.config.complete_config()
+
+ message_list = []
+
+ def noop(message):
+ message_list.append(message)
+
+ messagefn = noop
+ self.context = dict(messagefn=messagefn, messages=message_list)
+
+ def get_config(self, **kwargs):
+ file_name = self.custom_app_config(
+ dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ return config
+
+ def test_init_raises_error_if_default_config_is_invalid(self):
+ invalid_config = self.get_config(port="not_valid")
+ with self.assertRaises(ConfigurationError):
+ invalid_config.complete_config()
+
+ @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+ def test_complete_config_checks_all_attr(self, mock_check_attrs):
+ mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+ self.server_config.complete_config(self.context)
+ self.assertEqual(mock_check_attrs.call_count, 40)
+
+ def test_handle_app__throws_error_if_port_doesnt_exist(self):
+ config = self.get_config(port=99999999)
+ with self.assertRaises(ConfigurationError):
+ config.server_config.handle_app(self.context)
+
+ @patch("server.common.config.server_config.discover_s3_region_name")
+ def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
+ mock_discover_region_name.return_value = None
+ # Default config
+ self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
+ # hard coded
+ config = self.get_config()
+ self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
+ # incorrectly formatted
+ dataroot = {
+ "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+ "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
+ }
+ file_name = self.custom_app_config(
+ dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ with self.assertRaises(ConfigurationError):
+ config.server_config.handle_data_locator()
+
+ @patch("server.common.config.server_config.discover_s3_region_name")
+ def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
+ mock_discover_region_name.return_value = "us-west-2"
+ dataroot = {
+ "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+ "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
+ }
+ file_name = self.custom_app_config(
+ dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ config.server_config.handle_data_locator()
+ self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
+ mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
+
+ def test_handle_app___can_use_envar_port(self):
+ config = self.get_config(port=24)
+ self.assertEqual(config.server_config.app__port, 24)
+
+ # Note if the port is set in the config file it will NOT be overwritten by a different envvar
+ os.environ["CXG_SERVER_PORT"] = "4008"
+ self.config = AppConfig()
+ self.config.server_config.handle_app(self.context)
+ self.assertEqual(self.config.server_config.app__port, 4008)
+
+ def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
+ config = self.get_config(flask_secret_key="KEY_FROM_FILE")
+ self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
+
+ os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
+ config.server_config.handle_app(self.context)
+ self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
+
+ def test_handle_app__sets_web_base_url(self):
+ config = self.get_config(web_base_url="anything.com")
+ self.assertEqual(config.server_config.app__web_base_url, "anything.com")
+
+ def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
+ config = self.get_config(client_secret="KEY_FROM_FILE")
+ config.server_config.handle_authentication()
+ self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
+
+ os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
+ config.server_config.handle_authentication()
+
+ self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
+
+ def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
+ file_name = self.custom_app_config(
+ dataroot=f"{FIXTURES_ROOT}",
+ config_file_name="two_data_roots.yml",
+ dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ with self.assertRaises(ConfigurationError):
+ config.server_config.handle_data_source()
+
+ file_name = self.custom_app_config(config_file_name="zero_roots.yml")
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ with self.assertRaises(ConfigurationError):
+ config.server_config.handle_data_source()
+
+ def test_get_api_base_url_works(self):
+
+ # test the api_base_url feature, and that it can contain a path
+ config = AppConfig()
+ backend_port = find_available_port("localhost", 10000)
+ config.update_server_config(
+ app__api_base_url=f"http://localhost:{backend_port}/additional/path",
+ multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
+ )
+
+ config.complete_config()
+
+ with test_server(["-p", str(backend_port)], app_config=config) as server:
+ session = requests.Session()
+ self.assertEqual(server, f"http://localhost:{backend_port}")
+ response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
+ self.assertEqual(response.status_code, 200)
+ data_config = response.json()
+ self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+
+ # test the health check at the correct url
+ response = session.get(f"{server}/additional/path/health")
+ assert response.json()["status"] == "pass"
+
+ # also check that the old URL still works.
+ # NOTE: this old URL location will soon be deprecated, and when that happens
+ # this check can be removed.
+ response = session.get(f"{server}/health")
+ assert response.json()["status"] == "pass"
+
+ def test_get_web_base_url_works(self):
+ config = self.get_config(web_base_url="www.thisisawebsite.com")
+ web_base_url = config.server_config.get_web_base_url()
+ self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+ config = self.get_config(web_base_url="local", port=12)
+ web_base_url = config.server_config.get_web_base_url()
+ self.assertEqual(web_base_url, "http://localhost:12")
+
+ config = self.get_config(web_base_url="www.thisisawebsite.com/")
+ web_base_url = config.server_config.get_web_base_url()
+ self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+ config = self.get_config(api_base_url="www.api_base.com/")
+ web_base_url = config.server_config.get_web_base_url()
+ self.assertEqual(web_base_url, "www.api_base.com")
+
+ def test_config_for_single_dataset(self):
+ file_name = self.custom_app_config(
+ config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ config.server_config.handle_single_dataset(self.context)
+ self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
+
+ file_name = self.custom_app_config(
+ config_file_name="single_dataset_with_about.yml",
+ about="www.cziscience.com",
+ dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
+ )
+ config = AppConfig()
+ config.update_from_config_file(file_name)
+ with self.assertRaises(ConfigurationError):
+ config.server_config.handle_single_dataset(self.context)
+
+ def test_multi_dataset_raises_error_for_illegal_routes(self):
+ # test for illegal url_dataroots
+ for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+ self.config.update_server_config(
+ multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ )
+ with self.assertRaises(ConfigurationError):
+ self.config.complete_config()
+
+ def test_multidataset_works_for_legal_routes(self):
+ # test for legal url_dataroots
+ for legal in ("d", "this.is-okay_", "a/b"):
+ self.config.update_server_config(
+ multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ )
+ self.config.complete_config()
+
+ def test_mulitdatasets_work_e2e(self):
+ # test that multi dataroots work end to end
+ self.config.update_server_config(
+ multi_dataset__dataroot=dict(
+ s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+ s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+ s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+ )
+ )
+
+ # Change this default to test if the dataroot overrides below work.
+ self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+ # specialize the configs for set1
+ self.config.add_dataroot_config(
+ "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+ )
+
+ # specialize the configs for set2
+ self.config.add_dataroot_config(
+ "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+ )
+
+ # no specializations for set3 (they get the default dataset config)
+ self.config.complete_config()
+
+ with test_server(app_config=self.config) as server:
+ session = requests.Session()
+
+ response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+ data_config = response.json()
+ assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+ assert data_config["config"]["parameters"]["annotations"] is False
+ assert data_config["config"]["parameters"]["disable-diffexp"] is False
+ assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+ response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+ data_config = response.json()
+ assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+ assert data_config["config"]["parameters"]["annotations"] is True
+ assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+ response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+ data_config = response.json()
+ assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+ assert data_config["config"]["parameters"]["annotations"] is True
+ assert data_config["config"]["parameters"]["disable-diffexp"] is False
+ assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+ response = session.get(f"{server}/health")
+ assert response.json()["status"] == "pass"
+
+ @patch("server.common.config.server_config.diffexp_tiledb.set_config")
+ def test_handle_diffexp(self, mock_tiledb_config):
+ custom_config_file = self.custom_app_config(
+ dataroot=f"{FIXTURES_ROOT}",
+ cpu_multiplier=3,
+ diffexp_max_workers=1,
+ target_workunit=4,
+ config_file_name=self.config_file_name,
+ )
+ config = AppConfig()
+ config.update_from_config_file(custom_config_file)
+ config.server_config.handle_diffexp()
+ # called with the min of diffexp_max_workers and cpus*cpu_multiplier
+ mock_tiledb_config.assert_called_once_with(1, 4)
+
+ @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
+ def test_handle_adaptor(self, mock_tiledb_context):
+ custom_config = self.custom_app_config(
+ dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
+ )
+ config = AppConfig()
+ config.update_from_config_file(custom_config)
+ config.server_config.handle_adaptor()
+ mock_tiledb_context.assert_called_once_with(
+ {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
+ )
+
+ @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
+ @patch("server.common.config.get_secret_key")
+ def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
+ mock_get_secret_key.return_value = {
+ "flask_secret_key": "mock_flask_secret",
+ "oauth_client_secret": "mock_oauth_secret",
+ "db_uri": "mock_db_uri",
+ }
+
+ config = AppConfig()
+
+ with self.assertLogs(level="INFO") as logger:
+ from server.common.config import handle_config_from_secret
+
+ # should not throw error
+ # "AttributeError: 'XConfig' object has no attribute 'x'"
+ handle_config_from_secret(config)
+
+ # should log 3 lines (one for each var set from a secret)
+ self.assertEqual(len(logger.output), 3)
+ self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
+ self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
+ self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
+ self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
+ self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
+ self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
diff --git a/server/test/unit/common/test_api.py b/server/test/unit/common/test_api.py
index ee979b1f..7f524f7d 100644
--- a/server/test/unit/common/test_api.py
+++ b/server/test/unit/common/test_api.py
@@ -8,8 +8,14 @@ import requests
import server.test.unit.decode_fbs as decode_fbs
from server.data_common.matrix_loader import MatrixDataType
-from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server,
- stop_test_server)
+from server.test import (
+ data_with_tmp_annotations,
+ make_fbs,
+ PROJECT_ROOT,
+ FIXTURES_ROOT,
+ start_test_server,
+ stop_test_server,
+)
from server.test.fixtures.fixtures import pbmc3k_colors
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
@@ -381,11 +387,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
@classmethod
def setUpClass(cls):
- cls._setupClass(cls, [
- f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
- "--disable-annotations",
- "--experimental-enable-reembedding",
- ])
+ cls._setupClass(
+ cls,
+ [
+ f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+ "--disable-annotations",
+ "--experimental-enable-reembedding",
+ ],
+ )
@classmethod
def tearDownClass(cls):
@@ -403,10 +412,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
@classmethod
def setUpClass(cls):
- cls._setupClass(cls, [
- f"{FIXTURES_ROOT}/pbmc3k.cxg",
- "--disable-annotations",
- ])
+ cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
@classmethod
def tearDownClass(cls):
@@ -423,7 +429,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
MatrixDataType.H5AD, annotations_fixture=True
)
- cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ])
+ cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
@@ -439,11 +445,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
@classmethod
def setUpClass(cls):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
- cls._setupClass(cls, [
- "--annotations-file",
- cls.annotations.output_file,
- cls.data.get_location(),
- ])
+ cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py
deleted file mode 100644
index c3dd1436..00000000
--- a/server/test/unit/common/test_app_config.py
+++ /dev/null
@@ -1,249 +0,0 @@
-import os
-import unittest
-from unittest import mock
-from unittest.mock import patch
-import tempfile
-
-import requests
-
-from server.common.app_config import AppConfig
-from server.common.errors import ConfigurationError
-from server.common.utils.utils import find_available_port
-from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
-
-
-# NOTE, there are more tests that should be written for AppConfig.
-# this is just a start.
-
-def mockenv(**envvars):
- return mock.patch.dict(os.environ, envvars)
-
-
-class AppConfigTest(unittest.TestCase):
- def test_update(self):
- config = AppConfig()
- config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
- vars = config.server_config.changes_from_default()
- self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
-
- config = AppConfig()
- config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
- vars = config.server_config.changes_from_default()
- self.assertCountEqual(vars, [])
-
- config = AppConfig()
- config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
- vars = config.default_dataset_config.changes_from_default()
- self.assertCountEqual(vars, [])
-
- config = AppConfig()
- config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
- vars = config.default_dataset_config.changes_from_default()
- self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
-
- def test_multi_dataset(self):
-
- config = AppConfig()
- # test for illegal url_dataroots
- for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
- config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
- )
- with self.assertRaises(ConfigurationError):
- config.complete_config()
-
- # test for legal url_dataroots
- for legal in ("d", "this.is-okay_", "a/b"):
- config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
- )
- config.complete_config()
-
- # test that multi dataroots work end to end
- config.update_server_config(
- multi_dataset__dataroot=dict(
- s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
- s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
- s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
- )
- )
-
- # Change this default to test if the dataroot overrides below work.
- config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
-
- # specialize the configs for set1
- config.add_dataroot_config(
- "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
- )
-
- # specialize the configs for set2
- config.add_dataroot_config(
- "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
- )
-
- # no specializations for set3 (they get the default dataset config)
- config.complete_config()
-
- with test_server(app_config=config) as server:
- session = requests.Session()
-
- response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
- data_config = response.json()
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is False
- assert data_config["config"]["parameters"]["disable-diffexp"] is False
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
-
- response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
- data_config = response.json()
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is True
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
-
- response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
- data_config = response.json()
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is True
- assert data_config["config"]["parameters"]["disable-diffexp"] is False
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
-
- response = session.get(f"{server}/health")
- assert response.json()["status"] == "pass"
-
- @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
- @patch('server.common.aws_secret_utils.get_secret_key')
- def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
- mock_get_secret_key.return_value = {
- "flask_secret_key": "mock_flask_secret",
- "oauth_client_secret": "mock_oauth_secret",
- "db_uri": "mock_db_uri"
- }
-
- config = AppConfig()
-
- with self.assertLogs(level="INFO") as logger:
- from server.common.aws_secret_utils import handle_config_from_secret
- # should not throw error
- # "AttributeError: 'XConfig' object has no attribute 'x'"
- handle_config_from_secret(config)
-
- # should log 3 lines (one for each var set from a secret)
- self.assertEqual(len(logger.output), 3)
- self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0])
- self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1])
- self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2])
- self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
- self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
- self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
-
- def test_api_base_url(self):
-
- # test the api_base_url feature, and that it can contain a path
- config = AppConfig()
- backend_port = find_available_port("localhost", 10000)
- config.update_server_config(
- app__api_base_url=f"http://localhost:{backend_port}/additional/path",
- multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset"
- )
-
- config.complete_config()
-
- with test_server(["-p", str(backend_port)], app_config=config) as server:
- session = requests.Session()
- self.assertEqual(server, f"http://localhost:{backend_port}")
- response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
- self.assertEqual(response.status_code, 200)
- data_config = response.json()
- self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
-
- # test the health check at the correct url
- response = session.get(f"{server}/additional/path/health")
- assert response.json()["status"] == "pass"
-
- # also check that the old URL still works.
- # NOTE: this old URL location will soon be deprecated, and when that happens
- # this check can be removed.
- response = session.get(f"{server}/health")
- assert response.json()["status"] == "pass"
-
- def test_configfile_with_specialization(self):
- # test that per_dataset_config config load the default config, then the specialized config
-
- with tempfile.TemporaryDirectory() as tempdir:
- configfile = os.path.join(tempdir, "config.yaml")
- with open(configfile, "w") as fconfig:
- config = """
- server:
- multi_dataset:
- dataroot:
- test:
- base_url: test
- dataroot: fake_dataroot
-
- dataset:
- user_annotations:
- enable: false
- type: hosted_tiledb_array
- hosted_tiledb_array:
- db_uri: fake_db_uri
- hosted_file_directory: fake_dir
-
- per_dataset_config:
- test:
- user_annotations:
- enable: true
- """
- fconfig.write(config)
-
- app_config = AppConfig()
- app_config.update_from_config_file(configfile)
-
- test_config = app_config.dataroot_config["test"]
-
- # test config from default
- self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
- self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
-
- # test config from specialization
- self.assertTrue(test_config.user_annotations__enable)
-
- def test_configfile_no_dataset_section(self):
- # test a config file without a dataset section
-
- with tempfile.TemporaryDirectory() as tempdir:
- configfile = os.path.join(tempdir, "config.yaml")
- with open(configfile, "w") as fconfig:
- config = """
- server:
- multi_dataset:
- dataroot: test_dataroot
-
- """
- fconfig.write(config)
-
- app_config = AppConfig()
- app_config.update_from_config_file(configfile)
- server_changes = app_config.server_config.changes_from_default()
- dataset_changes = app_config.default_dataset_config.changes_from_default()
- self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
- self.assertEqual(dataset_changes, [])
-
- def test_configfile_no_server_section(self):
- # test a config file without a dataset section
-
- with tempfile.TemporaryDirectory() as tempdir:
- configfile = os.path.join(tempdir, "config.yaml")
- with open(configfile, "w") as fconfig:
- config = """
- dataset:
- user_annotations:
- enable: false
- """
- fconfig.write(config)
-
- app_config = AppConfig()
- app_config.update_from_config_file(configfile)
- server_changes = app_config.server_config.changes_from_default()
- dataset_changes = app_config.default_dataset_config.changes_from_default()
- self.assertEqual(server_changes, [])
- self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
diff --git a/server/test/unit/common/test_corpora.py b/server/test/unit/common/test_corpora.py
index 1a0222dc..5f26dae9 100644
--- a/server/test/unit/common/test_corpora.py
+++ b/server/test/unit/common/test_corpora.py
@@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase):
def setCorporaFields(cls, path):
adata = anndata.read_h5ad(path)
corpora_props = {
- "version": {
- "corpora_schema_version": "1.0.0",
- "corpora_encoding_version": "0.1.0"
- },
+ "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "PBMC3K",
- "contributors": json.dumps([
- {"name": "name"}
- ]),
- "layer_descriptions": {
- "X": "raw counts"
- },
+ "contributors": json.dumps([{"name": "name"}]),
+ "layer_descriptions": {"X": "raw counts"},
"organism": "human",
"organism_ontology_term_id": "unknown",
"project_name": "test project",
"project_description": "test description",
- "project_links": json.dumps([
- {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
- ]),
+ "project_links": json.dumps(
+ [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
+ ),
"default_embedding": "X_tsne",
}
adata.uns.update(corpora_props)
diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py
index f650fecf..9d2f5f1d 100644
--- a/server/test/unit/common/test_writable_annotation.py
+++ b/server/test/unit/common/test_writable_annotation.py
@@ -27,7 +27,7 @@ class auth(object):
class WritableTileDBStoredAnnotationTest(unittest.TestCase):
def setUp(self):
- self.user_id = '1234'
+ self.user_id = "1234"
self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
self.data.dataset_config.user_annotations = self.annotations
self.db = self.annotations.db
@@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
}
self.fbs = make_fbs(self.test_dict)
self.df = pd.DataFrame(self.test_dict)
- self.app = Flask('fake_app')
+ self.app = Flask("fake_app")
self.app.__setattr__("auth", auth)
def tearDown(self):
@@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
self.annotations.write_labels(self.df, self.data)
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
- Annotation,
- [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
+ Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
)
# retrieve tiledb array
df = tiledb.open(annotation.tiledb_uri)
@@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
with self.app.test_request_context():
- new_name = 'new_dataset/location'
+ new_name = "new_dataset/location"
self.data.get_location = MagicMock(return_value=new_name)
num_datasets = len(self.db.query([CellxGeneDataset]))
self.annotation_put_fbs(self.fbs)
@@ -130,15 +129,14 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
with self.assertRaises(KeyError):
self.annotation_put_fbs(fbs_bad)
- @patch('server.common.annotations.hosted_tiledb.current_app')
+ @patch("server.common.annotations.hosted_tiledb.current_app")
def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
- mock_user_id.auth.get_user_id.return_value = '1234'
+ mock_user_id.auth.get_user_id.return_value = "1234"
self.annotations.write_labels(self.df, self.data)
# get uri
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
- Annotation,
- [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)]
+ Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
)
df = tiledb.open(annotation.tiledb_uri)
diff --git a/server/test/unit/common/utils/test_cxg_generation_utils.py b/server/test/unit/common/utils/test_cxg_generation_utils.py
index 57893913..b9043c33 100644
--- a/server/test/unit/common/utils/test_cxg_generation_utils.py
+++ b/server/test/unit/common/utils/test_cxg_generation_utils.py
@@ -8,8 +8,12 @@ import numpy as np
import tiledb
from pandas import Series, DataFrame
-from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
- convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
+from server.common.utils.cxg_generation_utils import (
+ convert_dictionary_to_cxg_group,
+ convert_dataframe_to_cxg_array,
+ convert_ndarray_to_cxg_dense_array,
+ convert_matrix_to_cxg_array,
+)
PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
@@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase):
dictionary_name = "favorite_desserts"
expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
- convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
- group_metadata_name=dictionary_name)
+ convert_dictionary_to_cxg_group(
+ self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
+ )
array = tiledb.open(expected_array_directory)
actual_stored_metadata = dict(array.meta.items())
@@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase):
random_dataframe_name = f"random_dataframe_{uuid4()}"
random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
- convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
- "int_category", tiledb.Ctx())
+ convert_dataframe_to_cxg_array(
+ self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
+ )
expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
expected_array_metadata = {
- "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
- "index": "int_category"})}
+ "cxg_schema": json.dumps(
+ {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
+ )
+ }
actual_stored_dataframe_array = tiledb.open(expected_array_directory)
actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
@@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase):
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[:, :][''].size == 0)
+ self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
matrix = np.zeros([3, 3])
@@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase):
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[0, 0][''] == 1)
- self.assertTrue(actual_stored_array[1, 1][''] == 1)
- self.assertTrue(actual_stored_array[2, 2][''] == 2)
- self.assertTrue(actual_stored_array[:, :][''].size == 3)
+ self.assertTrue(actual_stored_array[0, 0][""] == 1)
+ self.assertTrue(actual_stored_array[1, 1][""] == 1)
+ self.assertTrue(actual_stored_array[2, 2][""] == 2)
+ self.assertTrue(actual_stored_array[:, :][""].size == 3)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase):
# a matrix of zeros which is sparse.
column_shift = np.ones((3, 2))
- convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
- column_shift_for_sparse_encoding=column_shift)
+ convert_matrix_to_cxg_array(
+ matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+ )
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[:, :][''].size == 0)
+ self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase):
# Only column shift the first column of ones.
column_shift = np.array([[1, 0], [1, 0]])
- convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
- column_shift_for_sparse_encoding=column_shift)
+ convert_matrix_to_cxg_array(
+ matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+ )
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[0, 1][''] == 1)
- self.assertTrue(actual_stored_array[1, 1][''] == 1)
- self.assertTrue(actual_stored_array[:, :][''].size == 2)
+ self.assertTrue(actual_stored_array[0, 1][""] == 1)
+ self.assertTrue(actual_stored_array[1, 1][""] == 1)
+ self.assertTrue(actual_stored_array[:, :][""].size == 2)
diff --git a/server/test/unit/common/utils/test_matrix_utils.py b/server/test/unit/common/utils/test_matrix_utils.py
index ffda1045..9cc9daf5 100644
--- a/server/test/unit/common/utils/test_matrix_utils.py
+++ b/server/test/unit/common/utils/test_matrix_utils.py
@@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_
class TestMatrixUtils(unittest.TestCase):
-
def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
matrix = np.array([1, 2, 3])
diff --git a/server/test/unit/common/utils/test_sanitization_utils.py b/server/test/unit/common/utils/test_sanitization_utils.py
index 8ef04218..e209be95 100644
--- a/server/test/unit/common/utils/test_sanitization_utils.py
+++ b/server/test/unit/common/utils/test_sanitization_utils.py
@@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani
class TestSanitizationUtils(unittest.TestCase):
-
def test__sanitize_values_in_list__not_strings_raises_exception(self):
keys_to_sanitize = [1, 2, 3]
diff --git a/server/test/unit/common/utils/test_type_conversion_utils.py b/server/test/unit/common/utils/test_type_conversion_utils.py
index f1653697..ade2d999 100644
--- a/server/test/unit/common/utils/test_type_conversion_utils.py
+++ b/server/test/unit/common/utils/test_type_conversion_utils.py
@@ -5,12 +5,17 @@ from unittest.mock import patch
import numpy as np
from pandas import Series, DataFrame
-from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
- get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, convert_pandas_series_to_numpy
+from server.common.utils.type_conversion_utils import (
+ can_cast_to_float32,
+ can_cast_to_int32,
+ get_dtype_of_array,
+ get_schema_type_hint_of_array,
+ get_dtypes_and_schemas_of_dataframe,
+ convert_pandas_series_to_numpy,
+)
class TestTypeConversionUtils(unittest.TestCase):
-
def test__can_cast_to_float32__string_is_false(self):
array_to_convert = Series(data=["1", "2", "3"], dtype=str)
@@ -97,8 +102,9 @@ class TestTypeConversionUtils(unittest.TestCase):
expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
for test_type_index in range(len(types)):
- with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}",
- i=test_type_index):
+ with self.subTest(
+ f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index
+ ):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
@@ -123,8 +129,9 @@ class TestTypeConversionUtils(unittest.TestCase):
expected_dtypes = [np.float32, np.int32]
for test_type_index in range(len(types)):
- with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}",
- i=test_type_index):
+ with self.subTest(
+ f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index
+ ):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
@@ -141,8 +148,9 @@ class TestTypeConversionUtils(unittest.TestCase):
expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
for test_type_index in range(len(types)):
- with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}",
- i=test_type_index):
+ with self.subTest(
+ f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index
+ ):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
@@ -160,8 +168,9 @@ class TestTypeConversionUtils(unittest.TestCase):
for test_type_index in range(len(types)):
with self.subTest(
- f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
- i=test_type_index):
+ f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
+ i=test_type_index,
+ ):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
@@ -171,8 +180,10 @@ class TestTypeConversionUtils(unittest.TestCase):
dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
- expected_schema_type_hints_dict = {"float_array": {"type": "float32"},
- "category_array": {"type": "categorical", "categories": ["a", "b"]}}
+ expected_schema_type_hints_dict = {
+ "float_array": {"type": "float32"},
+ "category_array": {"type": "categorical", "categories": ["a", "b"]},
+ }
actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
@@ -201,5 +212,6 @@ class TestTypeConversionUtils(unittest.TestCase):
with self.assertLogs(level="ERROR") as logger:
convert_pandas_series_to_numpy(int_series, np.int32)
- self.assertIn("Cannot convert a pandas Series object to an integer dtype if it contains NaNs",
- logger.output[0])
+ self.assertIn(
+ "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0]
+ )
diff --git a/server/test/unit/converters/test_h5ad_data_file.py b/server/test/unit/converters/test_h5ad_data_file.py
index f8587ebd..99ad40af 100644
--- a/server/test/unit/converters/test_h5ad_data_file.py
+++ b/server/test/unit/converters/test_h5ad_data_file.py
@@ -16,7 +16,6 @@ PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
class TestH5ADDataFile(unittest.TestCase):
-
def setUp(self):
self.sample_anndata = self._create_sample_anndata_dataset()
self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
@@ -40,8 +39,12 @@ class TestH5ADDataFile(unittest.TestCase):
def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
with self.assertLogs(level="WARN") as logger:
- H5ADDataFile(self.sample_h5ad_filename, dataset_title="My Awesome Dataset",
- dataset_about="http://www.awesomedataset.com", use_corpora_schema=False)
+ H5ADDataFile(
+ self.sample_h5ad_filename,
+ dataset_title="My Awesome Dataset",
+ dataset_about="http://www.awesomedataset.com",
+ use_corpora_schema=False,
+ )
self.assertIn("will override any metadata that is extracted", logger.output[0])
@@ -49,10 +52,12 @@ class TestH5ADDataFile(unittest.TestCase):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
- self.assertEqual(h5ad_file.anndata.obs.sort_index(inplace=True),
- self.sample_anndata.obs.sort_index(inplace=True))
- self.assertEqual(h5ad_file.anndata.var.sort_index(inplace=True),
- self.sample_anndata.var.sort_index(inplace=True))
+ self.assertEqual(
+ h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True)
+ )
+ self.assertEqual(
+ h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True)
+ )
for key in h5ad_file.anndata.obsm.keys():
self.assertIn(key, self.sample_anndata.obsm.keys())
@@ -73,8 +78,12 @@ class TestH5ADDataFile(unittest.TestCase):
self.assertIn("name_0", h5ad_file.var.columns)
def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
- obs_index_column_name="float_category", vars_index_column_name="int_category")
+ h5ad_file = H5ADDataFile(
+ self.sample_h5ad_filename,
+ use_corpora_schema=False,
+ obs_index_column_name="float_category",
+ vars_index_column_name="int_category",
+ )
self.assertNotIn("name_0", h5ad_file.obs.columns)
self.assertNotIn("name_0", h5ad_file.var.columns)
@@ -82,15 +91,23 @@ class TestH5ADDataFile(unittest.TestCase):
def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
with self.assertRaises(Exception) as exception_context:
- H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
- obs_index_column_name="float_category", vars_index_column_name="bool_category")
+ H5ADDataFile(
+ self.sample_h5ad_filename,
+ use_corpora_schema=False,
+ obs_index_column_name="float_category",
+ vars_index_column_name="bool_category",
+ )
self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
with self.assertRaises(Exception) as exception_context:
- H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
- obs_index_column_name="unknown_category", vars_index_column_name="i_dont_exist")
+ H5ADDataFile(
+ self.sample_h5ad_filename,
+ use_corpora_schema=False,
+ obs_index_column_name="unknown_category",
+ vars_index_column_name="i_dont_exist",
+ )
self.assertIn("does not exist", str(exception_context.exception))
@@ -101,8 +118,9 @@ class TestH5ADDataFile(unittest.TestCase):
self.assertEqual(h5ad_file.dataset_about, "www.link.com")
def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename, dataset_about="override_about",
- dataset_title="override_title")
+ h5ad_file = H5ADDataFile(
+ self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title"
+ )
self.assertEqual(h5ad_file.dataset_title, "override_title")
self.assertEqual(h5ad_file.dataset_about, "override_about")
@@ -145,8 +163,11 @@ class TestH5ADDataFile(unittest.TestCase):
remove(sparse_with_column_shift_filename)
def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False):
- expected_directories, expected_obs_files, expected_var_files = \
- self._get_expected_generated_list_of_tiledb_files()
+ (
+ expected_directories,
+ expected_obs_files,
+ expected_var_files,
+ ) = self._get_expected_generated_list_of_tiledb_files()
for directory in expected_directories:
self.assertTrue(path.isdir(directory))
@@ -187,8 +208,18 @@ class TestH5ADDataFile(unittest.TestCase):
var_files.append("bool_category.tdb")
var_files.append("int_category.tdb")
- return [metadata_directory, main_x_directory, overall_embedding_directory, specific_embedding_directory,
- obs_directory, var_directory], obs_files, var_files
+ return (
+ [
+ metadata_directory,
+ main_x_directory,
+ overall_embedding_directory,
+ specific_embedding_directory,
+ obs_directory,
+ var_directory,
+ ],
+ obs_files,
+ var_files,
+ )
def _write_anndata_to_file(self, anndata):
temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad"
@@ -204,7 +235,8 @@ class TestH5ADDataFile(unittest.TestCase):
random_string_category = Series(data=["a", "b", "b"], dtype="category")
random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
obs_dataframe = DataFrame(
- data={"string_category": random_string_category, "float_category": random_float_category})
+ data={"string_category": random_string_category, "float_category": random_float_category}
+ )
obs = obs_dataframe
# Create vars
@@ -230,6 +262,7 @@ class TestH5ADDataFile(unittest.TestCase):
# Set project links to be a dictionary
uns["project_links"] = json.dumps(
- [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}])
+ [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]
+ )
return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)
diff --git a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py
index 35ccb886..b3db3cb3 100644
--- a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py
+++ b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py
@@ -3,7 +3,7 @@ import json
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
from server.test import PROJECT_ROOT
diff --git a/server/test/unit/data_common/test_matrix_loader.py b/server/test/unit/data_common/test_matrix_loader.py
index d365b001..9631de32 100644
--- a/server/test/unit/data_common/test_matrix_loader.py
+++ b/server/test/unit/data_common/test_matrix_loader.py
@@ -4,7 +4,7 @@ import tempfile
import time
import unittest
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
from server.common.errors import DatasetAccessError
from server.data_common.matrix_loader import MatrixDataCacheManager
from server.test import FIXTURES_ROOT
@@ -38,7 +38,7 @@ class MatrixCacheTest(unittest.TestCase):
result = {}
for k, v in datasets.items():
# filter out the dirname and the .cxg from the name
- newk = int(k[1][len(dirname) + 1: -4])
+ newk = int(k[1][len(dirname) + 1 : -4])
result[newk] = v
return result
diff --git a/server/test/unit/eb/test_eb.py b/server/test/unit/eb/test_eb.py
index b43fda24..c148f6bc 100644
--- a/server/test/unit/eb/test_eb.py
+++ b/server/test/unit/eb/test_eb.py
@@ -3,7 +3,7 @@ import tempfile
import requests
import subprocess
from server.test import PROJECT_ROOT, FIXTURES_ROOT
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
from contextlib import contextmanager
import time
@@ -36,9 +36,7 @@ class Elastic_Beanstalk_Test(unittest.TestCase):
c = AppConfig()
# test that eb works
- c.update_server_config(
- multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame"
- )
+ c.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
c.complete_config()
c.write_config(f"{tempdirname}/config.yaml")
From 7bee09cd166620a7e11990fd281cb3b4cf48e5c9 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Tue, 29 Sep 2020 15:00:56 -0700
Subject: [PATCH 12/73] Add blueprint eslint plugin (#1892)
* add bp3 eslint plugin
* first eslint runthrough + manual changes
* small fixes
* update snapshots
* update h1 to h4
Co-authored-by: czimergebot <35308261+czimergebot@users.noreply.github.com>
---
.../__snapshots__/e2eAnnotations.test.js.snap | 4 +-
client/configuration/eslint/eslint.js | 1 +
client/package-lock.json | 435 ++++++++++++++++++
client/package.json | 1 +
client/src/actions/embedding.js | 24 +-
.../src/components/autosave/filenameDialog.js | 11 +-
.../components/brushableHistogram/index.js | 12 +-
.../components/categorical/category/index.js | 22 +-
.../src/components/categorical/value/index.js | 13 +-
.../components/categorical/value/occupancy.js | 12 +-
client/src/components/embedding/index.js | 9 +-
.../src/components/geneExpression/addGenes.js | 7 +-
client/src/components/menubar/authButtons.js | 6 +-
client/src/components/menubar/clip.js | 23 +-
client/src/components/menubar/infoMenu.js | 15 +-
client/src/components/menubar/undoRedo.js | 11 +-
client/src/components/miniHistogram/index.js | 1 -
client/src/components/miniStackedBar/index.js | 2 -
18 files changed, 536 insertions(+), 73 deletions(-)
diff --git a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
index 01ed89a8..56a349b2 100644
--- a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
+++ b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
@@ -3,14 +3,14 @@
exports[`annotations stacked bar graph renders 1`] = `
Array [
"",
- "",
+ "",
]
`;
exports[`annotations stacked bar graph renders 2`] = `
Array [
"",
- "",
+ "",
]
`;
diff --git a/client/configuration/eslint/eslint.js b/client/configuration/eslint/eslint.js
index bafb7d31..51b619db 100644
--- a/client/configuration/eslint/eslint.js
+++ b/client/configuration/eslint/eslint.js
@@ -4,6 +4,7 @@ module.exports = {
extends: [
"airbnb",
"plugin:eslint-comments/recommended",
+ "plugin:@blueprintjs/recommended",
"plugin:compat/recommended",
"plugin:prettier/recommended",
"prettier/react",
diff --git a/client/package-lock.json b/client/package-lock.json
index c94ba503..68eb0080 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -4156,6 +4156,216 @@
"tslib": "~1.10.0"
}
},
+ "@blueprintjs/eslint-plugin": {
+ "version": "0.3.0",
+ "resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.0.tgz",
+ "integrity": "sha512-bQEdE4ApEHxCDV8hT9uIxeRbDFKOtRLBT3/Zy3Ku+nowDAYl/8jwZKp6lJuR/nqvsfuIXTnVef6ivwdBEieQfA==",
+ "dev": true,
+ "requires": {
+ "@typescript-eslint/experimental-utils": "^4.2.0",
+ "eslint": "^7.9.0"
+ },
+ "dependencies": {
+ "@typescript-eslint/experimental-utils": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.3.0.tgz",
+ "integrity": "sha512-cmmIK8shn3mxmhpKfzMMywqiEheyfXLV/+yPDnOTvQX/ztngx7Lg/OD26J8gTZfkLKUmaEBxO2jYP3keV7h2OQ==",
+ "dev": true,
+ "requires": {
+ "@types/json-schema": "^7.0.3",
+ "@typescript-eslint/scope-manager": "4.3.0",
+ "@typescript-eslint/types": "4.3.0",
+ "@typescript-eslint/typescript-estree": "4.3.0",
+ "eslint-scope": "^5.0.0",
+ "eslint-utils": "^2.0.0"
+ }
+ },
+ "@typescript-eslint/typescript-estree": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.3.0.tgz",
+ "integrity": "sha512-ZAI7xjkl+oFdLV/COEz2tAbQbR3XfgqHEGy0rlUXzfGQic6EBCR4s2+WS3cmTPG69aaZckEucBoTxW9PhzHxxw==",
+ "dev": true,
+ "requires": {
+ "@typescript-eslint/types": "4.3.0",
+ "@typescript-eslint/visitor-keys": "4.3.0",
+ "debug": "^4.1.1",
+ "globby": "^11.0.1",
+ "is-glob": "^4.0.1",
+ "lodash": "^4.17.15",
+ "semver": "^7.3.2",
+ "tsutils": "^3.17.1"
+ }
+ },
+ "acorn": {
+ "version": "7.4.0",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+ "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
+ "dev": true
+ },
+ "cross-spawn": {
+ "version": "7.0.3",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+ "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+ "dev": true,
+ "requires": {
+ "path-key": "^3.1.0",
+ "shebang-command": "^2.0.0",
+ "which": "^2.0.1"
+ }
+ },
+ "eslint": {
+ "version": "7.10.0",
+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz",
+ "integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==",
+ "dev": true,
+ "requires": {
+ "@babel/code-frame": "^7.0.0",
+ "@eslint/eslintrc": "^0.1.3",
+ "ajv": "^6.10.0",
+ "chalk": "^4.0.0",
+ "cross-spawn": "^7.0.2",
+ "debug": "^4.0.1",
+ "doctrine": "^3.0.0",
+ "enquirer": "^2.3.5",
+ "eslint-scope": "^5.1.1",
+ "eslint-utils": "^2.1.0",
+ "eslint-visitor-keys": "^1.3.0",
+ "espree": "^7.3.0",
+ "esquery": "^1.2.0",
+ "esutils": "^2.0.2",
+ "file-entry-cache": "^5.0.1",
+ "functional-red-black-tree": "^1.0.1",
+ "glob-parent": "^5.0.0",
+ "globals": "^12.1.0",
+ "ignore": "^4.0.6",
+ "import-fresh": "^3.0.0",
+ "imurmurhash": "^0.1.4",
+ "is-glob": "^4.0.0",
+ "js-yaml": "^3.13.1",
+ "json-stable-stringify-without-jsonify": "^1.0.1",
+ "levn": "^0.4.1",
+ "lodash": "^4.17.19",
+ "minimatch": "^3.0.4",
+ "natural-compare": "^1.4.0",
+ "optionator": "^0.9.1",
+ "progress": "^2.0.0",
+ "regexpp": "^3.1.0",
+ "semver": "^7.2.1",
+ "strip-ansi": "^6.0.0",
+ "strip-json-comments": "^3.1.0",
+ "table": "^5.2.3",
+ "text-table": "^0.2.0",
+ "v8-compile-cache": "^2.0.3"
+ },
+ "dependencies": {
+ "eslint-scope": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+ "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+ "dev": true,
+ "requires": {
+ "esrecurse": "^4.3.0",
+ "estraverse": "^4.1.1"
+ }
+ }
+ }
+ },
+ "eslint-visitor-keys": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+ "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+ "dev": true
+ },
+ "espree": {
+ "version": "7.3.0",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
+ "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
+ "dev": true,
+ "requires": {
+ "acorn": "^7.4.0",
+ "acorn-jsx": "^5.2.0",
+ "eslint-visitor-keys": "^1.3.0"
+ }
+ },
+ "esrecurse": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+ "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+ "dev": true,
+ "requires": {
+ "estraverse": "^5.2.0"
+ },
+ "dependencies": {
+ "estraverse": {
+ "version": "5.2.0",
+ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+ "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+ "dev": true
+ }
+ }
+ },
+ "glob-parent": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+ "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+ "dev": true,
+ "requires": {
+ "is-glob": "^4.0.1"
+ }
+ },
+ "globals": {
+ "version": "12.4.0",
+ "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+ "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+ "dev": true,
+ "requires": {
+ "type-fest": "^0.8.1"
+ }
+ },
+ "ignore": {
+ "version": "4.0.6",
+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+ "dev": true
+ },
+ "path-key": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+ "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+ "dev": true
+ },
+ "semver": {
+ "version": "7.3.2",
+ "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+ "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
+ "dev": true
+ },
+ "shebang-command": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+ "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+ "dev": true,
+ "requires": {
+ "shebang-regex": "^3.0.0"
+ }
+ },
+ "shebang-regex": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+ "dev": true
+ },
+ "which": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+ "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+ "dev": true,
+ "requires": {
+ "isexe": "^2.0.0"
+ }
+ }
+ }
+ },
"@blueprintjs/icons": {
"version": "3.19.0",
"resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.19.0.tgz",
@@ -4184,6 +4394,76 @@
"minimist": "^1.2.0"
}
},
+ "@eslint/eslintrc": {
+ "version": "0.1.3",
+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz",
+ "integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==",
+ "dev": true,
+ "requires": {
+ "ajv": "^6.12.4",
+ "debug": "^4.1.1",
+ "espree": "^7.3.0",
+ "globals": "^12.1.0",
+ "ignore": "^4.0.6",
+ "import-fresh": "^3.2.1",
+ "js-yaml": "^3.13.1",
+ "lodash": "^4.17.19",
+ "minimatch": "^3.0.4",
+ "strip-json-comments": "^3.1.1"
+ },
+ "dependencies": {
+ "acorn": {
+ "version": "7.4.0",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+ "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
+ "dev": true
+ },
+ "ajv": {
+ "version": "6.12.5",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
+ "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
+ "dev": true,
+ "requires": {
+ "fast-deep-equal": "^3.1.1",
+ "fast-json-stable-stringify": "^2.0.0",
+ "json-schema-traverse": "^0.4.1",
+ "uri-js": "^4.2.2"
+ }
+ },
+ "eslint-visitor-keys": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+ "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+ "dev": true
+ },
+ "espree": {
+ "version": "7.3.0",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
+ "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
+ "dev": true,
+ "requires": {
+ "acorn": "^7.4.0",
+ "acorn-jsx": "^5.2.0",
+ "eslint-visitor-keys": "^1.3.0"
+ }
+ },
+ "globals": {
+ "version": "12.4.0",
+ "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+ "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+ "dev": true,
+ "requires": {
+ "type-fest": "^0.8.1"
+ }
+ },
+ "ignore": {
+ "version": "4.0.6",
+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+ "dev": true
+ }
+ }
+ },
"@hapi/address": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@@ -5137,6 +5417,32 @@
}
}
},
+ "@nodelib/fs.scandir": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz",
+ "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==",
+ "dev": true,
+ "requires": {
+ "@nodelib/fs.stat": "2.0.3",
+ "run-parallel": "^1.1.9"
+ }
+ },
+ "@nodelib/fs.stat": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz",
+ "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==",
+ "dev": true
+ },
+ "@nodelib/fs.walk": {
+ "version": "1.2.4",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz",
+ "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==",
+ "dev": true,
+ "requires": {
+ "@nodelib/fs.scandir": "2.1.3",
+ "fastq": "^1.6.0"
+ }
+ },
"@npmcli/move-file": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz",
@@ -5489,6 +5795,22 @@
"eslint-utils": "^2.0.0"
}
},
+ "@typescript-eslint/scope-manager": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.3.0.tgz",
+ "integrity": "sha512-cTeyP5SCNE8QBRfc+Lgh4Xpzje46kNUhXYfc3pQWmJif92sjrFuHT9hH4rtOkDTo/si9Klw53yIr+djqGZS1ig==",
+ "dev": true,
+ "requires": {
+ "@typescript-eslint/types": "4.3.0",
+ "@typescript-eslint/visitor-keys": "4.3.0"
+ }
+ },
+ "@typescript-eslint/types": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.3.0.tgz",
+ "integrity": "sha512-Cx9TpRvlRjOppGsU6Y6KcJnUDOelja2NNCX6AZwtVHRzaJkdytJWMuYiqi8mS35MRNA3cJSwDzXePfmhU6TANw==",
+ "dev": true
+ },
"@typescript-eslint/typescript-estree": {
"version": "2.34.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz",
@@ -5512,6 +5834,24 @@
}
}
},
+ "@typescript-eslint/visitor-keys": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.3.0.tgz",
+ "integrity": "sha512-xZxkuR7XLM6RhvLkgv9yYlTcBHnTULzfnw4i6+z2TGBLy9yljAypQaZl9c3zFvy7PNI7fYWyvKYtohyF8au3cw==",
+ "dev": true,
+ "requires": {
+ "@typescript-eslint/types": "4.3.0",
+ "eslint-visitor-keys": "^2.0.0"
+ },
+ "dependencies": {
+ "eslint-visitor-keys": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
+ "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
+ "dev": true
+ }
+ }
+ },
"@webassemblyjs/ast": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz",
@@ -8796,6 +9136,15 @@
}
}
},
+ "dir-glob": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
+ "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
+ "dev": true,
+ "requires": {
+ "path-type": "^4.0.0"
+ }
+ },
"doctrine": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -10187,6 +10536,31 @@
"integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==",
"dev": true
},
+ "fast-glob": {
+ "version": "3.2.4",
+ "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz",
+ "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==",
+ "dev": true,
+ "requires": {
+ "@nodelib/fs.stat": "^2.0.2",
+ "@nodelib/fs.walk": "^1.2.3",
+ "glob-parent": "^5.1.0",
+ "merge2": "^1.3.0",
+ "micromatch": "^4.0.2",
+ "picomatch": "^2.2.1"
+ },
+ "dependencies": {
+ "glob-parent": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+ "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+ "dev": true,
+ "requires": {
+ "is-glob": "^4.0.1"
+ }
+ }
+ }
+ },
"fast-json-stable-stringify": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -10202,6 +10576,15 @@
"resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz",
"integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw=="
},
+ "fastq": {
+ "version": "1.8.0",
+ "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz",
+ "integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==",
+ "dev": true,
+ "requires": {
+ "reusify": "^1.0.4"
+ }
+ },
"favicons": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz",
@@ -11159,6 +11542,28 @@
"resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
"integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA=="
},
+ "globby": {
+ "version": "11.0.1",
+ "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz",
+ "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==",
+ "dev": true,
+ "requires": {
+ "array-union": "^2.1.0",
+ "dir-glob": "^3.0.1",
+ "fast-glob": "^3.1.1",
+ "ignore": "^5.1.4",
+ "merge2": "^1.3.0",
+ "slash": "^3.0.0"
+ },
+ "dependencies": {
+ "array-union": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
+ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
+ "dev": true
+ }
+ }
+ },
"got": {
"version": "6.7.1",
"resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
@@ -11691,6 +12096,12 @@
"integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
"dev": true
},
+ "ignore": {
+ "version": "5.1.8",
+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
+ "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==",
+ "dev": true
+ },
"ignore-walk": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
@@ -14189,6 +14600,12 @@
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
},
+ "merge2": {
+ "version": "1.4.1",
+ "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+ "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
+ "dev": true
+ },
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
@@ -15251,6 +15668,12 @@
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=",
"dev": true
},
+ "path-type": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
+ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
+ "dev": true
+ },
"pbkdf2": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
@@ -17065,6 +17488,12 @@
"resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
"integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="
},
+ "reusify": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+ "dev": true
+ },
"rgb-regex": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz",
@@ -17100,6 +17529,12 @@
"resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
"integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA=="
},
+ "run-parallel": {
+ "version": "1.1.9",
+ "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz",
+ "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==",
+ "dev": true
+ },
"run-queue": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",
diff --git a/client/package.json b/client/package.json
index 9e6f20ef..feec7fad 100644
--- a/client/package.json
+++ b/client/package.json
@@ -84,6 +84,7 @@
"@babel/preset-react": "^7.10.4",
"@babel/register": "^7.10.5",
"@babel/runtime": "^7.10.5",
+ "@blueprintjs/eslint-plugin": "^0.3.0",
"@sentry/webpack-plugin": "^1.12.0",
"babel-eslint": "^10.1.0",
"babel-jest": "^26.1.0",
diff --git a/client/src/actions/embedding.js b/client/src/actions/embedding.js
index 615bcd68..7b781a0d 100644
--- a/client/src/actions/embedding.js
+++ b/client/src/actions/embedding.js
@@ -5,20 +5,23 @@ action creators related to embeddings choice
import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
-export async function _switchEmbedding(prevAnnoMatrix, prevCrossfilter, newEmbeddingName) {
+export async function _switchEmbedding(
+ prevAnnoMatrix,
+ prevCrossfilter,
+ newEmbeddingName
+) {
/*
DRY helper used by this and reembedding action creators
*/
const base = prevAnnoMatrix.base();
const embeddingDf = await base.fetch("emb", newEmbeddingName);
const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
- const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix, prevCrossfilter.obsCrossfilter).select(
- "emb",
- newEmbeddingName,
- {
- mode: "all",
- }
- );
+ const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+ annoMatrix,
+ prevCrossfilter.obsCrossfilter
+ ).select("emb", newEmbeddingName, {
+ mode: "all",
+ });
return [annoMatrix, obsCrossfilter];
}
@@ -30,7 +33,10 @@ export const layoutChoiceAction = (newLayoutChoice) => async (
On layout choice, make sure we have selected all on the previous layout, AND the new
layout.
*/
- const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
+ const {
+ annoMatrix: prevAnnoMatrix,
+ obsCrossfilter: prevCrossfilter,
+ } = getState();
const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
prevAnnoMatrix,
prevCrossfilter,
diff --git a/client/src/components/autosave/filenameDialog.js b/client/src/components/autosave/filenameDialog.js
index 3937ecee..08fe1462 100644
--- a/client/src/components/autosave/filenameDialog.js
+++ b/client/src/components/autosave/filenameDialog.js
@@ -3,11 +3,12 @@ import { connect } from "react-redux";
import {
Button,
- Tooltip,
- InputGroup,
- Dialog,
Classes,
+ Code,
Colors,
+ Dialog,
+ InputGroup,
+ Tooltip,
} from "@blueprintjs/core";
@connect((state) => ({
@@ -145,9 +146,9 @@ class FilenameDialog extends React.Component {
Your annotations are stored in this file:
-
+
{filenameText}-{idhash}.csv
-
+
(We added a unique ID to your filename)
diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js
index 71175b2d..a18608b2 100644
--- a/client/src/components/brushableHistogram/index.js
+++ b/client/src/components/brushableHistogram/index.js
@@ -5,12 +5,13 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
https://bl.ocks.org/mbostock/3019563
*/
import React, { useEffect, useRef, useState, useCallback } from "react";
-import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core";
import { connect } from "react-redux";
import * as d3 from "d3";
import { interpolateCool } from "d3-scale-chromatic";
import Async from "react-async";
import memoize from "memoize-one";
+import { IconNames } from "@blueprintjs/icons";
import * as globals from "../../globals";
import actions from "../../actions";
import { histogramContinuous } from "../../util/dataframe/histogram";
@@ -26,7 +27,7 @@ function maybeScientific(x) {
const _ticks = x.ticks(4);
if (x.domain().some((n) => Math.abs(n) >= 10000)) {
- /*
+ /*
heuristic: if the last tick d3 wants to render has one significant
digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
4.20e+n
@@ -99,7 +100,7 @@ const HistogramFooter = React.memo(
pvalAdj,
}) => {
/*
- Footer of each histogram. Will render range, title, and optionally
+ Footer of each histogram. Will render range, title, and optionally
differential expression info.
Required props:
@@ -214,10 +215,7 @@ const HistogramHeader = React.memo(
>
{onScatterPlotXClick && onScatterPlotYClick ? (
-
+
{
alignItems: "flex-start",
}}
>
-
+
-
+
-
+
-
+
@@ -461,14 +460,12 @@ class CategoryValue extends React.Component {
return (
diff --git a/client/src/components/categorical/value/occupancy.js b/client/src/components/categorical/value/occupancy.js
index 934822a9..dae4a0ed 100644
--- a/client/src/components/categorical/value/occupancy.js
+++ b/client/src/components/categorical/value/occupancy.js
@@ -3,10 +3,10 @@ import React from "react";
import { connect } from "react-redux";
import * as d3 from "d3";
import {
+ Classes,
Popover,
PopoverInteractionKind,
Position,
- Classes,
} from "@blueprintjs/core";
@connect((state) => ({
@@ -18,8 +18,8 @@ class Occupancy extends React.PureComponent {
_HEIGHT = 11;
createHistogram = () => {
- /*
- Knowing that colorScale is based off continous data,
+ /*
+ Knowing that colorScale is based off continous data,
createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
It then seperates that data into 50 bins for drawing the mini-histogram
*/
@@ -75,8 +75,8 @@ class Occupancy extends React.PureComponent {
};
createOccupancyStack = () => {
- /*
- Knowing that the color scale is based off of catagorical data,
+ /*
+ Knowing that the color scale is based off of catagorical data,
createOccupancyStack obtains a map showing the number if cells per colored value
Using the colorScale a stack of colored bars is drawn representing the map
*/
@@ -155,7 +155,7 @@ class Occupancy extends React.PureComponent {
popoverClassName={Classes.POPOVER_CONTENT_SIZING}
>
- Embedding Choice
+ Embedding Choice
There are {schema?.dataframe?.nObs} cells in the entire dataset.
diff --git a/client/src/components/geneExpression/addGenes.js b/client/src/components/geneExpression/addGenes.js
index bc3e2f82..2d1ca1b4 100644
--- a/client/src/components/geneExpression/addGenes.js
+++ b/client/src/components/geneExpression/addGenes.js
@@ -6,11 +6,12 @@ import fuzzysort from "fuzzysort";
import { connect } from "react-redux";
import { Suggest } from "@blueprintjs/select";
import {
- MenuItem,
Button,
+ ControlGroup,
FormGroup,
InputGroup,
- ControlGroup,
+ Intent,
+ MenuItem,
} from "@blueprintjs/core";
import * as globals from "../../globals";
import actions from "../../actions";
@@ -278,7 +279,7 @@ class AddGenes extends React.Component {
popoverProps={{ minimal: true }}
/>
this.handleClick(activeItem)}
diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js
index 7eb4c4c0..9eaa3ba5 100644
--- a/client/src/components/menubar/authButtons.js
+++ b/client/src/components/menubar/authButtons.js
@@ -1,5 +1,5 @@
import React from "react";
-import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
import * as globals from "../../globals";
import styles from "./menubar.css";
@@ -9,7 +9,7 @@ const Auth = React.memo((props) => {
if (!auth || (auth && !auth.requires_client_login)) return null;
return (
-
+
{
{!userinfo.is_authenticated ? "Log In" : "Log Out"}
-
+
);
});
diff --git a/client/src/components/menubar/clip.js b/client/src/components/menubar/clip.js
index 5b866bd0..5bc1d18d 100644
--- a/client/src/components/menubar/clip.js
+++ b/client/src/components/menubar/clip.js
@@ -1,12 +1,16 @@
import React from "react";
import {
- Position,
Button,
- Popover,
- NumericInput,
+ ButtonGroup,
Icon,
+ Intent,
+ NumericInput,
+ Popover,
+ Position,
Tooltip,
} from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+
import { tooltipHoverOpenDelay } from "../../globals";
import styles from "./menubar.css";
@@ -28,13 +32,13 @@ const Clip = React.memo((props) => {
pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin;
const clipMax =
pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax;
- const activeClipClass =
+ const intent =
clipPercentileMin > 0 || clipPercentileMax < 100
- ? " bp3-intent-warning"
- : "";
+ ? Intent.INTENT_WARNING
+ : Intent.NONE;
return (
-
}
/>
-
+
);
});
diff --git a/client/src/components/menubar/infoMenu.js b/client/src/components/menubar/infoMenu.js
index d136ae0c..a97b8b63 100644
--- a/client/src/components/menubar/infoMenu.js
+++ b/client/src/components/menubar/infoMenu.js
@@ -1,6 +1,13 @@
// jshint esversion: 6
import React from "react";
-import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
+import {
+ Button,
+ ButtonGroup,
+ Menu,
+ MenuItem,
+ Popover,
+ Position,
+} from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import styles from "./menubar.css";
@@ -11,7 +18,7 @@ const handleClick = (dispatch) => {
const InformationMenu = React.memo((props) => {
const { libraryVersions, tosURL, privacyURL, dispatch } = props;
return (
-
+
@@ -64,13 +71,13 @@ const InformationMenu = React.memo((props) => {
>
-
+
);
});
diff --git a/client/src/components/menubar/undoRedo.js b/client/src/components/menubar/undoRedo.js
index 601dba32..8591505c 100644
--- a/client/src/components/menubar/undoRedo.js
+++ b/client/src/components/menubar/undoRedo.js
@@ -1,12 +1,13 @@
import React from "react";
-import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
import { tooltipHoverOpenDelay } from "../../globals";
import styles from "./menubar.css";
const UndoRedo = React.memo((props) => {
const { undoDisabled, redoDisabled, dispatch } = props;
return (
-
+
{
>
{
dispatch({ type: "@@undoable/undo" });
@@ -32,7 +33,7 @@ const UndoRedo = React.memo((props) => {
>
{
dispatch({ type: "@@undoable/redo" });
@@ -43,7 +44,7 @@ const UndoRedo = React.memo((props) => {
data-testid="redo"
/>
-
+
);
});
diff --git a/client/src/components/miniHistogram/index.js b/client/src/components/miniHistogram/index.js
index eb57f832..59153d91 100644
--- a/client/src/components/miniHistogram/index.js
+++ b/client/src/components/miniHistogram/index.js
@@ -72,7 +72,6 @@ export default class MiniHistogram extends React.PureComponent {
popoverClassName={Classes.POPOVER_CONTENT_SIZING}
>
Date: Tue, 29 Sep 2020 15:32:21 -0700
Subject: [PATCH 13/73] Make sure there are more than 1 values in a category
before rendering it (#1871)
---
client/src/components/categorical/index.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/client/src/components/categorical/index.js b/client/src/components/categorical/index.js
index 99ccbfcd..5003ab8c 100644
--- a/client/src/components/categorical/index.js
+++ b/client/src/components/categorical/index.js
@@ -185,7 +185,9 @@ class Categories extends React.Component {
{/* READ ONLY CATEGORICAL FIELDS */}
{/* this is duplicative but flat, could be abstracted */}
{allCategoryNames.map((catName) =>
- !schema.annotations.obsByName[catName].writable ? (
+ !schema.annotations.obsByName[catName].writable &&
+ (schema.annotations.obsByName[catName].categories?.length > 1 ||
+ !schema.annotations.obsByName[catName].categories) ? (
Date: Tue, 29 Sep 2020 18:31:59 -0500
Subject: [PATCH 14/73] remove AppFeature and all references to it in the
code/tests (#1893)
* remove AppFeature and all references to it in the code/tests
Co-authored-by: bmccandless
---
server/common/config/client_config.py | 4 ---
server/data_common/data_adaptor.py | 25 -------------------
.../unit/common/config/test_app_config.py | 8 +++---
server/test/unit/common/test_api.py | 1 -
.../unit/common/test_writable_annotation.py | 17 -------------
.../unit/data_anndata/test_anndata_adaptor.py | 17 -------------
6 files changed, 4 insertions(+), 68 deletions(-)
diff --git a/server/common/config/client_config.py b/server/common/config/client_config.py
index 9fec9042..ccddf2c1 100644
--- a/server/common/config/client_config.py
+++ b/server/common/config/client_config.py
@@ -17,9 +17,6 @@ def get_client_config(app_config, data_adaptor):
# make sure the configuration has been checked.
app_config.check_config()
- # features
- features = [f.todict() for f in data_adaptor.get_features(annotation)]
-
# display_names
title = app_config.get_title(data_adaptor)
about = app_config.get_about(data_adaptor)
@@ -75,7 +72,6 @@ def get_client_config(app_config, data_adaptor):
# gather it all together
client_config = {}
config = client_config["config"] = {}
- config["features"] = features
config["displayNames"] = display_names
config["library_versions"] = library_versions
config["links"] = links
diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py
index 3945ea7a..36af8339 100644
--- a/server/data_common/data_adaptor.py
+++ b/server/data_common/data_adaptor.py
@@ -155,17 +155,6 @@ class DataAdaptor(metaclass=ABCMeta):
"""
pass
- def get_features(self, annotations=None):
- """Return list of features, to return as part of the config route"""
- features = [
- AppFeature("/cluster/", method="POST", available=False),
- AppFeature("/layout/obs", method="GET", available=self.get_embedding_names() is not None),
- AppFeature("/layout/obs", method="PUT", available=self.dataset_config.embeddings__enable_reembedding),
- AppFeature("/diffexp/", method="POST", available=self.dataset_config.diffexp__enable),
- AppFeature("/annotations/obs", method="PUT", available=annotations is not None),
- ]
- return features
-
def update_parameters(self, parameters):
parameters.update(self.parameters)
@@ -388,17 +377,3 @@ class DataAdaptor(metaclass=ABCMeta):
except RuntimeError:
lastmod = None
return lastmod
-
-
-class AppFeature(object):
- def __init__(self, path, available=False, method="POST", extra={}):
- self.path = path
- self.available = available
- self.method = method
- self.extra = extra
- [setattr(self, key, value) for key, value in extra.items()]
-
- def todict(self):
- d = dict(available=self.available, method=self.method, path=self.path)
- d.update(self.extra)
- return d
diff --git a/server/test/unit/common/config/test_app_config.py b/server/test/unit/common/config/test_app_config.py
index 726ff86f..2dc3273d 100644
--- a/server/test/unit/common/config/test_app_config.py
+++ b/server/test/unit/common/config/test_app_config.py
@@ -40,11 +40,11 @@ class AppConfigTest(ConfigTests):
expected_config = yaml.load(default_config, Loader=yaml.Loader)
- server_config = app_default_config['server']
- dataset_config = app_default_config['dataset']
+ server_config = app_default_config["server"]
+ dataset_config = app_default_config["dataset"]
- expected_server_config = expected_config['server']
- expected_dataset_config = expected_config['dataset']
+ expected_server_config = expected_config["server"]
+ expected_dataset_config = expected_config["dataset"]
self.assertDictEqual(app_default_config, expected_config)
self.assertDictEqual(server_config, expected_server_config)
diff --git a/server/test/unit/common/test_api.py b/server/test/unit/common/test_api.py
index 7f524f7d..b7dca258 100644
--- a/server/test/unit/common/test_api.py
+++ b/server/test/unit/common/test_api.py
@@ -49,7 +49,6 @@ class EndPoints(object):
result_data = result.json()
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
- self.assertEqual(len(result_data["config"]["features"]), 5)
def test_get_layout_fbs(self):
endpoint = "layout/obs"
diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py
index 9d2f5f1d..0e369e56 100644
--- a/server/test/unit/common/test_writable_annotation.py
+++ b/server/test/unit/common/test_writable_annotation.py
@@ -268,20 +268,3 @@ class WritableAnnotationTest(unittest.TestCase):
all_col_schema["cat_B"],
{"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
)
-
- def test_config(self):
- features = self.data.get_features(self.annotations)
-
- # test each for singular presence and accuracy of available flag
- def check_feature(method, path, available):
- feature = list(
- filter(lambda f: f.method == method and f.path == path and f.available == available, features)
- )
- self.assertIsNotNone(feature)
- self.assertEqual(len(feature), 1)
-
- check_feature("POST", "/cluster/", False)
- check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable)
- check_feature("GET", "/layout/obs", True)
- check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding)
- check_feature("PUT", "/annotations/obs", True)
diff --git a/server/test/unit/data_anndata/test_anndata_adaptor.py b/server/test/unit/data_anndata/test_anndata_adaptor.py
index d4a3a778..d0df5db1 100644
--- a/server/test/unit/data_anndata/test_anndata_adaptor.py
+++ b/server/test/unit/data_anndata/test_anndata_adaptor.py
@@ -94,23 +94,6 @@ class AdaptorTest(unittest.TestCase):
with pytest.raises(TypeError):
self.data._create_schema()
- def test_config(self):
- features = self.data.get_features(annotations=None)
-
- # test each for singular presence and accuracy of available flag
- def check_feature(method, path, available):
- feature = list(
- filter(lambda f: f.method == method and f.path == path and f.available == available, features)
- )
- self.assertIsNotNone(feature)
- self.assertEqual(len(feature), 1)
-
- check_feature("POST", "/cluster/", False)
- check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable)
- check_feature("GET", "/layout/obs", True)
- check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding)
- check_feature("PUT", "/annotations/obs", False)
-
def test_layout(self):
fbs = self.data.layout_to_fbs_matrix(fields=None)
layout = decode_fbs.decode_matrix_FBS(fbs)
From 998fa4762d987fb3ec99b5013e74b327b732f1d9 Mon Sep 17 00:00:00 2001
From: Madison Dunitz
Date: Wed, 30 Sep 2020 11:16:13 -0500
Subject: [PATCH 15/73] run black formatter on repo (#1891)
* add black to lint make cmd
* add black dependency to installation to push test pipeline
---
.github/workflows/push_tests.yml | 3 ++-
Makefile | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml
index b4cfe599..70fcf947 100644
--- a/.github/workflows/push_tests.yml
+++ b/.github/workflows/push_tests.yml
@@ -31,9 +31,10 @@ jobs:
- name: Install dependencies
run: |
pip install flake8
+ pip install black
cd client
npm install
- - name: Lint with flake8
+ - name: Format with black and lint with flake8
run: |
make lint-server
- name: Lint src with eslint
diff --git a/Makefile b/Makefile
index 56dccf59..ca8754f7 100644
--- a/Makefile
+++ b/Makefile
@@ -82,7 +82,7 @@ fmt-py:
lint: lint-server lint-client
.PHONY: lint-server
-lint-server:
+lint-server: fmt-py
flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821'
From 04a3c3c6b6cb9b836445ddab03537432e0044943 Mon Sep 17 00:00:00 2001
From: Colin Megill
Date: Wed, 30 Sep 2020 11:45:10 -0700
Subject: [PATCH 16/73] Partial fix for 1830 (#1863)
* Remove door icon from log in button
* Move log in and info buttons from the top bar to in line with the cellxgene icon and dataset name
* Hover over on login button should say "Log in to cellxgene"
* Show email
closes #1830
---
client/src/components/categorical/index.js | 2 +-
.../leftSidebar/topLeftLogoAndTitle.js | 110 ++++++++++++------
client/src/components/menubar/authButtons.js | 3 +-
client/src/components/menubar/index.js | 13 +--
client/src/components/menubar/infoMenu.js | 35 +++++-
server/auth/auth_test.py | 1 +
6 files changed, 107 insertions(+), 57 deletions(-)
diff --git a/client/src/components/categorical/index.js b/client/src/components/categorical/index.js
index 5003ab8c..63db6cac 100644
--- a/client/src/components/categorical/index.js
+++ b/client/src/components/categorical/index.js
@@ -187,7 +187,7 @@ class Categories extends React.Component {
{allCategoryNames.map((catName) =>
!schema.annotations.obsByName[catName].writable &&
(schema.annotations.obsByName[catName].categories?.length > 1 ||
- !schema.annotations.obsByName[catName].categories) ? (
+ !schema.annotations.obsByName[catName].categories) ? (
({
datasetTitle: state.config?.displayNames?.dataset ?? "",
+ auth: state.config?.authentication,
+ userinfo: state.userinfo,
+ libraryVersions: state.config?.["library_versions"],
+ aboutLink: state.config?.links?.["about-dataset"],
+ tosURL: state.config?.parameters?.["about_legal_tos"],
+ privacyURL: state.config?.parameters?.["about_legal_privacy"],
}))
class LeftSideBar extends React.Component {
handleClick = () => {
@@ -20,7 +28,16 @@ class LeftSideBar extends React.Component {
};
render() {
- const { datasetTitle } = this.props;
+ const {
+ datasetTitle,
+ auth,
+ userinfo,
+ libraryVersions,
+ aboutLink,
+ privacyURL,
+ tosURL,
+ dispatch,
+ } = this.props;
return (
-
-
- cell
+
+
- Ã
-
- gene
-
-
-
-
- {datasetTitle}
+ cell
+
+ Ã
-
-
-
+ gene
+
+
+
+
+
+
+ {datasetTitle}
+
+
+
+
+
+ {!userinfo.is_authenticated ? (
+
+ ) : null}
+
);
}
diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js
index 9eaa3ba5..985b3da3 100644
--- a/client/src/components/menubar/authButtons.js
+++ b/client/src/components/menubar/authButtons.js
@@ -11,7 +11,7 @@ const Auth = React.memo((props) => {
return (
@@ -19,7 +19,6 @@ const Auth = React.memo((props) => {
type="button"
data-testid="auth-button"
disabled={false}
- icon={!userinfo.is_authenticated ? "log-in" : "log-out"}
href={!userinfo.is_authenticated ? auth.login : auth.logout}
>
{!userinfo.is_authenticated ? "Log In" : "Log Out"}
diff --git a/client/src/components/menubar/index.js b/client/src/components/menubar/index.js
index 3fd840c0..b4271302 100644
--- a/client/src/components/menubar/index.js
+++ b/client/src/components/menubar/index.js
@@ -6,8 +6,7 @@ import * as globals from "../../globals";
import styles from "./menubar.css";
import actions from "../../actions";
import Clip from "./clip";
-import AuthButtons from "./authButtons";
-import InformationMenu from "./infoMenu";
+
import Subset from "./subset";
import UndoRedoReset from "./undoRedo";
import DiffexpButtons from "./diffexpButtons";
@@ -204,7 +203,6 @@ class MenuBar extends React.PureComponent {
render() {
const {
dispatch,
- libraryVersions,
disableDiffexp,
undoDisabled,
redoDisabled,
@@ -212,17 +210,12 @@ class MenuBar extends React.PureComponent {
clipPercentileMin,
clipPercentileMax,
graphInteractionMode,
- aboutLink,
showCentroidLabels,
- privacyURL,
- tosURL,
categoricalSelection,
colorAccessor,
subsetPossible,
subsetResetPossible,
enableReembedding,
- auth,
- userinfo,
} = this.props;
const { pendingClipPercentiles } = this.state;
@@ -248,10 +241,6 @@ class MenuBar extends React.PureComponent {
zIndex: 3,
}}
>
-
-
{
@@ -16,7 +16,14 @@ const handleClick = (dispatch) => {
};
const InformationMenu = React.memo((props) => {
- const { libraryVersions, tosURL, privacyURL, dispatch } = props;
+ const {
+ libraryVersions,
+ tosURL,
+ privacyURL,
+ auth,
+ userinfo,
+ dispatch,
+ } = props;
return (
{
handleClick(dispatch)}
- icon={IconNames.INFO_SIGN}
+ icon="info-sign"
text="Dataset Overview"
/>
{
href={privacyURL}
target="_blank"
text="Privacy Policy"
+ rel="noopener"
/>
) : null}
+
+ {auth?.["requires_client_login"] &&
+ userinfo?.["is_authenticated"] ? (
+ <>
+
+
+ >
+ ) : null}
}
position={Position.BOTTOM_RIGHT}
+ modifiers={{
+ preventOverflow: { enabled: false },
+ hide: { enabled: false },
+ }}
>
Date: Wed, 30 Sep 2020 12:53:51 -0700
Subject: [PATCH 17/73] add-menu-test-id (#1895)
---
client/src/components/menubar/infoMenu.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/client/src/components/menubar/infoMenu.js b/client/src/components/menubar/infoMenu.js
index 8cfb7739..06b6bb60 100644
--- a/client/src/components/menubar/infoMenu.js
+++ b/client/src/components/menubar/infoMenu.js
@@ -93,6 +93,7 @@ const InformationMenu = React.memo((props) => {
}}
>
Date: Wed, 30 Sep 2020 15:24:51 -0700
Subject: [PATCH 18/73] Bump bl from 4.0.2 to 4.0.3 in /client (#1810)
Bumps [bl](https://github.com/rvagg/bl) from 4.0.2 to 4.0.3.
- [Release notes](https://github.com/rvagg/bl/releases)
- [Commits](https://github.com/rvagg/bl/compare/v4.0.2...v4.0.3)
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Severiano Badajoz
---
client/package-lock.json | 16 +++-------------
1 file changed, 3 insertions(+), 13 deletions(-)
diff --git a/client/package-lock.json b/client/package-lock.json
index 68eb0080..de8aa501 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -6727,9 +6727,9 @@
}
},
"bl": {
- "version": "4.0.2",
- "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
- "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
+ "version": "4.0.3",
+ "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
+ "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
"dev": true,
"requires": {
"buffer": "^5.5.0",
@@ -6737,16 +6737,6 @@
"readable-stream": "^3.4.0"
},
"dependencies": {
- "buffer": {
- "version": "5.5.0",
- "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz",
- "integrity": "sha512-9FTEDjLjwoAkEwyMGDjYJQN2gfRgOKBKRfiglhvibGbpeeU/pQn1bJxQqm32OD/AIeEuHxU9roxXxg34Byp/Ww==",
- "dev": true,
- "requires": {
- "base64-js": "^1.0.2",
- "ieee754": "^1.1.4"
- }
- },
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
From 8bd4cbd1e5abaea09b7ae3112fef69576735f8f8 Mon Sep 17 00:00:00 2001
From: Timmy Huang
Date: Thu, 1 Oct 2020 12:29:59 -0700
Subject: [PATCH 19/73] 1807-authN-smoke-test (#1898)
This PR does the following:
1. Add `login` and `logout` helper functions in `client/__tests__/e2e/cellxgeneActions.js`
2. Add conditional AuthN integration test in `client/__tests__/e2e/e2e.test.js`. The test will only run if env variable `TEST_AUTH_INTEGRATION` is `"true"`, which is only set in `single-cell-infra`'s Github Action flow. Corresponding PR [here](https://github.com/chanzuckerberg/single-cell-infra/pull/198)
---
client/__tests__/e2e/cellxgeneActions.js | 74 ++++++++++++++++++++++++
client/__tests__/e2e/e2e.test.js | 16 +++++
client/__tests__/e2e/puppeteer.setup.js | 6 +-
3 files changed, 93 insertions(+), 3 deletions(-)
diff --git a/client/__tests__/e2e/cellxgeneActions.js b/client/__tests__/e2e/cellxgeneActions.js
index eee4714c..9fda69b1 100644
--- a/client/__tests__/e2e/cellxgeneActions.js
+++ b/client/__tests__/e2e/cellxgeneActions.js
@@ -13,8 +13,11 @@ import {
getTestClass,
getTestId,
isElementPresent,
+ goToPage,
} from "./puppeteerUtils";
+import { appUrlBase } from "./config";
+
export async function drag(testId, start, end, lasso = false) {
const layout = await waitByID(testId);
const elBox = await layout.boxModel();
@@ -312,4 +315,75 @@ export async function assertCategoryDoesNotExist(categoryName) {
await expect(result).toBe(false);
}
+
+export async function login() {
+ const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
+ const password = "Test1111";
+
+ await goToPage(appUrlBase);
+
+ await clickOn("auth-button");
+
+ // (thuang): Auth0 form is unstable and unsafe for input until verified
+ await waitUntilFormFieldStable('[name="email"]');
+
+ await expect(page).toFillForm("form", {
+ email,
+ password,
+ });
+
+ await Promise.all([
+ page.waitForNavigation({ waitUntil: "networkidle0" }),
+ expect(page).toClick('[name="submit"]'),
+ ]);
+
+ expect(page.url()).toContain(appUrlBase);
+}
+
+export async function logout() {
+ await clickOnUntil("menu", async () => {
+ await expect(page).toMatch("Log Out");
+
+ await Promise.all([
+ page.waitForNavigation({ waitUntil: "networkidle0" }),
+ expect(page).toClick("a", { text: "Log Out" }),
+ ]);
+ });
+
+ await expect(page).toMatch("Log In");
+}
+
+async function waitUntilFormFieldStable(selector) {
+ const MAX_RETRY = 10;
+ const WAIT_FOR_MS = 200;
+
+ const EXPECTED_VALUE = "aaa";
+
+ let retry = 0;
+
+ while (retry < MAX_RETRY) {
+ try {
+ await expect(page).toFill(selector, EXPECTED_VALUE);
+
+ const fieldHandle = await expect(page).toMatchElement(selector);
+
+ const fieldValue = await page.evaluate(
+ (input) => input.value,
+ fieldHandle
+ );
+
+ expect(fieldValue).toBe(EXPECTED_VALUE);
+
+ break;
+ } catch (error) {
+ retry += 1;
+
+ await page.waitFor(WAIT_FOR_MS);
+ }
+ }
+
+ if (retry === MAX_RETRY) {
+ throw Error("clickOnUntil() assertion failed!");
+ }
+}
/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
diff --git a/client/__tests__/e2e/e2e.test.js b/client/__tests__/e2e/e2e.test.js
index 982a076a..05f52314 100644
--- a/client/__tests__/e2e/e2e.test.js
+++ b/client/__tests__/e2e/e2e.test.js
@@ -31,6 +31,8 @@ import {
runDiffExp,
selectCategory,
subset,
+ login,
+ logout,
} from "./cellxgeneActions";
const data = datasets[DATASET];
@@ -518,4 +520,18 @@ test("lasso moves after pan", async () => {
expect(panCount).toBe(initialCount);
});
+
+const conditionalDescribe =
+ process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
+
+conditionalDescribe("AuthN Integration", () => {
+ it("logs in", async () => {
+ await login();
+ });
+
+ it("logs out", async () => {
+ await login();
+ await logout();
+ });
+});
/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
diff --git a/client/__tests__/e2e/puppeteer.setup.js b/client/__tests__/e2e/puppeteer.setup.js
index 40453777..6c593b93 100644
--- a/client/__tests__/e2e/puppeteer.setup.js
+++ b/client/__tests__/e2e/puppeteer.setup.js
@@ -17,7 +17,9 @@ setDefaultOptions({ timeout: 20 * 1000 });
jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
-(async () => {
+beforeEach(async () => {
+ await jestPuppeteer.resetBrowser();
+
const userAgent = await browser.userAgent();
await page.setUserAgent(`${userAgent}bot`);
@@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
}
}
});
-})().catch((error) => {
- console.error("puppeteer.setup.js error", error);
});
From 1f9bba6f00bf5bf35e738ed41d57fba1a4fca74c Mon Sep 17 00:00:00 2001
From: evanbiederstedt
Date: Thu, 1 Oct 2020 22:44:59 -0400
Subject: [PATCH 20/73] readme correction (#1896)
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index e2cb3215..ef313e53 100644
--- a/README.md
+++ b/README.md
@@ -81,7 +81,7 @@ If you believe you have found a security issue, we would appreciate notification
# Inspiration
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
From 3718e894edc8a8f6e7776946695ab37c5c96ec9f Mon Sep 17 00:00:00 2001
From: Leslie
Date: Fri, 2 Oct 2020 12:10:55 -0700
Subject: [PATCH 21/73] Removed legacy landing page and updated cxg readme
(#1897)
---
dev_docs/cxg.md | 2 +
docs/_config.yml | 4 +-
docs/_site/index.html | 18 +++---
docs/_site/posts/annotations.html | 18 +++---
docs/_site/posts/contact.html | 18 +++---
docs/_site/posts/contribute.html | 18 +++---
docs/_site/posts/demo-data.html | 18 +++---
docs/_site/posts/gallery.html | 20 +++----
docs/_site/posts/hosted.html | 59 ++++++++++---------
docs/_site/posts/hosted.md | 39 +++++++-----
docs/_site/posts/install.html | 18 +++---
docs/_site/posts/launch.html | 18 +++---
docs/_site/posts/methods.html | 18 +++---
docs/_site/posts/prepare.html | 18 +++---
docs/_site/posts/roadmap.html | 18 +++---
docs/_site/posts/troubleshooting.html | 18 +++---
.../cellxgene_cziscience_com.md | 0
docs/posts/gallery.md | 2 +-
18 files changed, 167 insertions(+), 157 deletions(-)
rename docs/{posts => deprecated}/cellxgene_cziscience_com.md (100%)
diff --git a/dev_docs/cxg.md b/dev_docs/cxg.md
index e851692f..05bf465f 100644
--- a/dev_docs/cxg.md
+++ b/dev_docs/cxg.md
@@ -1,3 +1,5 @@
+## UPDATE (9/30/2020): Starting today, the name Corpora will only be used as the internal project name, with cellxgene Data Portal being the official product name
+
# CXG Data Format Specification
Document Status: _draft_
diff --git a/docs/_config.yml b/docs/_config.yml
index 8ac27944..f70c05b5 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -15,6 +15,8 @@ nav:
url: posts/gallery
- title: Demo datasets
url: posts/demo-data
+ - title: All other datasets
+ url: https://cellxgene.cziscience.com/
- title: Preparing your data
url: posts/prepare
- title: Launching cellxgene
@@ -33,5 +35,3 @@ nav:
url: posts/contribute
- title: Contact & finding help
url: posts/contact
- - title: cellxgene.cziscience.com
- url: posts/cellxgene_cziscience_com
diff --git a/docs/_site/index.html b/docs/_site/index.html
index 2a5ed4f1..42326a08 100644
--- a/docs/_site/index.html
+++ b/docs/_site/index.html
@@ -7,19 +7,19 @@
Index | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html
index 2a76ec9f..725ab7db 100644
--- a/docs/_site/posts/annotations.html
+++ b/docs/_site/posts/annotations.html
@@ -7,19 +7,19 @@
annotations | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html
index 33c31b51..2bbefffa 100644
--- a/docs/_site/posts/contact.html
+++ b/docs/_site/posts/contact.html
@@ -7,19 +7,19 @@
Contact | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html
index 494d2ca3..9727cffb 100644
--- a/docs/_site/posts/contribute.html
+++ b/docs/_site/posts/contribute.html
@@ -7,19 +7,19 @@
Code of conduct | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html
index 479e6249..4b05dfbe 100644
--- a/docs/_site/posts/demo-data.html
+++ b/docs/_site/posts/demo-data.html
@@ -7,19 +7,19 @@
demo-data | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html
index 2321d81e..5b19ba47 100644
--- a/docs/_site/posts/gallery.html
+++ b/docs/_site/posts/gallery.html
@@ -7,19 +7,19 @@
Gallery | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
@@ -130,7 +130,7 @@ Check out the cool data that our users are using cellxgene to explore!
-
+
Want us to link to your dataset here? Just send us a note!
diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html
index 0e2d2b6a..b85e5d96 100644
--- a/docs/_site/posts/hosted.html
+++ b/docs/_site/posts/hosted.html
@@ -7,19 +7,19 @@
Hosting cellxgene on the web | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
@@ -139,35 +139,36 @@
Deploying cellxgene with Heroku
-Quickstart
+Heroku Support
-Clicking on the following button will forward you to Heroku to begin the deployment process:
+The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.
-
-
-
+While we no longer directly support Heroku, it is still possible to create a Heroku app via our provided Dockerfile here and Herokuâs documentation .
-If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
+You may have to tweak the Dockerfile like so:
-Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
+FROM ubuntu:bionic
-Default settings
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
-
- App name: the unique name for your deployment
- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
- App owner: Who will own this app. Either you personally or an organization/team
- Region: Location of the server where the app will be deployed (EU or US)
-
+RUN apt-get update && \
+ apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+ pip3 install cellxgene
-Configuration
+# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT
+
-
- DATASET: A publicly accessible URL pointing to a .h5ad file to view
- This defaults to pbm3k.h5ad
-
+and provide a heroku.yml file similar to this:
-After filling out the settings and pressing the Deploy app button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
+build :
+ docker :
+ web : Dockerfile
+run :
+ web :
+ command :
+ - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
+
What is Heroku?
diff --git a/docs/_site/posts/hosted.md b/docs/_site/posts/hosted.md
index 1168bdf0..073ede26 100644
--- a/docs/_site/posts/hosted.md
+++ b/docs/_site/posts/hosted.md
@@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list.
# Deploying cellxgene with Heroku
-## Quickstart
+## Heroku Support
-Clicking on the following button will forward you to Heroku to begin the deployment process:
+The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.
-
-
-
+While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml).
-If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
+You may have to tweak the `Dockerfile` like so:
-Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
+```Dockerfile
+FROM ubuntu:bionic
-### Default settings
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
-- `App name`: the unique name for your deployment
-- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
-- `App owner`: Who will own this app. Either you personally or an organization/team
-- `Region`: Location of the server where the app will be deployed (EU or US)
+RUN apt-get update && \
+ apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+ pip3 install cellxgene
-### Configuration
+# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT
+```
-- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view
-- This defaults to pbm3k.h5ad
+and provide a `heroku.yml` file similar to this:
-After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
+```yml
+build:
+ docker:
+ web: Dockerfile
+run:
+ web:
+ command:
+ - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
+```
## What is Heroku?
diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html
index 2eb470d8..649209dc 100644
--- a/docs/_site/posts/install.html
+++ b/docs/_site/posts/install.html
@@ -7,19 +7,19 @@
Install | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html
index 4aed95b3..86c8dd3b 100644
--- a/docs/_site/posts/launch.html
+++ b/docs/_site/posts/launch.html
@@ -7,19 +7,19 @@
demo-data | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html
index 8c97a84c..f27e270e 100644
--- a/docs/_site/posts/methods.html
+++ b/docs/_site/posts/methods.html
@@ -7,19 +7,19 @@
Methods | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html
index 56dce0c2..2b3b0f05 100644
--- a/docs/_site/posts/prepare.html
+++ b/docs/_site/posts/prepare.html
@@ -7,19 +7,19 @@
prepare | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html
index 34dc26b9..837a3bb7 100644
--- a/docs/_site/posts/roadmap.html
+++ b/docs/_site/posts/roadmap.html
@@ -7,19 +7,19 @@
roadmap | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html
index a936237a..c43ee9af 100644
--- a/docs/_site/posts/troubleshooting.html
+++ b/docs/_site/posts/troubleshooting.html
@@ -7,19 +7,19 @@
Troubleshooting | cellxgene
-
+
-
-
+
+
+{"url":"http://localhost:4000/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -50,6 +50,10 @@
+ All other datasets
+
+
+
Preparing your data
@@ -85,10 +89,6 @@
Contact & finding help
-
- cellxgene.cziscience.com
-
-
Code
diff --git a/docs/posts/cellxgene_cziscience_com.md b/docs/deprecated/cellxgene_cziscience_com.md
similarity index 100%
rename from docs/posts/cellxgene_cziscience_com.md
rename to docs/deprecated/cellxgene_cziscience_com.md
diff --git a/docs/posts/gallery.md b/docs/posts/gallery.md
index 13a7fc24..01f45301 100644
--- a/docs/posts/gallery.md
+++ b/docs/posts/gallery.md
@@ -39,6 +39,6 @@ Check out the cool data that our users are using cellxgene to explore!
### [Melanoma](https://melanoma.cellgeni.sanger.ac.uk/)
-### [CZI's own cellxgene site](cellxgene_cziscience_com)
+### [CZI's own cellxgene site](https://cellxgene.cziscience.com/)
_Want us to link to your dataset here? [Just send us a note!](contact)_
From b048fd8d9a8102f479214e7b8aff85362665c88c Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Tue, 6 Oct 2020 12:59:07 -0700
Subject: [PATCH 22/73] Allow columns encoded in float64 to be rendered as part
of continuous value histograms. (#1905)
---
client/src/components/continuous/continuous.js | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/client/src/components/continuous/continuous.js b/client/src/components/continuous/continuous.js
index ad5c908d..f1119507 100644
--- a/client/src/components/continuous/continuous.js
+++ b/client/src/components/continuous/continuous.js
@@ -14,7 +14,12 @@ class Continuous extends React.PureComponent {
if (!schema) return null;
const obsIndex = schema.annotations.obs.index;
const allContinuousNames = schema.annotations.obs.columns
- .filter((col) => col.type === "int32" || col.type === "float32")
+ .filter(
+ (col) =>
+ col.type === "int32" ||
+ col.type === "float32" ||
+ col.type === "float64"
+ )
.filter((col) => col.name !== obsIndex)
.filter((col) => !col.writable) // skip user annotations - they will be treated as categorical
.map((col) => col.name);
From b386ca3425201771f97dcadcbc39c946dc6f94a3 Mon Sep 17 00:00:00 2001
From: Leslie
Date: Tue, 6 Oct 2020 13:16:59 -0700
Subject: [PATCH 23/73] Move link to cellxgene data portal higher (#1909)
---
docs/_config.yml | 4 ++--
docs/_site/index.html | 16 ++++++++--------
docs/_site/posts/annotations.html | 16 ++++++++--------
docs/_site/posts/contact.html | 16 ++++++++--------
docs/_site/posts/contribute.html | 16 ++++++++--------
docs/_site/posts/demo-data.html | 16 ++++++++--------
docs/_site/posts/gallery.html | 16 ++++++++--------
docs/_site/posts/hosted.html | 16 ++++++++--------
docs/_site/posts/install.html | 16 ++++++++--------
docs/_site/posts/launch.html | 16 ++++++++--------
docs/_site/posts/methods.html | 16 ++++++++--------
docs/_site/posts/prepare.html | 16 ++++++++--------
docs/_site/posts/roadmap.html | 16 ++++++++--------
docs/_site/posts/troubleshooting.html | 16 ++++++++--------
14 files changed, 106 insertions(+), 106 deletions(-)
diff --git a/docs/_config.yml b/docs/_config.yml
index f70c05b5..6e29ea64 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -13,10 +13,10 @@ nav:
url: posts/install
- title: Gallery
url: posts/gallery
+ - title: Cellxgene data portal
+ url: https://cellxgene.cziscience.com/
- title: Demo datasets
url: posts/demo-data
- - title: All other datasets
- url: https://cellxgene.cziscience.com/
- title: Preparing your data
url: posts/prepare
- title: Launching cellxgene
diff --git a/docs/_site/index.html b/docs/_site/index.html
index 42326a08..49e763a0 100644
--- a/docs/_site/index.html
+++ b/docs/_site/index.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html
index 725ab7db..09941877 100644
--- a/docs/_site/posts/annotations.html
+++ b/docs/_site/posts/annotations.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html
index 2bbefffa..8040eed7 100644
--- a/docs/_site/posts/contact.html
+++ b/docs/_site/posts/contact.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html
index 9727cffb..2af2e3e9 100644
--- a/docs/_site/posts/contribute.html
+++ b/docs/_site/posts/contribute.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html
index 4b05dfbe..cc0c5958 100644
--- a/docs/_site/posts/demo-data.html
+++ b/docs/_site/posts/demo-data.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html
index 5b19ba47..ff0f5a81 100644
--- a/docs/_site/posts/gallery.html
+++ b/docs/_site/posts/gallery.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html
index b85e5d96..214b5b8a 100644
--- a/docs/_site/posts/hosted.html
+++ b/docs/_site/posts/hosted.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html
index 649209dc..c6e0daf5 100644
--- a/docs/_site/posts/install.html
+++ b/docs/_site/posts/install.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html
index 86c8dd3b..62ccbaf8 100644
--- a/docs/_site/posts/launch.html
+++ b/docs/_site/posts/launch.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html
index f27e270e..265b5263 100644
--- a/docs/_site/posts/methods.html
+++ b/docs/_site/posts/methods.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html
index 2b3b0f05..b1d6d619 100644
--- a/docs/_site/posts/prepare.html
+++ b/docs/_site/posts/prepare.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html
index 837a3bb7..8a3b177b 100644
--- a/docs/_site/posts/roadmap.html
+++ b/docs/_site/posts/roadmap.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html
index c43ee9af..8a7a082a 100644
--- a/docs/_site/posts/troubleshooting.html
+++ b/docs/_site/posts/troubleshooting.html
@@ -12,14 +12,14 @@
-
-
+
+
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"}
-
+
@@ -46,14 +46,14 @@
+ Cellxgene data portal
+
+
+
Demo datasets
- All other datasets
-
-
-
Preparing your data
From eb108feb37fc4e244b717bb9bc8a1bf14ed55f17 Mon Sep 17 00:00:00 2001
From: Madison Dunitz
Date: Wed, 7 Oct 2020 12:36:02 -0500
Subject: [PATCH 24/73] Performance test annotations (#1908)
* make testing plan
* create annotaions sets for different num categories/dataset size
* annotation creation testing
* create scale and perf tests for annotations
* create make commands for tests
* get cell count if not set in test_datasets dict
---
Makefile | 2 +-
server/Makefile | 8 +
.../performance_test_annotations_backend.py | 215 ++++++++++++++++++
.../performance/scale_test_annotations.py | 46 ++++
4 files changed, 270 insertions(+), 1 deletion(-)
create mode 100644 server/test/performance/performance_test_annotations_backend.py
create mode 100644 server/test/performance/scale_test_annotations.py
diff --git a/Makefile b/Makefile
index ca8754f7..8f9a4083 100644
--- a/Makefile
+++ b/Makefile
@@ -83,7 +83,7 @@ lint: lint-server lint-client
.PHONY: lint-server
lint-server: fmt-py
- flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821'
+ flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
.PHONY: lint-client
diff --git a/server/Makefile b/server/Makefile
index 700b9a27..016f89cc 100644
--- a/server/Makefile
+++ b/server/Makefile
@@ -39,3 +39,11 @@ create-test-db:
clean-test-db:
-docker stop test_db
-docker rm test_db
+
+.PHONY: test-annotations-performance
+test-annotations-performance:
+ python test/performance/performance_test_annotations_backend.py
+
+.PHONY: test-annotations-scale
+test-annotations-scale:
+ locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
diff --git a/server/test/performance/performance_test_annotations_backend.py b/server/test/performance/performance_test_annotations_backend.py
new file mode 100644
index 00000000..d0548523
--- /dev/null
+++ b/server/test/performance/performance_test_annotations_backend.py
@@ -0,0 +1,215 @@
+import json
+import string
+from contextlib import contextmanager
+from timeit import default_timer
+import concurrent.futures
+import numpy as np
+import requests
+import sys
+from server.data_common.fbs.matrix import encode_matrix_fbs
+import pandas as pd
+import random
+
+"""
+Before running, sign into the dataportal, copy the cookie and paste it below. To test in staging or prod update the
+url base below. It is also possible to configure the number of categories created and the number of unique labels per
+category.
+"""
+
+cookie = ""
+
+test_datasets = {
+ "smallest": {
+ "dataset_url": "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
+ "name": "smallest",
+ "num_cells": 5270,
+ },
+ "10k": {
+ "dataset_url": "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
+ "name": "10k",
+ "num_cells": 9409,
+ },
+ "80k": {
+ "dataset_url": "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", # noqa E501
+ "name": "80k",
+ "num_cells": 81736,
+ },
+ "140k": {"dataset_url": "Single_cell_drug_screening_a549-42-remixed.cxg", "name": "140k", "num_cells": 143015},
+ "largest": {"dataset_url": "human_cell_landscape.cxg", "name": "largest", "num_cells": 599926},
+ "1million": {"dataset_url": None, "name": "1million", "num_cells": 1000000},
+ "4million": {"dataset_url": None, "name": "4million", "num_cells": 4000000},
+}
+
+url_base = "https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/"
+annotations_category_count = [1, 10, 50]
+max_labels = [5, 50, 100]
+
+
+class PerformanceTestingAnnotations:
+ def __init__(
+ self,
+ datasets=test_datasets,
+ annotations_category_count=annotations_category_count,
+ max_labels=max_labels,
+ url_base=url_base,
+ ):
+ self.test_datasets = datasets
+ self.annotations_category_count = annotations_category_count
+ self.max_labels = max_labels
+ self.url_base = url_base
+ self.test_notes = self.create_info_dict()
+
+ def set_cell_count(self, dataset_name):
+ dataset_url = self.test_datasets[dataset_name]["dataset_url"]
+ headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
+ response = self.client.get(f"{self.url_base}{dataset_url}/api/v0.2/schema", headers=headers)
+ cell_count = json.loads(response._content)["schema"]["dataframe"]["nObs"]
+ self.test_datasets[dataset_name]["cell_count"] = cell_count
+
+ def create_info_dict(self):
+ request_info = {}
+ for dataset in self.test_datasets.keys():
+ request_info[dataset] = {}
+ for cat_count in self.annotations_category_count:
+ request_info[dataset][f"num_categories_{cat_count}"] = {}
+ for unique_labels in self.max_labels:
+ request_info[dataset][f"num_categories_{cat_count}"][f"max_label_{unique_labels}"] = {}
+ return request_info
+
+ def create_annotations_dict_multi_process(self, dataset_name, category_count, label_max):
+ annotation_dict = {}
+ futures = []
+ categories = [f"Category{i}" for i in range(category_count)]
+ if not self.test_datasets[dataset_name]["num_cells"]:
+ self.set_cell_count(dataset_name)
+ with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
+ for category in categories:
+ futures.append(
+ executor.submit(
+ self.build_array_for_category,
+ category,
+ self.test_datasets[dataset_name]["num_cells"],
+ label_max,
+ )
+ )
+ for future in concurrent.futures.as_completed(futures):
+ try:
+ result = future.result()
+ category_name, cells = result
+ annotation_dict[category_name] = pd.Series(cells, dtype="category")
+ except Exception as e:
+ print(f"Issue creating the annotations dict: {e}")
+ return annotation_dict
+
+ def build_array_for_category(self, category_name, cell_count, label_max):
+ unique_label_count = label_max
+ labels = self.generate_labels(unique_label_count)
+ cells_per_label = int(cell_count / len(labels))
+ extra = cell_count % len(labels)
+ cells = []
+ for label in labels:
+ cells.extend([label] * cells_per_label)
+ cells.extend(["extra"] * extra)
+ rng = np.random.default_rng()
+ rng.shuffle(cells)
+ return category_name, cells
+
+ @staticmethod
+ def convert_to_fbs(annotation_dict):
+ df = pd.DataFrame(annotation_dict)
+ return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+
+ @staticmethod
+ def generate_labels(unique_label_count):
+ labels = ["undefined"]
+ for i in range(unique_label_count):
+ length = random.randrange(10, 20)
+ labels.append(f"{i}__" + "".join(random.choice(string.ascii_letters) for z in range(length)))
+ return labels
+
+ @contextmanager
+ def elapsed_timer(self):
+ start = default_timer()
+ elapser = lambda: default_timer() - start # noqa E731
+ yield lambda: elapser()
+ end = default_timer()
+ elapser = lambda: end - start # noqa E731
+
+ def create_matrix(self, dataset_name, num_cat, max_labels):
+ with self.elapsed_timer() as elapsed:
+ annon_dict = self.create_annotations_dict_multi_process(dataset_name, num_cat, max_labels)
+ dict_size = sum(sys.getsizeof(value) for value in annon_dict.values()) / 1024 ** 2
+ self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["annotation_dict"] = {
+ "creation_time": str(elapsed()),
+ "size": f"{dict_size} mb",
+ }
+ df = pd.DataFrame(annon_dict)
+ df_size = sys.getsizeof(df) / 1024 ** 2
+ self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["data_frame"] = {
+ "creation_time": str(elapsed()),
+ "size": f"{df_size} mb",
+ }
+ try:
+ matrix = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+ matrix_size = sys.getsizeof(matrix) / 1024 ** 2
+ self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["fbs_matrix"] = {
+ "creation_time": str(elapsed()),
+ "size": f"{matrix_size} mb",
+ }
+ return matrix
+ except Exception as e:
+ print(f"Issue creating fbs matrix: {e}, for {dataset_name}")
+ return []
+
+ def send_put_request(self, dataset_url, data):
+ url = self.url_base + f"{dataset_url}/api/v0.2/annotations/obs"
+ with self.elapsed_timer() as elapsed:
+ try:
+ headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
+ response = requests.put(url=url, data=data, headers=headers)
+ except Exception as e:
+ print(f"Issue with put request: {e}")
+ return None, elapsed()
+ return response, elapsed()
+
+ def test_categories_max_label_matrix(self, dataset_name):
+ for unique_labels in self.max_labels:
+ for category_count in self.annotations_category_count:
+ print(f"Starting dataset: {dataset_name}, categories: {category_count}, labels: {unique_labels}")
+ fbs_matrix = self.create_matrix(dataset_name, category_count, unique_labels)
+ if self.test_datasets[dataset_name]["dataset_url"] and fbs_matrix:
+ response, response_time = self.send_put_request(
+ self.test_datasets[dataset_name]["dataset_url"], fbs_matrix
+ )
+ if response is None:
+ self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
+ "put_request"
+ ] = {"response_status": "failed", "request_time": str(response_time)}
+ else:
+ self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
+ "put_request"
+ ] = {"response_status": response.status_code, "request_time": str(response_time)}
+
+
+def test_all_datasets():
+ """
+ Run time is dependent on number of datasets, dataset size, number of categories/number being tested and number of
+ unique label counts being tested. However it generally takes a long time. I recommend running this in tmux
+ """
+ perf_test = PerformanceTestingAnnotations()
+ for dataset_name in perf_test.test_datasets.keys():
+ print(f"Testing annotation creation for: {dataset_name}")
+ try:
+ perf_test.test_categories_max_label_matrix(dataset_name)
+ except Exception as e:
+ print(f"something went wrong with {dataset_name}: {e}")
+ return perf_test.test_notes
+
+
+def main():
+ notes = test_all_datasets()
+ print(notes)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/server/test/performance/scale_test_annotations.py b/server/test/performance/scale_test_annotations.py
new file mode 100644
index 00000000..5aa2ac6e
--- /dev/null
+++ b/server/test/performance/scale_test_annotations.py
@@ -0,0 +1,46 @@
+import time
+import random
+
+from locust import HttpUser, between, task
+
+random.seed(time.time())
+"""
+To run this script sign into cellxgene in the desired environment and grab the returned cookie, update the cookie
+variable below with your cookie and run the following command to see results in the terminal:
+locust -f server/test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+
+Or if you want to use the locust gui run:
+locust -f server/test/performance/scale_test_annotations.py -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/
+
+If you want to test staging you'll need to substitute staging for dev in the host url
+To test prod you'll need to replace dev.single-cell.czi.technology with cziscience.com
+If you'd like to test additional datasets you'll need to add them to the dataset_urls array
+
+Todo @mdunitz update script to retrieve different annotation categories -- may need to create them to ensure the
+categories are shared across datasets for a given user.
+"""
+cookie = ""
+
+
+class WebsiteUser(HttpUser):
+ wait_time = between(1, 2)
+ dataset_urls = [
+ "human_cell_landscape.cxg",
+ "Single_cell_drug_screening_a549-42-remixed.cxg",
+ "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
+ "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
+ "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg",
+ ]
+
+ @task
+ def get_annotations(self):
+ dataset_url = random.choice(self.dataset_urls)
+ url = f"{dataset_url}/api/v0.2/annotations/obs?annotation-name=cell_type"
+ headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
+ self.client.get(url, headers=headers)
+
+ @task
+ def get_schema(self):
+ dataset_url = random.choice(self.dataset_urls)
+ headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
+ self.client.get(f"{dataset_url}/api/v0.2/schema", headers=headers)
From cf77a8da9e4138af1e95a41071227140df064e4b Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Wed, 7 Oct 2020 12:17:23 -0700
Subject: [PATCH 25/73] Add "picture" to the /userinfo endpoint. (#1914)
* Add "picture" to the /userinfo endpoint.
This may be null or a URL.
add picture for the test authentication method
---
server/auth/auth.py | 4 ++++
server/auth/auth_oauth.py | 16 +++++++---------
server/auth/auth_test.py | 6 ++++++
server/common/config/client_config.py | 1 +
server/test/unit/auth/test_auth.py | 7 +++++++
5 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/server/auth/auth.py b/server/auth/auth.py
index 145616cc..03184e8d 100644
--- a/server/auth/auth.py
+++ b/server/auth/auth.py
@@ -43,6 +43,10 @@ class AuthTypeBase(ABC):
"""Return the name of the user (string)"""
pass
+ def get_user_picture(self):
+ """Return the location to the user's picture"""
+ return None
+
class AuthTypeClientBase(AuthTypeBase):
"""Base type for all authentication types that require the client to login"""
diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py
index d7162318..b2724c57 100644
--- a/server/auth/auth_oauth.py
+++ b/server/auth/auth_oauth.py
@@ -146,21 +146,19 @@ class AuthTypeOAuth(AuthTypeClientBase):
def get_user_id(self):
payload = self.get_userinfo()
- if payload and payload.get("sub"):
- return payload.get("sub")
- return None
+ return payload.get("sub") if payload else None
def get_user_name(self):
payload = self.get_userinfo()
- if payload and payload.get("name"):
- return payload.get("name")
- return None
+ return payload.get("name") if payload else None
def get_user_email(self):
payload = self.get_userinfo()
- if payload and payload.get("email"):
- return payload.get("email")
- return None
+ return payload.get("email") if payload else None
+
+ def get_user_picture(self):
+ payload = self.get_userinfo()
+ return payload.get("picture") if payload else None
def update_response(self, response):
response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
diff --git a/server/auth/auth_test.py b/server/auth/auth_test.py
index 4594dae0..06b92cc7 100644
--- a/server/auth/auth_test.py
+++ b/server/auth/auth_test.py
@@ -10,12 +10,14 @@ class AuthTypeTest(AuthTypeClientBase):
CXGUID = "cxguid_test"
CXGUNAME = "cxguname_test"
CXGUEMAIL = "cxguemail_test"
+ CXGUPICTURE = "cxgupicture_test"
def __init__(self, app_config):
super().__init__()
self.user_name = "test_account"
self.user_id = "id0001"
self.user_email = "test_account@test.com"
+ self.user_picture = None
def is_valid_authentication_type(self):
return True
@@ -42,12 +44,16 @@ class AuthTypeTest(AuthTypeClientBase):
def get_user_email(self):
return session.get(self.CXGUEMAIL)
+ def get_user_picture(self):
+ return session.get(self.CXGUPICTURE)
+
def login(self):
args = request.args
return_to = args.get("dataset", "/")
session[self.CXGUID] = args.get("userid", self.user_id)
session[self.CXGUNAME] = args.get("username", self.user_name)
session[self.CXGUEMAIL] = args.get("email", self.user_email)
+ session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
return redirect(return_to)
def logout(self):
diff --git a/server/common/config/client_config.py b/server/common/config/client_config.py
index ccddf2c1..8a70c0b6 100644
--- a/server/common/config/client_config.py
+++ b/server/common/config/client_config.py
@@ -117,5 +117,6 @@ def get_client_userinfo(app_config, data_adaptor):
"username": auth.get_user_name(),
"user_id": auth.get_user_id(),
"email": auth.get_user_email(),
+ "picture": auth.get_user_picture(),
}
return userinfo
diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py
index f7f6f358..0b8c0bf1 100644
--- a/server/test/unit/auth/test_auth.py
+++ b/server/test/unit/auth/test_auth.py
@@ -82,6 +82,7 @@ class AuthTest(unittest.TestCase):
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
+ self.assertEqual(userinfo["userinfo"]["picture"], None)
self.assertTrue(config["config"]["parameters"]["annotations"])
r = session.get(f"{server}/{logout_uri}")
@@ -100,6 +101,12 @@ class AuthTest(unittest.TestCase):
self.assertIsNone(userinfo)
self.assertFalse(config["config"]["parameters"]["annotations"])
+ # login with a picture
+ r = session.get(f"{server}/{login_uri}&picture=myimage.png")
+ userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+ self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+ self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
+
def test_auth_test_single(self):
c = AppConfig()
c.update_server_config(
From 1c4c501c43b24504b26b289c3668b5d544dd8292 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Wed, 7 Oct 2020 15:02:55 -0700
Subject: [PATCH 26/73] Auth UI tweaks (#1915)
* remove auth buttons and dataset info from info menu
* add auth buttons to menubar
* remove auth from top left
* new auth buttons
* move infomenu to lsb dir
* styling fixes
* feedback
* more feedback
Co-authored-by: Timmy Huang
---
client/src/components/leftSidebar/infoMenu.js | 72 ++++++++++++
.../leftSidebar/topLeftLogoAndTitle.js | 14 +--
client/src/components/menubar/authButtons.js | 87 +++++++++++---
client/src/components/menubar/index.js | 4 +
client/src/components/menubar/infoMenu.js | 108 ------------------
5 files changed, 148 insertions(+), 137 deletions(-)
create mode 100644 client/src/components/leftSidebar/infoMenu.js
delete mode 100644 client/src/components/menubar/infoMenu.js
diff --git a/client/src/components/leftSidebar/infoMenu.js b/client/src/components/leftSidebar/infoMenu.js
new file mode 100644
index 00000000..6ab9bdde
--- /dev/null
+++ b/client/src/components/leftSidebar/infoMenu.js
@@ -0,0 +1,72 @@
+// jshint esversion: 6
+import React from "react";
+import { Button, Menu, MenuItem, Popover, Position } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+
+const InformationMenu = React.memo((props) => {
+ const { libraryVersions, tosURL, privacyURL } = props;
+ return (
+
+
+
+
+
+
+ {tosURL && (
+
+ )}
+ {privacyURL && (
+
+ )}
+
+ }
+ position={Position.BOTTOM_RIGHT}
+ modifiers={{
+ preventOverflow: { enabled: false },
+ hide: { enabled: false },
+ }}
+ >
+
+
+ );
+});
+
+export default InformationMenu;
diff --git a/client/src/components/leftSidebar/topLeftLogoAndTitle.js b/client/src/components/leftSidebar/topLeftLogoAndTitle.js
index edfc8cef..e4f3d5a7 100644
--- a/client/src/components/leftSidebar/topLeftLogoAndTitle.js
+++ b/client/src/components/leftSidebar/topLeftLogoAndTitle.js
@@ -7,15 +7,12 @@ import * as globals from "../../globals";
import Logo from "../framework/logo";
import Truncate from "../util/truncate";
import InfoDrawer from "../infoDrawer/infoDrawer";
-import AuthButtons from "../menubar/authButtons";
-import InformationMenu from "../menubar/infoMenu";
+import InformationMenu from "./infoMenu";
const DATASET_TITLE_FONT_SIZE = 14;
@connect((state) => ({
datasetTitle: state.config?.displayNames?.dataset ?? "",
- auth: state.config?.authentication,
- userinfo: state.userinfo,
libraryVersions: state.config?.["library_versions"],
aboutLink: state.config?.links?.["about-dataset"],
tosURL: state.config?.parameters?.["about_legal_tos"],
@@ -30,8 +27,6 @@ class LeftSideBar extends React.Component {
render() {
const {
datasetTitle,
- auth,
- userinfo,
libraryVersions,
aboutLink,
privacyURL,
@@ -79,7 +74,7 @@ class LeftSideBar extends React.Component {
gene
-
+
- {!userinfo.is_authenticated ? (
-
- ) : null}
);
diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js
index 985b3da3..60ace82b 100644
--- a/client/src/components/menubar/authButtons.js
+++ b/client/src/components/menubar/authButtons.js
@@ -1,30 +1,83 @@
import React from "react";
-import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import {
+ AnchorButton,
+ Button,
+ MenuItem,
+ Tooltip,
+ Popover,
+ Menu,
+} from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+
import * as globals from "../../globals";
import styles from "./menubar.css";
+const BASE_EMOJI = [0x1f9d1, 0x1f468, 0x1f469];
+const SKIN_TONES = [0x1f3fb, 0x1f3fc, 0x1f3fd, 0x1f3fe, 0x1f3ff];
+const MICROSCOPE = 0x1f52c;
+const ZERO_WIDTH_JOINER = 0x0200d;
+
const Auth = React.memo((props) => {
const { auth, userinfo } = props;
- if (!auth || (auth && !auth.requires_client_login)) return null;
+ const randomInt = Math.random() * 15;
+ const sexIndex = Math.floor(randomInt / 5);
+ const skinToneIndex = Math.floor(randomInt % 5);
+
+ const scientist = String.fromCodePoint(
+ BASE_EMOJI[sexIndex],
+ SKIN_TONES[skinToneIndex],
+ ZERO_WIDTH_JOINER,
+ MICROSCOPE
+ );
+
+ if (!auth?.["requires_client_login"]) return null;
+
+ if (userinfo?.["is_authenticated"]) {
+ const PopoverContent = (
+
+
+
+
+ );
+
+ return (
+
+
+ {userinfo?.picture ? (
+
+ ) : (
+ {scientist}
+ )}
+
+
+ );
+ }
return (
-
-
+
-
- {!userinfo.is_authenticated ? "Log In" : "Log Out"}
-
-
-
+ Log In
+
+
);
});
diff --git a/client/src/components/menubar/index.js b/client/src/components/menubar/index.js
index b4271302..115eb6ff 100644
--- a/client/src/components/menubar/index.js
+++ b/client/src/components/menubar/index.js
@@ -7,6 +7,7 @@ import styles from "./menubar.css";
import actions from "../../actions";
import Clip from "./clip";
+import AuthButtons from "./authButtons";
import Subset from "./subset";
import UndoRedoReset from "./undoRedo";
import DiffexpButtons from "./diffexpButtons";
@@ -216,6 +217,8 @@ class MenuBar extends React.PureComponent {
subsetPossible,
subsetResetPossible,
enableReembedding,
+ userinfo,
+ auth,
} = this.props;
const { pendingClipPercentiles } = this.state;
@@ -241,6 +244,7 @@ class MenuBar extends React.PureComponent {
zIndex: 3,
}}
>
+
{
- dispatch({ type: "toggle dataset drawer" });
-};
-
-const InformationMenu = React.memo((props) => {
- const {
- libraryVersions,
- tosURL,
- privacyURL,
- auth,
- userinfo,
- dispatch,
- } = props;
- return (
-
-
- handleClick(dispatch)}
- icon="info-sign"
- text="Dataset Overview"
- />
-
-
-
-
-
-
- {tosURL ? (
-
- ) : null}
- {privacyURL ? (
-
- ) : null}
-
- {auth?.["requires_client_login"] &&
- userinfo?.["is_authenticated"] ? (
- <>
-
-
- >
- ) : null}
-
- }
- position={Position.BOTTOM_RIGHT}
- modifiers={{
- preventOverflow: { enabled: false },
- hide: { enabled: false },
- }}
- >
-
-
-
- );
-});
-
-export default InformationMenu;
From 6c1756f85260f81754921536a849ca14e34a4f0b Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Wed, 7 Oct 2020 15:38:42 -0700
Subject: [PATCH 27/73] Enhance the AppConfig with external config sources.
(#1904)
* Enhance the AppConfig with external config sources.
The external config sources are currently environment variables
and AWS secrets manager.
The config file can be augmented with a section describing how
environmen variables and secrets can update config parameters.
benefits:
- it will enable the config to draw from more than one secret. This is useful
for shared secrets between cellxgene and data portal, as well as auth0 secrets.
- it will make it very straightforward to check the config before a deployment.
Part of #1859
---
server/common/aws_secret_utils.py | 2 +-
server/common/config/app_config.py | 70 +++++-
server/common/config/base_config.py | 19 ++
server/common/config/dataset_config.py | 24 +-
server/common/config/external_config.py | 96 ++++++++
server/common/config/server_config.py | 30 +--
server/common/utils/type_conversion_utils.py | 14 ++
server/default_config.py | 55 +++++
server/test/__init__.py | 8 +-
server/test/unit/common/config/__init__.py | 51 +++-
.../unit/common/config/test_app_config.py | 74 ++++++
.../unit/common/config/test_dataset_config.py | 4 +-
.../common/config/test_external_config.py | 231 ++++++++++++++++++
.../unit/common/config/test_server_config.py | 9 +-
14 files changed, 632 insertions(+), 55 deletions(-)
create mode 100644 server/common/config/external_config.py
create mode 100644 server/test/unit/common/config/test_external_config.py
diff --git a/server/common/aws_secret_utils.py b/server/common/aws_secret_utils.py
index 070ed160..ce40794d 100644
--- a/server/common/aws_secret_utils.py
+++ b/server/common/aws_secret_utils.py
@@ -18,6 +18,6 @@ def get_secret_key(region_name, secret_name):
return secret
except Exception as e:
logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
- raise SecretKeyRetrievalError
+ raise SecretKeyRetrievalError(str(e))
return None
diff --git a/server/common/config/app_config.py b/server/common/config/app_config.py
index 422dcb8e..5cd4c4a5 100644
--- a/server/common/config/app_config.py
+++ b/server/common/config/app_config.py
@@ -4,6 +4,7 @@ from flatten_dict import unflatten
from server.default_config import get_default_config
from server.common.config.dataset_config import DatasetConfig
from server.common.config.server_config import ServerConfig
+from server.common.config.external_config import ExternalConfig
from server.common.errors import ConfigurationError
@@ -44,6 +45,9 @@ class AppConfig(object):
# dataroot config
self.dataroot_config = {}
+ # external config
+ self.external_config = ExternalConfig(self, self.default_config["external"])
+
# Set to true when config_completed is called
self.is_completed = False
@@ -61,6 +65,7 @@ class AppConfig(object):
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
+ self.external_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
@@ -73,6 +78,51 @@ class AppConfig(object):
value.update(**kw)
self.is_complete = False
+ def update_single_config_from_path_and_value(self, path, value):
+ """Update a single config parameter with the value.
+ Path is a list of string, that gives a path to the config parameter to be updated.
+ For example, path may be ["server","app","port"].
+ """
+ self.is_complete = False
+ if not isinstance(path, list):
+ raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+ for part in path:
+ if not isinstance(part, str):
+ raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+
+ if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
+ raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
+
+ if path[0] == "server":
+ attr = "__".join(path[1:])
+ try:
+ self.update_server_config(**{attr: value})
+ except ConfigurationError:
+ raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+ elif path[0] == "dataset":
+ attr = "__".join(path[1:])
+ try:
+ self.update_default_dataset_config(**{attr: value})
+ except ConfigurationError:
+ raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+ elif path[0] == "per_dataset_config":
+ if len(path) < 2:
+ raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
+ dataroot = path[1]
+ if dataroot not in self.dataroot_config:
+ dataroots = str(list(self.dataroot_config.keys()))
+ raise ConfigurationError(
+ f"unknown dataroot when using per_dataset_config: got '{path}',"
+ f" dataroots specified in config are {dataroots}"
+ )
+
+ attr = "__".join(path[2:])
+ try:
+ self.dataroot_config[dataroot].update(**{attr: value})
+ except ConfigurationError:
+ raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
def update_from_config_file(self, config_file):
try:
with open(config_file) as yml_file:
@@ -94,12 +144,16 @@ class AppConfig(object):
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
+ if config.get("external"):
+ self.external_config.update_from_config(config["external"], "external")
+
self.is_complete = False
- def write_config(self, config_file):
- """output the config to a yaml file"""
+ def config_to_dict(self):
+ """return the configuration as an unflattened dict"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+ external = self.external_config.create_mapping(self.external_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
@@ -111,15 +165,23 @@ class AppConfig(object):
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+ for attrname in external.keys():
+ config["external__" + attrname] = getattr(self.external_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
+ return config
+
+ def write_config(self, config_file):
+ """output the config to a yaml file"""
+ config = self.config_to_dict()
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
- diff = dict(server=diff_server, dataset=diff_dataset)
+ diff_external = self.external.changes_from_default()
+ diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
@@ -154,6 +216,8 @@ class AppConfig(object):
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
+ # complete config for external_config first, since this may update values in the other sections
+ self.external_config.complete_config(context)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
diff --git a/server/common/config/base_config.py b/server/common/config/base_config.py
index a0b46f1b..6b9087e7 100644
--- a/server/common/config/base_config.py
+++ b/server/common/config/base_config.py
@@ -80,8 +80,27 @@ class BaseConfig(object):
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
+ """Update the attributes defined in kw with their new values."""
for key, value in kw.items():
if not hasattr(self, key):
+
+ # check if the key is setting into a dictval entry.
+ found_dictval = False
+ for dictval in self.dictval_cases:
+ dictvalname = "__".join(dictval)
+ if dictvalname + "__" in key:
+ dictkey = key[len(dictvalname) + 2 :]
+ curdictval = getattr(self, dictvalname)
+ if curdictval is None:
+ setattr(self, dictvalname, dict(dictkey=value))
+ else:
+ curdictval[dictkey] = value
+
+ found_dictval = True
+ break
+
+ if found_dictval:
+ continue
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
diff --git a/server/common/config/dataset_config.py b/server/common/config/dataset_config.py
index 8c8231ac..7586ec2f 100644
--- a/server/common/config/dataset_config.py
+++ b/server/common/config/dataset_config.py
@@ -30,22 +30,18 @@ class DatasetConfig(BaseConfig):
self.user_annotations__type = default_config["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
"directory"
- ] # noqa E501
+ ]
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
"obo_location"
- ] # noqa E501
+ ]
self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
"hosted_tiledb_array"
- ][
- "db_uri"
- ] # noqa E501
+ ]["db_uri"]
self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
"hosted_tiledb_array"
- ][
- "hosted_file_directory"
- ] # noqa E501
+ ]["hosted_file_directory"]
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
@@ -98,20 +94,20 @@ class DatasetConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__directory", (type(None), str)
- ) # noqa E501
+ )
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__file", (type(None), str)
- ) # noqa E501
+ )
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__ontology__obo_location", (type(None), str)
- ) # noqa E501
+ )
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
- ) # noqa E501
+ )
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
- ) # noqa E501
+ )
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
@@ -166,7 +162,7 @@ class DatasetConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", str
- ) # noqa E501
+ )
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
diff --git a/server/common/config/external_config.py b/server/common/config/external_config.py
new file mode 100644
index 00000000..bebfbfa9
--- /dev/null
+++ b/server/common/config/external_config.py
@@ -0,0 +1,96 @@
+import os
+
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError
+from server.common.config import get_secret_key
+from server.common.errors import SecretKeyRetrievalError
+from server.common.utils.type_conversion_utils import convert_string_to_value
+
+
+class ExternalConfig(BaseConfig):
+ """Manages the config attribute associated with external configuration sources, such as
+ environment variables or the AWS Secrets Manager."""
+
+ def __init__(self, app_config, default_config):
+ super().__init__(app_config, default_config)
+ try:
+ self.environment = default_config["environment"]
+ self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
+ self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
+
+ except KeyError as e:
+ raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+ def complete_config(self, context):
+ self.handle_environment(context)
+ self.handle_aws_secrets_manager(context)
+
+ def handle_environment(self, context):
+ """For each environment variable defined, get the value (if it is set),
+ and set the specified config parameter"""
+ self.validate_correct_type_of_configuration_attribute("environment", list)
+ for envdict in self.environment:
+ name = envdict.get("name")
+ if name is None:
+ raise ConfigurationError("environment: 'name' is missing")
+ required = envdict.get("required", False)
+ if type(required) != bool:
+ raise ConfigurationError("environment: 'required' must be a bool")
+ path = envdict.get("path")
+ if path is None:
+ raise ConfigurationError("environment: 'path' is missing")
+
+ value = os.environ.get(name)
+ if value is None:
+ if required:
+ raise ConfigurationError(f"required environment variable '{name}' not set")
+ else:
+ value = convert_string_to_value(value)
+ self.app_config.update_single_config_from_path_and_value(path, value)
+
+ def handle_aws_secrets_manager(self, context):
+ """For each aws secret defined, get the key/values, and set the specified config parameter"""
+ self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
+ self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
+
+ if not self.aws_secrets_manager__secrets:
+ return
+
+ self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
+
+ for secret in self.aws_secrets_manager__secrets:
+ secret_name = secret.get("name")
+ if secret_name is None:
+ raise ConfigurationError("aws_secrets_manager: 'name' is missing")
+ if not isinstance(secret_name, str):
+ raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
+
+ try:
+ secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
+ except SecretKeyRetrievalError as e:
+ raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
+
+ values = secret.get("values")
+ if values is None:
+ raise ConfigurationError("aws_secrets_manager: 'values' is missing")
+ if not isinstance(values, list):
+ raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
+
+ for value in values:
+ key = value.get("key")
+ if key is None:
+ raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
+ path = value.get("path")
+ if path is None:
+ raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
+ required = value.get("required", False)
+ if type(required) != bool:
+ raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
+
+ secret_value = secret_dict.get(key)
+ if secret_value is None:
+ if required:
+ raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
+ else:
+ secret_value = convert_string_to_value(secret_value)
+ self.app_config.update_single_config_from_path_and_value(path, secret_value)
diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py
index 508b3d4f..6c4b5570 100644
--- a/server/common/config/server_config.py
+++ b/server/common/config/server_config.py
@@ -44,17 +44,17 @@ class ServerConfig(BaseConfig):
self.authentication__type = default_config["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
"oauth_api_base_url"
- ] # noqa E501
+ ]
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
"client_secret"
- ] # noqa E501
+ ]
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
"jwt_decode_options"
- ] # noqa E501
+ ]
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
"session_cookie"
- ] # noqa E501
+ ]
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
@@ -62,10 +62,10 @@ class ServerConfig(BaseConfig):
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
"max_datasets"
- ] # noqa E501
+ ]
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
"timelimit_s"
- ] # noqa E501
+ ]
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
@@ -151,11 +151,6 @@ class ServerConfig(BaseConfig):
if not self.app__verbose:
sys.tracebacklimit = 0
- # secret key:
- # first, from CXG_SECRET_KEY environment variable
- # second, from config file
- self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
-
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
@@ -178,25 +173,20 @@ class ServerConfig(BaseConfig):
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__oauth_api_base_url", ptypes
- ) # noqa E501
+ )
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
- ) # noqa E501
+ )
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__cookie", (type(None), dict)
- ) # noqa E501
+ )
else:
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
- # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
- # second, from config file
- self.authentication__params_oauth__client_secret = os.environ.get(
- "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
- )
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
@@ -286,7 +276,7 @@ class ServerConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
self.validate_correct_type_of_configuration_attribute(
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
- ) # noqa E501
+ )
if self.multi_dataset__dataroot is None:
return
diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py
index 4bc6bf88..ccda1177 100644
--- a/server/common/utils/type_conversion_utils.py
+++ b/server/common/utils/type_conversion_utils.py
@@ -151,3 +151,17 @@ def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
return series_to_convert.to_numpy(dtype)
+
+
+def convert_string_to_value(value: str):
+ """convert a string to value with the most appropriate type"""
+ if value.lower() == "true":
+ return True
+ if value.lower() == "false":
+ return False
+ if value == "null":
+ return None
+ try:
+ return eval(value)
+ except: # noqa E722
+ return value
diff --git a/server/default_config.py b/server/default_config.py
index 88e96fe5..20922ef9 100644
--- a/server/default_config.py
+++ b/server/default_config.py
@@ -204,6 +204,61 @@ dataset:
enable: true
lfc_cutoff: 0.01
top_n: 10
+
+external:
+ # You can retrieve configuration parameters from this config file, the environment,
+ # the AWS secrets manager, or from the "cellxgene launch" command line arguments.
+ # They are applied in that order, meaning that if a parameter is defined in more
+ # than one location, the last one applied takes effect.
+
+ # environment variables:
+ # This section describes how to map environment variables to configuration parameters.
+ # The format is a list defining an environment variable.
+ # Each entry in the list is a dictionary with three entries:
+ # name: the name of the environment variable
+ # path: the path within the cellxgene configuration to update.
+ # required: (default=False) a boolean. If true, then it is an error if the environment variable is not set.
+
+ environment:
+ - name: CXG_SECRET_KEY
+ path: [server, app, flask_secret_key]
+ required: false
+ - name: CXG_OAUTH_CLIENT_SECRET
+ path: [server, authentication, params_oauth, client_secret]
+ required: false
+
+ # AWS Secrets Manager
+ # This section describes how to map aws secrets to configuration parameters.
+ # The format is the region for the secrets manager, then a list of secrets.
+ # each secret has a name, and a list of values.
+ # Each entry in the list of values is a dictionary with three entries:
+ # key: the key of the aws secret.
+ # path: the path within the cellxgene configuration to update.
+ # required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret.
+ #
+ # example:
+ # aws_secrets_manager:
+ # region: us-west-2
+ # - name: my_first_secret
+ # values:
+ # - key: flask_secret_key
+ # path: [server, app, flask_secret_key]
+ # required: true
+ # - key: db_uri
+ # path: [dataset, user_annotations, hosted_tiledb_array, db_uri]
+ # required: true
+ # - name: my_auth_secret
+ # values:
+ # - key: client_secret
+ # path: [server, authentication, params_oauth, client_secret]
+ # required: true
+ # - key: client_id
+ # path: [server, authentication, params_oauth, client_id]
+ # required: true
+
+ aws_secrets_manager:
+ region: null
+ secrets: []
"""
diff --git a/server/test/__init__.py b/server/test/__init__.py
index bbb1de0b..586aa798 100644
--- a/server/test/__init__.py
+++ b/server/test/__init__.py
@@ -117,7 +117,7 @@ def random_string(n):
return "".join(random.choice(string.ascii_letters) for _ in range(n))
-def start_test_server(command_line_args=[], app_config=None):
+def start_test_server(command_line_args=[], app_config=None, env=None):
"""
Command line arguments can be passed in, as well as an app_config.
This function is meant to be used like this, for example:
@@ -155,7 +155,7 @@ def start_test_server(command_line_args=[], app_config=None):
command.extend(["-c", config_file])
server = f"http://localhost:{port}"
- ps = Popen(command)
+ ps = Popen(command, env=env)
for _ in range(10):
try:
@@ -178,10 +178,10 @@ def stop_test_server(ps):
@contextmanager
-def test_server(command_line_args=[], app_config=None):
+def test_server(command_line_args=[], app_config=None, env=None):
"""A context to run the cellxgene server."""
- ps, server = start_test_server(command_line_args, app_config)
+ ps, server = start_test_server(command_line_args, app_config, env)
try:
yield server
finally:
diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py
index fa730df2..7c06b311 100644
--- a/server/test/unit/common/config/__init__.py
+++ b/server/test/unit/common/config/__init__.py
@@ -3,6 +3,7 @@ import shutil
import unittest
import random
from unittest import mock
+import yaml
from server.test import FIXTURES_ROOT
@@ -133,6 +134,9 @@ class ConfigTests(unittest.TestCase):
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
+ environment=None,
+ aws_secrets_manager_region=None,
+ aws_secrets_manager_secrets=[],
config_file_name="app_config.yml",
):
random_num = random.randrange(999999)
@@ -201,13 +205,17 @@ class ConfigTests(unittest.TestCase):
top_n=top_n,
config_file_name=f"temp_dataset_config_{random_num}.yml",
)
- with open(server_config) as server_config:
- with open(dataset_config) as dataset_config:
- with open(configfile, "w") as app_config_file:
- for line in server_config:
- app_config_file.write(line)
- for line in dataset_config:
- app_config_file.write(line)
+ external_config = self.custom_external_config(
+ environment=environment,
+ aws_secrets_manager_region=aws_secrets_manager_region,
+ aws_secrets_manager_secrets=aws_secrets_manager_secrets,
+ config_file_name=f"temp_external_config_{random_num}.yml",
+ )
+
+ with open(configfile, "w") as app_config_file:
+ app_config_file.write(open(server_config).read())
+ app_config_file.write(open(dataset_config).read())
+ app_config_file.write(open(external_config).read())
return configfile
@@ -244,3 +252,32 @@ class ConfigTests(unittest.TestCase):
dataset_config_file.write(dataset_config)
return configfile
+
+ def custom_external_config(
+ self,
+ environment=None,
+ aws_secrets_manager_region=None,
+ aws_secrets_manager_secrets=[],
+ config_file_name="external_config.yaml",
+ ):
+ # set to the default if environment is None
+ if environment is None:
+ environment = [
+ dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
+ dict(
+ name="CXG_OAUTH_CLIENT_SECRET",
+ path=["server", "authentication", "params_oauth", "client_secret"],
+ required=False,
+ ),
+ ]
+ external_config = {
+ "external": {
+ "environment": environment,
+ "aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
+ }
+ }
+
+ configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+ with open(configfile, "w") as external_config_file:
+ yaml.dump(external_config, external_config_file)
+ return configfile
diff --git a/server/test/unit/common/config/test_app_config.py b/server/test/unit/common/config/test_app_config.py
index 2dc3273d..362ab330 100644
--- a/server/test/unit/common/config/test_app_config.py
+++ b/server/test/unit/common/config/test_app_config.py
@@ -138,3 +138,77 @@ class AppConfigTest(ConfigTests):
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
+
+ def test_simple_update_single_config_from_path_and_value(self):
+ """Update a simple config parameter"""
+
+ config = AppConfig()
+ config.server_config.multi_dataset__dataroot = dict(
+ s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
+ s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
+ )
+ config.add_dataroot_config("s1")
+ config.add_dataroot_config("s2")
+
+ # test simple value in server
+ config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
+ self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
+
+ # test simple value in default dataset
+ config.update_single_config_from_path_and_value(
+ ["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi",
+ )
+ self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+ self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+ self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+
+ # test simple value in specific dataset
+ config.update_single_config_from_path_and_value(
+ ["per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri"], "s1dburi"
+ )
+ self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+ self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
+ self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+
+ # error checking
+ bad_paths = [
+ (
+ ["dataset", "does", "not", "exist"],
+ "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
+ ),
+ (["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
+ ([], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
+ (["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'"),
+ (
+ ["per_dataset_config", "unknown"],
+ "unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
+ " dataroots specified in config are ['s1', 's2']",
+ ),
+ ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
+ ("string", "path must be a list of strings, got 'string'"),
+ ]
+ for bad_path, error_message in bad_paths:
+ with self.assertRaises(ConfigurationError) as config_error:
+ config.update_single_config_from_path_and_value(bad_path, "value")
+
+ self.assertEqual(config_error.exception.message, error_message)
+
+ def test_dict_update_single_config_from_path_and_value(self):
+ """Update a config parameter that has a value of dict"""
+
+ # the path leads to a dict config param, set the config parameter to the new value
+ config = AppConfig()
+ config.update_single_config_from_path_and_value(
+ ["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100)
+ )
+ self.assertEqual(config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100))
+
+ # the path leads to an entry within a dict config param, the value is simple
+ config = AppConfig()
+ config.server_config.authentication__params_oauth__cookie = dict(key="mykey1", max_age=100)
+ config.update_single_config_from_path_and_value(
+ ["server", "authentication", "params_oauth", "cookie", "httponly"], True,
+ )
+ self.assertEqual(
+ config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True)
+ )
diff --git a/server/test/unit/common/config/test_dataset_config.py b/server/test/unit/common/config/test_dataset_config.py
index a32d1f87..5ad66e44 100644
--- a/server/test/unit/common/config/test_dataset_config.py
+++ b/server/test/unit/common/config/test_dataset_config.py
@@ -155,7 +155,7 @@ class TestDatasetConfig(ConfigTests):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
config.complete_config()
@@ -163,7 +163,7 @@ class TestDatasetConfig(ConfigTests):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
config.complete_config()
diff --git a/server/test/unit/common/config/test_external_config.py b/server/test/unit/common/config/test_external_config.py
new file mode 100644
index 00000000..5e3825e6
--- /dev/null
+++ b/server/test/unit/common/config/test_external_config.py
@@ -0,0 +1,231 @@
+import os
+from unittest.mock import patch
+
+import requests
+
+from server.common.errors import ConfigurationError
+from server.common.config.app_config import AppConfig
+from server.test import test_server, FIXTURES_ROOT
+from server.common.utils.type_conversion_utils import convert_string_to_value
+from server.test.unit.common.config import ConfigTests
+
+
+class TestExternalConfig(ConfigTests):
+ def test_type_convert(self):
+ # The values from environment variables and aws secrets are returned as strings.
+ # These values need to be converted to the proper types.
+
+ self.assertEqual(convert_string_to_value("1"), int(1))
+ self.assertEqual(convert_string_to_value("1.1"), float(1.1))
+ self.assertEqual(convert_string_to_value("string"), "string")
+ self.assertEqual(convert_string_to_value("true"), True)
+ self.assertEqual(convert_string_to_value("True"), True)
+ self.assertEqual(convert_string_to_value("false"), False)
+ self.assertEqual(convert_string_to_value("False"), False)
+ self.assertEqual(convert_string_to_value("null"), None)
+ self.assertEqual(convert_string_to_value("None"), None)
+ self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
+
+ def test_environment_variable(self):
+ configfile = self.custom_external_config(
+ environment=[
+ dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
+ dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
+ ],
+ config_file_name="environment_external_config.yaml",
+ )
+
+ env = os.environ
+ env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+ env["DIFFEXP"] = "False"
+ with test_server(command_line_args=["-c", configfile], env=env) as server:
+ session = requests.Session()
+ response = session.get(f"{server}/api/v0.2/config")
+ data_config = response.json()
+ self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+ self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
+
+ env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
+ env["DIFFEXP"] = "True"
+ with test_server(command_line_args=["-c", configfile], env=env) as server:
+ session = requests.Session()
+ response = session.get(f"{server}/api/v0.2/config")
+ data_config = response.json()
+ self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
+ self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
+
+ def test_environment_variable_errors(self):
+
+ # no name
+ app_config = AppConfig()
+ app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
+
+ # required has wrong type
+ app_config = AppConfig()
+ app_config.external_config.environment = [
+ dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
+
+ # no path
+ app_config = AppConfig()
+ app_config.external_config.environment = [dict(name="myenvar", required=True)]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
+
+ # required environment variable is not set
+ app_config = AppConfig()
+ app_config.external_config.environment = [
+ dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
+
+ @patch("server.common.config.external_config.get_secret_key")
+ def test_aws_secrets_manager(self, mock_get_secret_key):
+ mock_get_secret_key.return_value = {
+ "oauth_client_secret": "mock_oauth_secret",
+ "db_uri": "mock_db_uri",
+ }
+ configfile = self.custom_external_config(
+ aws_secrets_manager_region="us-west-2",
+ aws_secrets_manager_secrets=[
+ dict(
+ name="my_secret",
+ values=[
+ dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False),
+ dict(
+ key="db_uri",
+ path=["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
+ required=True,
+ ),
+ dict(
+ key="oauth_client_secret",
+ path=["server", "authentication", "params_oauth", "client_secret"],
+ required=True,
+ ),
+ ],
+ )
+ ],
+ config_file_name="secret_external_config.yaml",
+ )
+
+ app_config = AppConfig()
+ app_config.update_from_config_file(configfile)
+ app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+ app_config.server_config.app__flask_secret_key = "original"
+ app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+
+ app_config.complete_config()
+
+ self.assertEqual(app_config.server_config.app__flask_secret_key, "original")
+ self.assertEqual(app_config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
+ self.assertEqual(app_config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
+
+ @patch("server.common.config.external_config.get_secret_key")
+ def test_aws_secrets_manager_error(self, mock_get_secret_key):
+ mock_get_secret_key.return_value = {
+ "oauth_client_secret": "mock_oauth_secret",
+ "db_uri": "mock_db_uri",
+ }
+
+ # no region
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = None
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(
+ config_error.exception.message,
+ "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
+ )
+
+ # missing secret name
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
+
+ # secret name wrong type
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
+
+ # missing values name
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
+
+ # values wrong type
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
+
+ # entry missing key
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
+
+ # entry required is wrong type
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
+
+ # entry missing path
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(name="mysecret", values=[dict(key="db_uri", required=True)])
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
+
+ # secret missing required key
+ app_config = AppConfig()
+ app_config.external_config.aws_secrets_manager__region = "us-west-2"
+ app_config.external_config.aws_secrets_manager__secrets = [
+ dict(
+ name="mysecret",
+ values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
+ )
+ ]
+ with self.assertRaises(ConfigurationError) as config_error:
+ app_config.complete_config()
+ self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py
index 78ee9570..65974523 100644
--- a/server/test/unit/common/config/test_server_config.py
+++ b/server/test/unit/common/config/test_server_config.py
@@ -105,13 +105,14 @@ class TestServerConfig(ConfigTests):
self.config = AppConfig()
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
+ del os.environ["CXG_SERVER_PORT"]
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
- config.server_config.handle_app(self.context)
+ config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
def test_handle_app__sets_web_base_url(self):
@@ -124,7 +125,7 @@ class TestServerConfig(ConfigTests):
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
- config.server_config.handle_authentication()
+ config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
@@ -215,7 +216,7 @@ class TestServerConfig(ConfigTests):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
self.config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
self.config.complete_config()
@@ -224,7 +225,7 @@ class TestServerConfig(ConfigTests):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
self.config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+ multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
self.config.complete_config()
From b5ec43c4b124e4b64449d41ea0f7cc6bd404196f Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Thu, 8 Oct 2020 08:44:09 -0700
Subject: [PATCH 28/73] Add a function to check the configuration for errors.
(#1919)
This can be used as a sanity check before a deployment:
chanzuckerberg/single-cell#63
---
server/common/config/server_config.py | 2 +-
server/eb/check_config.py | 39 ++++++++++++
server/test/__init__.py | 8 ++-
server/test/unit/auth/test_auth.py | 60 ++++++++++---------
server/test/unit/auth/test_oauth.py | 2 +
server/test/unit/common/config/__init__.py | 4 +-
.../unit/common/config/test_app_config.py | 8 ++-
.../unit/common/config/test_base_config.py | 2 +
.../unit/common/config/test_dataset_config.py | 10 +++-
.../unit/common/config/test_server_config.py | 3 +
server/test/unit/eb/test_eb.py | 39 ++++++++++--
11 files changed, 136 insertions(+), 41 deletions(-)
create mode 100644 server/eb/check_config.py
diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py
index 6c4b5570..14eac613 100644
--- a/server/common/config/server_config.py
+++ b/server/common/config/server_config.py
@@ -114,7 +114,7 @@ class ServerConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
- self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str))
+ self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
diff --git a/server/eb/check_config.py b/server/eb/check_config.py
new file mode 100644
index 00000000..6886e101
--- /dev/null
+++ b/server/eb/check_config.py
@@ -0,0 +1,39 @@
+import sys
+import argparse
+import yaml
+
+from server.common.config.app_config import AppConfig
+
+
+def main():
+ parser = argparse.ArgumentParser("A script to check hosted configuration files")
+ parser.add_argument("config_file", help="the configuration file")
+ parser.add_argument(
+ "-s",
+ "--show",
+ default=False,
+ action="store_true",
+ help="print the configuration. NOTE: this may print secret values to stdout",
+ )
+
+ args = parser.parse_args()
+
+ app_config = AppConfig()
+ app_config.update_from_config_file(args.config_file)
+ try:
+ app_config.complete_config()
+ except Exception as e:
+ print(f"Error: {str(e)}")
+ print("FAIL:", args.config_file)
+ sys.exit(1)
+
+ if args.show:
+ yaml_config = app_config.config_to_dict()
+ yaml.dump(yaml_config, sys.stdout)
+
+ print("PASS:", args.config_file)
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/server/test/__init__.py b/server/test/__init__.py
index 586aa798..1b96f7c1 100644
--- a/server/test/__init__.py
+++ b/server/test/__init__.py
@@ -34,7 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
- multi_dataset__dataroot=data_locator.path, authentication__type="test",
+ app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test",
)
config.update_default_dataset_config(
embeddings__names=["umap"],
@@ -64,7 +64,10 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
- single_dataset__obs_names=None, single_dataset__var_names=None, single_dataset__datapath=data_locator.path
+ app__flask_secret_key="secret",
+ single_dataset__obs_names=None,
+ single_dataset__var_names=None,
+ single_dataset__datapath=data_locator.path,
)
config.update_default_dataset_config(
embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
@@ -97,6 +100,7 @@ def skip_if(condition, reason: str):
def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
config = AppConfig()
config.update_server_config(
+ app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
adaptor__anndata_adaptor__backed=backed,
diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py
index 0b8c0bf1..2caba6b1 100644
--- a/server/test/unit/auth/test_auth.py
+++ b/server/test/unit/auth/test_auth.py
@@ -11,13 +11,14 @@ class AuthTest(unittest.TestCase):
self.dataset_dataroot = FIXTURES_ROOT
def test_auth_none(self):
- c = AppConfig()
- c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
- c.update_default_dataset_config(user_annotations__enable=False)
+ app_config = AppConfig()
+ app_config.update_server_config(app__flask_secret_key="secret")
+ app_config.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
+ app_config.update_default_dataset_config(user_annotations__enable=False)
- c.complete_config()
+ app_config.complete_config()
- with test_server(app_config=c) as server:
+ with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
@@ -25,12 +26,13 @@ class AuthTest(unittest.TestCase):
self.assertIsNone(userinfo)
def test_auth_session(self):
- c = AppConfig()
- c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
- c.update_default_dataset_config(user_annotations__enable=True)
- c.complete_config()
+ app_config = AppConfig()
+ app_config.update_server_config(app__flask_secret_key="secret")
+ app_config.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
+ app_config.update_default_dataset_config(user_annotations__enable=True)
+ app_config.complete_config()
- with test_server(app_config=c) as server:
+ with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
@@ -40,9 +42,10 @@ class AuthTest(unittest.TestCase):
self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def test_auth_test(self):
- c = AppConfig()
- c.update_server_config(authentication__type="test")
- c.update_server_config(
+ app_config = AppConfig()
+ app_config.update_server_config(app__flask_secret_key="secret")
+ app_config.update_server_config(authentication__type="test")
+ app_config.update_server_config(
multi_dataset__dataroot=dict(
a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
@@ -50,12 +53,12 @@ class AuthTest(unittest.TestCase):
)
# specialize the configs
- c.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
- c.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
+ app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
+ app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
- c.complete_config()
+ app_config.complete_config()
- with test_server(app_config=c) as server:
+ with test_server(app_config=app_config) as server:
session = requests.Session()
# auth datasets
@@ -102,20 +105,21 @@ class AuthTest(unittest.TestCase):
self.assertFalse(config["config"]["parameters"]["annotations"])
# login with a picture
- r = session.get(f"{server}/{login_uri}&picture=myimage.png")
+ session.get(f"{server}/{login_uri}&picture=myimage.png")
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
def test_auth_test_single(self):
- c = AppConfig()
- c.update_server_config(
+ app_config = AppConfig()
+ app_config.update_server_config(app__flask_secret_key="secret")
+ app_config.update_server_config(
authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
)
- c.complete_config()
+ app_config.complete_config()
- with test_server(app_config=c) as server:
+ with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
@@ -130,10 +134,10 @@ class AuthTest(unittest.TestCase):
self.assertEqual(login_uri, "/login")
self.assertEqual(logout_uri, "/logout")
- r = session.get(f"{server}/{login_uri}")
+ response = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
- self.assertEqual(r.history[0].status_code, 302)
- self.assertEqual(r.url, f"{server}/")
+ self.assertEqual(response.history[0].status_code, 302)
+ self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
@@ -141,10 +145,10 @@ class AuthTest(unittest.TestCase):
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertTrue(config["config"]["parameters"]["annotations"])
- r = session.get(f"{server}/{logout_uri}")
+ response = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
- self.assertEqual(r.history[0].status_code, 302)
- self.assertEqual(r.url, f"{server}/")
+ self.assertEqual(response.history[0].status_code, 302)
+ self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index 8b250ed6..f5a85cb5 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -160,12 +160,14 @@ class AuthTest(unittest.TestCase):
def test_auth_oauth_session(self):
# test with session cookies
app_config = AppConfig()
+ app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
self.auth_flow(app_config)
def test_auth_oauth_cookie(self):
# test with specified cookie
app_config = AppConfig()
+ app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__params_oauth__session_cookie=False,
authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60),
diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py
index 7c06b311..8a48be5e 100644
--- a/server/test/unit/common/config/__init__.py
+++ b/server/test/unit/common/config/__init__.py
@@ -31,7 +31,7 @@ class ConfigTests(unittest.TestCase):
port="null",
open_browser="false",
force_https="false",
- flask_secret_key="null",
+ flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
@@ -82,7 +82,7 @@ class ConfigTests(unittest.TestCase):
port="null",
open_browser="false",
force_https="false",
- flask_secret_key="null",
+ flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
diff --git a/server/test/unit/common/config/test_app_config.py b/server/test/unit/common/config/test_app_config.py
index 362ab330..ea8fe047 100644
--- a/server/test/unit/common/config/test_app_config.py
+++ b/server/test/unit/common/config/test_app_config.py
@@ -15,6 +15,7 @@ class AppConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
+ self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
@@ -106,6 +107,8 @@ class AppConfigTest(ConfigTests):
with open(configfile, "w") as fconfig:
config = """
server:
+ app:
+ flask_secret_key: secret
multi_dataset:
dataroot: test_dataroot
@@ -116,7 +119,10 @@ class AppConfigTest(ConfigTests):
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
- self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
+ self.assertEqual(
+ server_changes,
+ [("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)],
+ )
self.assertEqual(dataset_changes, [])
def test_configfile_no_server_section(self):
diff --git a/server/test/unit/common/config/test_base_config.py b/server/test/unit/common/config/test_base_config.py
index d41082cf..d5d538cd 100644
--- a/server/test/unit/common/config/test_base_config.py
+++ b/server/test/unit/common/config/test_base_config.py
@@ -10,6 +10,7 @@ class BaseConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
+ self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
@@ -47,6 +48,7 @@ class BaseConfigTest(ConfigTests):
server_changes,
[
("app__verbose", True, False),
+ ("app__flask_secret_key", "secret", None),
("multi_dataset__dataroot", FIXTURES_ROOT, None),
("multi_dataset__matrix_cache__timelimit_s", 5, 30),
("data_locator__s3__region_name", "us-east-1", True),
diff --git a/server/test/unit/common/config/test_dataset_config.py b/server/test/unit/common/config/test_dataset_config.py
index 5ad66e44..b65c7d83 100644
--- a/server/test/unit/common/config/test_dataset_config.py
+++ b/server/test/unit/common/config/test_dataset_config.py
@@ -19,6 +19,7 @@ class TestDatasetConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
+ self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.dataset_config = self.config.default_dataset_config
self.config.complete_config()
@@ -155,7 +156,8 @@ class TestDatasetConfig(ConfigTests):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
+ app__flask_secret_key="secret",
+ multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
with self.assertRaises(ConfigurationError):
config.complete_config()
@@ -163,17 +165,19 @@ class TestDatasetConfig(ConfigTests):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
+ app__flask_secret_key="secret",
+ multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
+ app__flask_secret_key="secret",
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
- )
+ ),
)
# Change this default to test if the dataroot overrides below work.
diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py
index 65974523..f862a40f 100644
--- a/server/test/unit/common/config/test_server_config.py
+++ b/server/test/unit/common/config/test_server_config.py
@@ -23,6 +23,7 @@ class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
+ self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
@@ -103,6 +104,7 @@ class TestServerConfig(ConfigTests):
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
+ self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
@@ -152,6 +154,7 @@ class TestServerConfig(ConfigTests):
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
+ app__flask_secret_key="secret",
app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
)
diff --git a/server/test/unit/eb/test_eb.py b/server/test/unit/eb/test_eb.py
index c148f6bc..61cf1867 100644
--- a/server/test/unit/eb/test_eb.py
+++ b/server/test/unit/eb/test_eb.py
@@ -6,6 +6,7 @@ from server.test import PROJECT_ROOT, FIXTURES_ROOT
from server.common.config.app_config import AppConfig
from contextlib import contextmanager
import time
+import os
@contextmanager
@@ -34,12 +35,12 @@ class Elastic_Beanstalk_Test(unittest.TestCase):
tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/server")
tempdirname = tempdir.name
- c = AppConfig()
+ config = AppConfig()
# test that eb works
- c.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
+ config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
- c.complete_config()
- c.write_config(f"{tempdirname}/config.yaml")
+ config.complete_config()
+ config.write_config(f"{tempdirname}/config.yaml")
subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/server/eb", shell=True)
subprocess.check_call(["make", "build"], cwd=tempdirname)
@@ -50,3 +51,33 @@ class Elastic_Beanstalk_Test(unittest.TestCase):
r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
data_config = r.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+
+ def test_config(self):
+ check_config_script = os.path.join(PROJECT_ROOT, "server", "eb", "check_config.py")
+ with tempfile.TemporaryDirectory() as tempdir:
+ configfile = os.path.join(tempdir, "config.yaml")
+ app_config = AppConfig()
+ app_config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}")
+ app_config.write_config(configfile)
+
+ command = ["python", check_config_script, configfile]
+
+ # test failure mode (flask_secret_key not set)
+ env = os.environ.copy()
+ env.pop("CXG_SECRET_KEY", None)
+ with self.assertRaises(subprocess.CalledProcessError) as exception_context:
+ subprocess.check_output(command, env=env)
+ output = str(exception_context.exception.stdout, "utf-8")
+ self.assertTrue(
+ output.startswith(
+ "Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType"
+ )
+ )
+ self.assertEqual(exception_context.exception.returncode, 1)
+
+ # test passing case
+ env = os.environ.copy()
+ env["CXG_SECRET_KEY"] = "secret"
+ output = subprocess.check_output(command, env=env)
+ output = str(output, "utf-8")
+ self.assertTrue(output.startswith("PASS"))
From c4c48b9a5742e4fa07623002640200dc1eaeac43 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Thu, 8 Oct 2020 11:02:40 -0600
Subject: [PATCH 29/73] create e2e test for auth buttons (#1907)
This PR adds a few helpful additions regarding authentication.
Changes:
* e2e tests are now run on test_oauth via a passed config.yaml
* node dev server correctly handles `/login` and `/logout` endpoints to make developing for auth easier
* Introduced auth e2e tests to check that buttons display and work
---
client/Makefile | 4 +-
client/__tests__/e2e/cellxgeneActions.js | 11 +++--
client/__tests__/e2e/e2e.test.js | 12 +++++
client/__tests__/e2e/test_config.yaml | 47 ++++++++++++++++++++
client/server/development.js | 30 +++++++++----
client/src/components/menubar/authButtons.js | 6 ++-
6 files changed, 94 insertions(+), 16 deletions(-)
create mode 100644 client/__tests__/e2e/test_config.yaml
diff --git a/client/Makefile b/client/Makefile
index 3b36337a..eb5c1a78 100644
--- a/client/Makefile
+++ b/client/Makefile
@@ -3,6 +3,8 @@ include ../common.mk
ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
+
# Packaging
.PHONY: clean
clean:
@@ -31,7 +33,7 @@ start-frontend:
.PHONY: smoke-test
smoke-test:
start_server_and_test \
- 'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+ 'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
$(CXG_SERVER_PORT) \
'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
diff --git a/client/__tests__/e2e/cellxgeneActions.js b/client/__tests__/e2e/cellxgeneActions.js
index 9fda69b1..7ffcc8cf 100644
--- a/client/__tests__/e2e/cellxgeneActions.js
+++ b/client/__tests__/e2e/cellxgeneActions.js
@@ -322,7 +322,7 @@ export async function login() {
await goToPage(appUrlBase);
- await clickOn("auth-button");
+ await clickOn("log-in");
// (thuang): Auth0 form is unstable and unsafe for input until verified
await waitUntilFormFieldStable('[name="email"]');
@@ -341,16 +341,15 @@ export async function login() {
}
export async function logout() {
- await clickOnUntil("menu", async () => {
- await expect(page).toMatch("Log Out");
-
+ await clickOnUntil("user-info", async () => {
+ await waitByID("log-out");
await Promise.all([
page.waitForNavigation({ waitUntil: "networkidle0" }),
- expect(page).toClick("a", { text: "Log Out" }),
+ clickOn("log-out"),
]);
});
- await expect(page).toMatch("Log In");
+ await waitByID("log-in");
}
async function waitUntilFormFieldStable(selector) {
diff --git a/client/__tests__/e2e/e2e.test.js b/client/__tests__/e2e/e2e.test.js
index 05f52314..2bd8ea8c 100644
--- a/client/__tests__/e2e/e2e.test.js
+++ b/client/__tests__/e2e/e2e.test.js
@@ -17,6 +17,7 @@ import {
goToPage,
typeInto,
waitByID,
+ clickOnUntil,
} from "./puppeteerUtils";
import {
@@ -521,6 +522,17 @@ test("lasso moves after pan", async () => {
expect(panCount).toBe(initialCount);
});
+describe("auth buttons", () => {
+ test("login then logout", async () => {
+ await goToPage(appUrlBase);
+ await clickOnUntil("log-in", async () => {
+ await page.waitForNavigation({ waitUntil: "networkidle0" });
+ await waitByID("user-info");
+ });
+ await logout();
+ });
+});
+
const conditionalDescribe =
process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
diff --git a/client/__tests__/e2e/test_config.yaml b/client/__tests__/e2e/test_config.yaml
new file mode 100644
index 00000000..11576878
--- /dev/null
+++ b/client/__tests__/e2e/test_config.yaml
@@ -0,0 +1,47 @@
+server:
+ app:
+ force_https: true
+
+ # By default, cellxgene will serve api requests from the same base url as the webpage.
+ # In general api_base_url and web_base_url will not need to be set.
+ # There are two reasons to set these parameters:
+ # 1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
+ # which then redirects back to the web_base_url. If the web_base_url is not set, it will default to
+ # the api_base_url. If oauth authentication is used, the api_base_url must be set.
+ # For a local test (where the server runs on "http://localhost:"), then the api_base_url may be
+ # set to the string "local".
+ # 2. The cellxgene deploymnent is in an environment where the webpage and api have
+ # different base urls. In this case both api_base_url and web_base_url must be set.
+ # It is up to the server admin to ensure that the networking is setup correctly for this environment.
+ api_base_url: http://localhost:5005
+ web_base_url: http://localhost:3000
+
+ authentication:
+ # The authentication types may be "none", "session", "oauth"
+ # none: No authentication support, features like user_annotations must not be enabled.
+ # session: A session based userid is automatically generated. (no params needed)
+ # oauth: oauth2 is used for authentication; parameters are defined in params_oauth.
+ type: test
+
+dataset:
+ app:
+ about_legal_tos: null
+ about_legal_privacy: null
+
+ presentation:
+ max_categories: 1000
+ custom_colors: true
+
+ user_annotations:
+ enable: false
+ type: local_file_csv
+ local_file_csv:
+ directory: null
+ file: null
+ ontology:
+ enable: false
+ obo_location: null
+
+ embeddings:
+ names: []
+ enable_reembedding: false
diff --git a/client/server/development.js b/client/server/development.js
index 0b3e3811..1530856e 100644
--- a/client/server/development.js
+++ b/client/server/development.js
@@ -1,5 +1,3 @@
-const path = require("path");
-const historyApiFallback = require("connect-history-api-fallback");
const chalk = require("chalk");
const express = require("express");
const favicon = require("serve-favicon");
@@ -11,35 +9,51 @@ const utils = require("./utils");
process.env.NODE_ENV = "development";
const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
+const { CXG_SERVER_PORT } = process.env;
+
+const API = {
+ prefix: `http://localhost:${CXG_SERVER_PORT}/`,
+};
// Set up compiler
const compiler = webpack(config);
-compiler.plugin("invalid", () => {
+compiler.hooks.invalid.tap("invalid", () => {
utils.clearConsole();
console.log("Compiling...");
});
-compiler.plugin("done", (stats) => {
+compiler.hooks.done.tap("done", (stats) => {
utils.formatStats(stats, CLIENT_PORT);
});
// Launch server
const app = express();
-app.use(historyApiFallback({ verbose: false }));
-
app.use(
devMiddleware(compiler, {
logLevel: "warn",
publicPath: config.output.publicPath,
+ index: true,
})
);
app.use(favicon("./favicon.png"));
-app.get("*", (req, res) => {
- res.sendFile(path.resolve("index.html"));
+app.get("/login", async (req, res) => {
+ try {
+ res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
+ } catch (err) {
+ console.error(err);
+ }
+});
+
+app.get("/logout", async (req, res) => {
+ try {
+ res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
+ } catch (err) {
+ console.error(err);
+ }
});
app.listen(CLIENT_PORT, (err) => {
diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js
index 60ace82b..129125ec 100644
--- a/client/src/components/menubar/authButtons.js
+++ b/client/src/components/menubar/authButtons.js
@@ -51,7 +51,11 @@ const Auth = React.memo((props) => {
return (
-
+
{userinfo?.picture ? (
) : (
From 6677d0de568da3046babfb35bc3054cef756a487 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Thu, 8 Oct 2020 12:16:02 -0600
Subject: [PATCH 30/73] disable profile picture (#1923)
---
client/src/components/menubar/authButtons.js | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js
index 129125ec..a9ad8176 100644
--- a/client/src/components/menubar/authButtons.js
+++ b/client/src/components/menubar/authButtons.js
@@ -56,8 +56,9 @@ const Auth = React.memo((props) => {
className={styles.menubarButton}
style={{ padding: 0 }}
>
- {userinfo?.picture ? (
-
+ {/* eslint-disable-next-line no-constant-condition -- disable profile picture until CSP is tweaked */}
+ {userinfo?.picture && false ? (
+
) : (
{scientist}
)}
From c01a2c72b6334b21240df898a8a9543e78e74dad Mon Sep 17 00:00:00 2001
From: Timmy Huang
Date: Thu, 8 Oct 2020 16:57:53 -0700
Subject: [PATCH 31/73] thuang-1840-authn-prompt (#1911)
---
client/src/components/menubar/authButtons.js | 96 +++++++++++++++++++-
client/src/components/termsPrompt/index.js | 27 +-----
client/src/components/util/localStorage.js | 22 +++++
3 files changed, 117 insertions(+), 28 deletions(-)
create mode 100644 client/src/components/util/localStorage.js
diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js
index a9ad8176..91729679 100644
--- a/client/src/components/menubar/authButtons.js
+++ b/client/src/components/menubar/authButtons.js
@@ -1,4 +1,5 @@
-import React from "react";
+import React, { useState } from "react";
+
import {
AnchorButton,
Button,
@@ -6,20 +7,36 @@ import {
Tooltip,
Popover,
Menu,
+ Elevation,
+ PopoverPosition,
+ Checkbox,
+ Card,
} from "@blueprintjs/core";
+
import { IconNames } from "@blueprintjs/icons";
import * as globals from "../../globals";
+
import styles from "./menubar.css";
+import { storageGet, storageSet, KEYS } from "../util/localStorage";
+
const BASE_EMOJI = [0x1f9d1, 0x1f468, 0x1f469];
const SKIN_TONES = [0x1f3fb, 0x1f3fc, 0x1f3fd, 0x1f3fe, 0x1f3ff];
const MICROSCOPE = 0x1f52c;
const ZERO_WIDTH_JOINER = 0x0200d;
+const LOGIN_PROMPT_OFF = "off";
+
const Auth = React.memo((props) => {
+ const [isPromptOpen, setIsPromptOpen] = useState(shouldShowPrompt());
+
const { auth, userinfo } = props;
+ const isAuthenticated = userinfo && userinfo.is_authenticated;
+
+ window.userinfo = userinfo;
+
const randomInt = Math.random() * 15;
const sexIndex = Math.floor(randomInt / 5);
const skinToneIndex = Math.floor(randomInt % 5);
@@ -31,9 +48,9 @@ const Auth = React.memo((props) => {
MICROSCOPE
);
- if (!auth?.["requires_client_login"]) return null;
+ if (!shouldShowAuth()) return null;
- if (userinfo?.["is_authenticated"]) {
+ if (isAuthenticated) {
const PopoverContent = (
{
);
}
- return (
+ const LoginButton = (
{
@@ -84,6 +100,76 @@ const Auth = React.memo((props) => {
);
+
+ if (isPromptOpen) {
+ return (
+ }
+ onInteraction={setIsPromptOpen}
+ >
+ {LoginButton}
+
+ );
+ }
+
+ return LoginButton;
+
+ function shouldShowAuth() {
+ return auth && auth.requires_client_login;
+ }
+
+ function shouldShowPrompt() {
+ if (storageGet(KEYS.LOGIN_PROMPT) === LOGIN_PROMPT_OFF) return false;
+
+ return shouldShowAuth && !isAuthenticated;
+ }
});
+function PromptContent({ setIsPromptOpen }) {
+ const [isChecked, setIsChecked] = useState(false);
+
+ function handleOKClick() {
+ if (isChecked) {
+ storageSet(KEYS.LOGIN_PROMPT, LOGIN_PROMPT_OFF);
+ }
+
+ setIsPromptOpen(false);
+ }
+
+ function handleCheckboxChange() {
+ setIsChecked(!isChecked);
+ }
+
+ return (
+
+
+ Logging in will enable you to create your own categories and labels.
+ Logging in later will reset cellxgene to the default view and cause you
+ to lose progress.
+
+
+ Do not show me this message again
+
+
+
+ Acknowledge
+
+
+
+ );
+}
+
export default Auth;
diff --git a/client/src/components/termsPrompt/index.js b/client/src/components/termsPrompt/index.js
index c6209a00..9aef013c 100644
--- a/client/src/components/termsPrompt/index.js
+++ b/client/src/components/termsPrompt/index.js
@@ -8,26 +8,7 @@ import {
Colors,
Icon,
} from "@blueprintjs/core";
-
-const CookieDecision = "cxg.cookieDecision";
-
-function storageGet(key, defaultValue = null) {
- try {
- const val = window.localStorage.getItem(key);
- if (val === null) return defaultValue;
- return val;
- } catch (e) {
- return defaultValue;
- }
-}
-
-function storageSet(key, value) {
- try {
- window.localStorage.setItem(key, value);
- } catch {
- // continue
- }
-}
+import { storageGet, storageSet, KEYS } from "../util/localStorage";
@connect((state) => ({
tosURL: state.config?.parameters?.["about_legal_tos"],
@@ -37,7 +18,7 @@ class TermsPrompt extends React.PureComponent {
constructor(props) {
super(props);
const { tosURL, privacyURL } = this.props;
- const cookieDecision = storageGet(CookieDecision, null);
+ const cookieDecision = storageGet(KEYS.COOKIE_DECISION, null);
const hasDecided = cookieDecision !== null;
this.state = {
hasDecided,
@@ -55,7 +36,7 @@ class TermsPrompt extends React.PureComponent {
handleOK = () => {
this.setState({ isOpen: false });
- storageSet(CookieDecision, "yes");
+ storageSet(KEYS.COOKIE_DECISION, "yes");
if (window.cookieDecisionCallback instanceof Function) {
try {
window.cookieDecisionCallback();
@@ -67,7 +48,7 @@ class TermsPrompt extends React.PureComponent {
handleNo = () => {
this.setState({ isOpen: false });
- storageSet(CookieDecision, "no");
+ storageSet(KEYS.COOKIE_DECISION, "no");
};
renderTos() {
diff --git a/client/src/components/util/localStorage.js b/client/src/components/util/localStorage.js
new file mode 100644
index 00000000..5766d9f9
--- /dev/null
+++ b/client/src/components/util/localStorage.js
@@ -0,0 +1,22 @@
+export const KEYS = {
+ COOKIE_DECISION: "cxg.cookieDecision",
+ LOGIN_PROMPT: "cxg.LOGIN_PROMPT",
+};
+
+export function storageGet(key, defaultValue = null) {
+ try {
+ const val = window.localStorage.getItem(key);
+ if (val === null) return defaultValue;
+ return val;
+ } catch (e) {
+ return defaultValue;
+ }
+}
+
+export function storageSet(key, value) {
+ try {
+ window.localStorage.setItem(key, value);
+ } catch {
+ // continue
+ }
+}
From 86ff48ae36d55944aa3ba66b5025a5fba51c8183 Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Fri, 9 Oct 2020 10:09:32 -0700
Subject: [PATCH 32/73] Revert "Allow columns encoded in float64 to be rendered
as part of continuous value histograms. (#1905)" (#1925)
This reverts commit b048fd8d9a8102f479214e7b8aff85362665c88c.
---
client/src/components/continuous/continuous.js | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/client/src/components/continuous/continuous.js b/client/src/components/continuous/continuous.js
index f1119507..ad5c908d 100644
--- a/client/src/components/continuous/continuous.js
+++ b/client/src/components/continuous/continuous.js
@@ -14,12 +14,7 @@ class Continuous extends React.PureComponent {
if (!schema) return null;
const obsIndex = schema.annotations.obs.index;
const allContinuousNames = schema.annotations.obs.columns
- .filter(
- (col) =>
- col.type === "int32" ||
- col.type === "float32" ||
- col.type === "float64"
- )
+ .filter((col) => col.type === "int32" || col.type === "float32")
.filter((col) => col.name !== obsIndex)
.filter((col) => !col.writable) // skip user annotations - they will be treated as categorical
.map((col) => col.name);
From beb46bf3df3f69eac17b527be7c3afe4d5015944 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Fri, 9 Oct 2020 12:42:02 -0600
Subject: [PATCH 33/73] add and check system arg to state auth type in e2e
test(#1924)
* add and check system arg to state auth type
* add tolower
Co-authored-by: czimergebot <35308261+czimergebot@users.noreply.github.com>
---
client/Makefile | 2 +-
client/__tests__/e2e/e2e.test.js | 7 ++++++-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/client/Makefile b/client/Makefile
index eb5c1a78..1a8d3a81 100644
--- a/client/Makefile
+++ b/client/Makefile
@@ -35,7 +35,7 @@ smoke-test:
start_server_and_test \
'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
$(CXG_SERVER_PORT) \
- 'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
+ 'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
# start an instance of cellxgene and run the end-to-end annotations tests
.PHONY: smoke-test-annotations
diff --git a/client/__tests__/e2e/e2e.test.js b/client/__tests__/e2e/e2e.test.js
index 2bd8ea8c..7e044c40 100644
--- a/client/__tests__/e2e/e2e.test.js
+++ b/client/__tests__/e2e/e2e.test.js
@@ -522,7 +522,12 @@ test("lasso moves after pan", async () => {
expect(panCount).toBe(initialCount);
});
-describe("auth buttons", () => {
+const describeIfCalledByMakeFileTarget =
+ process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
+ ? describe
+ : describe.skip;
+
+describeIfCalledByMakeFileTarget("auth buttons", () => {
test("login then logout", async () => {
await goToPage(appUrlBase);
await clickOnUntil("log-in", async () => {
From 5325495123f426f0e22f4949b8eab54281b79679 Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Mon, 12 Oct 2020 11:20:58 -0700
Subject: [PATCH 34/73] Speed up dataset drawer rendering (#1926)
---
.../src/components/infoDrawer/infoDrawer.js | 98 ++++---------------
.../src/components/infoDrawer/infoFormat.js | 88 ++++++-----------
2 files changed, 53 insertions(+), 133 deletions(-)
diff --git a/client/src/components/infoDrawer/infoDrawer.js b/client/src/components/infoDrawer/infoDrawer.js
index 57bfd77b..9f41ef14 100644
--- a/client/src/components/infoDrawer/infoDrawer.js
+++ b/client/src/components/infoDrawer/infoDrawer.js
@@ -1,13 +1,9 @@
import React, { PureComponent } from "react";
-import { connect, shallowEqual } from "react-redux";
+import { connect } from "react-redux";
import { Drawer } from "@blueprintjs/core";
-import Async from "react-async";
import InfoFormat from "./infoFormat";
-import {
- selectableCategoryNames,
- createCategorySummaryFromDfCol,
-} from "../../util/stateManager/controlsHelpers";
+import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers";
@connect((state) => {
return {
@@ -20,44 +16,6 @@ import {
};
})
class InfoDrawer extends PureComponent {
- static watchAsync(props, prevProps) {
- return !shallowEqual(props.watchProps, prevProps.watchProps);
- }
-
- fetchAsyncProps = async (props) => {
- const { schema } = props.watchProps;
- const { annoMatrix } = this.props;
-
- const allCategoryNames = selectableCategoryNames(schema).sort();
-
- const nonUserAnnoCategories = allCategoryNames.map((catName) => {
- const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
- if (!isUserAnno) return annoMatrix.fetch("obs", catName);
- return null;
- });
- const singleValueCategories = (
- await Promise.all(nonUserAnnoCategories)
- ).reduce((acc, categoryData, i) => {
- // Actually check to see if it is null(user anno)
- if (!categoryData) return acc;
- const catName = allCategoryNames[i];
-
- const column = categoryData.icol(0);
- const colSchema = schema.annotations.obsByName[catName];
-
- const categorySummary = createCategorySummaryFromDfCol(column, colSchema);
-
- const { numCategoryValues } = categorySummary;
- // Add to the array if the category has only one value
- if (numCategoryValues === 1) {
- acc.set(catName, categorySummary.allCategoryValues[0]);
- }
- return acc;
- }, new Map());
-
- return { singleValueCategories };
- };
-
handleClose = () => {
const { dispatch } = this.props;
@@ -74,45 +32,31 @@ class InfoDrawer extends PureComponent {
dataPortalProps,
} = this.props;
+ const allCategoryNames = selectableCategoryNames(schema).sort();
+ const singleValueCategories = new Map();
+
+ allCategoryNames.forEach((catName) => {
+ const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
+ const colSchema = schema.annotations.obsByName[catName];
+ if (!isUserAnno && colSchema.categories?.length === 1) {
+ singleValueCategories.set(catName, colSchema.categories[0]);
+ }
+ });
+
return (
-
-
-
-
-
- {(error) => {
- console.error(error);
- return Failed to load info ;
- }}
-
-
- {(asyncProps) => {
- const { singleValueCategories } = asyncProps;
- return (
-
- );
- }}
-
-
+
);
}
diff --git a/client/src/components/infoDrawer/infoFormat.js b/client/src/components/infoDrawer/infoFormat.js
index eacaa406..880a5f7d 100644
--- a/client/src/components/infoDrawer/infoFormat.js
+++ b/client/src/components/infoDrawer/infoFormat.js
@@ -1,15 +1,15 @@
-import { H3, H1, UL, Classes } from "@blueprintjs/core";
+import { H3, H1, UL } from "@blueprintjs/core";
import React from "react";
import Truncate from "../util/truncate";
-const renderContributors = (contributors, affiliations, skeleton) => {
+const renderContributors = (contributors, affiliations) => {
// eslint-disable-next-line no-constant-condition -- Temp removed contributor section to avoid publishing PII
if (!contributors || contributors.length === 0 || true) return null;
return (
<>
- Contributors
-
+
Contributors
+
{contributors.map((contributor) => {
const { email, name, institution } = contributor;
@@ -22,7 +22,7 @@ const renderContributors = (contributors, affiliations, skeleton) => {
);
})}
- {renderAffiliations(affiliations, skeleton)}
+ {renderAffiliations(affiliations)}
>
);
};
@@ -39,14 +39,14 @@ const buildAffiliations = (contributors = []) => {
return affiliations;
};
-const renderAffiliations = (affiliations, skeleton) => {
+const renderAffiliations = (affiliations) => {
if (affiliations.length === 0) return null;
return (
<>
- Affiliations
+ Affiliations
{affiliations.map((item, index) => (
-
+
{index + 1}
{" "}
{item}
@@ -57,12 +57,12 @@ const renderAffiliations = (affiliations, skeleton) => {
);
};
-const renderDOILink = (type, doi, skeleton) => {
+const renderDOILink = (type, doi) => {
if (!doi) return null;
return (
<>
-
{type}
-
+
{type}
+
{doi}
@@ -71,12 +71,12 @@ const renderDOILink = (type, doi, skeleton) => {
);
};
-const renderOrganism = (organism, skeleton) => {
+const renderOrganism = (organism) => {
if (!organism) return null;
return (
<>
-
Organism
-
{organism}
+
Organism
+
{organism}
>
);
};
@@ -85,11 +85,11 @@ const ONTOLOGY_KEY = "ontology_term_id";
const CAT_WIDTH = "30%";
const VAL_WIDTH = "35%";
// Render list of metadata attributes found in categorical field
-const renderSingleValueCategories = (singleValueCategories, skeleton) => {
+const renderSingleValueCategories = (singleValueCategories) => {
if (singleValueCategories.size === 0) return null;
return (
<>
-
Dataset Metadata
+
Dataset Metadata
{Array.from(singleValueCategories).reduce((elems, pair) => {
const [category, value] = pair;
@@ -115,11 +115,7 @@ const renderSingleValueCategories = (singleValueCategories, skeleton) => {
} else {
// Create the list item
elems.push(
-
+
{`${category}:`}
@@ -138,20 +134,17 @@ const renderSingleValueCategories = (singleValueCategories, skeleton) => {
// Renders any links found in the config where link_type is not "SUMMARY"
// If there are no links in the config, render the aboutURL
-const renderLinks = (projectLinks, aboutURL, skeleton) => {
+const renderLinks = (projectLinks, aboutURL) => {
if (!projectLinks && !aboutURL) return null;
if (projectLinks)
return (
<>
- Project Links
+ Project Links
{projectLinks.map((link) => {
if (link.link_type === "SUMMARY") return null;
return (
-
+
{link.link_name}
@@ -164,14 +157,9 @@ const renderLinks = (projectLinks, aboutURL, skeleton) => {
return (
<>
- More Info
+ More Info
-
+
{aboutURL}
@@ -179,24 +167,14 @@ const renderLinks = (projectLinks, aboutURL, skeleton) => {
);
};
-const NUM_CATEGORIES = 8;
-
-// Generates arbitrary placeholder array for singleValueCategories skeleton shape
-const singleValueCategoriesPlaceholder = Array.from(Array(NUM_CATEGORIES)).map(
- (_, index) => {
- return [index, index];
- }
-);
-
const InfoFormat = React.memo(
({
datasetTitle,
- singleValueCategories = new Map(singleValueCategoriesPlaceholder),
- aboutURL = "thisisabouthtelengthofaurl",
+ singleValueCategories,
+ aboutURL,
dataPortalProps = {},
- skeleton = false,
}) => {
- if (dataPortalProps.corpora_schema_version === "1.0.0") {
+ if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
dataPortalProps = {};
}
const {
@@ -212,15 +190,13 @@ const InfoFormat = React.memo(
return (
-
- {title ?? datasetTitle}
-
- {renderContributors(contributors, affiliations, skeleton)}
- {renderDOILink("DOI", doi, skeleton)}
- {renderDOILink("Preprint DOI", preprintDOI, skeleton)}
- {renderOrganism(organism, skeleton)}
- {renderSingleValueCategories(singleValueCategories, skeleton)}
- {renderLinks(projectLinks, aboutURL, skeleton)}
+ {title ?? datasetTitle}
+ {renderContributors(contributors, affiliations)}
+ {renderDOILink("DOI", doi)}
+ {renderDOILink("Preprint DOI", preprintDOI)}
+ {renderOrganism(organism)}
+ {renderSingleValueCategories(singleValueCategories)}
+ {renderLinks(projectLinks, aboutURL)}
);
}
From 798976e4c1f44f35be7acc4d8a95ea2beb5b825c Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Tue, 13 Oct 2020 15:47:56 -0700
Subject: [PATCH 35/73] Fix custom color handling (#1929)
---
client/src/components/infoDrawer/infoFormat.js | 7 +------
client/src/util/stateManager/colorHelpers.js | 10 +++++++---
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/client/src/components/infoDrawer/infoFormat.js b/client/src/components/infoDrawer/infoFormat.js
index 880a5f7d..6909e7cf 100644
--- a/client/src/components/infoDrawer/infoFormat.js
+++ b/client/src/components/infoDrawer/infoFormat.js
@@ -168,12 +168,7 @@ const renderLinks = (projectLinks, aboutURL) => {
};
const InfoFormat = React.memo(
- ({
- datasetTitle,
- singleValueCategories,
- aboutURL,
- dataPortalProps = {},
- }) => {
+ ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
dataPortalProps = {};
}
diff --git a/client/src/util/stateManager/colorHelpers.js b/client/src/util/stateManager/colorHelpers.js
index 4eaea2b7..d324dd37 100644
--- a/client/src/util/stateManager/colorHelpers.js
+++ b/client/src/util/stateManager/colorHelpers.js
@@ -106,15 +106,19 @@ export function loadUserColorConfig(userColors) {
return -1;
})
.reduce(
- (acc, label, i) => {
+ (acc, label) => {
const color = parseRGB(userColors[category][label]);
acc[0][label] = color;
- acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
+ acc[1][label] = d3.rgb(
+ 255 * color[0],
+ 255 * color[1],
+ 255 * color[2]
+ );
return acc;
},
[{}, {}]
);
- const scale = (i) => scaleMap[i];
+ const scale = (label) => scaleMap[label];
convertedUserColors[category] = { colors, scale };
});
return convertedUserColors;
From 242546371b643031d99f3be8fab3ba05f80bc1f4 Mon Sep 17 00:00:00 2001
From: Madison Dunitz
Date: Wed, 14 Oct 2020 12:46:24 -0500
Subject: [PATCH 36/73] Remove Continuous vars with 1 value from histogram, add
to info drawer (#1927)
* remove single val continous metadata from histogram, add to info drawer
* refactor to save singleContinuous values in state
* fix edge case, single continuous values reappeard in rsb when clipped
---
.../components/brushableHistogram/index.js | 36 ++++++++++++++++---
.../src/components/infoDrawer/infoDrawer.js | 18 +++++++---
.../src/components/infoDrawer/infoFormat.js | 10 +++---
client/src/reducers/index.js | 3 +-
client/src/reducers/singleContinuousValue.js | 14 ++++++++
5 files changed, 65 insertions(+), 16 deletions(-)
create mode 100644 client/src/reducers/singleContinuousValue.js
diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js
index a18608b2..e7c6432c 100644
--- a/client/src/components/brushableHistogram/index.js
+++ b/client/src/components/brushableHistogram/index.js
@@ -450,6 +450,7 @@ const Histogram = ({
isScatterplotYYaccessor: state.controls.scatterplotYYaccessor === field,
continuousSelectionRange: state.continuousSelection[myName],
isColorAccessor: state.colors.colorAccessor === field,
+ singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
};
})
class HistogramBrush extends React.PureComponent {
@@ -608,18 +609,44 @@ class HistogramBrush extends React.PureComponent {
};
fetchAsyncProps = async () => {
- const { annoMatrix } = this.props;
+ const { annoMatrix, field, dispatch, singleContinuousValues } = this.props;
const { isClipped } = annoMatrix;
-
+ if (singleContinuousValues.has(field)) {
+ return {
+ histogram: undefined,
+ range: undefined,
+ unclippedRange: undefined,
+ unclippedRangeColor: globals.blue,
+ isSingleValue: true,
+ OK2Render: false,
+ };
+ }
const query = this.createQuery();
const df = await annoMatrix.fetch(...query);
const column = df.icol(0);
- // if we are clipped, fetch both our value and our unclipped value,
- // as we need the absolute min/max range, not just the clipped min/max.
const summary = column.summarize();
const range = [summary.min, summary.max];
+ if (summary.min === summary.max && !isClipped) {
+ dispatch({
+ type: "add single continuous value",
+ field,
+ value: summary.min,
+ });
+ return {
+ histogram: undefined,
+ range,
+ unclippedRange: range,
+ unclippedRangeColor: globals.blue,
+ isSingleValue: true,
+ OK2Render: false,
+ };
+ }
+
+ const isSingleValue = summary.min === summary.max;
+ // if we are clipped, fetch both our value and our unclipped value,
+ // as we need the absolute min/max range, not just the clipped min/max.
let unclippedRange = [...range];
if (isClipped) {
const parent = await annoMatrix.viewOf.fetch(...query);
@@ -643,7 +670,6 @@ class HistogramBrush extends React.PureComponent {
this.height
);
- const isSingleValue = summary.min === summary.max;
const nonFiniteExtent =
summary.min === undefined ||
summary.max === undefined ||
diff --git a/client/src/components/infoDrawer/infoDrawer.js b/client/src/components/infoDrawer/infoDrawer.js
index 9f41ef14..aad395bc 100644
--- a/client/src/components/infoDrawer/infoDrawer.js
+++ b/client/src/components/infoDrawer/infoDrawer.js
@@ -1,7 +1,6 @@
import React, { PureComponent } from "react";
-import { connect } from "react-redux";
+import { connect, shallowEqual } from "react-redux";
import { Drawer } from "@blueprintjs/core";
-
import InfoFormat from "./infoFormat";
import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers";
@@ -13,9 +12,14 @@ import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers
aboutURL: state.config?.links?.["about-dataset"],
isOpen: state.controls.datasetDrawer,
dataPortalProps: state.config?.["corpora_props"] ?? {},
+ singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
};
})
class InfoDrawer extends PureComponent {
+ static watchAsync(props, prevProps) {
+ return !shallowEqual(props.watchProps, prevProps.watchProps);
+ }
+
handleClose = () => {
const { dispatch } = this.props;
@@ -30,18 +34,22 @@ class InfoDrawer extends PureComponent {
schema,
isOpen,
dataPortalProps,
+ singleContinuousValues,
} = this.props;
const allCategoryNames = selectableCategoryNames(schema).sort();
- const singleValueCategories = new Map();
+ const allSingleValues = new Map();
allCategoryNames.forEach((catName) => {
const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
const colSchema = schema.annotations.obsByName[catName];
if (!isUserAnno && colSchema.categories?.length === 1) {
- singleValueCategories.set(catName, colSchema.categories[0]);
+ allSingleValues.set(catName, colSchema.categories[0]);
}
});
+ singleContinuousValues.forEach((value, catName) => {
+ allSingleValues.set(catName, value);
+ });
return (
diff --git a/client/src/components/infoDrawer/infoFormat.js b/client/src/components/infoDrawer/infoFormat.js
index 6909e7cf..04de063b 100644
--- a/client/src/components/infoDrawer/infoFormat.js
+++ b/client/src/components/infoDrawer/infoFormat.js
@@ -85,13 +85,13 @@ const ONTOLOGY_KEY = "ontology_term_id";
const CAT_WIDTH = "30%";
const VAL_WIDTH = "35%";
// Render list of metadata attributes found in categorical field
-const renderSingleValueCategories = (singleValueCategories) => {
- if (singleValueCategories.size === 0) return null;
+const renderSingleValues = (singleValues) => {
+ if (singleValues.size === 0) return null;
return (
<>
Dataset Metadata
- {Array.from(singleValueCategories).reduce((elems, pair) => {
+ {Array.from(singleValues).reduce((elems, pair) => {
const [category, value] = pair;
// If the value is empty skip it
if (!value) return elems;
@@ -168,7 +168,7 @@ const renderLinks = (projectLinks, aboutURL) => {
};
const InfoFormat = React.memo(
- ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
+ ({ datasetTitle, allSingleValues, aboutURL, dataPortalProps = {} }) => {
if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
dataPortalProps = {};
}
@@ -190,7 +190,7 @@ const InfoFormat = React.memo(
{renderDOILink("DOI", doi)}
{renderDOILink("Preprint DOI", preprintDOI)}
{renderOrganism(organism)}
- {renderSingleValueCategories(singleValueCategories)}
+ {renderSingleValues(allSingleValues)}
{renderLinks(projectLinks, aboutURL)}
);
diff --git a/client/src/reducers/index.js b/client/src/reducers/index.js
index f40dd946..5305706b 100644
--- a/client/src/reducers/index.js
+++ b/client/src/reducers/index.js
@@ -21,7 +21,7 @@ import centroidLabels from "./centroidLabels";
import pointDialation from "./pointDilation";
import { reembedController } from "./reembed";
import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
-
+import singleContinuousValue from "./singleContinuousValue";
import undoableConfig from "./undoableConfig";
const Reducer = undoable(
@@ -32,6 +32,7 @@ const Reducer = undoable(
["ontology", ontology],
["annotations", annotations],
["layoutChoice", layoutChoice],
+ ["singleContinuousValue", singleContinuousValue],
["categoricalSelection", categoricalSelection],
["continuousSelection", continuousSelection],
["graphSelection", graphSelection],
diff --git a/client/src/reducers/singleContinuousValue.js b/client/src/reducers/singleContinuousValue.js
new file mode 100644
index 00000000..7bea9dfe
--- /dev/null
+++ b/client/src/reducers/singleContinuousValue.js
@@ -0,0 +1,14 @@
+const initialState = {
+ singleContinuousValues: new Map(),
+};
+const singleContinuousValue = (state = initialState, action) => {
+ switch (action.type) {
+ case "add single continuous value":
+ state.singleContinuousValues.set(action.field, action.value);
+ return state;
+ default:
+ return state;
+ }
+};
+
+export default singleContinuousValue;
From c9f95491182d134332edf8fe218b8ff2554ecff0 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Fri, 16 Oct 2020 11:59:28 -0700
Subject: [PATCH 37/73] Adopt JS standards once `userinfo` data is in frontend
(#1930)
---
client/src/actions/index.js | 8 ++++----
client/src/components/autosave/filenameDialog.js | 6 +++---
client/src/components/categorical/index.js | 8 ++++----
client/src/components/menubar/authButtons.js | 12 ++++++------
client/src/components/menubar/index.js | 6 +++---
client/src/reducers/index.js | 4 ++--
client/src/reducers/{userinfo.js => userInfo.js} | 5 ++---
7 files changed, 24 insertions(+), 25 deletions(-)
rename client/src/reducers/{userinfo.js => userInfo.js} (83%)
diff --git a/client/src/actions/index.js b/client/src/actions/index.js
index 8730730e..a71149ee 100644
--- a/client/src/actions/index.js
+++ b/client/src/actions/index.js
@@ -43,12 +43,12 @@ async function configFetch(dispatch) {
async function userInfoFetch(dispatch) {
return fetchJson("userinfo").then((response) => {
- const { userinfo } = response || {};
+ const { userinfo: userInfo } = response || {};
dispatch({
- type: "userinfo load complete",
- userinfo,
+ type: "userInfo load complete",
+ userInfo,
});
- return userinfo;
+ return userInfo;
});
}
diff --git a/client/src/components/autosave/filenameDialog.js b/client/src/components/autosave/filenameDialog.js
index 08fe1462..3ff7e500 100644
--- a/client/src/components/autosave/filenameDialog.js
+++ b/client/src/components/autosave/filenameDialog.js
@@ -15,7 +15,7 @@ import {
idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
annotations: state.annotations,
auth: state.config?.authentication,
- userinfo: state.userinfo,
+ userInfo: state.userInfo,
writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
}))
class FilenameDialog extends React.Component {
@@ -97,7 +97,7 @@ class FilenameDialog extends React.Component {
writableCategoriesEnabled,
annotations,
idhash,
- userinfo,
+ userInfo,
} = this.props;
const { filenameText } = this.state;
@@ -105,7 +105,7 @@ class FilenameDialog extends React.Component {
annotations.promptForFilename &&
!annotations.dataCollectionNameIsReadOnly &&
!annotations.dataCollectionName &&
- userinfo.is_authenticated ? (
+ userInfo.is_authenticated ? (
Create new category
diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js
index 91729679..91af0b42 100644
--- a/client/src/components/menubar/authButtons.js
+++ b/client/src/components/menubar/authButtons.js
@@ -31,11 +31,11 @@ const LOGIN_PROMPT_OFF = "off";
const Auth = React.memo((props) => {
const [isPromptOpen, setIsPromptOpen] = useState(shouldShowPrompt());
- const { auth, userinfo } = props;
+ const { auth, userInfo } = props;
- const isAuthenticated = userinfo && userinfo.is_authenticated;
+ const isAuthenticated = userInfo && userInfo.is_authenticated;
- window.userinfo = userinfo;
+ window.userInfo = userInfo;
const randomInt = Math.random() * 15;
const sexIndex = Math.floor(randomInt / 5);
@@ -55,7 +55,7 @@ const Auth = React.memo((props) => {
{
style={{ padding: 0 }}
>
{/* eslint-disable-next-line no-constant-condition -- disable profile picture until CSP is tweaked */}
- {userinfo?.picture && false ? (
-
+ {userInfo?.picture && false ? (
+
) : (
{scientist}
)}
diff --git a/client/src/components/menubar/index.js b/client/src/components/menubar/index.js
index 115eb6ff..cbfeec5e 100644
--- a/client/src/components/menubar/index.js
+++ b/client/src/components/menubar/index.js
@@ -42,7 +42,7 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
celllist2: state.differential.celllist2,
libraryVersions: state.config?.["library_versions"],
auth: state.config?.authentication,
- userinfo: state.userinfo,
+ userInfo: state.userInfo,
undoDisabled: state["@@undoable/past"].length === 0,
redoDisabled: state["@@undoable/future"].length === 0,
aboutLink: state.config?.links?.["about-dataset"],
@@ -217,7 +217,7 @@ class MenuBar extends React.PureComponent {
subsetPossible,
subsetResetPossible,
enableReembedding,
- userinfo,
+ userInfo,
auth,
} = this.props;
const { pendingClipPercentiles } = this.state;
@@ -244,7 +244,7 @@ class MenuBar extends React.PureComponent {
zIndex: 3,
}}
>
-
+
{
switch (action.type) {
case "initial data load start":
@@ -7,12 +6,12 @@ const UserInfo = (state = {}, action) => {
loading: true,
error: null,
};
- case "userinfo load complete":
+ case "userInfo load complete":
return {
...state,
loading: false,
error: null,
- ...action.userinfo,
+ ...action.userInfo,
};
case "initial data load error":
return {
From 6a741956e1ee92496c09943f5f8fd8d19f1462a8 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Fri, 16 Oct 2020 14:02:05 -0700
Subject: [PATCH 38/73] Update readme for eb server. (#1928)
* Update readme for eb server.
Update the README with new way of handling secrets.
Update portions that were out of date.
Add a section for Authentication and a placeholder for User Annotations.
Also remove an obsolete function that processes the AWS secrets.
#1522
Co-authored-by: Madison Dunitz
---
server/common/config/__init__.py | 64 +------
server/eb/README.md | 165 +++++++++++++-----
server/eb/app.py | 20 +--
server/eb/check_config.py | 2 +-
.../unit/common/config/test_server_config.py | 27 ---
5 files changed, 123 insertions(+), 155 deletions(-)
diff --git a/server/common/config/__init__.py b/server/common/config/__init__.py
index fb2439e7..8a74427b 100644
--- a/server/common/config/__init__.py
+++ b/server/common/config/__init__.py
@@ -1,66 +1,4 @@
-import logging
-import os
-import sys
-
-from server.common.aws_secret_utils import get_secret_key
-from server.common.data_locator import discover_s3_region_name
+from server.common.aws_secret_utils import get_secret_key # noqa F504
DEFAULT_SERVER_PORT = 5005
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
-
-
-def handle_config_from_secret(app_config):
- """Update configuration from the secret manager"""
- secret_name = os.getenv("CXG_AWS_SECRET_NAME")
- if not secret_name:
- return
-
- # need to find the secret manager region.
- # 1. from CXG_AWS_SECRET_REGION_NAME
- # 2. discover from dataroot location (if on s3)
- # 3. discover from config file location (if on s3)
- secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
- if secret_region_name is None:
- secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
- if not secret_region_name:
- from server.eb.app import config_file
-
- secret_region_name = discover_s3_region_name(config_file)
- if not secret_region_name:
- logging.error("Could not determine the AWS Secret Manager region")
- sys.exit(1)
-
- secrets = get_secret_key(secret_region_name, secret_name)
-
- if not secrets:
- return
-
- server_attrs = (
- ("flask_secret_key", "app__flask_secret_key"),
- ("oauth_client_secret", "authentication__params_oauth__client_secret"),
- )
- default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
-
- # update server configuration attributes
- for key, attr in server_attrs:
- cur_val = getattr(app_config.server_config, attr)
- if cur_val:
- continue
-
- # replace the attr with the secret if it is not set
- val = secrets.get(key)
- if val:
- logging.info(f"set {attr} from secret")
- app_config.update_server_config(**{attr: val})
-
- # update default dataset configuration attributes
- for key, attr in default_dataset_attrs:
- cur_val = getattr(app_config.default_dataset_config, attr)
- if cur_val:
- continue
-
- # replace the attr with the secret if it is not set
- val = secrets.get(key)
- if val:
- logging.info(f"set {attr} from secret")
- app_config.update_default_dataset_config(**{attr: val})
diff --git a/server/eb/README.md b/server/eb/README.md
index f4209f15..306eda00 100644
--- a/server/eb/README.md
+++ b/server/eb/README.md
@@ -1,12 +1,12 @@
# AWS Elastic Beanstalk
-This directory contains script to aid in creating and deploying cellxgene on
-an AWS Elastic Beanstalk instance.
+This directory contains scripts to aid in creating and deploying cellxgene on
+AWS Elastic Beanstalk.
This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3.
-All datasets must be in the new CXG (tiledb) format - see the converter script cxgtool.py
-in server/converters - and located in a single S3 prefix, which is accessible to the instance.
-In the current incarnation, no access control or authentication support is available
+All datasets must be in the CXG (tiledb) format (see `cellxene convert --help`),
+and located under a single S3 prefix, which is accessible to the instance.
+In the current incarnation, no access control is available
(outside of anything you configure yourself), so this is most appropriate for public datasets.
This is early development work, and will change significantly in the near future.
@@ -17,10 +17,10 @@ We would love feedback on it, but please assume it will change.
1. Some familiarity with AWS EB, S3, and IAM are needed.
2. Install the awsebcli.
-Instruction are here:
-https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
+ Instruction are here:
+ https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
-3. In the top level directory, run ```make build-client``` to create the client static assets.
+3. In the top level directory, run `make build-client` to create the client static assets.
## Steps
@@ -31,20 +31,21 @@ There are many more options to these commands that may be important or necessary
The following choices are known to work.
-* S3 Bucket.
-* POSIX filesystem (such as Lustre)
-* Lustre filesystem backed by S3
+- S3 Bucket.
+- POSIX filesystem (such as Lustre)
+- Lustre filesystem backed by S3
S3 is convenient and the relatively inexpensive option.
Lustre is higher performance, but more expensive, and slightly more complex to setup and manage.
-AWS supports a feature to back the Lustre filesystem with S3, which give an easy to manage and high
+AWS supports a feature to back the Lustre filesystem with S3, which gives an easy to manage, high
performance option.
-Once the storage is in place, the next step is to copy your matrix files to that location.
-Currently cellxgene supports a flat file organization. Each matrix file is located from
-the same s3 prefix or filesystem directory. This location is specified in the configuration as the dataroot.
+Once the storage is in place, the next step is to copy your data files to that location.
+Currently cellxgene supports a flat file organization. Each matrix file is located under
+the same s3 prefix or filesystem directory. This location is specified in the configuration
+as the dataroot.
-### 2. Create an elastic beanstalk application. For example:
+### 2. Create an elastic beanstalk application. For example:
```
EB_APP=cellxgene-app
@@ -54,9 +55,9 @@ eb init -p python-3.6 $EB_APP
### 3. Configuring cellxgene
All the cellxgene configuration options can be set from a configuration file.
-This file can be generated like this:
+A yaml config file containing all of the default configuration options can be generated like this:
-```cellxgene launch --dump-default-config > myconfig.yaml```
+`cellxgene launch --dump-default-config > myconfig.yaml`
The config file may then be customized before the app is deployed.
@@ -66,18 +67,14 @@ First, if your config file is named "config.yaml" and exists in `customize/confi
then it will be bundled with the application zip file and installed along
side the app on the EB servers.
-Second, a potentially more flexible approach is to place your config file in a location accessible to the EB
-servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml.
+Second, a potentially more flexible approach is to place your config file in a location accessible
+to the EB servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml.
Set the CXG_CONFIG_FILE environment variable to specify this location.
-Another option is to set the CXG_DATAROOT environment variable. The dataroot
+Another option is to set the CXG_DATAROOT environment variable. The dataroot
is the location where the matrix files are located.
This environment variable will override the dataroot in the config file (if specified).
-- Note: Certain features, such as user annotations, are automatically disabled by the EB app,
-and cannot be enabled using configuration. They may be enabled manually by modifying app.py, however
-this is not supported or recommended at this time.
-
### 4. Customization
The deployment can be customized in several ways, by adding files to a directory called
@@ -93,15 +90,15 @@ The cellxgene server can serve additional static webpages that will be associate
These include the about_legal_tos (terms of service), and about_legal_privacy, for example.
To use this feature, do the following:
-* In this directory, create a sub directory called "customize/deploy/".
-* Copy the files you want to serve into this directory
-* modify your configuration file to set the location to these file: /static/cellxgene/deploy/
+- In this directory, create a sub directory called "customize/deploy/".
+- Copy the files you want to serve into this directory
+- modify your configuration file to set the location to these file: /static/cellxgene/deploy/
-Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene.
+Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene.
Assume files called "tos.html" and "privacy.html" exist.
```
-$ mkdir static
+$ mkdir -p customize/deploy
$ cp /tos.html customize/deploy/tos.html
$ cp /privacy.html customize/deploy/privacy.html
@@ -116,14 +113,14 @@ about_legal_privacy: /static/cellxgene/deploy/privacy.html
Additional scripts can be added using the server/inline_scripts config parameters.
To include these scripts in the deployment, use the following steps:
-* In this directory, create a sub directory called "customize/inline_scripts".
-* Copy the script files into this directory
-* modify your configuration file to set the location to these file (leaving off customize/inline_scripts)
+- In this directory, create a sub directory called "customize/inline_scripts".
+- Copy the script files into this directory
+- Modify your configuration file to set the location to these file (leaving off customize/inline_scripts)
For example, to add an inline script called "myscript.js":
```
-$ mkdir scripts
+$ mkdir -p customize/inline_scripts
$ cp /myscript.js customize/inline_scripts/myscript.js
# edit the config.yaml
$ grep inline_scripts config.yaml
@@ -135,14 +132,14 @@ $ grep inline_scripts config.yaml
Optionally, you can add plugins to the server python code. To include a plugin in the deployment use the following steps:
```
-$ mkdir plugins
+$ mkdir -p customize/plugins
$ cp /.py customize/plugins/.py
```
#### ebextensions
Any additional config files intended for the `.ebextensions` directory of the artifact can be added
-to the `customize/ebextensions` directory. Any file found here will be copied over.
+to the `customize/ebextensions` directory. Any file found here will be copied over.
#### requirements.txt
@@ -152,6 +149,7 @@ This is useful to ensure that the dependencies do not change from one deployment
Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1).
This file can be generated the first time using a process like this:
+
```
# assume you are running in this directory
$ virtualenv temp
@@ -168,6 +166,20 @@ If a future cellxgene version updates its requirements by modifying a module ver
or adding a new dependency, then the `make build` process will detect any
incompatibilities and raise an error.
+#### File structure for customizations
+
+The following diagram shows the file structure for the customization directory.
+
+```
+customization
++-- config.yaml
++-- deploy/
++-- inline_scripts/
++-- plugins/
++-- ebextensions/
++-- requirements.txt
+```
+
### 5. Create the artifact.zip file for the application
```
@@ -176,19 +188,13 @@ $ make build
### 6. Flask secret key
-The application requires as secret key to be provided to flask, the web framework used by cellxgene.
+The application requires a secret key to be provided to flask, the web framework used by cellxgene.
There are three ways to provide the secret key:
-- In the configuration file: update the server/flask_secret_key attribute.
+- In the configuration file, update the server/flask_secret_key attribute.
+- In the configuration file, update the external/aws_secrets_manager section to set the
+ secret name and key that defines the flask secret key.
- An environment variable: `CXG_SECRET_KEY`
-- Managed by the AWS Secret Manager
-
-If using the AWS Secret Manager, then the secret name is passed as an environment variable: CXG_AWS_SECRET_NAME.
-The secret must contain a key with the name "flask_secret_key".
-The region name for the AWS Secret Manager must be specified (e.g. us-east-1).
-The most straightforward way is to specified it with the CXG_AWS_SECRET_REGION_NAME environment variable.
-If this environment variable is not defined, then the app attempts to determine the region from the
-dataroot (if in s3), or the config file location (if in s3).
### 7. Create an environment
@@ -203,7 +209,8 @@ $ EB_INSTANCE=m5.large
$ CXG_DATAROOT=
$ CXG_CONFIG_FILE=
-# Potentially also set envvars for the secret key.
+# Potentially also set an environment variable for the flask secret key,
+# and other environemet variable described in the configuration file.
$ eb create $EB_ENV --instance-type $EB_INSTANCE \
--envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE
@@ -227,3 +234,67 @@ $ eb deploy $EB_ENV
```
$ eb open $EB_ENV
```
+
+## Advanced Features
+
+### Authentication
+
+Authentication can be configured in the configuration file. Authentication is required
+for User Annotations (see below). User Annotations is a feature where annotations can be
+created by the user
+, and
+then associated with the user's id.
+When the user revisits the site, their annotations will be available.
+
+There are three main authentication modes: null, session, or oauth.
+In the configuration file specify the authentication mode by setting
+`server / authentication / type`.
+
+#### null
+
+Authentication is disabled: user annotations cannot be enabled.
+
+#### session
+
+The user is associated with their client browser session. This approach is
+simple to setup, but not recommended for hosted cellxgene, since the user will not have access to
+their annotations when running from a different browser, or if their cookies get cleared.
+
+#### oauth
+
+A user logs into cellxgene using an identity provider (like Google), or logs in using
+an email/password. This is the best option, but requires making use of an oauth service and
+additional configuration of the cellxgene server.
+
+To see what this looks like, please look at https://cellxgene.cziscience.com/,
+and view one of the cellxgene datasets.
+For this server, Auth0 (auth0.com) is used for authentication, but there are other options.
+There are good sources of documentation online that describe how to use one of these
+services.
+
+The `params_oauth` section in the configuration file describes characteristics of the
+authentication service, like "client_id" and "client_secret".
+For security, the client_secret needs to be protected. One option is to
+store it in the AWS Secrets Manager.
+
+### User Annotations
+
+User annotations can be configured in the configuration file both generally and for a specific data route. The annotations feature is only available when Authorization is enabled.
+To enable Annotations, it is necessary to create a relational database and add the database uri (typically `postgresql://[user[:password]@][netloc][:port][/dbname]`) to the secrets manager under `DB_URI`.
+The hosted version of cellxgene runs on AWS's [Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.AuroraPostgreSQL.html) but any sqlalchemy compatible relational database should work.
+Once the database is set up apply the cellxgene schema to your database by running the following inside the cellxgene repo
+`PROJECT_ROOT=$(git rev-parse --show-toplevel)`
+`python3`
+Inside the python console
+`from sqlalchemy import create_engine`
+`from server.db.cellxgene_orm import Base`
+`uri = "[DB_URI]â`
+`engine = create_engine(uri)`
+
+ Base.metadata.create_all(engine)`
+
+To check the schema was properly applied (or just to check what is in the database at any point)
+ssh into your database. For a postgres database this entails running:
+`psql [DB_URI]`
+
+You'll also need to update your IAM policies to allow the instance to write to the s3 bucket.
diff --git a/server/eb/app.py b/server/eb/app.py
index 0844f599..4b2212f5 100644
--- a/server/eb/app.py
+++ b/server/eb/app.py
@@ -9,8 +9,6 @@ from flask import json
import logging
from flask_talisman import Talisman
from flask_cors import CORS
-from server.common.config import handle_config_from_secret
-from server.common.errors import SecretKeyRetrievalError
if os.path.isdir("/opt/python/log"):
@@ -165,26 +163,14 @@ try:
logging.info("Configuration from CXG_DATAROOT")
app_config.update_server_config(multi_dataset__dataroot=dataroot)
- # update from secret manager
- try:
- handle_config_from_secret(app_config)
- except SecretKeyRetrievalError:
- sys.exit(1)
-
- # features are unsupported in the current hosted server
+ # overwrite configuration for the eb app
app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
+
+ # complete config
app_config.complete_config(logging.info)
- if not app_config.server_config.app__flask_secret_key:
- logging.critical(
- "flask_secret_key is not provided. Either set in config file, CXG_SECRET_KEY environment variable, "
- "or in AWS Secret Manager"
- )
- sys.exit(1)
-
server = WSGIServer(app_config)
-
debug = False
application = server.app
diff --git a/server/eb/check_config.py b/server/eb/check_config.py
index 6886e101..2ea1c25e 100644
--- a/server/eb/check_config.py
+++ b/server/eb/check_config.py
@@ -19,8 +19,8 @@ def main():
args = parser.parse_args()
app_config = AppConfig()
- app_config.update_from_config_file(args.config_file)
try:
+ app_config.update_from_config_file(args.config_file)
app_config.complete_config()
except Exception as e:
print(f"Error: {str(e)}")
diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py
index f862a40f..69257bfd 100644
--- a/server/test/unit/common/config/test_server_config.py
+++ b/server/test/unit/common/config/test_server_config.py
@@ -310,30 +310,3 @@ class TestServerConfig(ConfigTests):
mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
)
-
- @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
- @patch("server.common.config.get_secret_key")
- def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
- mock_get_secret_key.return_value = {
- "flask_secret_key": "mock_flask_secret",
- "oauth_client_secret": "mock_oauth_secret",
- "db_uri": "mock_db_uri",
- }
-
- config = AppConfig()
-
- with self.assertLogs(level="INFO") as logger:
- from server.common.config import handle_config_from_secret
-
- # should not throw error
- # "AttributeError: 'XConfig' object has no attribute 'x'"
- handle_config_from_secret(config)
-
- # should log 3 lines (one for each var set from a secret)
- self.assertEqual(len(logger.output), 3)
- self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
- self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
- self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
- self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
- self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
- self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
From 377e4bccaa51d12142b1e643c8fdc14ff94acc31 Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Mon, 19 Oct 2020 10:31:36 -0700
Subject: [PATCH 39/73] Remove errornous checking for converting float64 to
float32. In reality the slight difference by downcasting is totally fine.
(#1935)
---
server/common/utils/type_conversion_utils.py | 23 ++++++--------------
1 file changed, 7 insertions(+), 16 deletions(-)
diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py
index ccda1177..762dccb2 100644
--- a/server/common/utils/type_conversion_utils.py
+++ b/server/common/utils/type_conversion_utils.py
@@ -88,24 +88,15 @@ def get_schema_type_hint_from_dtype(dtype, array_values=None):
def can_cast_to_float32(dtype, array_values):
"""
- A dtype can be cast to float32 if it is a float type and converting it to float32 presents the same output as the
- original values. Note that NaNs fail equality (i.e. np.NaN != np.NaN) so we use np.testing.assert_equal to ensure
- that the arrays are equal minus NaNs.
+ Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
+ a float.
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
- Since NaNs are floating points in numpy, we upcast the integer array to float32.
+ Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
"""
if dtype.kind == "f":
- # Try to convert the array to float32
- converted_float32_values = array_values.to_numpy(np.float32)
- original_values = array_values.to_numpy()
-
- # Verify that the two arrays are equal except for NaNs (which will equate to be unequal).
- if not ((converted_float32_values != original_values) == np.isnan(original_values)).all():
- return False
-
- if dtype != np.float32:
+ if not np.can_cast(dtype, np.float32):
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
return True
@@ -138,9 +129,9 @@ def can_cast_to_int32(dtype, array_values=None):
return True
ii32 = np.iinfo(np.int32)
if (
- not ordered_array_values.empty
- and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
- or ordered_array_values.empty
+ not ordered_array_values.empty
+ and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+ or ordered_array_values.empty
):
return True
return False
From 9793398737bff4a6c9337ba74eec7c80726eccbb Mon Sep 17 00:00:00 2001
From: maniarathi
Date: Wed, 21 Oct 2020 09:16:13 -0700
Subject: [PATCH 40/73] Add in missing previous crossfilter which was causing
the re-embedding feature to fail. (#1936)
---
client/src/actions/reembed.js | 3 ++-
server/requirements.txt | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/client/src/actions/reembed.js b/client/src/actions/reembed.js
index 2212f380..11a01839 100644
--- a/client/src/actions/reembed.js
+++ b/client/src/actions/reembed.js
@@ -79,10 +79,11 @@ export function requestReembed() {
type: "reembed: request completed",
});
- const { annoMatrix: prevAnnoMatrix } = getState();
+ const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
const base = prevAnnoMatrix.base().addEmbedding(schema);
const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
base,
+ prevCrossfilter,
schema.name
);
dispatch({
diff --git a/server/requirements.txt b/server/requirements.txt
index d188764b..2e2de8b6 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -11,6 +11,7 @@ flask-talisman>=0.7.0
flatbuffers>=1.11.0
flatten-dict>=0.2.0
fsspec>=0.4.4,<0.8.0
+gunicorn>=20.0.4
numba>=0.49.1
numpy>=1.16.0
packaging>=20.0
@@ -18,7 +19,6 @@ pandas>=0.24.2
PyYAML>=5.3
scipy>=1.3.0
requests>=2.22.0
-sqlalchemy>=1.3.18
tiledb>=0.5.9,>=0.6.2
s3fs==0.4.2
-gunicorn>=20.0.4
+sqlalchemy>=1.3.18
From f41a023418be3285ace3763b884e42e67885f97a Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Thu, 22 Oct 2020 17:04:08 -0700
Subject: [PATCH 41/73] Minor changes to eb server to use Docker (#1938)
part of #1866
---
server/eb/Makefile | 3 +++
server/eb/app.py | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/server/eb/Makefile b/server/eb/Makefile
index 3c9c5b4a..05b24f83 100644
--- a/server/eb/Makefile
+++ b/server/eb/Makefile
@@ -27,6 +27,9 @@ build: clean
if [ -f customize/config.yaml ] ; then \
cp customize/config.yaml artifact.dir; \
fi ; \
+ if [ -f customize/Dockerfile ] ; then \
+ cp customize/Dockerfile artifact.dir; \
+ fi ; \
if [ -f customize/requirements.txt ] ; then \
pip install requirements-parser ; \
pip install packaging ; \
diff --git a/server/eb/app.py b/server/eb/app.py
index 4b2212f5..7de161b7 100644
--- a/server/eb/app.py
+++ b/server/eb/app.py
@@ -185,7 +185,7 @@ else:
if __name__ == "__main__":
try:
- application.run(debug=debug, threaded=not debug, use_debugger=False)
+ application.run(host=app_config.server_config.app__host, debug=debug, threaded=not debug, use_debugger=False)
except Exception:
logging.critical("Caught exception during initialization", exc_info=True)
sys.exit(1)
From 2fa206f2adb801b4e128bb67b0cec6a76ac15304 Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Fri, 23 Oct 2020 14:51:48 -0700
Subject: [PATCH 42/73] Add token invalidation tests to oauth tests (#1941)
* add tests
* run black
* run black and add disclaimer that tweaked errors on server
* lint
* change to get so it will return None
* tweak existing token instead of new one
* Trigger
* token is dict
* jsonify dict before encoding
* json dump instead of jsonify
* encode into bytes object
* use correct id token
* decode byte to string
---
server/test/unit/auth/test_oauth.py | 26 ++++++++++++++++++++++++--
1 file changed, 24 insertions(+), 2 deletions(-)
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index f5a85cb5..1096ef1a 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -59,7 +59,12 @@ def logout():
@mock_oauth_app.route("/.well-known/jwks.json")
def jwks():
- data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
+ data = dict(
+ alg="RS256",
+ kty="RSA",
+ use="sig",
+ kid="fake_kid",
+ )
return make_response(jsonify(dict(keys=[data])))
@@ -147,6 +152,21 @@ class AuthTest(unittest.TestCase):
self.assertNotEqual(access_token_before, access_token_after)
self.assertNotEqual(id_token_before, id_token_after)
+ # invalid cookie fails
+ # (THIS CURRENTLY 500's ON THE SERVER, BUT SHOULD RETURN EMPTY USERINFO OR 401)
+ session.cookies.set(cookie_key, "TEST_" + cookie)
+ userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+ self.assertIsNone(userinfo.get("userinfo"))
+
+ # invalid id_token fails
+ test_token = token
+ test_token["id_token"] = "TEST_" + id_token_after
+ encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
+ session.cookies.set(cookie_key, encoded_cookie)
+ userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+ self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+ self.assertIsNone(userinfo["userinfo"]["username"])
+
r = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
@@ -161,7 +181,9 @@ class AuthTest(unittest.TestCase):
# test with session cookies
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
+ app_config.update_server_config(
+ authentication__params_oauth__session_cookie=True,
+ )
self.auth_flow(app_config)
def test_auth_oauth_cookie(self):
From c106ebc525eec6f8d62d51fc1fc0af73cc514983 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Fri, 23 Oct 2020 15:14:31 -0700
Subject: [PATCH 43/73] smnall fix to the test suite. (#1944)
I noticed a few tests failed when run individually, but not as a suite.
#1942
---
server/test/unit/cli/test_launch.py | 10 +++++-----
.../data_anndata/test_anndata_adaptor_data_load.py | 2 ++
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/server/test/unit/cli/test_launch.py b/server/test/unit/cli/test_launch.py
index ac388cc0..5569cfae 100644
--- a/server/test/unit/cli/test_launch.py
+++ b/server/test/unit/cli/test_launch.py
@@ -21,8 +21,8 @@ class CLIPLaunchTests(unittest.TestCase):
shutil.rmtree(cls.tmp_dir)
-def test_dump_default_config(self):
- os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
- with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
- expected_config.write(yaml.dump(default_config))
- filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")
+ def test_dump_default_config(self):
+ os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+ with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+ expected_config.write(yaml.dump(default_config))
+ filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")
diff --git a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py
index b3db3cb3..9724eadb 100644
--- a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py
+++ b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py
@@ -16,6 +16,7 @@ class DataLoadAdaptorTest(unittest.TestCase):
self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
config = AppConfig()
config.update_server_config(single_dataset__datapath=self.data_file.path)
+ config.update_server_config(app__flask_secret_key="secret")
config.complete_config()
self.data = AnndataAdaptor(self.data_file, config)
@@ -45,6 +46,7 @@ class DataLocatorAdaptorTest(unittest.TestCase):
config.update_server_config(
single_dataset__obs_names=None, single_dataset__var_names=None,
)
+ config.update_server_config(app__flask_secret_key="secret")
config.update_default_dataset_config(
embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
)
From 7e9353c5f1e706826727104968c9d0b678ea7ba1 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Mon, 26 Oct 2020 09:39:06 -0700
Subject: [PATCH 44/73] Fix bug in oauth. (#1949)
* Fix bug in oauth.
The error checking was too specific, and missed a case.
Make the error checking catch all exceptions.
#1947
* Add logging when the cookie cannot be processed
---
server/auth/auth_oauth.py | 23 ++++++++++++----------
server/test/unit/auth/test_oauth.py | 30 ++++++++++++++---------------
2 files changed, 28 insertions(+), 25 deletions(-)
diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py
index b2724c57..efc79c30 100644
--- a/server/auth/auth_oauth.py
+++ b/server/auth/auth_oauth.py
@@ -218,22 +218,24 @@ class AuthTypeOAuth(AuthTypeClientBase):
try:
if self.session_cookie:
- tokensdict = session.get(self.CXG_TOKENS)
- if tokensdict:
- g.tokens = Tokens(**tokensdict)
+ value = session.get(self.CXG_TOKENS)
+ if value:
+ g.tokens = Tokens(**value)
else:
return None
else:
value = request.cookies.get(self.cookie_params["key"])
- value = base64.b64decode(value)
- try:
- tokensdict = json.loads(value)
- g.tokens = Tokens(**tokensdict)
- except (TypeError, KeyError, json.decoder.JSONDecodeError):
- g.pop("tokens", None)
+ if value is None:
return None
+ value = base64.b64decode(value)
+ value = json.loads(value)
+ g.tokens = Tokens(**value)
- except (TypeError, KeyError):
+ except Exception:
+ # there are many types of exceptions that can be raise in the above section.
+ # It is impractical to list all the exceptions here, since that would be brittle.
+ # If an exception occurs, then return None, meaning that no token could be retrieved.
+ current_app.logger.warning(f"auth cookie is in the wrong format: {str(value)}")
g.pop("tokens", None)
return None
@@ -331,6 +333,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
# if there is no id_token, return None (user is not authenticated)
tokens = self.get_tokens()
+
if tokens is None or tokens.id_token is None:
return None
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index 1096ef1a..43bdf1c2 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -59,12 +59,7 @@ def logout():
@mock_oauth_app.route("/.well-known/jwks.json")
def jwks():
- data = dict(
- alg="RS256",
- kty="RSA",
- use="sig",
- kid="fake_kid",
- )
+ data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
return make_response(jsonify(dict(keys=[data])))
@@ -152,18 +147,25 @@ class AuthTest(unittest.TestCase):
self.assertNotEqual(access_token_before, access_token_after)
self.assertNotEqual(id_token_before, id_token_after)
- # invalid cookie fails
- # (THIS CURRENTLY 500's ON THE SERVER, BUT SHOULD RETURN EMPTY USERINFO OR 401)
+ # invalid cookie is rejected
session.cookies.set(cookie_key, "TEST_" + cookie)
- userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
- self.assertIsNone(userinfo.get("userinfo"))
+ self.assertTrue(cookie_key in session.cookies)
+ response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
+ # this is not an error, the invalid cookie is just ignored.
+ self.assertEqual(response.status_code, 200)
+ userinfo = response.json()
+ self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+ self.assertIsNone(userinfo["userinfo"]["username"])
- # invalid id_token fails
+ # invalid id_token is rejected
test_token = token
test_token["id_token"] = "TEST_" + id_token_after
encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
session.cookies.set(cookie_key, encoded_cookie)
- userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+ response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
+ # this is not an error, the invalid id_token is just ignored.
+ self.assertEqual(response.status_code, 200)
+ userinfo = response.json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
@@ -181,9 +183,7 @@ class AuthTest(unittest.TestCase):
# test with session cookies
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(
- authentication__params_oauth__session_cookie=True,
- )
+ app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
self.auth_flow(app_config)
def test_auth_oauth_cookie(self):
From 946a910ef420250506b76d58af51036920e2803c Mon Sep 17 00:00:00 2001
From: Madison Dunitz
Date: Mon, 26 Oct 2020 17:23:55 -0500
Subject: [PATCH 45/73] Fix compatibility test (#1948)
* update anndata version and warning about version
* update compatibility tests
---
.github/workflows/compatibility_tests.yml | 2 +-
server/data_anndata/anndata_adaptor.py | 6 +++---
server/requirements.txt | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml
index f0e55f38..99a3698d 100644
--- a/.github/workflows/compatibility_tests.yml
+++ b/.github/workflows/compatibility_tests.yml
@@ -29,7 +29,7 @@ jobs:
strategy:
matrix:
python-version: [3.6, 3.7, 3.8]
- anndata-version: [0.6.22.post1, 0.7.1]
+ anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4]
test-suite: [smoke-test, smoke-test-annotations]
steps:
- uses: actions/checkout@v2
diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py
index 3145a833..1312a8f6 100644
--- a/server/data_anndata/anndata_adaptor.py
+++ b/server/data_anndata/anndata_adaptor.py
@@ -177,10 +177,10 @@ class AnndataAdaptor(DataAdaptor):
)
def _validate_and_initialize(self):
- if anndata_version_is_pre_070() and self.server_config.adaptor__anndata_adaptor__backed:
+ if anndata_version_is_pre_070():
warnings.warn(
- "Use of --backed mode with anndata versions older than 0.7 will have serious "
- "performance issues. Please update to at least anndata 0.7 or later."
+ "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
+ "least anndata 0.7 or later."
)
# var and obs column names must be unique
diff --git a/server/requirements.txt b/server/requirements.txt
index 2e2de8b6..d288aee8 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -1,4 +1,4 @@
-anndata>=0.6.20
+anndata>=0.7.0
boto3>=1.12.18
click>=7.1.2
fastobo>=0.6.1
From 924b518492322a64a6bc0ce343b7aa3f362f049d Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Tue, 27 Oct 2020 17:12:13 -0700
Subject: [PATCH 46/73] Revert "Remove Continuous vars with 1 value from
histogram, add to info drawer (#1927)" (#1953)
This reverts commit 242546371b643031d99f3be8fab3ba05f80bc1f4.
---
.../components/brushableHistogram/index.js | 36 +++----------------
.../src/components/infoDrawer/infoDrawer.js | 18 +++-------
.../src/components/infoDrawer/infoFormat.js | 10 +++---
client/src/reducers/index.js | 3 +-
client/src/reducers/singleContinuousValue.js | 14 --------
5 files changed, 16 insertions(+), 65 deletions(-)
delete mode 100644 client/src/reducers/singleContinuousValue.js
diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js
index e7c6432c..a18608b2 100644
--- a/client/src/components/brushableHistogram/index.js
+++ b/client/src/components/brushableHistogram/index.js
@@ -450,7 +450,6 @@ const Histogram = ({
isScatterplotYYaccessor: state.controls.scatterplotYYaccessor === field,
continuousSelectionRange: state.continuousSelection[myName],
isColorAccessor: state.colors.colorAccessor === field,
- singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
};
})
class HistogramBrush extends React.PureComponent {
@@ -609,44 +608,18 @@ class HistogramBrush extends React.PureComponent {
};
fetchAsyncProps = async () => {
- const { annoMatrix, field, dispatch, singleContinuousValues } = this.props;
+ const { annoMatrix } = this.props;
const { isClipped } = annoMatrix;
- if (singleContinuousValues.has(field)) {
- return {
- histogram: undefined,
- range: undefined,
- unclippedRange: undefined,
- unclippedRangeColor: globals.blue,
- isSingleValue: true,
- OK2Render: false,
- };
- }
+
const query = this.createQuery();
const df = await annoMatrix.fetch(...query);
const column = df.icol(0);
+ // if we are clipped, fetch both our value and our unclipped value,
+ // as we need the absolute min/max range, not just the clipped min/max.
const summary = column.summarize();
const range = [summary.min, summary.max];
- if (summary.min === summary.max && !isClipped) {
- dispatch({
- type: "add single continuous value",
- field,
- value: summary.min,
- });
- return {
- histogram: undefined,
- range,
- unclippedRange: range,
- unclippedRangeColor: globals.blue,
- isSingleValue: true,
- OK2Render: false,
- };
- }
-
- const isSingleValue = summary.min === summary.max;
- // if we are clipped, fetch both our value and our unclipped value,
- // as we need the absolute min/max range, not just the clipped min/max.
let unclippedRange = [...range];
if (isClipped) {
const parent = await annoMatrix.viewOf.fetch(...query);
@@ -670,6 +643,7 @@ class HistogramBrush extends React.PureComponent {
this.height
);
+ const isSingleValue = summary.min === summary.max;
const nonFiniteExtent =
summary.min === undefined ||
summary.max === undefined ||
diff --git a/client/src/components/infoDrawer/infoDrawer.js b/client/src/components/infoDrawer/infoDrawer.js
index aad395bc..9f41ef14 100644
--- a/client/src/components/infoDrawer/infoDrawer.js
+++ b/client/src/components/infoDrawer/infoDrawer.js
@@ -1,6 +1,7 @@
import React, { PureComponent } from "react";
-import { connect, shallowEqual } from "react-redux";
+import { connect } from "react-redux";
import { Drawer } from "@blueprintjs/core";
+
import InfoFormat from "./infoFormat";
import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers";
@@ -12,14 +13,9 @@ import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers
aboutURL: state.config?.links?.["about-dataset"],
isOpen: state.controls.datasetDrawer,
dataPortalProps: state.config?.["corpora_props"] ?? {},
- singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
};
})
class InfoDrawer extends PureComponent {
- static watchAsync(props, prevProps) {
- return !shallowEqual(props.watchProps, prevProps.watchProps);
- }
-
handleClose = () => {
const { dispatch } = this.props;
@@ -34,22 +30,18 @@ class InfoDrawer extends PureComponent {
schema,
isOpen,
dataPortalProps,
- singleContinuousValues,
} = this.props;
const allCategoryNames = selectableCategoryNames(schema).sort();
- const allSingleValues = new Map();
+ const singleValueCategories = new Map();
allCategoryNames.forEach((catName) => {
const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
const colSchema = schema.annotations.obsByName[catName];
if (!isUserAnno && colSchema.categories?.length === 1) {
- allSingleValues.set(catName, colSchema.categories[0]);
+ singleValueCategories.set(catName, colSchema.categories[0]);
}
});
- singleContinuousValues.forEach((value, catName) => {
- allSingleValues.set(catName, value);
- });
return (
diff --git a/client/src/components/infoDrawer/infoFormat.js b/client/src/components/infoDrawer/infoFormat.js
index 04de063b..6909e7cf 100644
--- a/client/src/components/infoDrawer/infoFormat.js
+++ b/client/src/components/infoDrawer/infoFormat.js
@@ -85,13 +85,13 @@ const ONTOLOGY_KEY = "ontology_term_id";
const CAT_WIDTH = "30%";
const VAL_WIDTH = "35%";
// Render list of metadata attributes found in categorical field
-const renderSingleValues = (singleValues) => {
- if (singleValues.size === 0) return null;
+const renderSingleValueCategories = (singleValueCategories) => {
+ if (singleValueCategories.size === 0) return null;
return (
<>
Dataset Metadata
- {Array.from(singleValues).reduce((elems, pair) => {
+ {Array.from(singleValueCategories).reduce((elems, pair) => {
const [category, value] = pair;
// If the value is empty skip it
if (!value) return elems;
@@ -168,7 +168,7 @@ const renderLinks = (projectLinks, aboutURL) => {
};
const InfoFormat = React.memo(
- ({ datasetTitle, allSingleValues, aboutURL, dataPortalProps = {} }) => {
+ ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
dataPortalProps = {};
}
@@ -190,7 +190,7 @@ const InfoFormat = React.memo(
{renderDOILink("DOI", doi)}
{renderDOILink("Preprint DOI", preprintDOI)}
{renderOrganism(organism)}
- {renderSingleValues(allSingleValues)}
+ {renderSingleValueCategories(singleValueCategories)}
{renderLinks(projectLinks, aboutURL)}
);
diff --git a/client/src/reducers/index.js b/client/src/reducers/index.js
index b4a47153..64a4a48b 100644
--- a/client/src/reducers/index.js
+++ b/client/src/reducers/index.js
@@ -21,7 +21,7 @@ import centroidLabels from "./centroidLabels";
import pointDialation from "./pointDilation";
import { reembedController } from "./reembed";
import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
-import singleContinuousValue from "./singleContinuousValue";
+
import undoableConfig from "./undoableConfig";
const Reducer = undoable(
@@ -32,7 +32,6 @@ const Reducer = undoable(
["ontology", ontology],
["annotations", annotations],
["layoutChoice", layoutChoice],
- ["singleContinuousValue", singleContinuousValue],
["categoricalSelection", categoricalSelection],
["continuousSelection", continuousSelection],
["graphSelection", graphSelection],
diff --git a/client/src/reducers/singleContinuousValue.js b/client/src/reducers/singleContinuousValue.js
deleted file mode 100644
index 7bea9dfe..00000000
--- a/client/src/reducers/singleContinuousValue.js
+++ /dev/null
@@ -1,14 +0,0 @@
-const initialState = {
- singleContinuousValues: new Map(),
-};
-const singleContinuousValue = (state = initialState, action) => {
- switch (action.type) {
- case "add single continuous value":
- state.singleContinuousValues.set(action.field, action.value);
- return state;
- default:
- return state;
- }
-};
-
-export default singleContinuousValue;
From 727af831529c846baf486ff7c026320ea736cdbf Mon Sep 17 00:00:00 2001
From: Severiano Badajoz
Date: Wed, 28 Oct 2020 15:35:28 -0700
Subject: [PATCH 47/73] remove conditional rendering cases from color legend
(#1952)
* Revert "Remove Continuous vars with 1 value from histogram, add to info drawer (#1927)"
This reverts commit 242546371b643031d99f3be8fab3ba05f80bc1f4.
* remove conditional rendering cases
* ignore pointer events
Co-authored-by: Madison Dunitz
---
.../src/components/continuousLegend/index.js | 83 ++++++-------------
1 file changed, 25 insertions(+), 58 deletions(-)
diff --git a/client/src/components/continuousLegend/index.js b/client/src/components/continuousLegend/index.js
index c1153b22..38de34f1 100644
--- a/client/src/components/continuousLegend/index.js
+++ b/client/src/components/continuousLegend/index.js
@@ -11,20 +11,20 @@ import {
// create continuous color legend
// http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
-const continuous = (selectorId, colorscale, colorAccessor) => {
- const legendheight = 200;
- const legendwidth = 80;
+const continuous = (selectorId, colorScale, colorAccessor) => {
+ const legendHeight = 200;
+ const legendWidth = 80;
const margin = { top: 10, right: 60, bottom: 10, left: 2 };
const canvas = d3
.select(selectorId)
- .style("height", `${legendheight}px`)
- .style("width", `${legendwidth}px`)
+ .style("height", `${legendHeight}px`)
+ .style("width", `${legendWidth}px`)
.append("canvas")
- .attr("height", legendheight - margin.top - margin.bottom)
+ .attr("height", legendHeight - margin.top - margin.bottom)
.attr("width", 1)
- .style("height", `${legendheight - margin.top - margin.bottom}px`)
- .style("width", `${legendwidth - margin.left - margin.right}px`)
+ .style("height", `${legendHeight - margin.top - margin.bottom}px`)
+ .style("width", `${legendWidth - margin.left - margin.right}px`)
.style("position", "absolute")
.style("top", `${margin.top + 1}px`)
.style("left", `${margin.left + 1}px`)
@@ -37,18 +37,18 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
const ctx = canvas.getContext("2d");
- const legendscale = d3
+ const legendScale = d3
.scaleLinear()
- .range([1, legendheight - margin.top - margin.bottom])
+ .range([1, legendHeight - margin.top - margin.bottom])
.domain([
- colorscale.domain()[1],
- colorscale.domain()[0],
+ colorScale.domain()[1],
+ colorScale.domain()[0],
]); /* we flip this to make viridis colors dark if high in the color scale */
// image data hackery based on http://bl.ocks.org/mbostock/048d21cf747371b11884f75ad896e5a5
- const image = ctx.createImageData(1, legendheight);
- d3.range(legendheight).forEach((i) => {
- const c = d3.rgb(colorscale(legendscale.invert(i)));
+ const image = ctx.createImageData(1, legendHeight);
+ d3.range(legendHeight).forEach((i) => {
+ const c = d3.rgb(colorScale(legendScale.invert(i)));
image.data[4 * i] = c.r;
image.data[4 * i + 1] = c.g;
image.data[4 * i + 2] = c.b;
@@ -66,20 +66,20 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
});
*/
- const legendaxis = d3
- .axisRight(legendscale)
+ const legendAxis = d3
+ .axisRight(legendScale)
.ticks(6)
.tickFormat(
d3.format(
- legendscale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+ legendScale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
)
);
const svg = d3
.select(selectorId)
.append("svg")
- .attr("height", `${legendheight}px`)
- .attr("width", `${legendwidth}px`)
+ .attr("height", `${legendHeight}px`)
+ .attr("width", `${legendWidth}px`)
.style("position", "absolute")
.style("left", "0px")
.style("top", "0px");
@@ -89,16 +89,16 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
.attr("class", "axis")
.attr(
"transform",
- `translate(${legendwidth - margin.left - margin.right + 3},${margin.top})`
+ `translate(${legendWidth - margin.left - margin.right + 3},${margin.top})`
)
- .call(legendaxis);
+ .call(legendAxis);
// text label for the y axis
svg
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 2)
- .attr("x", 0 - legendheight / 2)
+ .attr("x", 0 - legendHeight / 2)
.attr("dy", "1em")
.style("text-anchor", "middle")
.style("fill", "white")
@@ -110,24 +110,7 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
colors: state.colors,
}))
class ContinuousLegend extends React.Component {
- constructor(props) {
- super(props);
- this.ref = null;
- this.state = {
- colorAccessor: null,
- colorScale: null,
- };
- }
-
- componentDidMount() {
- this.updateState(null);
- }
-
- componentDidUpdate(prevProps) {
- this.updateState(prevProps);
- }
-
- async updateState(prevProps) {
+ async componentDidUpdate(prevProps) {
const { annoMatrix, colors } = this.props;
if (!colors || !annoMatrix) return;
@@ -161,35 +144,19 @@ class ContinuousLegend extends React.Component {
);
}
}
-
- this.setState({
- colorAccessor,
- colorScale: colorTable.scale,
- });
}
}
render() {
- const { colorAccessor, colorScale } = this.state;
- if (
- colorScale?.domain &&
- colorScale.domain()[1] === colorScale.domain()[0]
- ) {
- /* it's a single value, not a distribution, min max are the same */
- return null;
- }
return (
{
- this.ref = ref;
- }}
style={{
- display: colorAccessor ? "inherit" : "none",
position: "absolute",
left: 8,
top: 35,
zIndex: 1,
+ pointerEvents: "none",
}}
/>
);
From 6a1e5f71be03e6dc9b3f5d4028bf0b2cb94897c5 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Thu, 29 Oct 2020 11:05:23 -0700
Subject: [PATCH 48/73] fix race condition in test_oauth (#1956)
---
server/common/utils/type_conversion_utils.py | 6 +++---
server/test/unit/auth/test_oauth.py | 14 ++++++++------
server/test/unit/cli/test_launch.py | 1 -
3 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py
index 762dccb2..90ceabbb 100644
--- a/server/common/utils/type_conversion_utils.py
+++ b/server/common/utils/type_conversion_utils.py
@@ -129,9 +129,9 @@ def can_cast_to_int32(dtype, array_values=None):
return True
ii32 = np.iinfo(np.int32)
if (
- not ordered_array_values.empty
- and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
- or ordered_array_values.empty
+ not ordered_array_values.empty
+ and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+ or ordered_array_values.empty
):
return True
return False
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index 43bdf1c2..b5214767 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -73,13 +73,15 @@ def launch_mock_oauth():
class AuthTest(unittest.TestCase):
- def setUp(self):
- self.dataset_dataroot = FIXTURES_ROOT
- self.mock_oauth_process = Process(target=launch_mock_oauth)
- self.mock_oauth_process.start()
+ @classmethod
+ def setUpClass(cls):
+ cls.dataset_dataroot = FIXTURES_ROOT
+ cls.mock_oauth_process = Process(target=launch_mock_oauth)
+ cls.mock_oauth_process.start()
- def tearDown(self):
- self.mock_oauth_process.terminate()
+ @classmethod
+ def tearDownClass(cls):
+ cls.mock_oauth_process.terminate()
def auth_flow(self, app_config, cookie_key=None):
diff --git a/server/test/unit/cli/test_launch.py b/server/test/unit/cli/test_launch.py
index 5569cfae..09906abe 100644
--- a/server/test/unit/cli/test_launch.py
+++ b/server/test/unit/cli/test_launch.py
@@ -20,7 +20,6 @@ class CLIPLaunchTests(unittest.TestCase):
def tearDownClass(cls) -> None:
shutil.rmtree(cls.tmp_dir)
-
def test_dump_default_config(self):
os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
From 3b6c46ba8699bd9dabd627b8f4b3cd191ea03f4b Mon Sep 17 00:00:00 2001
From: Madison Dunitz
Date: Fri, 30 Oct 2020 10:47:12 -0500
Subject: [PATCH 49/73] Fix dependency issues in compatibility tests (#1951)
* update reqs
* pin scanpy
* merge in fix for race conditions
---
.github/workflows/compatibility_tests.yml | 2 +-
server/requirements-dev.txt | 1 -
server/requirements-prepare.txt | 1 -
server/requirements.txt | 7 ++++---
server/test/unit/auth/test_oauth.py | 1 -
5 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml
index 99a3698d..fff70efa 100644
--- a/.github/workflows/compatibility_tests.yml
+++ b/.github/workflows/compatibility_tests.yml
@@ -28,7 +28,7 @@ jobs:
continue-on-error: true
strategy:
matrix:
- python-version: [3.6, 3.7, 3.8]
+ python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4]
test-suite: [smoke-test, smoke-test-annotations]
steps:
diff --git a/server/requirements-dev.txt b/server/requirements-dev.txt
index 0882128a..9f4fd724 100644
--- a/server/requirements-dev.txt
+++ b/server/requirements-dev.txt
@@ -6,6 +6,5 @@ parameterized>=0.7.0
psycopg2-binary>=2.8.5
pytest>=3.6.3
python-jose>=3.2.0
-scanpy>=1.4.6
twine>=1.12.1
-r requirements.txt
diff --git a/server/requirements-prepare.txt b/server/requirements-prepare.txt
index 630015df..254f827b 100644
--- a/server/requirements-prepare.txt
+++ b/server/requirements-prepare.txt
@@ -1,3 +1,2 @@
-scanpy>=1.3.7
python-igraph
louvain>=0.6
diff --git a/server/requirements.txt b/server/requirements.txt
index d288aee8..7aeec71a 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -13,12 +13,13 @@ flatten-dict>=0.2.0
fsspec>=0.4.4,<0.8.0
gunicorn>=20.0.4
numba>=0.49.1
-numpy>=1.16.0
+numpy>=1.15.0
packaging>=20.0
-pandas>=0.24.2
+pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
PyYAML>=5.3
-scipy>=1.3.0
+scipy>=1.0
requests>=2.22.0
tiledb>=0.5.9,>=0.6.2
s3fs==0.4.2
+scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
sqlalchemy>=1.3.18
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index b5214767..be6ab850 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -62,7 +62,6 @@ def jwks():
data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
return make_response(jsonify(dict(keys=[data])))
-
# The port that the mock oauth server will listen on
PORT = random.randint(10000, 12000)
From b9e132a00cc4b3901fcf7b896e751eccbdcaae50 Mon Sep 17 00:00:00 2001
From: bmccandless
Date: Sun, 1 Nov 2020 12:36:38 -0800
Subject: [PATCH 50/73] Updates due dependency version changes. (#1960)
* Updates due dependency version changes.
h5py recently changes and now values once returned as str are now returned as bytes.
This would have caused a much larger change, so instead the version is restricted to <3.0.0.
This caused the bulk of the testing failues.
A few other changes were needed to make a few other tests pass.
#1959
---
server/common/annotations/hosted_tiledb.py | 12 +++++--
server/requirements.txt | 1 +
server/test/unit/auth/test_oauth.py | 36 +++++++++++++++----
.../unit/common/test_writable_annotation.py | 8 +++--
4 files changed, 44 insertions(+), 13 deletions(-)
diff --git a/server/common/annotations/hosted_tiledb.py b/server/common/annotations/hosted_tiledb.py
index 54f7dac2..235f8a16 100644
--- a/server/common/annotations/hosted_tiledb.py
+++ b/server/common/annotations/hosted_tiledb.py
@@ -34,6 +34,12 @@ class AnnotationsHostedTileDB(Annotations):
f"{unsanitary_original_category_names} are not valid category names, please resubmit"
)
+ def get_user_name(self):
+ return current_app.auth.get_user_name()
+
+ def get_user_id(self):
+ return current_app.auth.get_user_id()
+
def is_safe_collection_name(self, name):
"""
return true if this is a safe collection name
@@ -48,7 +54,7 @@ class AnnotationsHostedTileDB(Annotations):
self.CXG_ANNO_COLLECTION = name
def read_labels(self, data_adaptor):
- user_id = current_app.auth.get_user_id()
+ user_id = self.get_user_id()
if user_id is None:
return
dataset_name = data_adaptor.get_location()
@@ -103,8 +109,8 @@ class AnnotationsHostedTileDB(Annotations):
return new_df
def write_labels(self, df, data_adaptor):
- auth_user_id = current_app.auth.get_user_id()
- user_name = current_app.auth.get_user_name()
+ auth_user_id = self.get_user_id()
+ user_name = self.get_user_name()
timestamp = time.time()
dataset_location = data_adaptor.get_location()
dataset_id = self.db.get_or_create_dataset(dataset_location)
diff --git a/server/requirements.txt b/server/requirements.txt
index 7aeec71a..537abb04 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -12,6 +12,7 @@ flatbuffers>=1.11.0
flatten-dict>=0.2.0
fsspec>=0.4.4,<0.8.0
gunicorn>=20.0.4
+h5py<3.0.0 # h5py returns bytes instead of str, which breaks many assumptions
numba>=0.49.1
numpy>=1.15.0
packaging>=20.0
diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py
index be6ab850..caa13968 100644
--- a/server/test/unit/auth/test_oauth.py
+++ b/server/test/unit/auth/test_oauth.py
@@ -62,22 +62,44 @@ def jwks():
data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
return make_response(jsonify(dict(keys=[data])))
-# The port that the mock oauth server will listen on
-PORT = random.randint(10000, 12000)
-
# function to launch the mock oauth server
-def launch_mock_oauth():
- mock_oauth_app.run(port=PORT)
+def launch_mock_oauth(mock_port):
+ mock_oauth_app.run(port=mock_port)
class AuthTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
+ # The port that the mock oauth server will listen on
+ cls.mock_port = random.randint(10000, 12000)
cls.dataset_dataroot = FIXTURES_ROOT
- cls.mock_oauth_process = Process(target=launch_mock_oauth)
+ cls.mock_oauth_process = Process(target=launch_mock_oauth, args=(cls.mock_port,))
cls.mock_oauth_process.start()
+ # Verify that the mock oauth server is ready (accepting requests) before starting the tests.
+
+ # The following lines are polling until the mock server is ready.
+ # The issue is we are starting a mock oauth server, then we are starting a cellxgene server,
+ # which will start making requests to the mock oauth server.
+ # So there is a race condition because the mock oauth server needs to be ready before it gets requests.
+ # We check to see if it is ready, and if not we wait 1 second, then try again.
+ # If it gets to 5 seconds, which is shouldn't, we assume something has gone wrong and fail the test.
+ server_okay = False
+ for _ in range(5):
+ try:
+ response = requests.get(f"http://localhost:{cls.mock_port}/.well-known/jwks.json")
+ if response.status_code == 200:
+ server_okay = True
+ break
+ except: # noqa: E722
+ pass
+
+ # wait one second and try again
+ time.sleep(1)
+
+ assert(server_okay)
+
@classmethod
def tearDownClass(cls):
cls.mock_oauth_process.terminate()
@@ -87,7 +109,7 @@ class AuthTest(unittest.TestCase):
app_config.update_server_config(
app__api_base_url="local",
authentication__type="oauth",
- authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}",
+ authentication__params_oauth__oauth_api_base_url=f"http://localhost:{self.mock_port}",
authentication__params_oauth__client_id="mock_client_id",
authentication__params_oauth__client_secret="mock_client_secret",
authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py
index 0e369e56..fda910aa 100644
--- a/server/test/unit/common/test_writable_annotation.py
+++ b/server/test/unit/common/test_writable_annotation.py
@@ -129,9 +129,11 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
with self.assertRaises(KeyError):
self.annotation_put_fbs(fbs_bad)
- @patch("server.common.annotations.hosted_tiledb.current_app")
- def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
- mock_user_id.auth.get_user_id.return_value = "1234"
+ @patch("server.common.annotations.hosted_tiledb.AnnotationsHostedTileDB.get_user_id")
+ @patch("server.common.annotations.hosted_tiledb.AnnotationsHostedTileDB.get_user_name")
+ def test_write_labels_stores_df_as_tiledb_array(self, mock_user_name, mock_user_id):
+ mock_user_id.return_value = "1234"
+ mock_user_name.return_value = "user1234"
self.annotations.write_labels(self.df, self.data)
# get uri
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
From 78176f971159804da93a01aa60012a34d670bd62 Mon Sep 17 00:00:00 2001
From: Marcus Kinsella
Date: Mon, 2 Nov 2020 08:26:37 -0800
Subject: [PATCH 51/73] Add schema subcommand (#1939)
Add the `cellxgene schema apply` and `cellxgene schema validate` subcommands.
The first takes an h5ad file and a yaml with config information and produces a new h5ad that follows the cellxgene data integration schema.
The second takes an h5ad and checks if it follows the schema version written into its metadata.
Both are currently marked as "experimental" as the primary intended users are still at CZI.
---
MANIFEST.in | 2 +
dev_docs/schema_guide.md | 175 +++++++
server/cli/cli.py | 2 +
server/cli/schema.py | 72 +++
server/converters/schema/__init__.py | 0
server/converters/schema/gene_symbol.py | 208 +++++++++
.../schema/hgnc_complete_set.txt.gz | Bin 0 -> 3562345 bytes
server/converters/schema/ontology.py | 86 ++++
server/converters/schema/remix.py | 264 +++++++++++
.../schema/schema_definitions/1_0_0.yaml | 95 ++++
server/converters/schema/validate.py | 236 ++++++++++
server/test/fixtures/hgnc_example.txt.gz | Bin 0 -> 1490 bytes
.../schema_test_data/generate_test_data.sh | 139 ++++++
server/test/fixtures/test_bad_config.yaml | 34 ++
server/test/fixtures/test_config.yaml | 34 ++
.../test/unit/converters/schema/__init__.py | 0
.../converters/schema/test_gene_symbol.py | 56 +++
.../unit/converters/schema/test_ontology.py | 129 ++++++
.../test/unit/converters/schema/test_remix.py | 257 +++++++++++
.../unit/converters/schema/test_validate.py | 435 ++++++++++++++++++
20 files changed, 2224 insertions(+)
create mode 100644 dev_docs/schema_guide.md
create mode 100644 server/cli/schema.py
create mode 100644 server/converters/schema/__init__.py
create mode 100644 server/converters/schema/gene_symbol.py
create mode 100644 server/converters/schema/hgnc_complete_set.txt.gz
create mode 100644 server/converters/schema/ontology.py
create mode 100644 server/converters/schema/remix.py
create mode 100644 server/converters/schema/schema_definitions/1_0_0.yaml
create mode 100644 server/converters/schema/validate.py
create mode 100644 server/test/fixtures/hgnc_example.txt.gz
create mode 100755 server/test/fixtures/schema_test_data/generate_test_data.sh
create mode 100644 server/test/fixtures/test_bad_config.yaml
create mode 100644 server/test/fixtures/test_config.yaml
create mode 100644 server/test/unit/converters/schema/__init__.py
create mode 100644 server/test/unit/converters/schema/test_gene_symbol.py
create mode 100644 server/test/unit/converters/schema/test_ontology.py
create mode 100644 server/test/unit/converters/schema/test_remix.py
create mode 100644 server/test/unit/converters/schema/test_validate.py
diff --git a/MANIFEST.in b/MANIFEST.in
index 7d059a83..90913b0c 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,3 +3,5 @@ recursive-include server/common/web/static *
include server/requirements.txt
include server/requirements-prepare.txt
+include server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/schema_definitions *
diff --git a/dev_docs/schema_guide.md b/dev_docs/schema_guide.md
new file mode 100644
index 00000000..cd70611a
--- /dev/null
+++ b/dev_docs/schema_guide.md
@@ -0,0 +1,175 @@
+# Cellxgene Schema Guide
+
+Datasets included in the [data portal](https://cellxgene.cziscience.com/) and hosted cellxgene need to follow the schema
+described [here](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md). That
+schema defines some required fields, requirements about feature labels, and some optional fields that mostly help with
+presentation.
+
+The number of fields is rather low, and we expect that information needed to populate those fields should either already
+be present in datasets prepared by a submitter or be easy to obtain. However, this still leaves the task of actually
+manipulating the dataset so that it follows the schema: adjusting field names, ensuring proper ontologies are used,
+converting gene symbols to a common set, etc. This can be tedious and error-prone, and at the beginning of the hosted
+cellxgene project, this was always done with engineering support. As we increase the rate at which we add data, we want
+to eliminate the need for engineering support so that ultimately submitters themselves can create files that follow the
+schema.
+
+## `cellxgene schema apply`
+
+To enable this, we have a new cellxgene subcommand, `cellxgene schema`, that handles applying and verifying the schema.
+Its first subcommand, `cellxgene schema apply`, takes three inputs:
+
+1. A source h5ad file. The input needs to be an AnnData file, so if a submitter has, say, a serialized Seurat or
+ SingleCellExperiment object, it needs to be converted to AnnData first. This can be done with
+ [sceasy](https://github.com/cellgeni/sceasy) or via
+ [Seurat](https://satijalab.org/seurat/v3.1/conversion_vignette.html).
+2. A configuration yaml file that describes the fields to add and conversions to apply (see below).
+3. A name for the new h5ad file that should follow the schema.
+
+### Configuration yaml
+
+The configuration yaml file describes how to apply the schema. This is an example of a "skeleton" yaml that has all the
+fields required for the 1.0.0 schema but is not yet filled in with any logic:
+
+```
+uns:
+ version:
+ corpora_schema_version: 1.0.0
+ corpora_encoding_version: 0.1.0
+ contributors:
+ title:
+ layer_descriptions:
+ preprint_doi:
+ publication_doi:
+ organism_ontology_term_id:
+obs:
+ tissue_ontology_term_id:
+ assay_ontology_term_id:
+ disease_ontology_term_id:
+ cell_type_ontology_term_id:
+ sex:
+ ethnicity_ontology_term_id:
+ development_stage_ontology_term_id:
+fixup_gene_symbols:
+```
+
+#### Unstructured metadata
+The first section is `uns`, which includes metadata fields that describe the whole dataset (see
+[here](https://anndata.readthedocs.io/en/latest/) for further description of `uns` and `obs`.).
+
+The first line is `version`, which is required for most of our tooling to work. The schema version is set at
+1.0.0 in the example above, but of course for future versions that should be changed.
+
+Next is `contributors` which describes who is adding the dataset to the portal. If you consult the schema, you see that
+contributors is a list where each element can have `name`, `email`, and `institution`. So when filled out, the
+`contributors` field should look like this:
+
+```
+contributors:
+ - name: Mary B. Scientist
+ email: mbs@singlecell.edu
+ institution: Single-Cell University
+ - name: Robert J. Scientist
+ email: rjs@usingle.edu
+ institution: University of Single Cell
+```
+
+`title` is the name of the dataset, and is just a string that gets displayed in the portal and cellxgene to identify the
+dataset.
+
+`layer_descriptions` is free text descriptions of the different
+[layers](https://anndata.readthedocs.io/en/latest/anndata.AnnData.layers.html) of the AnnData file. It should look like
+this when complete, depending on what layers are present:
+```
+layer_descriptions:
+ X: CPM and logged
+ raw.X: raw
+```
+Note that one of the layers needs to be "raw", that is, the AnnData file must contain raw counts.
+
+The two DOI fields are optional but can be included if the dataset is associated with a publication or preprint. Note
+that the DOI should be a full url:
+```
+publication_doi: https://doi.org/10.1073%2Fpnas.83.15.5372
+```
+
+Finally, the `organism_ontology_term_id` field is the species of the donor organism from the NCBITaxon ontology. The
+value for _Homo sapiens_ is `NCBITaxon:9606`:
+```
+organism_ontology_term_id: NCBITaxon:9606
+```
+Note that the schema also requires a human-readable `organism` field, but this doesn't need to be included in the yaml.
+When the `cellxgene schema apply` script encounters an ontology field, it looks up the label for the term(s) and inserts it
+into the appropriate field.
+
+
+#### Observation metadata
+The next section is `obs`, which is metadata than can vary for each observation (and "observation" usually means cell).
+These fields are all ontology fields except for `sex`, which has its own enumerated set of permitted values.
+
+There are two ways to fill in the `obs` fields. The first is useful when there is only one value for all the
+observations in the dataset. This is not uncommon, for example all cells often come from the same assay. In that case
+just insert the ontology term:
+```
+assay_ontology_term_id: EFO:0009922
+```
+
+The second is for when there is an existing field in the dataset that needs to be mapped to the schema field. For
+example, the submitter may have included cell type annotations in a field called `CellType`, and those annotations may
+just be free text. This doesn't follow the schema because it needs to be in `cell_type_ontology_term_id` and
+`cell_type`, and it needs ontology terms and labels, not just any text. In that case the field can be a dictionary:
+
+```
+cell_type_ontology_term_id:
+ CellType:
+ t-cell: CL:0000084
+ b-cell: CL:0000236
+```
+
+This will look at the `obs.CellType` field in the dataset, and where it has the value "t-cell", it will insert
+`CL:0000084` into `cell_type_ontology_term_id` and its label `T cell` into `cell_type`.
+
+Now there are often situations where there is no valid ontology term for some field. For example, the dataset may have
+been produced via an assay not present in `EFO`. Or, a particular cell type may have no entry in `CL`. In that case, a
+free text description can be used in the `ontology_term_id` field:
+
+```
+assay_ontology_term_id: Sci-Plex
+cell_type_ontology_term_id:
+ CellType:
+ t-cell: CL:0000084
+ b-cell: CL:0000236
+ new cell type: new cell type
+```
+
+In these cases, the `cellxgene schema apply` script will leave the ontology field blank and move the free text
+description into the label field. So the `assay_ontology_term_id` in the new dataset would be `""` but `assay` would be
+`Sci-Plex`.
+
+
+#### Gene symbol harmonization
+
+The last section describes how gene symbol conversion should be applied to each of the layers. This is similar to the
+`layer_descriptions` field above, but there are only three permitted values: `raw`, `log1p`, and `sqrt`:
+
+```
+fixup_gene_symbols:
+ X: log1p
+ raw.X: raw
+```
+
+This tells the script how each each layer was transformed from raw values that can be directly summed. `raw` means that
+the layer contains raw counts or some linear tranformation of raw counts. `log1p` means that the layer has `log(X + 1)`
+for each the raw `X` values. `sqrt` means `sqrt(X)` (this is not common). For layers produced by Seurat's normalization
+or SCTransform functions, the correct choice is usually `log1p`.
+
+
+### `cellxgene schema validate`
+
+The next `cellxgene schema` subcommand is `cellxgene schema validate`, and it validates that a given h5ad follows a
+version of the schema. It accepts two parameters:
+
+1. The h5ad file to check
+2. The version of the schema to check against.
+
+If the validation succeeds, the command will have a zero exit code. If it does not, it will have a non-zero exit code
+and will print validation failure messages.
diff --git a/server/cli/cli.py b/server/cli/cli.py
index dc3e5837..f8f7fde6 100644
--- a/server/cli/cli.py
+++ b/server/cli/cli.py
@@ -4,6 +4,7 @@ from .convert_to_cxg import convert_to_cxg
from .launch import launch
from .prepare import prepare
from .upgrade import log_upgrade_check
+from .schema import schema_cli
from .. import __version__
@@ -31,3 +32,4 @@ def cli(upgrade_check):
cli.add_command(launch)
cli.add_command(prepare)
cli.add_command(convert_to_cxg)
+cli.add_command(schema_cli)
diff --git a/server/cli/schema.py b/server/cli/schema.py
new file mode 100644
index 00000000..5f16ec64
--- /dev/null
+++ b/server/cli/schema.py
@@ -0,0 +1,72 @@
+import click
+
+from server.converters.schema import remix, validate
+
+
+@click.group(
+ name="schema",
+ subcommand_metavar="COMMAND ",
+ short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
+ context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+def schema_cli():
+ try:
+ import scanpy # noqa: F401
+ except ImportError:
+ raise click.ClickException(
+ "[cellxgene] cellxgene schema requires scanpy"
+ )
+
+
+@click.command(
+ name="apply",
+ short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
+ help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
+ "h5ad file, apply the schema changes and create a new, conforming h5ad.",
+)
+@click.option(
+ "--source-h5ad",
+ help="Input h5ad file.",
+ nargs=1,
+ required=True,
+ type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+ "--remix-config",
+ help="Config yaml with information on how to apply the schema.",
+ nargs=1,
+ required=True,
+ type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+ "--output-filename",
+ help="Filename for the new, schema-conforming h5ad file.",
+ required=True,
+ nargs=1
+)
+def schema_apply(source_h5ad, remix_config, output_filename):
+ remix.apply_schema(source_h5ad, remix_config, output_filename)
+
+
+@click.command(
+ name="validate",
+ short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
+)
+@click.argument(
+ "h5ad",
+ nargs=1,
+ type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+ "--shallow",
+ help="When true, just check that the correct version information is present.",
+ default=False,
+ show_default=True,
+ is_flag=True,
+)
+def schema_validate(h5ad, shallow):
+ validate.validate(h5ad, shallow)
+
+
+schema_cli.add_command(schema_apply)
+schema_cli.add_command(schema_validate)
diff --git a/server/converters/schema/__init__.py b/server/converters/schema/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/server/converters/schema/gene_symbol.py b/server/converters/schema/gene_symbol.py
new file mode 100644
index 00000000..00dd386c
--- /dev/null
+++ b/server/converters/schema/gene_symbol.py
@@ -0,0 +1,208 @@
+"""Helpers for converting and checking HGNC gene symbols."""
+
+import argparse
+import enum
+import logging
+import os
+import re
+import numpy as np
+import pandas as pd
+
+
+def get_upgraded_var_index(var, hgnc_path=None):
+ """Given an anndata var dataframe, return a new index for the dataframe
+ where human gene symbols have been upgraded to the current HGNC set.
+ """
+
+ if not hgnc_path:
+ hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
+
+ hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
+
+ return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
+
+
+class SymbolStatus(enum.Enum):
+ """The status of a symbol in the HGNC database.
+
+ APPROVED: Currently a valid symbol
+ WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
+ not to exist _unless_ that symbol is also approved
+ AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
+ multiple approved symbols
+ UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
+ symbol
+ UNKNOWN: A symbol that does not appear in HGNC
+ """
+
+ APPROVED = 1
+ WITHDRAWN = 2
+ AMBIGUOUS = 3
+ UPGRADABLE = 4
+ UNKNOWN = 5
+
+
+class HGNCSymbolChecker:
+ """Handle checking and correcting HGNC symbols."""
+
+ def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
+ self.approved_symbols = approved_symbols
+ self.withdrawn_symbols = withdrawn_symbols
+ self.ambiguous_symbols = ambiguous_symbols
+ self.symbol_map = symbol_map
+
+ def print_symbol_map(self):
+ """Print out a map from old symbol to new symbol."""
+
+ for symbol_pair in self.symbol_map.items():
+ print("\t".join(symbol_pair))
+
+ def check_symbol(self, symbol):
+ """See if a symbol if approved or something else."""
+ if symbol in self.approved_symbols:
+ return SymbolStatus.APPROVED
+
+ if symbol in self.withdrawn_symbols:
+ return SymbolStatus.WITHDRAWN
+
+ if symbol in self.ambiguous_symbols:
+ return SymbolStatus.AMBIGUOUS
+
+ if symbol in self.symbol_map:
+ return SymbolStatus.UPGRADABLE
+
+ return SymbolStatus.UNKNOWN
+
+ def upgrade_symbol(self, symbol):
+ """Return the approved symbol for the given symbol.
+
+ If the symbol cannot be upgraded, just return the original symbol.
+ """
+
+ fixed_symbol, stripped_symbol = format_symbol(symbol)
+
+ if fixed_symbol in self.approved_symbols:
+ return fixed_symbol
+ elif fixed_symbol in self.symbol_map:
+ return self.symbol_map[fixed_symbol]
+ elif stripped_symbol in self.approved_symbols:
+ return stripped_symbol
+ elif stripped_symbol in self.symbol_map:
+ return self.symbol_map[stripped_symbol]
+
+ return symbol
+
+ @classmethod
+ def from_hgnc_records(cls, hgnc_dataset_path):
+ """Parse a hgnc database download into a HGNCSymbolChecker object."""
+
+ def all_symbols(record):
+ """Get all the symbols associated with an HGNC record including previous, alias,
+ and approved."""
+ yield format_symbol(record["symbol"])[0]
+ for symbol in alias_and_previous_symbols(record):
+ yield symbol
+
+ def alias_and_previous_symbols(record):
+ """Get alias and previous symbols from an HGNC record."""
+ for field in ("alias_symbol", "prev_symbol"):
+ if record[field] is not np.nan:
+ for symbol in record[field].split("|"):
+ yield format_symbol(symbol)[0]
+
+ hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
+
+ # Get all symbols that are currently approved.
+ approved_symbols = set()
+ for record in hgnc_records:
+ if record["status"] == "Approved":
+ approved_symbols.add(format_symbol(record["symbol"])[0])
+
+ # Get all symbols that have been withdrawn
+ withdrawn_symbols = set()
+ for record in hgnc_records:
+ if record["status"] == "Entry Withdrawn":
+ for symbol in all_symbols(record):
+ withdrawn_symbols.add(symbol)
+
+ # If a symbol is both approved and withdrawn, be optimistic and call it approved
+ logging.warning(
+ f"Some symbols are simulaneously withdrawn and approved\n"
+ f"We will treat them at approved:\n"
+ f"{withdrawn_symbols.intersection(approved_symbols)}"
+ )
+ withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
+
+ # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
+ alias_previous_to_approved = {}
+ ambiguous_symbols = set()
+
+ for record in hgnc_records:
+ if record["status"] == "Approved":
+
+ # The approved symbol is what we'll map to
+ approved_symbol = format_symbol(record["symbol"])[0]
+
+ for symbol in alias_and_previous_symbols(record):
+
+ # If the alias or previous symbol is also an approved symbol,
+ # we'll just leave it alone
+ if symbol in approved_symbols:
+ continue
+
+ # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
+ if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
+ ambiguous_symbols.add(symbol)
+ else:
+ alias_previous_to_approved[symbol] = approved_symbol
+
+ # Remove all the ambiguous symbols from the map
+ for ambiguous_symbol in ambiguous_symbols:
+ alias_previous_to_approved.pop(ambiguous_symbol)
+
+ return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
+
+
+def format_symbol(symbol):
+ """HGNC rules say symbols should all be upper case except for C#orf#. However, case is
+ variable in both alias and previous symbols as well as in the symbols we get in
+ submissions. So, upper case everything except for the one situation where mixed-case
+ is allowed, which are the genes like C2orf157.
+
+ Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
+ names persist throughout the life of the object. They won't match against the HGNC database
+ and we want to merge them, so we need to strip off the suffix and try matching again.
+
+ This function takes a symbol and returns the symbol with the fixed case and also with the
+ seurat/scanpy suffix stripped off.
+ """
+
+ match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
+
+ if match:
+ fixed_case = f"C{match.group(2)}orf{match.group(4)}"
+ else:
+ fixed_case = symbol.upper()
+
+ suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
+
+ return fixed_case, suffix_stripped
+
+
+def main():
+ """When called as main, parse a given hgnc download and print out a map from old to new
+ symbol.
+ """
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
+ )
+ args = parser.parse_args()
+
+ hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
+
+ hgnc_symbol_checker.print_symbol_map()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/server/converters/schema/hgnc_complete_set.txt.gz b/server/converters/schema/hgnc_complete_set.txt.gz
new file mode 100644
index 0000000000000000000000000000000000000000..29c3c7a97cf51e8515a8becdf01c49f30ac2390d
GIT binary patch
literal 3562345
zcmV(*K;FL}iwFpqBUxVp188S%V_#!$ZE$R5bY)+2Wppldcys`?TU&D)N4I^R^D8Kx
zoGOF+e#sj`w&Lg#Y}wzJPtVX!E
zx(X|v7F9W4WaH^*nT_voag}}gGP{m%1YzWpa+FPh;Jun|t|MSz)S^r0!_4vUNq>N>B
zc;}!sn%oad!l)R}vUKbWA%@wgMGFivZv8o4-n7!UbiBG7F0)>ebjJNHmt7?{gZRU}
z^Brb7lHr43Tz8o5a9ozXK07~R^uq;LoMpd!`0&%m({##7%`D{0n7CbBb#<1VonCOs
ztTS1^e|XM?wA0Rz6#f7o>=-_h-Ep&t~NsBb>H)3Ghfk6z4KL?xt+D?VKwHP*;@6rJF9x}@bVw_UB~xc*$xt7o+Km~35#_&
z?+8sp(P6H`!%&huh0QdRTDpxKg>la7hjMha9E+~Z4*EkWQ6c-`lCiRpY&kU0d`*1FDz%d+}zb!NAmcZJ*x+P(gpRquST
z!qKSkTd&gO{WtBnV!&quW-@BK=>S6p)V#NPR^Gq~c@t3;N_ozGKox30m;xfsWW&$AmA=cu
zlDCnyNvkb{Pq#&{BTk>`bRHmzpZ1=|KH@hXUKl+D`2rJEJOVE}f;Wd;L0KFAT`0}j
zj$bfNI3EPzzl{EXAc8*-0F*!B4-zuj2CW2~l+w`GCw#wBwkG3G1@QX^brb$u!#r1W
zZW(Y1+Uc{t1ZG#Q!S90(P5H95e>qp>G!Ib2Ta9P56d%2ZUOQhJdeZOUMu05hVFX!q
z5Y-4|({R4r7%g`|7EXDewEgZ&7YH2#G7pDL!Bi;se^c*CgC@NelM0`Z8-$gBibzhNQ52il_lJ&yEmm0)++008`-so1ekaFSs#2%5x$3p4jXX
zam6*D_wuNB@iI
zGjt>z7%mr9BNT5urc=^wiv?kugYH5}C7(}jNl60_fZa1(9H@YSCt%q}c|7gNXkAXq@~RaQ{vWwkDZSPC9Gnje%Z{Pgs
zIbXK6ky#B|C14n7dSZf`vq61GsRZ$N@_1T$Wt&NB@Mk;h(yHL%j41w`
z)Y`+Xk|}9b&@W==gZ@T!+)6O89r?Y_n&>Jiv|$PStlPB%lpY_ayKj&z<;(SqVH8kc`F3cM3#>OJrK&6oKrjn3bQ$)!hQ`(i+}W<+NOs
z)1notU$H#bT8x(nY5lBR-V{}KH~Taz|D0v7XUpoo^-Ap37Fh)a;s
ztk>(E!(?(;T;8%~OHURKXZ&M7od*{&XtkEVH6y~$)m7rku39nU94
zr?<-SLc<*NsZEy+Z*b*eoFf;yssPuptd>A#(i+bech>-ET+D{|lqrFEji=+h_2S_(
ze^`C{bCO>VMJ&YsSbYS8(af)l2GWA|1)l;}V@cvciiY6{(H~<1
z^MF8Xq@EN2=G!}fgN7q6TlsK&)e7DiVqd;&T^7sy-&8-rktt#UeW)LzLK#Ctr(gDB
zOvd!z9}=K*LPg4+29$ks0rs{!|MN*_42?#?Vi68?MmIH)1D~;L#+H?hCG&sp7t0vHE+9-qxH&c@Is06+#Q_{B}
z_lU&l7}s|4#M48?V;!Uo#i5uR`3~%7Ye=k0QJn*pMep6nm6#H{m`f^_rV5c_Qa`1S
z8#rq6f^{GE`dMDxPiWbZDP9%v6`sdbLK2jYZA`N2zdqN!P$qkHy4H(gu?!UHhZwUB
z`6Mcz#Pm9JM;aAPE}OhYS<4^r8fy}-`BFe5Pox0hSSK!#dP?c6%?3h{MmuJtW40~^
zYiN3;n-c0sq$U7{L
z>$+;p3E|saqKNn?OcPS6l-12ju?I<(JFses0|{0#^n*S#+QLf!Si!6`Is!|)@tqkk
z52R^F)mu_@y=-lX#_cN34~_@$;%>JJ_u~zUl;Oz5Rg$
z8-YxqX`xC`1@VCn+GwE&bb^T|N;{4B$u{qqc%ERg)*ktH+9RJ)y?`@~<%=-bE&*`o
z>{^pd-MiecN$~6xz8f_KA_n^&7#cgnPs2@3fiuZ1i|qw2pnvc5e?I<(sn!Jq$Jg2e
z)ymeuJYGm3{5DX!bcgyoDAm6QrT+>jLqS_2)iMnxhB&FY+xNf_ee=6(@;}MTfe1;K
z(l?}L5lWFIL)47mTi37+&(^>^38mW@qK{6QV++@wv^E#KinXFASJ|$0P*4&YI8=j!XwI98$C;;I>ox^p
z9t4}}g5vJ#&(NG@2pV3c;sKRT0{@qAma&tf1(E!B?o9vAhyub@c%p!ye@GbQiaVwa
zGWNs;CA2*y0>UeoCGZW%zur~Tw&)rPzkU@5w!Kwr5UFlM4GPK%j(P{Flt`_qQ9hkw
zbd}GS5X|K`REk>9-Tk3&saGC;2XYQlVODdgHhuAWTR|$XnM8Byp`{vFQwS0Pcv}b-
zL_0Dak>7rzcGd~(-lhr;zN?ckFi6dU6sRymQZR!<8yez5Qm2fhU@d@MH^dcpzAm|$
zn1m5*VB*e24g_b7Y-?}KCY+Pqc+7`;kD(z01nR}29MV>utJbISjFk7gTad9^k+F1a
zAHVJO@D^@$c>Lz~xpC$gL_~@W-ZJ3Lw=*2*scJS2;Wq3}olJP6(vFSA^&C_64j9s1
z%hNst&tshofFWdwQ}Nb?od(t_!;#)9i+}Y_1LmD^>-1J@5!mH1bq|eV`f&nJIMyu<
z5xno|AviqjJ2=wCc+mv+ba~^Yd)QU|Zgu9jT*1ogV|(!75FoCo=}Vfq>K`*hR@dje
ztgd0VjnWV2t#V$>T2+yUNw2FMXJcr3*z_?-WP>c9jmBkDRMYXO%9mwz-x?N^$#eaVKyC_~xLxOjF`we(QP%aI8e&3Q
zWxTl$M_4C!>%^M)YtaACH=iiOKcE7A+J{3Kl})Z*O6#Jpw^}kF2-uv?U>u4Kccii(yhZ_e26)`x9J#MnSmBC!atwrW6wE()&LUhq;Xeq&}Y0WQziqmoT8c
zGerjU^lDNunp8XrB6LyW4u*B#ST_X$gWt)(sDvNMp7^6GA&8I2{gqvdAw^sSNEH=X
zD{jcy!!9tZ!*4L56kUa*ge~pOF+xM;Y&&YPLiu$4!|)l0}k{jcUwb{&}M^GQvYO|!An6q{(?~dx~8-`Mis5=
zs3K_t#XJ9-q$a!3!V1gRg$k{7vL&jgQZP0=Wa5w75y>jW5$RNr*dbNOBeet_%B#^h
ze5l65n?3t-p6Q?*RV?vttR9^F(gy*QAeg;9tLQXV(b=<#zJnIP$27FS?Nb7$*Q6_K
zMvS6UWOmy_8wyS;OKWOmUz+Wv7)E)+V@c$`yoT)By#v}|$o=Y|&uLeuYS-3@d=@EJ
zaFfczJHJyWuYbontk+&ipBiciLW0+&r@GwzSa2cvC1T4+j^{iG@_y1;QW>OC<^2AA
zUWU4ke0^;9Pd|vW6(sfkyuDVNFx_N43RAso4eyub90?%p9j9eIvZ_XdF78VRgg9=c
z*$h*=SzrmxrUY}lJB_SmDut=yeg+qDC`o1sf5i3(KJ^^3f%33}%xN5a+9nM&wPAH+
zQcjA{W(Mp2`ZjaAA+(p_OJS1H62T|9SIyEjRSO%gRIt=W^A6VqSq>hT<~JqQ>Q=aJ
zNG-!L3r~Oi;aiWcGjO@_Yx^Y>(ni|%P_Sr7xO#X-~G9$cWuZ(EL_}qO}ntYPnVp!N-rj-jnAN)ZM*(=lU5Xv}aoA;5<+s2Yc?-k4iR8VgEhbe>@07BE-oan_
zxba;(kaWse4nDkwj17pJGO}g~1~OP@EJm}ehK%xKE*>wv6TGS7r+o5Dhvb;z;G5jg
zuW>6?I@E}*Hxq4t*nov#Y5Z5VT%|_>-s_zo{nw^XEaj@WxexV894cDevx;?q
zjfLOsv-g8@@7?O;w{sEtu2A>(-#1>>>Hw!Z$z330fDOZrv55G$P3^{?PXu$V5rTaq
zRgl7~W_3tgCCu(-XX?lYc5-t$qb~g$jZ7da6_^HGb;{dxN`Z~DDFvoc+5oMged`f5
z@YRWct-{J3*eGW!oTVN<-)ZJj<3@A4rl*_?JW40Db9Xr^5AEDkhusJ?r5K9l40jI(
zAHG(+o%v0N0H7IX{0wq%|bpD=m8~p;xfideA=k1@7yo)yXj(+G%rTXT5^)
zu(fvqtN$nN%6{8Mp7gW+3xN-d1Q^GC9}6rNmXyO2J5hYhB=hovBFnKR@)21|CIkNQ
zTUFgnQWPE2w1Ity$k^tg{;I3%_^NvG^A?x(ud~(lmjx`lOAM_qXZSey_3rItI=z@I
zuW#^=HUx1q+|GPO-3iJm>PpItF$W`CTOeR!h8U#${`E7)U|tE~0h_QuXlnwR+B$R=
zPe(@@ZVPCDzZe~U^gm>=_&QF3mt*49@JOcNLQ3bG341c}YG9=zX2r>1*qhv3->j}{
zu}7L<)8+hT)yH`HVsdr{`ffV80x$bvnOPC%S2~u@+iCl}UJ>pMk(cU!F!U=c1CQp4
zu311J>`s(fBtsKXD=R%?cP4akXnN&frN~uKVJEMu!vHxNBQv0%N>rNJRiI|s)WS_5%|lY;Y3I=x>}wmEC^t|Al0cTlH%fOj$!dvAh;sb
z(qIW|q=!pkV9tSp`D8fJzTjp8^a=w}*eKHnYX-a=om-
zNT27$!)UFVtfCXWoPAj@LR|QE{ds*gU-fNFYk}A72{h9y;^WMhp>p5zp!};jc#K7C
zqKAH$4VL}cd_qB8EU8QsKIKKMgaVM6jYcNucT64qb@;*cGH;~{1EEAw5
z9FK@{Z!uk+$iWy03jl$_5w@~QTz~$1HeKzXA(*0Wf<_($xp;pZN1AYT_7m5%k5%9p
zMlZ>wk=)n(>slRh{DdbIF>_*O@mQkGC?#ilWQFyPHCkWvcT%b(ofukeE)`9lJ(ylF
z%6kXz-$fZ+!9p__9*6_JQxaHe%8<0=!<1%-mh$ps|J}!9SmdhD-1%4?!*H)D+&71j
zT7o``e{{Svzk(Tn4_?skvr1ZC7Fxr_8Ys@t!~n~*qB$Ie(Ka=QN9EFcI|Tm|6Yofp
zk=$>V$u&-Q8pLGn<6xfjR9Lq^uVH3AWj>yp8}d}rF?)@V%kgvP1g3NSx%5c<5fy-7
zO8K4_qg!oUmX!&c%HSqg3uw3K8SRy%)Q+HewD5S3#pB8AQSSs3Zg9^tm|rLk8N3S5-;54MC!nd|
zjdtp9^c%j?PVr99@Pg;RRSB~Nq0Xk98js2Q0<5ZUhIE@+S(QNVp1@L{rwdfSzr&J=
zGbPw_a7C!C0*|Wq-h;sb#<>HmtP_~4m+;3Xo!E2|YppA48oE}t
zMnFDJ@5$nb^JQn4zdSwx7v5-@9}Et`@5t!O$PUoRLdUsE5kh?eiE29(W4#sI!3`a?
z-OW$X&!^Ve0vZNhOwe{%hw#CJDdKt<((sZecZi!JIKjXcinByw*xp347?@3%5~yM1
zw(>9$BSm*mtucB8$&ivUEFQC0uMYFQqr;KNhwqO@;Hn>kZ~A4jJX@7&?--OJSfzvE
zQGPV$KF9yVGS-A&Stl`Dq9MA1)oHPFdo!D21yBBP$cgk6rX%In?!vrEMDWhk=uVhU
z5h26F&%Rw?{@Xb;szPT9sRQ59!qggrZvl3hSDv9Y5}7wjs06rIAU%ic&>7{ne|_Eu
zYq4MbX2UcXC*@%Dk7z7XKIq4fA1}h_>`)h?E4tfcfsKxK&*zKT?e6V#zH>GEal1SD
zJY8O{S1&GBmr+Vk&LVvr*Ht0wa^~xD7Be2`MZz6)YdO=8ad%X7ckl>Gz%z6$dL8Dy
zPM%9hMAT%UIF-s&r@mI;iq>fmlF3Men16xSIyEV(3BSa%5GF?_@`L;H&%Zz`JFhGg
zDbZ(&^-&-xq8G>0aG_a%0jwj(>NDT4IwGk%lA$}8)`*zLAtH9FxTOD~iUz|$^G@)X
zU&7;3$OPbFHF!iF9+Bb#e_#AS%?$9wKo$JXsRACX2YSb)-+%`_dL@gA8oF(S(ItFT
z)4FMR9ht;&uyn0$jg0Ny?kldF@d%Io*a-+*c=*WM+SZk^r3^8D%9x>1NGsfo09!B{
z0}zXEKPdh$u@hKOnA~$7XNmxO6QJ1t>yw}&Z1Cb0%Q;urj?3rZ(82PU3iN2@R3_nr
z24*jgF*x>$H{(kCr8qibDjeB2{vPI07|jiZ3W!!dGgF{+Z(vADXP3wKyfk|xVH
z#Atxdnuyg(7G_Dx`oSCuX&AVTE6xCw_ns)I8F~UFtOVPSnT
ze6SBkgn*`(ZiG+P;Y-~$f@kMj!{?t0K2UZbq#jF}I^gG9<5yF5weT<)h~OP3{MHng
zS3<(LQjX2ya2M*1;~YDMdBO!a)ZVCKIkLW(PZ}-MIH&|jqn}W%g^WNXK)egCf)52!
zc=pIwY;ZG2I*c<$$Kq!dFA_5>$@g9&2LC8TR*=uT(a4^dBzFhR)dv
z#ur1;D+eg?sDoe3+Q%i>6IxQngQE4F#Fyfdcgt^bZ*h(AwZ*!gB~+kdVi@?G2p*pV
zEX76_%XMi@sbEbh$`T3=wnkVZqz`fFIG7~0uB4Efy6oyLl61r)h%4vg_JHt>_BZ!2
zp>DV#CR8|PT048@|0dT(Nf~=gFIY30ZBc>YK<`%e6NKR
z`N2LWruK(NV$jRx(~H%`Ps-#>K{?H?=Br76e;-uUVI)gY#G~HUoDdt^?m^KPVZdxWguYv<|?6U`mdB=uwGh*qYLr)3;a;6{ixJ
zTVS}koDysr!+qeca$m5hU^(WZ;3;aHU=cZkZQ>D2q_85}^6Y`HM)g;
zfVQ;{(YEzKVmCbPc7HQNsX5<>Er`6)uzXedcG59lFV6;r154ECgP@v-ph_iWrQm(N
zaazh)Dl4?C)`nKKDmOuHc!~dLK2fh8kYFTIr>A&2Rx!r4Kgk6-N%cL43d2*(+LlnA73k>_dxO9iZm$J*p2MH;bCs!CphwTo|K
zWfE5ZV0#K!>!pfVDHzNwf)&e_OynMFA{>md&zSFJ!g+yyEcGLnJa&fzD2*?YEFo)&
z7$B6UTni(mxP;QyLq;kF^ArX12D&why89X!_~h}3_NYRxoyh40utiZgBYs*&GrH24
zzE8hvoQLU|S+5)NzmC-=|3WH;wMiB1l~bF!ez06@Xi&h}*EPlL4Jge(LA5ZieZ(Hz
z=m0r*#;iCie}?R4aVTGdr4*xuXeb$^J{Alj<+VvXXCsy9MIf7CxdQG%^nGWO`E2brAm(?Y|$@h-;PJ(M}yD5|y)V8r%a3lQKZ*`+qQkXhx
zf;$tUrE2#1Uy5im>3k~k0tY@~?l5h%2X`UKD7HUZkBH}&3X&|f`|ju$g2NvKnY
zBFQB7iprkEVjtKX<$`li?hmRa!z)mfvM31>M@ARRnyN=vY&|llT;iY69b9W!a*yC!
z8(P-F5rs(#$8jrx9U|WxhR^+z_s6G4V2^IpGr{kQ;?8ce=I&E9*y;{OkNb^$GqIdD
zy#$A>DY9FB%&EP`D{uf!Dtxy*Ra|S&!*&x!fg@R*`FI9CIjNu16S0}6#Ft)(`?MyN<~D7G{US9AuR)PqFlR>dOI4
zp%29R23xCBirB=&ml3g)%BPJ(v=h!0XSO(VVcD>56E|l#Ho#0qrw$n~7G2R>S<{I15=l)vzv9n+jlF-Ph|X
z0%h|RSUX4(rL7P&_DKgAq7da^s0g8K+Wcp`qZxJ&XsVY$NlEDz9*DV!r)&@!*+yt2
zHV25uUhgS{qAxpwA+su5-}df+zzQeS#wJ}{i&`Hcs~s6qIQByDdVBcVNNe7hTHytK
zmUGF{NUMgCR{u{UtvZghYB$oH>P&E3qsm}|EX(b|w&h^6Uk$c!Ml=t0$|5Qe2KWIX
zSN;eeodP)VNYrwZACF&;Lnuz<%PL|V)7t`s*<&@Z&
z=)UdZK9(8o)h+)Keqil?lHlWm{C$fqRd{&_8-*zMfyTb;_=e2AZbzbtqcnDIT#+V_@sVV>bHz
z7;zN&6h5&*-{JeclU_ERTum19=^E4Mgm0pE_hEWWumB7`U@~nO=^gPESrPrwRQ%km
zQ}|{gK}c*CQri&{Q>bKgfBkk&%wOp(O9XH)A^{xpv9MHNUpi+o*W+@JTffPY!wAoc
zy+yGCon|js?GK#qa77C94Ke$UyGj2P)Wu$HO02HionmJa#I4kofZ!Feu0
zHr`{iS>kuYaOH;wATao|ol(~7A}XSVgVrc3@;(}JXsKd1nieuw&pTiP+EgDhzr_Np
zPu%jfDFMTtL|H6Oc04vyu|%fC+;BBY?66$9%rQu#U5pZQ7V?He%)%=vDvDtX`g*cS
zMsOo
zq_MOq1gLO>lpvkV#Nf3tac^5^q&87veMMr80X?b9?Nm7DMW`z2I__k*%m!Xr=sZQW
z+?9TjTd56KPq68jW!02r9qQJCuOG2Bf{l&EZIpYRAESAL!7mG%Bntnq73HW=F52Zw
zF}r1Jdsoy$&c?;Cl03isLVRKlmWvWab}K0^>0+cA0Jc#K6uV1t5otedtU{abh0!Pc
zcvG;+)ss>>rADzlHC&!3#zXz+9T=(*^?J=W{WZhbR<;b;WEi`!S*e5}sd%Eu&Jx?B
z&q5on+v!bY7&C>ty#A7597m#8j!zVF$|4U&uR&o+(T{oC)?W4JoS|^erZnISh
zcvvhtM~#clI09j4(HolG{^9rtKT(mqeBQA_aB5`6a84OyXtzb3x9TK$eop{G;r%lDOb-urroF|`dZ;8Ni2Qn7l2RID|^)`AxNSeZzVL}ARhqkO-~g#b-yt-2$!Ea76+81F=kOSol(TXch4
zv}2c66<({w`?ouuwl&5Z%Xb)KOS$*z7;91AD*#j!k*uj*6cU>8&T&NV3?M7X?ikSI
zAX1!u=vt3u_bNGl!>RjuJZ5k9^YI(_I6gh%7)NlJ0g@(v^X3)LPu`6V!MVR(FV3&8
zelFI_*#wOA>FfquWVLP?wy$tS>zd71}cbs)Jv|Q0Nm{WJ2=H&(V
zlCILy46E-EW{gbs_vICK-97+Qm6fIBB3fCna@e>JyfAT}0E5PJIxBw$7WW6NF^A>E
zdclnJIm|etl`De|CT5MLbtZ0N%ero2%L+3b=>km0?N(7U#}b>eM~n7bsyXH+3?(Lp
zzO`sQf-iuInHNVC29XaBNBsR=BGfsG;9L4>tXj;>j90QQCmtXTaeOX;nl)vLp(DYz8XgfsLlEEkppR2gH(1B4BIn
z1cX&}27-whlejeB&p;SHo`Fzrp^Whda8!&$-D30OeLQsc!|uuN7vUq19(J|!v99f0
zjh&BWu|`~)N~nARO{n)6
z^7dv=_bR#uXnc2Q#4k(X(y3Q`qyK{fN6c+Q6Wepl&T)6Yy-zTfLa;b@a9`Gh2MciM
zYWZ8lrHEf)_n#2B0LKlh+u{HdM$Ms%hRUZ@qXDlkqP}Y%Z|AcAtTY#&-)EZ&9!NCl
zn2nEKjzCQa+#1QsJUo5;2sI~D>rO<4mrenT8wkcGWEugfHsc5NQ`2)+g^d)kToJ5@
z=oBy-!VEJq+HnKRm%vnJ>>6rt2YB6359PRMvSew@qq#xxqS3vYYxL;E=ErYfiP1;4
zcla*fI~a}guub-2w*Gp3_VaeNyuP{kiLu~=(fvS-z|%h%ZT5k?vyiD|G;i$bLtTw(
zlx8JD=Xa*g=jR|Se+lkV8#vr=Hp~@e;fsth=!ZEim=wy`943Yaf1}th&*QCP{R8Ms
z_~svX!1n$+`#;vMt+}bA**@!EsPsgtaBA+CM@znPj3Y1v!kfVkI57r{`4Uq4@uyeM
z%)V*MTaulJ?1^K^Qm?sm_jIpyG2ZR@=Ix8}oM8BO=lmKK%!@}Ud)~`e=@fi!nS!8K
zRw)RYHd-*@LgnT7?$YnxsJnav>r{G&`36&Oqx=JTwOH#SKi65?OH=A;SAC;$W%|1v
zAs?l)tMD3BS0Yu5LCOK^Hv|=Ff-EpG$bDH^Eaqb{J?pk;_&J?zr?#`*a4;`mG+900
z?;5L5^{?*XR#cT0RS#WbHFUA1S|Tm(3Ojet0^CqAGflE=X3h&|>#i`a4&L-_iY~|Y
z&Sc~2?To9}$&|=UWcoNWQ?G4P)he@7x?B0>EDm(GX)HG7eAN9mwNdoe+_`+K>TI^D
z+0Lee^>|8sKCa>Al30j=-&EUt9n4g23OfIB|KT`C7dFz38LqT+}yybao4kyjH`z&Dss;Z
z@tqut);{N0KEFp*4~tOb42yJ$FfO3N@I`z*eWoZtgjTJp{2r5=Xk1j8?RJmLaf)~p
zUrl#Ar9!t@s|Xx0P4{eX{r&g>y6Y&yVIEIBC-diXFBb3KoWaqazkNR60+hTtD{gbr
zD;w-X3l#}U{w~>OCWr^wCT`%K2PsR1f(kZYzIl_noS@-s(W{NF8HyaVKtbPvT-9hr
zXu7Dbh)()ZwzKpiMq4A*y;q9e!l}Xm?zlg4Z~a|<`|25!`J3_+P`S7uxL)!dt0k{J
z6wppKN6p-KZCbb52i0s#krCwp?GsfZi_NJ0`}H<;$BKyDUs_EflcdtoOs_-~0VaR6
z)tj)0nobn2nqbwxupG6TEdd
zAdC##>fDy2v$x4?Y~&Ip7X^D+>gDRMpTC;XTUuk8v=MB!A}g-pPGe3^6@5`}bp*jv
z)?Hu~lR`S9bCudYvRY8(SV^w$avIBX4(nhLv4aOA##vy<-j@M)5Nc`C)?mdj3t}6(1AuW
zyosvnz*Yh$Dzi2naFMuCaV09gWa%|3nfWCr$6R!EFkuqe@x5Izj?y$;p<1Uh3O1J5;9TPRVv{->-90ANMv4_C%|TxxUpvriTT?$$^(*
z%izp*_SI`~xLd1RRQ^zRHA|J2%9J|S|1yVru$jYM+p0X@<^l@}Msy-kcoBN`IadkW
z8J8%a4)p4ysIaqnPHY|yZV`tM_H?>RcJyb%Xs}T}tO3&D_{p22<5&Aj4$uHd(X};d
zz=7D6zVk6N?fiHC4K3~z5=dsIR!K1<-LUG?LV=O0MyVWvT$@}$gApCDb8=28NS}}w
z0Y`NKt_Hys%&GATGTxP!6jL;R$X=4@6%F&T+gsstX4P|M^-!&it{Oji_T%P?;)2H@
za570TUF0Xb9^__{t)9PqxtPvgax?vWK@qTK$78Yu)@yXX_q?e0d#=aO6yd8jLeSv5
zC2(3eQDkRtO9{L7Su+|vdOk01r%Kswl+2Lw`t(Jsf*st9z6wV@+^0&_$YzEny)8pi
z)y8m$T#+v7N5;jtr2N26$87H(@V7T-TFy614Q5`vm05eTnTLqa>MTBsCxc6FVbK8}
zcepiH&yezwS~@LT%qMei=|C0v8rc-v1UY)w{dGN;e)=2LR?jpftJ1kZk*-U$
z8swT-nl48W$S#EgnO7qU)VfuSt0vV892(Z94ZXA)vD@juaMs9jg7(m^jqKK)D(b7^
zj5q!J?A6)H=sThJ<(f%3w}Hhr1cX;}ZaJe&v!X*ouDq+YDsK>G&YNB1|$xa&F4TEKdIm2mu(J_XRZXPKzg4q`
z2HTxTwr3(0CT(xI{QYQ8j?Uk+3iLJLy!i6@ocoq;?&t{!DMAT@RPUUq@WECH;A+P4n_66A~
zD&=I>SBvC|%UW(a2KLHf5(lw(yIRo~*2{*3*Uzs~1BO&j$B|L
zH{}xB3O`|8H~<5mp=0t!UZa)bX!igb0X6Nm&IKe|lJVc);1ql-X>H_+WAc?{E}sL1
z^<2Pr8cmc}#)I=v;l*0I1+L%8TaWd6(Z)Z5SaHoIDmq`xw}%%Rfz>o!2kvZg7
znE{)ar{mKdER{Ae8Iu7MC%?dhUx}*@QVri8B
zeEgjDDr9g%K2}iU&sScSwzL=F)f8Q_$_A^+aYXnjgNdSACYGYqYMJ1gGuySjK=cwE
zj6qQ%2j4YbW(SO5pN_A8e*aRWzppOPum0icf_Jk=5HZ#zyI%&A+2Qey4U5y+@jh2`
z4t%}@lFs_q%OLUDZ*fL965F8GTq+7KY(VEu2aNHFri+9(Y^TVj=Oc0vPOm?
zKJ7EQ@UUOR>F*PXND-6Z_bYvMi%%ojVL_({z$QcvNTI-^CLhDoMQMNAP
zMc0=VQZb)l8k*FbI5PT%TU@hGZqa|e=xoNMoIKEBU;p$sAi(8gw61W6$od~A1&3^=(C
z6a*LfMT((oc|Yr7Hg@RjfO2^Sz?#{x*4eZJl5;ZP1B<8Qi_5R)*Wms9dHs;-|BZ#(
zMSilGK`Qnfl=+`f{@mPMcRpVK`0xW-_ZOGrO9V;ur#n=pqruN!iGkBnF+D3tgYSqO
zTGR6RI0x2QK{OBY3i`=Z!zZ*mdgV$+JmzdGEt*U#Ot6-^bI-AGen`UM*g4NJnkHqy-k#f
z!O*Iwh3zO#b7Fz&Nbe#(N^159)-dxN?)8oBV<@R$AJwy=Vsk;WTKNstKQYu=GhzSc
z?)K;S4K44B+w*^JmM28@S-NjOs2v%)%r21&QsijxpFID;<(~fdP%c5Lrc?gF$W?Ji
z10M%@Pzm16X?Iy%Jd4gAl5egS)BWehok05qA95E|eM)?}{S)cb#za-*e$ffCp
zJT{sD(~0J}=1sSURo63}Os3_BXt}f%8LmuA!Sqkmk4_IA?LQ`GpXh9IXf03VWIiW4
zEcySAF!k^3b6TiLr7)1+AZnFpV@H)$+b+klBgq()%mx}9_=Q8srtP%|L5asVYWX>N
zze{`aG59l+FleNenh*k2=}c;Z$h8pCeF(XtMF=@U*IsdrVcnK)Vyi73gjKG~8;5p;
z#2feRTXyh$kfrZA*mOgtxxV^^KT+@#iX$Oyt*fr39~Hy40I@xuJy6Jikb9FiQnNdR
z2>r%G5*I4xEmYn#tuwzt&2I0vm+xd9ELpasYX?iphNi$HZ6>Cxk8a6sxzP6F!xz}{
z51o>KY4?J&o1#{l{uH&oekLkY+ECvzf@W$|nFL;S-Sfe&S?IAS%vdDQLy48l?#}M)
z583*Ubqnn}78{LLzRUQ@>GQquhx40@tBdoy%le$PJG!b{7Wte_L&w~8;KUVe{<>}^
zz&VpN=bifduPbIyOBqt>S~7|&a+++5JkbKhsQEh#Zp3E#SQ8B~xg~#VF*}|%rx@f!
zb3vJ9vVTX*HE7qN^NY(<%#|sIn^Rh@-pXpZ`dp5$P+^B~{IwZX$DKE_%6T{wN0l+!
zldC5c3DadOy6E$K8p=gV`v6wilmpJs%50Gjp?YOXM8i-tYT3=$k7yjaBrPmSf|Bur7avKv=QE%IU|}DsoA<^h+9CP{ab1tAnTL=~iFV7hFQ94U(Rv|P3#p)ot9d6wyE
zY2OjXYrm4qu=>8maprqm{<$06@7?YrUHB|jrB6-|DbU+VD*XO2z|NZ0}Zl08`Y95BJ7vQUaHGnMtMBb#?Q+KX
za`gMb#pQP&D^Z$K(Ykulna3ioG%;mF^x+o=yU(QvyI}usy1B785b=v#J;HzX`{lPE
zmrw}!enme#zrDKqIi~lj3QBh6V(^kjC-s^G{b7xe!z;!uSXCo+2=v+QL17i@X5`~Tn@`D5}B^GE1^3bhEmjaT_rHIv$
zBU7K04hM$zXedL
z2C7;=bK3`$T@xr(4{D%X0m^L*l-mp_w>?mH1Dp(MvLZ^Hnn8?)8cSCW&sJ<3_#Lpr
z2ydbaE9s0(P%F^VHCpxA>iW=fgZrM(ygI({>STocd_#cz_7-=w5}Sow*BVM|PhI`H!?Ny8`Y+cTn0Hw=fYu{&Uzd46dwsqLl)?s^F`%P`l`ggb^
zY3bhuXO>uyxR9WjTiho$XtnOmn~{xcbH~cuq0ii5Yjej<%^kNlci4Ptd)ZWwQYhjU
zOSX2E)}R(j-e+qc!wOr+%GO}b&9;txwvJodT8d4LjZ&IztsS?T+)6deC7`6)afcvL
zj*8Y~@Svi;&)}h(oU5%|ZXc8{Q+hB{;ir!kFh_(>&`mA_R8e9cO=gIfeTkQ+op`x7
zT=tuV%hTa-d0L0d`6h4xJ1eDVRn>ak5#WGER_@|6=U0jULRtydpj@yibB&9((Br_;$MQWeB2~dfesbzG;O-E{bC*zD>h&j0kOLdwS
zt?s!VLdW3myI5lz=WMS8yY}w%7&Uy?sp#qaLt;bPT68i~`{(CmU(>cle_{tYkh8NR
zJ)P4}AN~2@Uk5K;Az3j#SFL!85!22lrHf_ctDpJQ_HMO_z8ZyU6!XXmQH`>O8!aba
zjZnoIj^EAcuSQE=hlq;^f+1x~I?=@3O63%ktVRhvw8Y!hG%k#DsiF{kmwF8P+b-yv
zB@|8KFDOqs0yDhmW~cX`tO;}5QE4OI_niBX6W%YeNAZ>wtp(Go$xFmviuVdABfY<$T|P_Un=o;@c8ng
z!S
z9DvA}#K3j^$H>SWkMlA7NaCrg9!yU6&yKS7Ihk1hwH9Sc2LYQ^=^6-F*`~=9VTo7Y
zfmtCdntp*e-R@LqkM{KBk(o*l4JSjBV$$7XvP7TFyBoj*YW!8M@yC+^n9dieJSeyma(>L>)$Yr)
zqsi!wmdE8C6x~iQzkk{}JJS6Cji2CMep&j688^a8H_1(XYiL8ONYa7^bBTC=Xg4QXYs<+tOgp%^{o`zag
zhC{AuFfh+mJ@M$=kNpDog#OwTa^~<%Z;d1&i@@XSi3@x*&H3>O&mqwMoa27H&
z?=NoBwWGA?ALmu;gHE^04gz7cFTRENureUEm2p8HUr@gAr_y0W>o+e{8;y11KA#V0
z=R5^1R9S$ZSEJ2Nep+${v2QitNQcZfT_aD?#ALeBZCnGN)(a<1K4&|Om5%>U+m$V+
zk!;&%{(|7ijK8ktUO090uY%0tpaXIFx8B~-+{b=F0I
z5<%`W3Z$8CjqLxY@w6Eq8%k+u25q+IceDWHD07c~p5ZGR3HHG6Rm9
z1WW|z<50>Jm#!&Jhs=eF$76(+weg$StZ2bC0dZ_Byyr0M@!R*JVJFaEl}_B#;{{k$
zs+tif&(Ew4h9=SsdbyNGvUZ=hhjMN&h@7{L1!f_9BXZv@{)qT%ZqRy4A
zbH0wp>Eb!9xRi@5T~}*22R3|7j=4qUC0o>&U!iWWs2wWtVwt7Nvc&(U0hqE%BUy}8
zG1A2-FGf-4fG!X`SZ550`IC9$K7nwJp27Pp(fj4L#M@R`@Sa#^+>ocY8)%Sk@>qd|
zz_+Ofi~7l)Z2pbDEaO**IddFnf3PG~HQgY$2~v`_G?*Vmw#M?ULQS5U+;F{Y4`>0A
z`$2F!OebPT#sGrLVkC}6G5+SPLSN=buf4>vZWw|^tYl`v$EwOs=-d?zWMus%eFV}9
z@(w%vToO*)77B_j~gBK5j^{NZ%P*HaO~5+AEjkL_pGN9#W7m;7(LdT;N=eZT7Y`T=WVYi!lj
zt7^Pu&X1<47m7EH{`+>wJ2&~wKmiN@6B=gwKyZ8rOx7yurci*92!s^~L|4U0&ZVnTmZC2zupH
znSEtqJo^Z^D3vu&ijF)abN^6ht1wW3yD}agF(TvOiG0SNHWd2~-aKL+@2KkB8k;*M5E|8vcrqoe%ui}f#0&sK@ML?@g}gkr#qeK?udhSj`Q
zjuz<#f?1}bb0EJgE)GrWSa_QiCw;)XZoWVE()4lYJmt6&^hj4!9d3Hn=FMP^JQY!Z8VoW;H$<&mU`5#<~`#M!?A0E%|eM79g-!H2oGbgl*T
z5}HW^7L@H^T0G|oy@$36zNvrAmv;YPIhxZ(I0Qh?*QYnNfI0@;$Kl!;eEx_G?&{6S
zpbr$7{%o-v&KKw~HE4wpwN?nTu@gd+2u*pM?+P?Q4W%G}|K%>PdoxBgDJcQY9Rrr%
z#y~qm$V&TMrDU-Vq?F{OWN3&Oh*to%&Jd}}Q}F9xkpwwAQYqlgY(dW)7Y=NVili(
zHskmVN$IeA^u6Q451Vt~ayFifE)HD}dYJU6U-xidzp-FEA$|-VX
zOW-x@l_+aKIGv58b*up)t6VjUrCa4wi*tFra|ZB
z_5XDHnQ_jWLtZ-{MY(3q=Am!5!^XDAZCo|IPUWv2vpRZ@SslGvGmW`l^b%kVRnh0B
z*(*f~9$TlyWDF3TBLMX!bWqTQ1&3LY|M&@c4A*kV^|>tV46;)gbDm!!SjTVmm+
zY?ESIsnZahC=Mtxsc4U`r!w{-C*o97yuMs~kyTt(dmu}0oRCoUz(TKXtz=y^1I$W9
zm6?vmAr~Yqt>EDBC`HRRB!UERRewAF1B;AKMU&W_RIZFF*$$Zel_!GHoOVtqh
zfi2vUhnBGVGaaKN#%F?i;`}>KDUh
z&ThZk#b$xnz(If3Z32|;p(Syfjn~%AXg>QsCX<(TB6TqPK4Y-H{%kb;=LcE5-S+Zo
z%;kZt@^OlPTYoB!K5#-eCh^&?R9P9Rk}Xxq;Ae0Yn*9(c$B5viSJ09_X>E%HiDOd8
zSQ;CnipF}4Jd+m)SA$L3k{Q#9c)#S8h*j(#&UK!5f>D2na*PG9`jLD
ztz=@>T~03kCO`5>m;^MRak*)5`MvT;q*zi7C$#{_WpLm^s*Shv3;hup2VIcKdy??Mukj#6zs;J)I3gSbv_RGd{&%
zDndvd^{QI;PgY2?(ZV^+NTkx~*#fzi-{>=?jQAbsFN(EFWNfa|D1s%TF{2ovq8v72
z9)VHFsn10!hLqa1;tX?1-o_-Cj3rElgJrTJrC1&Tpbz?t-nI!m8!d->kS@d97v-Sy
zh6F)}9ONPK((7#dV%fS!Ui@UNGp5AR-lQ|CV$hRcmgjksgIPudU$LB>aYZcf{)=##
ziUvKBiXZU_s^w4}Zbd>d&%jI?6^2>e!;~8EW!ueS(Obk4&wmiF^t(r`=8keP;u3M1
z{D9bcJ6Vp&t!qC!znZuId>%oPZl_
z3+~qQdx4A^{&=;{-ddyA+zZsl1@~ey_kP0_PM|
z8RQeLQ0PNuS=n_$Cw2%fT4_D=QnNg8uxAxPZVnegfJv5llqFEAA0kY8-+t$SEC)bc
zF}(FWPK+XZpd7)emITQyD-A&wu!HOmRQyD;gGBRq0L~T!>HoR=9-#81@)0~mlC~y4
zQTQwx!8FY{yUnkkKA<})2gjOD67&FfOF<0qmnJblhED7V66Z;J=-w3JHt|5|?c>VJ
z$fAiWlU(Fn9{5PviHFLr1|E{_sFo1bOqfLKQ#F;)CTQ`|?2eQW;d~FTB}VH^QbJUq
z+RVQq%OSTJyEyT1S9nvI&&pb9-VFfk-Hx=1Of~uRF5GjxO;ik;c78Q{dO}NnBaP4a
zwR22gwZiu?bi_RtMCGg5^mU9>XH#&`hpxI}ScP2^=y+XgpC{`%I3ITxXq9+x$vQHH
zyXXSsA1HVR(xDKD_D&6qu2jSo;(L-o_QRwZ5F&(w3E56pDfgzs;m{qr%25@|hXD9Z
zH^1LDAEX%&ZWIpA+;Z+?<{1=Kz$EXJ&Cy-9S7ngq@n$~+^fY8qZgUG_(W=!-?bSfw%
zJK999A4$6?x%}{AvKHwe{YxoNo8&12X{CCky+St>_R4!O8D=qp21L1Pc|ceTwz3vt
z%-3gp^U1cgu(+DdmwPS+KG>8f2c46oh3WxX*gud#$g1`hgw`yP2Nc2WD2o;b^
z%A)HzxnkFo?2pLagzYl+m66M-Q^}vy8q!9s&GVc?flM1Aiea1
zjLidN+U5=9ZA(QFIp`sKD%3{f_mQu&&S;ytI
z0}V`{mh%#=i9r#8o#m5&qaOYq56V&zh1z76_ypP{_=x<_8eU$`N9V)o&teShpA262
z2gS#R8)u+-qVMRuKY51=Q+khBb;+IMt|tUhXJeH6EN&-XhRadyCiF$84@(`w=Q(n~shl=eVS;8iwK98j57Gt+EMtK_
zK$}HeQg{lyK{Abdm|w(zN;*v|+(u(j_=0D~uwRB&rTLfF-9ecCN`=LX_J$nH;7v@v
zEcPf(Z|D){=ef~{HBDv_aY;=}1d4*?JsiChzje56Ro7p>Jqp~a(N
zLF74(cXpz_+)im13(^43U~rxkS}7one7iL0q|052*&va9i1W(QFgvA@2wMQ}8}2%I
zdn&>RZRen}IF=yoRd@!n){v!cOt#PR
z03tvCwm94woh>IK0ai2I1O8e)b+xKJydlG|)oH*@6wuhPRP(i^S^yc-PR-?eTb63_
z_W?vvZF3=aBX>=AMk2Vv2GwwM#wgSpMUtUCf@VKP-2{sF%phm_SfC%H;S`*)TEs$D
zI=X2O_hq$7bVvtk7re1eJeZynlBbQlF@`@SbBy>nV^PHG>6^F5{8h8?CGAaFNIcNS
zOyv4w@gW=fOJyUkQX03ex|iotH49peasft3-2C+eg;E~mq=oGh`Ro<)kdnI>g|}WQ
zo4k#k&sp%_5Q_N|4)~-{zTdH!8
zDp|CIW$}8iKoNnvUz%t&-2Fk%R}cD}PV#wBot`(-ZK=vuRrAv_1G9Of#MTED71>u_
z7Ts4WH>rXy%TdD2b{4=)KKb4sM`Ux;@<10vC#{T7L_k)vaI#Y=wK1(IAlhz?b(`2R
zbxd-s3t1<=EDa9xeh(ba$#F?_=$ojPpWJ1tCAToAL5J*SldT5b0m3Q0&j(?1e%C#y
z;QXhLqoelq$K$~pv3mUxMhzL#m$S*`xG418aF2M=Hrdys;i~Bi?f7*iQ$VbhWGs(Z
zzq29RYP8i=liNwRMipSuz0VuN)y}i{J0>Rsu5kaqi6dl46*$!w3NICJq0U)363Un73?a6l0lGQLL-Rv?W9
ziAe0iQ`#k98Ictu!F+CgGV4HMsd8IVu;BDdRlb_B&UkuvRGq1U`iq8+-=9B39QjWJ
zFGKO)4O#D9NruphO*ryuJC71hzEe2qn4k%&;V#k9+@#nOsI7@mWlJWhdX=MRIhFj8
zIaq*|#ZFe*++6`K7Mmr>*$DD-CIF7*r|t;Uzw}w;Q6%>SWgG>ag%)ui$-%R1qSMax
zXnLFMTq!%x_nL>u&g=(La1c#~+W&!5nOEE+q-O3>sSr+aTGfbP$a0o?4G=)Ys)x&2
zB%45AYaVlMDss~E_C@FWXM@T*8A2v0MQRJqYw{*l){65vQ$F9Ps+rM+7V
zzqtMmDxU7YGHC!K9@Gb^ciz@KNDjUE^qXmcevRo|3iS~M`eeZvO~>+Ka2vWI3Ai
zwo>c&Vpn20n#`_)GdCZd&u_=eYjm@IVZ6jAOUXn!*Ldv=mb07HV0>W_H^*^RPY1xduH>?;dJ~@y)AlpF&~lKXY=Lwd^l-e{8-$O=Qc0y}kU@X}J%MKhWA+#ZtePn2Z?uem{BV>M*4ZAYA0-Sr;)GSE9
zATAmUGW-6^)^K_`rYW8-NDgmt6y1&HESzv!?PKPL{`he1(07Sd|40BN`(|{r9AD6x
zcrgw`Mq+3k_0f{G`+3^WHsAcb*6ol5B@Q)wvMl#LhA6>MMKmLBiio76AF7)6H{|yo
zz?oRwT+xRGC^vl7tb~n!w&5|JC(t6rx)d`lXqjjXk*f+3D6%qLtsl?YCR@4#&_~!P
zuSeXE_ey~zFtkr9rK6oHqyUV&J3S)zLP+bBvw0D^^CeP(%R{RXCer{JR&0#k8W$BI
zZxY!<7|HW6mi6GX+$fhPZ+iYF)WXIq96s7RoLzN~-}R5^qv7zesE+TKm8uw@!R2$?
zMe`rtuB@qzYg<3}zfkp)PnF5-J$CQ=CWLJWU<-j0oR?D=+z6&i?0&AdGRYb$sCv5+{eZq>jkHYdQVuS%=Ip%J)
zBRw6NqhCKNKs0LlGB)8@xW)_DR=O;Rl(i$5Ysan74y(j#$7ow>e=8I@;P^v**3vW8
z7vb3IXWfv}79c%Gncb8A|S@Xn<{yNNhzXTqY|9EPV7O4hc*_et!9b;6xrH!ndeo#kPBSSpF
zJw4v|O%$ydp_`#1?Rvwgk~*RWDHG~LNe3SI{NsBr5J0*|lVMa@;f!g@)|A!>GZ5%;
zSrEAuWzp_TX^pqgkGNztfXagQK*2TIeG`&)=OAdN?RV}Sp;x1``}sfjTvpq{3H#}&
zLmA5D!!i5Rxo!68Bqr_p@A2uR^Za_nr@C*Mx6R(PJ*rNo)?~U34q26GDC|+SllkEn
zjK^
zFo2F>kdP@+>JEc6P_H^`oC#x2(0{FZV-ab{Xt?rnvG<d`3>|fboogUIkO=
zg3KZ>fU8>?KhsSJNOT}p4@X}sWXq!day%RhbW1e>j{&*OSLaiPRa%x+1Q!Bc{NwTS
z-<*Cwznl(kAJ4C+j}L9q3LUmKpd}_47A2ZA0Bv{x(r>%s3~byr;6YI0qbei`4|yrb
z?Mg_AWvpSwPV7tcs@7SR&cYZ*iA~N8cfZl_bfwaj5H+Yqz~dE`BLK5VornPd#~4&<
z3vH}i69G*tk7nwT(LT(>`O_UoF3vBfH?zO6rYBrpF?hA}_~pyfi>yBa7P`kSW&(}T
zXnQQ`cjKLU=gSkh;1VPhYu|=;p-=VBa6Vi3D5KPcZ|CC;N!4DR2t`mMc2%RqWG}m>x&8y@B_J^j(pgSCuqT}nUVNBUJFkV_{6gSng!n#S@$i9O35sxmpR-bYsbP5sB
zDR?88(Bi)XeAtO&{({5(?Fr@-^RG0I?i1bwXai!Z^&y@ORNoi7fRC7|trQVUWc^62RQwF9OEZ4hmh(hRq*j13Kntq0J$=xV7p
z^g;oFO^FW6rSAJM57%fyZP49S28$QRe6lmbl%CK?40`x55%uVFG%2~BIzD-Je|LFy
z+nU;S^S4){t<$a1;D%QCFaK#iQsynd7GAYiC@X>LF*to&Te|0uAieKsgd-1F=|J^G
z0uM+O($EdZ^s@kQo&_PFQoXrnf}2Qs@@=E^F%^2kI4Ntp+G_KNHY$q7Jhx8nPxM-4
z=@;skFMVD49&e
zMFPGUD`p%N^h0i84`TA-S5vq>;b6_?EaGwP527CieG%xNX>Qr6&iw^^X8vRI1>(p*YSTGa_{58zOp#kf9y$Jq;B={7U6FxP=?DdZEU<{B
zvD{`tVggQ+td^kjsCIv>h{<_C5DrI&?|3-eO#eJ1RX-0rBo+ooS^OZ3h;WkIp%0ID
zGM~%!$=jPpaxPf#;JCQv?yTwHV%may`T6bBqe+Bg3&{}6WcV7sAwp*W&=>70b3@i;
ze?tT2u~+MS{;w}!L>3{MBB(uzApBAUWUR6zJ0|By=qYR*p#dHgTUD%TFqbM`ky%0(
zitiUOYW6&}X`|`O*!irqDKOLD(_l`OFaOJu9-N(DT?~9&vR&Bw{5$|tT8ThyB~aRO
zb;0)^tb0HwoWDJ!7Zcg%D@Irjmp}9MLc@_LQXm_dy|Te{C>xv(>7%73P!C(HYKh9O
z9=E1&-vbjoV&frn{AL!hWpbTLXi1v&Z0kmn$M6bzqvZEA7
zSV}~5WjfM*-yg0}PX|S=?qbTd^wEB;-bIUD;w^=w5O2|mvCwwY^Op+~QwE>OfmPXX
z7e#xbLBXm*h7=v+N6vwwQIL-%Xt|22s3HiK2)%Y6xDLMx&}0gF8_hv5s353xHbUw%y$#F80vBd=2Pum&=7{N&e3e`wQTbvLJ)BQP{l^d$D0$Xj
zOO`7$O9fMI!McTk+5ypZwr}8;jOhYeDR*&kjU8$D2|YZ3jNQjNuUsCGCs7o`QCjfb
zqpO`<#Z_{bTnJ|p@3S#k5TPw-?U9}Ed^dyF2tC@HXrIT$dWHP;i^(7LzsR)KV;a?a
zpK*}e&9LqWxe0bj^pCd_y2vo}Fg$eVz@=e5^Ci`^nmP`ppw3Js(Uy#Td*IfKR*K}Y
zLk&SZJg9Y7wrUA;6A3h}7T_ADi|q>8+Lw}TmW>41>PguxVhFL6u3pQyjG(hrCHvm_
zgfJ~5vQx)t`?U=vLvI9m-$Aos9kKQHTkM5KN3%(#)m&rBW74+kT*Xq4JUaSRU
z@d;_`OME8JKHo=-qXAwhRXKFWe%TxQ2n}|LE@jU;N6;!Z^xixKS4(RVx&_I+4|SLk
zmdMkWS)cwPvj#wHAGolOu@%dzFrr?y+@s2?jLS|{F?9z^+7_U_S;N0A63#l=M1D?G
zEA;cz(3rA-J;?=d;UvEtdZp~umnUy{*^%}8VF+=NGt3-CsS-XHTB3$X5dYXKC+Xp3&`TOte9&mwv(eyPrtmF%e=1ae(*|h2|8dY=NLA
z2#Pvs!i88%p@7rwo$_yA&$u87(P+H!OdV?c^|qMNAL-EbE68w_=32jfyhKAC1R+^lp#DS0A2i
z(Lm7^&~Ru^TD~QvqDp{JwgX!dWETCRN*R!C^u^hohc-o&KLn5^=5=78m4Q)E117@v
zVUd?zAOZUcR7Ru%MZb}z79>hihgv;7Vfwn%?klS5MXE5N2>%o0iEC54q`7F;2`>F3
zV{oOVEH+q(9GIpX|Olr0H}#A2y_8*H?E}7m2KH
z@Cw(TEG1LW8C^fj+VA~(Kzr%k42ali6=-uVi1xVYoHj{)TXaB0tRxSkk2{AxZ(i5W
zzukEHL!l!wBk~p@sc)Q*j2bvPN7e7AdziJ1%2^a^7IIYvst~D=)(r+EU>ngFynMoL
zwJk`IstHl?>91a8!VCHHbd-X#7U7tW&dx8P)8RqwwQhb|w`PrTPtH5Jh_ZuWE4Ax4
zK`=EWM+N@6y!MqBC91Z3jVJO!zNrNUI_c)qezDq>;ttDD7(_Otky(@}+nsCrk|6(}
z3Q*Sh%hW1WsEtT(R3N=k+3pE23o$odae*&q>d~VmqMSCA++hRst;uw=(11X$5f|Exe`VjwDNVj_Es*_og=_HP
zib^L)Q}jefky<3R*YsCH{4rj}dN^$@ezfOrSoK_@eO096czAF~kI3%MuHnpfAF~90
zHod^>LAs<~e7)9R8$=5kn_ju3ycay{ltPo8u+T(aq@IzlQ<^3pk)eq(2T>WcbcIAu
z8Ib&Nh9V?k3YtU}^02Xgj|`bg$Y!FHS&%g7IZOk>r>ZUppK8Z+M6xzy!I#TLMQu>4+YKv(XUYFr0q(9ke
zuH7i*@Qj20x6RiK82Nh8y?SxfEPNzSMJ+DkOD`EZ6?uE@
z!BYVMs+WKlp~+oc{B?~Kp)ITP=+r0uUFznt9L+L0YlRcT%;U71)X
zQGv%WC>~@wTBLxu=zyk*66lTXD6A~v$l6d@`}>
zRr*>%NX~**y}zI?PF9(kIc*U7!J^PNX{ALb0trz1$tC86maFC6?Msl6Y@X2-Wzjx0
zhAi@KuGIBY;L%f%;_n+&j>n4efCUW_PRGUJe+0rNSPH7eAzO(slGhPvLU0izPB1A9
zeLkSM(Aj2aE-mRThi-}g&;`Pi?ckC|+eLT6E6Y(e;I+_(Uh3~waqu^aa}r;Tx$DRZ
zyt|w|&|Sc<1c`9WcZb7qeQ@~u*7$^vbl}`#?|HK?gX#U#{PKQw{xqM~*Ry{H
zH?%;XKVF~R5B`~QJ-z?@Z2olb8(`8qH210DuANV>uLu1nkj>VW)0qYW5MQd?`Y96&Hi)aV2v
z_pz=_MeA*oqO4-EhUBc+7#5B&3vFibK~$RzhLjz(4m_%{ToedxOV>{iGmf>qRGa<5
zaGa%^E~q8cfHb0SiYG*7I`DjGjqNwcI~@Y6w^Jm$PRY}gCgL57An=kL{n$EEq$ZL;
zu7ESmaN|hekVRBgGD%uSNi%%Q7{xxqnrBE_MM2NnUF+rIlpUik%e=S
z3Ng{Vxe(SV%ZeeXdF|(lC7MCHmKECJbrZPJRYddSXc_6HbkERBELYMf_atfPMa}$t
z{>CAUs#J55&B?;N@@v>KU7Re@MRQ4tX_nYB{W@fkfasn^+YqdT?Zgl1V*#E@g2s>+
zsJr9gC26|Y4@>4#$n(>-&5kR
zt}d?5H)uu?p&5lME)THm(Nhm-bL>B3`A*|)(2{9jx~0vslMo<}NA=rabrA@M#q{o;
zRx=c!JEXLIk+TYw0JJ!9%w!kAE6L5VMGX{VE6Rjk462@lC2g4~i=}6Qz0>loFxhVK@*^C;zy5nBCO@NTx*D$&Kk-JPSh&)Ig2ZDdnF0_2^c&qr!?EWTI
z3nVh0mhD>tAP16c)n};E_L&i7$rV6hTOk`OBH#mqw+bk2CD%#XKiyMB^sJYCqpz(keBYVBxnm?crs_
zxFD--EV$t%NBf=DCCKJLQQ5LtbzG&``I=<3GU}?;7fJIO3u)6AsY1F75EUp+EZWPC
zO{p&j18cH9j!Niv3;jF=3s<6x<|IEJD_MW|v~}|4c#Ayv6G^i^7obRjNELMSeIke3X=
zUb+f^$mrQPs@}iZE@>-=&;z}Is0?i=4ICR6IeJ-wSlX=3Te;MHUuxC>_gGARu<_pK
zD#EedcrT+w+Ku-{c2Me!@o=nMXbVuJXB3M|lsZg)6MnR?L-IGRjOW*lGr19lwr0P_
zA8EV{M?+a3kGIh0Cd4t2KJx70Zc2+^dR)<}g14!pA+0$=C?piluS!%pxBzEizLa=$B5^l_UqqM4P+N=zt3)<-%9_K$0+V
zFpvc6y;Yk-7A_x0=_lIgy5SAYYm%iwuFZu5~L9RN2=l
zSeK(QqzGFv|GT|Ak%2<;s4vmlgvks6ZTVu4d
zzEX`4BW%!R@`qF>Ukb2>R5z)|larmT_n%Z9QDyf5-OIuKnv2QNd*CKs~8G
zXd5k+VRFfv>6hrgl3yw#SR1O}yxXd!^=Gv00Ym)Ay#4Ks&Nve%tc}oo49C?V)D&41
zD{Dios|YHit!)LvOCP^ou8fybfk#wU%?a#0NngGI=H=n|*|i
zq91E*QAl~2P>fZQ<)PAw7wb(QiR24*CdP~@CbD0yN!v@`XB|klv#u7`5IJSgC!#vnAIAL5Y1?d?iA3W9kACKDgpa5^;sB|1Kb6JSJ#J=Q>*xAMZbIY
zP}Vgtf60%!r!Rkh!B9T02d7)tSGRwWB}TVg|Mx0Sa{*@B%DK=~lLCpQNjum|^(gpY
zcQ`*6^mKOR19jv&1*8bfhDejBO(0>wnai^ewjei{Y-xDO^d}){WH=iuA{4EXBDB?_
z7VSlYFd(#0g0lgF&(Y+1vQ$3n3LCtj?0a;HalM5Ar|LKgV26HkDbWQens5WC~PG&biiH
zvmK+X#cZ%G8Eo>8{SrzsTa#zb#I%0gdh5qMyg4O4}H5-x)`qiM&Lh8~sc%ec#+U?3~2cdFhcbw2-9
z@bm~BQ4)yGWQMHCwSf?g%APY6t0!4SWSWdwN-`NMOc+I(=~eY8cAFi=JHsiIA|>=031IB??fT_33aP^g5wzsjk2M`+ZITWAX6%w8-D&
zG!aIPUW%~uRHD`0cmZrfNf9t;9_l=b0(W%NB$XE4e##SBKj@pcOMy^Gc8~BN{9?7U
zu5#!V-$*;too({$cFdzRTLx%+F-^~;mVB%2!EaUKY!uxqeRJ1KSBx{(p$8_@X8IGw
z9)1J+>8Bsw1ELj7Dl+(h0as>gn6_YbO^LfyX&Ut8@PKp(>O@m^MBhD`9~H@I(ONZW
zKb7(2NZn+2CPg8v&M@xLnTGk^N#^%1f()5(|F<<;9W
z07?F*_xZ(D_#b@DbVh8X?$%IB(fsgkMdH=A*(;svqfYW}wMFrNV)o$;MQkb!9)hn(
z`np@)nc)zD?Ps6nb7~APNuJS
zw!XVLzT5in^?G*0cz%RRX>b(ijlOTX?n5eat}w0+(0+}wP&M<5
z%2GvHH;AT@Y0lXY>k5e*bd@V8bbKT4X!Vmn=pTJB=v$hDR0p+&VsxjO*Biu16`W_1
z6@vhL37rn>uraXgx$T>E)oH$6J#=z=M(=!lvvqg1b#-kqIv_g5X{
zU6Q=)DyyXFDTddvgU)G%hzleezLY|5zm%R}*iT^!PmT!ye-V3r-}myuv{%7hl^Gw=
zbC5;LDS5HfQkFR)Ak^`z?79lE-)etxavt5mD+F5u7866ay@LK`aU0abYF2TaJwP(+
z21r@JAR~ivyMtd3L{GrrKnSi*t6QHXY?FlLlEgcSHP@MWZku0qxVFb<*D6c*1zVk6
z!z2Hu2-M@gb&LKJvgf{4q^1WTLG(QU#kgu*wjQIO>xhA$J@0(!ToQv@R|syCs6Vwe
zmYcUbT8bZes%^3-#nsI_graR61u=x9AXV%V(j}SN481kv1{-F3ZPhUJ0zaS(flGS_
zKrtk1o0_mZu-{+0LnJs+`7y@%DR9oNe>>qNo6?o-aa~P?q)Q_K)#iu0?
zv<;~MQ`?{222MqaljY~X&kIYLR#s%Rcd{e7WCu0~Q}1j7@!zF~#z?qTg*#JPn5Ghn
zD3gxJ*Etj!E)&QOR)1kLBe#rkDqp`=Yid$$f~tUKrsigvOGi6NkMq-?DZGhP=G9pIC9IE|#}0hFM3D|2kBFKp-EOvCuKN0*Nju5Xl|e7ztwq
zzT%LdRwZU>GZL~eO?RV3CsNjdqXR>$l?sH`O6!5=VTISP^_mKNxZ|n{r?6@`g_Vq@
zI=}j8^%0oB?d~ml?A|Fr|I5v!xBr|Mmqi&cStsy$p;SIuy+pw^K6S73g6m?vs)7b*
z<0Q+9cZyrmPT-pn1>{kVc3|p&$J>*Ko*U$+<{R`XYxEKcJj);-bbv+jU;Yp44dbOEGSi5C3sf~nU~E~pcE
znKl;g$RXl2&FCGV<*M#c
z+cx#Y=>@0!FE38tDyEirbp7G_|K5^dPwxKr^zHv)!86^2QE|PlZF)hWnSk=7K$}7>
z9xTkL6kQiaH^Hw{9i=N$`o+~1X>&uBCr3dxH}qME|I(-ZF!FKq_q3UhOZzH72mPq>
zJMF9FboHCHF`}7s?eG4=7_8J4x}TZvN;3l~Ujf?zMOBaz6^+M&76v?O2;2v`o`&cJ
zxvd_8hJrF`F?|(Kp$sWVmvX9=F6~Q|bGUT3A&2~eR;F1#SC8cUkKdoj`LBEF=i1r>
z6mVU~ve0gTIXKXx;na6s9wZ$Xb)YBpGgQiTX>G)O=XY;{0Er=wEuwS_m{gc0dS^RV
zKq&TFPe(kqMlh2`TxDq^({Z$$S2kw7AI!az84L1t^8fw6ZGO|qfPqTG+Ob0m+8!D9
z*7ZGCrOAM&7VFz95RvR-%E>e%!wIdCB4AphWZmhQwZc*WT-T_n&{Eo`;Sa6Woc++M
ztkGRwqB<{8a>1e=UZSc}Vh6S&yDwBr*a^LS695w)@yJ(h^&@NiSR|OfY(RK|KYvik
z)g?IXj=87}6q*j!F6=@CPC~C64N%&wj*mANt!XtN<7G%T(CtoMk}6*SX+W00k<)6e
z5-!h0A;`5>H9@Z6K6jQ4;O(_tMsXd=h`cuZB5tNW+FdJb{T*?GE|;#T`k2R~hikoM
zT=8}!3x-J8v%-uK_+u>F_}s+nx0%OJh@u+usVr-s9>**^j#;hDex-5bSoBtj(t%(^2mKae4pE$e7oJLKN;Oe$6uwG@+D*Xl5RmEZk0kpp1I4{afP__
zb(53eMi-ZB!xthon$aLM9zChjBp)@mt#3ct0_8vc)c{{)8iT#ANc=Vu574C3$KiL?F+`QM(0J
z#vw738WPxYv$G)L!A(bZdHC_M7@G$BKCf>%
z;6ANR`~PJmCn=U4IvJpaZ7`SzN4+OQYr4JRF@DvjluK-*tW$6MJzJff?Zlw}z>i!K
zo!QjMMF&c*^q^!8bn=xw*PrM_n`SY*Kfbu=o@w(j0jX(w-G?;Ph+8)zE0j$r0&m+#
z08pzwSrHxIl~%*Weg{KGT-NuwjEam3uenp%;gX9Umt5&`>Fbmi6y;{gg3*O=T=E#>
zh!AuD^FgO|+4bWSYaCbqSpq?Z@sm-J)2hrM*SIU)f5xIFmt?T$rpc9l8j=WDE9JUG
z1g54E)NqJaL=fBEGZb;qL-#WZ+BFf!g5T3w1OlVM10uk4R}c_9_+B^jd2)PndUkx0
zb5a{kAJUVjKlk<@%U8Us_BtuBA9|nGb@&Q%K`Jv2B*8j3?>2(Psk35mQ0BV$KW7*b
zPKX6Q=Rhsc)>9c*aE>96)tvyH$S7BH47OxGQx;5zHPI#L1tr6FoI3
z*keWk7mXBX=MTNWO!q>#{Z!7@H>fAVe&$+_E9jgk^`9S4lb;mtJ?+6aQL?7mc|L
z!GC6PA2PnZ{0X?}V<)KWMoaaWls#dVTi;oR!RbB>9GSfADAL);PK89ZW86SS*Jm6l
zLMhVmSvqQ5IxAg^H>n7!nwun~7i!L;QsW|0BO)?mA~M~4Ri7ow(2}U3C54%8NoZ-!
zaM*83*v}*&5RM)f&PqoD#eSXt%ubgc6^Aa4RiZy!Ee_l1n$ioL#?pb*XXlBGj?aV~
zW33J@qUU+e0qGQy8t%=8bj-MP%ssqPnY~ikc%^1kKxV}Av|RzE+>nBo9x!{~a=Kif
z${uNWUb3+60FLJ1mcq2y2~N7sx>3UcXvs=gnT6xVh2!ocoFtke7&j^yx6CCUS~>Om
zFFxwvA;$azcp^XkM1K7z@}p1W*L|Y6?i1^
z{E1d3Kypi*sI77QRB`mFauURGyOAB;lFd`8jtSSMUs5!vIi*uB)>AGnG31y_7>osr
z4`6Na0mKp~K-{bRLZ+fz^C>6ODJR7=2h*7gcs}YEi0MKUutH|-IU6mtS>V=Sj_aKl
zQZxBjPJU~~x}vB-bmF8g8RfzuS!9-TF5XpRCYSk8@*!@*q+lYYeK7L)04KEFlfO@R
zMJIpc36^`E{H`BXHji4;2WfDQpL&UJv^wviuX&DAA6SxfMlWBJJ;=ohuk|@Gc
zw@+-tj1oQ2QB3mX903Ibdf8J;F;tc0M=ek7n-#vqnOU7MiPwowJYU;g#Wg`Zu4~!x
zQ$!yH8p#gbu+9cAmaK4~r3^l`FDo2zcQuQ4$y3|=lu=3+WYXaMwdlNi58AHMD;r~v
z#zK0nLrno$wXEn&$D&Y!{nVzym&w*vUgD0))&7`EtGQh7^|0*@5s=*UM^HUzf16(M
zC0eDk(upwguO764J>fo|?Q*aV!mj=j)P(MI&s`-ClVrv&J^UyaJhTAFSDE?HiEE
zK1x{US%WcSIbdErF$4;zOx>_)scwYnGekPmA5vA(BAzMBiQXvi3ER1Fn`8B%|_v
zq|^C9(nojt{WO4YCfTB~FNJO+MS-a{c2%yyB||M|c~iw`&(1gvXoFsn4YS@{#kl?WcNvmoVy`?i
zLk>&E|Bwny6DVkSK<7!=oH-qgS~U1*#^$+w1s1R@M=qNsv1=v#NN29Gb?OPPdy%(Y
zN;#MbNnmy}`E>c`<<;Mp3+-q6nq;#5m*TQ^xxHT#+RD@u`zJ4K`|H>hUq{Kmc)+k+
z=?$Jd+b!QGrh2*<{yrcEj8?4pT4+k;oY)JulNTy~Gqy^Snb6*>b9Yw2md|3YC79YNEGSwaU|RviQbx29n+2W;Cq!vw^Pq7)}gTmn_W
z3ak^G=|#I8O+FZe!e^3|d}Rgh=I_;B#jdi4Wu~?)Wz!RKLo-0%xjhvCSbzo6l+4kg
zKYjk1(SI%Koe3ohEVXP+JeloZfK+BDk$i6$g~A;Mteq-Qrzb@=FhENLD$BIBOh@+%
z-`N=c!5f1k3Y{YhU9C!tSy&^|0(C;ChU`fw{Sd=sX_>dD9~0Ia=rl4`gL5IlYB)e5
zy^L=hkFRL2wF(*3jaII$>|n;ly#73|KP{F)E>$Epm0r?WeSY^FiR*~oRIVel4hswu
z3opn_6`27m68L!9hgfCiip*q^853J4p#c*QFidjV5?|fEg^hUUr7q!}p6ne_yfO^LpMj0i1c`03k5#U95L#N-LFD(c4`>CL;v`r3_V^BG
z)^qU1=c%*A!pRN+d6h5YQ|rP6vZGki<(Z}zVF@K2q?82~P@Tr_$F{)Yw_?lTF>{lf
zF)QONnkj~l4*EX2UAd6~9SVJ_4jCD0SgQ}`U-Q5#{Ti_2GuaJ&;LC8dhr)ZF11Ha@
z#oL#lQt;5@LV{$f4P!vkqZhTXlS(h*+f8;)CVPtE5435+AE;sY18w@l_ssH_`@06@
zD!m}Lye(G+n6%#Lw#u7X{rwLwx)&1omzAeO88m?O~Rhdc%R
zh51nTqm{my0?trAbbP9eEme%!eMx4d+7_G_WTPEpA$I(>At;8Cu-I%m=^1
z)_3@*qV@EV4A{hIc%{~z_CX5DRUR@ev^5efiBh%)eihuIKCFs#k%uX@{OkH1&?O)5
z3N!(^`OY(Gyav82Vp$K6#_^D)5$U86bOp@he^$LUuQnKsPjm4tmnazrH@e(I$9n{<
zSGUrMoJ1aa|N1VS?Z!vZ5Yg&FLMaoFgF^#dhIVJb{X=arS0m}*LP8@&rBTK@+!&Bb
z;t_Pc#)_vk1L6E9zp$?S6x@1U#!m4;!}yjJ_Lg6#ds@&N@V2DWFJC8R
z=QP6sq9`)WN}Ho|GALI8$DA*KW70=8$pU1-FLTf;9uC*Y4opwv=36m{O(suX|3q8v
z-POtQ`PS|1ea4J_e@+(X=F|BV(l#Jjy3Ku)F(GO8w@Jg3Ej!E}z{l+(TX~&>S#|aC
zm`=^dfugtqcTZ<3ImFp5;Xv4EPg9qQ2`d
zXFn|EQedZV8PY2c`^TzTt0R>RxC4JdtwJQTYPVJB3#{B~W6%TG9(*-@>~yliy6L`O
zU>fdBVZo{l_7l)E*2hhg`t-PJ+2h`0dPOsRe7@fMkt62a+mCFJ0PmG`v|P+~cAnJa
znbO54)ivv%Sh$aBn{h^}S(7Y_JFoJ!o$gN?p@QZqY;0$O01H&REryoq0vLWBr!dBv
zW#bgV$>CahCCZje?{xG^mZU*HMJ0DO0TdzCs^(_8^9o(oH`j7!IyoV0anqXNn52K+
z`ffN?CV$xaIy=6-t4NbuJ6qrN-{bo^zs1NqG_U+&tN+T@3%8cMa9YE8TcCsnr=w(t
zMMsA|i;h6kMD}q1{QMbECxLcn_Wv!QO%jwi-5k;uc9?0!;ou4jT!_vz7(yxq@KB=@
zRxX8{_LT%Km9^+a3A%u8)?M8TjOZyZZ@V;bNSOyOI<4MX=gx>+r1SnBIcXQt_uL&@
zth*1V3P}hS`HD5V+g_rKNJ1Zdpw?ZcyPai<_qpdP-R-PWyzqXDboc&?w4HO3mb~oV
zfs2m%TqXJX7C04w1OBo|Q-sRL?x9IAOWu*&!h~j;tgsN%8nU82q^wt+IJ8)kyk3|2
zGo?K}LF>FyBf3``l%@<9rYKN&b(oq!={lxi)z|Qxe|zMY8_?VIqSh~OjsEg|71*f`
zgBQv#%#vYlD=F|4fJ!@0FtkoyR3Trr2((jd5i<5YxjzED_AonQ5|yTyB^EX6Sb3z$
zsZz;ndIi`Vqm*2t#nMO56hKF>^oSNW;QHzCQj(Z;)}$WOq#n|w9?_&8(xe{Iq+VZ>
zuDcaue^N-J4U9E;_vaS|`(sV&vKdt=A?`wx1wp{Gak6kQ^8@-nl#LMN=n
zWfY>N)kVh?(7h_WL3Fn%Z%|i%dra@9A5~~v_({GTE%gTgt~&v7<+3**nApixG1};z
zSpn0ECn-x4KAlhd{|Ny9Pf
z_NUS5+HS=oC$8TM9>@-qu4vlgF)nFT(b0s#4tQV6t(p-*vzXL?nC{c7C36!8u1}u7
z{)tmS50{myvhszap(U$Idu>Dr9q18fLf=4U>b{k
z`v6jN1C%QbP#cIzFBMa>q`YiX4=V>ij$X#qnP}^h3SvofvI#+{5ix}-rcgHAp&1a9
zxkuK+aMr_YCLwdLh2-&HI|?yTg@uHPr~qjqIH#5-Or#x_P+^*qv$fwm?YMpbD|qo>zZol*EhHWMV2>rsrV&%wW`}M-Oe%MS{0Lx-K%>$
zReS3>ReRG~Y>!CdjbRH>oMqd=nqVHl?g0(S{QuRu~N2V%uzT90mF5lCvC*lE#S>Htl3_Zp9V^GZzI+jU?{
zs3i^&wx$~fpHiYZe7FKfPRAcFgOL6QS{`ln8#{{jY^0tIwU_&c{u
zMkI?ca>BbtMls7sFUshCV}mM>4Q{kf^!=lZc_v4{&X_jNV=cQo*hep*r61N9Z`=RN`crB6KUHi
zkkmip0F*LfASt+MFKqI`OrMEUbE|&hJNS5-I<=Gv#%C1%U~^NF)*_%+D?Xw1;4n2|
zMag>@-|DjDKGmeQ@i-H@tu1dVxo~bo&2AI=0vM5Uh0@#{n${)(YV+TWD8>yb#*HY(
z^%P^c7NwV<;%V7|#1^3s2?Pp+RLi*g@HP=qNCjXkR*b^5W@1P1IJ|!P1C29My8yiQ
zoL4l;#ipH8}`
zbdVNv7_&)X$R>r6)g0C{Gf~G5CKF`WpQs8dlYHpY52s}`gbYngxN9nS%hn};m@@!*
zEszVA?M&^DN1sngl!Ga9lI6eD&{U9io%L0_AQ#F#Vw0XdojbQSWWOg(Ae0F5u8ovqL#&-eoW7*njmbRnfPPWaWGi+6_508?PA^qlItUHEEC-
z_o3mH&TVLB_UVEJYK-uy6`FPmcq|j|I+$`^hNH(Aaw7Y#y(uhEO1MfQWGN(x--K?R
z1)MKhaDVNxbHO+2N&fxOSkdrEZPc$$3rdumDQiT*SnVqu-O3tFjtwb@0fV^7tQi3?
z!l@2~hzDNj$XZD@#NabTC6vZ#Z0}vs+-6C#!7Q
z=uGIUGiTE*1%w7@ttBJ~EZ|1g$z&akmG+MGMk87(PiQHe(EDTiZC|z;i3}`BcS9t3
zlidl%%1)Oyi6r#Lh~e+0;i-4Gc{IQRb0u-5nNSUkBm1)Z=5cIGIRF^i_$p;x2#KV{
z^c0pb%hxQGZ#@#Q0e*Xxp8GRB=?*>n$?W>>>L#%;J)FfZbq1fu%Jn%
zB#eNQ-0OlYDP`ei1tE486}L*drj;*+!dGuU&Q9*0o&FC785WGYti{0eCcEPT-`&95
zN6J7qH&c?gnUmy6=qGotp6ynlntt;!!Y`Kh3TM1z
zE;)LKy>V6SeXGS!dhqikc|XnY$iu?ltA31cO*%A7K5^JcV0OjMx$F8KCzid*wf%p<
zGQGP8Y8K6o3|R_A3wN5=zs)TgMyqmA5k!@-Xf&q~@VTIcCP6j()&wV60k`>{mH
z0>yUt(9%g8!=zC~^-%d8)Xu0*9!?6cczAd_xu>G@6AqYX&(gP7$vGX?&a?TFe=BW#
zy2Hk3vkThNV#qL0Z7yRB>hvRE%O_~k;NKregLRzTm97k(so)(g2y|sAhU1))@-aKk
z&B@C7GGB{YLLF6+a#Dm1i#Db%x#*}CWtsH)aa;Cq=64PB`GkgsE3DV{Gqy}i6p#&<
z=VLjJ9v7WR&3L`QbPUuJKA%s{5CHrRY-(K(jg)mzNMOE{nq23}l&luI?B|iTft9gR
z1CqFq?+wAicMXHfccfVPn$x%eJ*h|5NGs!%44tja>Q>C_waa(EDMGgoKe
z;C=cOKKqO3a!z;C`nly()5Q3nXP4kD#reQTgPi~J>c46_WPr*NZ$m~7{>F>xveUcH
z4n@;Gy`xlEq^xyi91o-{d{-?X+
zyR)myt;ad2Ycp$p_;7cT|NYU`&HMi_iKaDry%9WmK_SuPwc@a9O?C5f%U9dGNPnXq
z`A6Vc6YU*ao4*jqzoCV9advqH(%Pi(P10g2r^VDLB1g7!on)Y1JY9UAFB1CnXq!31nND7m?r#DylDs)WUULW7w)vxwl`-^^`t4>T)SOOa!
z-Vm{U8zp+_Tl-WYBhvG-(P_f_^y=cTgn1x>%k%qz0pK%pFw%?eDFu_L>pE}+=Oj7f
zXqKUUrZ{8KQZ#A$sPIKAZMCpA%5q0h@6BPNO27Vab$k8c_->;mpC(Xsy|M?;d+#5%
zW;b8&KHOZLe7z%4pIlvBTwQ+82C-stn3gxKGP+Z_?esgtYhIWe_AVR4FE{Ubs={D`
zy?6wf5IfQlm4+vs7YJj7*W_eP!AKiLoYUGy8msoMYg3bi61tsa**TD%*#*Hn61;Ek
zhEgVE=FK;UYC$lY^saMeBwq()u6`(_DEF6go1>Woq)or8T!q|=eDmcm@^)Fuw$9-R
z+w_#%grXD_g)tb*HkhiK2IYd1VNPCyvqCgE5`~ttlp`U(L%T^982sM2TGEx0lLGsl
z$Mpa6e?H94E@n4dr{p0)9CmX3VRm`NvZ!}kSC{8s7r=~UYLTJP1FnKv__=2GxFoF`
zn5DSgLPeZL^8J-F@bDQfVY+I_1q?rrDXu!DZ+8o}?b8`-5{%UbMP&ZU^=9xusy+v3
zwwo0YUFeepqrxh4DfR+R$M1lbGu7!(g?L_ToAvKxz(y0m-I|iGNGpPl9}xxZt>*YC
zUI%!NNeDRk|EF;&Zf5V#ZlR=i