Merge branch 'main' into colinmegill/geneset-prototype

server/__init__.py

@@ -2,7 +2,7 @@ import logging
 import sys
 from server.common.utils.utils import import_plugins
 
-__version__ = "0.16.0"
+__version__ = "0.16.7"
 display_version = "cellxgene v" + __version__
 
 try:

server/app/app.py

@@ -240,7 +240,7 @@ class DatasetResource(Resource):
 
 class SchemaAPI(DatasetResource):
     # TODO @mdunitz separate dataset schema and user schema
-    @cache_control(no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.schema_get(data_adaptor)
@@ -261,7 +261,7 @@ class UserInfoAPI(DatasetResource):
 
 
 class AnnotationsObsAPI(DatasetResource):
-    @cache_control(public=True, no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.annotations_obs_get(request, data_adaptor)
@@ -434,11 +434,17 @@ class Server:
                 self.app.register_blueprint(dataroot_resources.blueprint)
 
                 self.app.add_url_rule(
-                    f"/{url_dataroot}/<dataset>/",
+                    f"/{url_dataroot}/<dataset>",
                     f"dataset_index_{url_dataroot}",
                     lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
                     methods=["GET"],
                 )
+                self.app.add_url_rule(
+                    f"/{url_dataroot}/<dataset>/",
+                    f"dataset_index_{url_dataroot}/",
+                    lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
+                    methods=["GET"],
+                )
                 self.app.add_url_rule(
                     f"/{url_dataroot}/<dataset>/static/<path:filename>",
                     f"static_assets_{url_dataroot}",

server/common/annotations/annotations.py

@@ -33,10 +33,10 @@ class Annotations(metaclass=ABCMeta):
             raise OntologyLoadFailure("Unable to find OBO ontology path") from e
 
         except SyntaxError as e:
-            raise OntologyLoadFailure("Syntax error loading OBO ontology") from e
+            raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
 
         except Exception as e:
-            raise OntologyLoadFailure("Error loading OBO file") from e
+            raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
 
     def get_schema(self, data_adaptor):
         schema = []

server/common/config/server_config.py

@@ -42,6 +42,9 @@ class ServerConfig(BaseConfig):
             self.app__web_base_url = default_config["app"]["web_base_url"]
 
             self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__insecure_test_environment = default_config["authentication"][
+                "insecure_test_environment"
+            ]
             self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
                 "oauth_api_base_url"
             ]
@@ -168,6 +171,10 @@ class ServerConfig(BaseConfig):
 
     def handle_authentication(self):
         self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
+
+        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
+            raise ConfigurationError("Test auth can only be used in an insecure test environment")
 
         # oauth
         ptypes = str if self.authentication__type == "oauth" else (type(None), str)

server/common/utils/type_conversion_utils.py

@@ -44,7 +44,7 @@ def get_dtype_from_dtype(dtype, array_values=None):
     if dtype_name == "bool":
         return np.uint8
     if dtype_name == "object" and dtype_kind == "O":
-        return np.unicode
+        return str
     if dtype_name == "category":
         return get_dtype_from_dtype(dtype.categories.dtype, array_values)
 

server/default_config.py

@@ -34,6 +34,7 @@ server:
     # session:  A session based userid is automatically generated. (no params needed)
     # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
     type: session
+    insecure_test_environment: false
 
     params_oauth:
        # url to the oauth server

server/requirements-dev.txt

@@ -8,3 +8,4 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability

server/requirements.txt

@@ -24,3 +24,4 @@ tiledb>=0.5.9,>=0.6.2,!=0.7.2
 s3fs==0.4.2
 scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
 sqlalchemy>=1.3.18
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

server/test/__init__.py

@@ -34,7 +34,10 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
     data_locator = DataLocator(fname)
     config = AppConfig()
     config.update_server_config(
-        app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test",
+        app__flask_secret_key="secret",
+        multi_dataset__dataroot=data_locator.path,
+        authentication__type="test",
+        authentication__insecure_test_environment=True,
     )
     config.update_default_dataset_config(
         embeddings__names=["umap"],

server/test/fixtures/server_config_outline.py

@@ -14,6 +14,7 @@ f"""server:
     web_base_url: {web_base_url}
   authentication:
     type: {auth_type}
+    insecure_test_environment: {insecure_test_environment}
     params_oauth:
       oauth_api_base_url: {oauth_api_base_url}
       client_id: {client_id}

server/test/locust/requirements-locust.txt

@@ -1 +1,2 @@
 locust
+-r ../../requirements.txt

server/test/unit/auth/test_auth.py

@@ -45,6 +45,7 @@ class AuthTest(unittest.TestCase):
         app_config = AppConfig()
         app_config.update_server_config(app__flask_secret_key="secret")
         app_config.update_server_config(authentication__type="test")
+        app_config.update_server_config(authentication__insecure_test_environment=True)
         app_config.update_server_config(
             multi_dataset__dataroot=dict(
                 a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
@@ -79,7 +80,7 @@ class AuthTest(unittest.TestCase):
             r = session.get(f"{server}/{login_uri}")
             # check that the login redirect worked
             self.assertEqual(r.history[0].status_code, 302)
-            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
 
             config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
             userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
@@ -91,7 +92,7 @@ class AuthTest(unittest.TestCase):
             r = session.get(f"{server}/{logout_uri}")
             # check that the logout redirect worked
             self.assertEqual(r.history[0].status_code, 302)
-            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
             config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
             userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
             self.assertFalse(userinfo["userinfo"]["is_authenticated"])
@@ -116,6 +117,7 @@ class AuthTest(unittest.TestCase):
         app_config.update_server_config(
             authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
         )
+        app_config.update_server_config(authentication__insecure_test_environment=True)
 
         app_config.complete_config()
 

server/test/unit/common/config/__init__.py

@@ -38,6 +38,7 @@ class ConfigTests(unittest.TestCase):
         api_base_url="null",
         web_base_url="null",
         auth_type="session",
+        insecure_test_environment="false",
         oauth_api_base_url="null",
         client_id="null",
         client_secret="null",

server/test/unit/common/config/test_server_config.py

@@ -53,7 +53,7 @@ class TestServerConfig(ConfigTests):
     def test_complete_config_checks_all_attr(self, mock_check_attrs):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.server_config.complete_config(self.context)
-        self.assertEqual(mock_check_attrs.call_count, 40)
+        self.assertEqual(mock_check_attrs.call_count, 41)
 
     def test_handle_app__throws_error_if_port_doesnt_exist(self):
         config = self.get_config(port=99999999)
@@ -278,6 +278,14 @@ class TestServerConfig(ConfigTests):
             response = session.get(f"{server}/health")
             assert response.json()["status"] == "pass"
 
+            # access a dataset (no slash)
+            response = session.get(f"{server}/set2/pbmc3k.cxg")
+            self.assertEqual(response.status_code, 200)
+
+            # access a dataset (with slash)
+            response = session.get(f"{server}/set2/pbmc3k.cxg/")
+            self.assertEqual(response.status_code, 200)
+
     @patch("server.common.config.server_config.diffexp_tiledb.set_config")
     def test_handle_diffexp(self, mock_tiledb_config):
         custom_config_file = self.custom_app_config(
@@ -304,3 +312,12 @@ class TestServerConfig(ConfigTests):
         mock_tiledb_context.assert_called_once_with(
             {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
         )
+
+    def test_test_auth_only_in_insecure(self):
+
+        config = self.get_config(auth_type="test")
+        with self.assertRaises(ConfigurationError):
+            config.complete_config()
+
+        config.update_server_config(authentication__insecure_test_environment=True)
+        config.complete_config()

server/test/unit/common/utils/test_type_conversion_utils.py

@@ -99,7 +99,7 @@ class TestTypeConversionUtils(unittest.TestCase):
 
     def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
         types = [np.float32, np.int32, np.bool_, str]
-        expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
+        expected_dtypes = [np.float32, np.int32, np.uint8, str]
 
         for test_type_index in range(len(types)):
             with self.subTest(
@@ -110,7 +110,7 @@ class TestTypeConversionUtils(unittest.TestCase):
 
     def test__get_dtype_of_array__categories_return_as_expected(self):
         array = Series(data=["a", "b", "c"], dtype="category")
-        expected_dtype = np.unicode
+        expected_dtype = str
 
         actual_dtype = get_dtype_of_array(array)
 
@@ -179,7 +179,7 @@ class TestTypeConversionUtils(unittest.TestCase):
         category_array = Series(data=["a", "b", "b"], dtype="category")
         dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
 
-        expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
+        expected_data_types_dict = {"float_array": np.float32, "category_array": str}
         expected_schema_type_hints_dict = {
             "float_array": {"type": "float32"},
             "category_array": {"type": "categorical", "categories": ["a", "b"]},