Add basic authentication in the server (#1670)

* Add basic authentication in the server

A pattern for creating authentication methods is introduced, with three
authentication types defined:
  none - no authentication
  session - like the current session based auth used for user annotations
  test - used to test the login/logout process end to end

The config endpoint now returns informations about the authentication, like if
the user is authenticated and their username.  The redirect uri's for login and
logout are also returned if the authentication type requires login

This is the first a several PRs for authentication.

*. Update server tests to avoid hardcoded ports

test_api and test_nan_rest now use a common function for starting a test server,
than will initially choose a random port.

server/app/app.py

@@ -85,6 +85,7 @@ def dataset_index(url_dataroot=None, dataset=None):
     try:
         cache_manager = current_app.matrix_data_cache_manager
         with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
+            data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
             dataset_title = app_config.get_title(data_adaptor)
             return render_template(
                 "index.html", datasetTitle=dataset_title, SCRIPTS=scripts, INLINE_SCRIPTS=inline_scripts
@@ -143,6 +144,7 @@ def rest_get_data_adaptor(func):
     def wrapped_function(self, dataset=None):
         try:
             with get_data_adaptor(self.url_dataroot, dataset) as data_adaptor:
+                data_adaptor.set_uri_path(f"{self.url_dataroot}/{dataset}")
                 return func(self, data_adaptor)
         except DatasetAccessError as e:
             return common_rest.abort_and_log(
@@ -160,6 +162,17 @@ def dataroot_test_index():
 
     config = current_app.app_config
     server_config = config.server_config
+
+    auth = server_config.auth
+    if auth.is_valid():
+        if server_config.auth.is_authenticated():
+            data += f"<p>Logged in as {auth.get_userid()} / {auth.get_username()}</p>"
+        if auth.requires_client_login():
+            if server_config.auth.is_authenticated():
+                data += "<p><a href='/logout'>Logout</a></p>"
+            else:
+                data += "<p><a href='/login'>Login</a></p>"
+
     datasets = []
     for dataroot_dict in server_config.multi_dataset__dataroot.values():
         dataroot = dataroot_dict["dataroot"]
@@ -338,10 +351,15 @@ class Server:
                     lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
                     methods=["GET"],
                 )
+
         else:
             bp_api = Blueprint("api", __name__, url_prefix=api_version)
             resources = get_api_resources(bp_api)
             self.app.register_blueprint(resources.blueprint)
 
+        self.app.auth = server_config.auth
+        if self.app.auth.requires_client_login():
+            self.app.auth.add_url_rules(self.app)
+
         self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
         self.app.app_config = app_config

server/auth/__init__.py

@@ -0,0 +1,6 @@
+
+# import the built in auth types so they can be registered
+
+import server.auth.auth_none  # noqa: F401
+import server.auth.auth_test  # noqa: F401
+import server.auth.auth_session  # noqa: F401

server/auth/auth.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+
+
+class AuthTypeBase(ABC):
+    """Base type for all authentication types."""
+
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def set_params(self, params):
+        """Set the parameters from app config.  raise ConfigurationError if any params are invalid"""
+        pass
+
+    @abstractmethod
+    def is_valid(self):
+        """Return True if the auth type can return user info (AuthTypeNone is the only one that cannot)"""
+        pass
+
+    def requires_client_login(self):
+        """Return True if the user needs to login from the client (e.g. Login button is shown)"""
+        return False
+
+    @abstractmethod
+    def is_authenticated(self):
+        """Return True if the user is authenticated"""
+        pass
+
+    @abstractmethod
+    def get_userid(self):
+        """Return the id for this user (string)"""
+        pass
+
+    @abstractmethod
+    def get_username(self):
+        """Return the name of the user (string)"""
+        pass
+
+
+class AuthTypeClientBase(AuthTypeBase):
+    """Base type for all authentication types that require the client to login"""
+
+    def __init__(self):
+        super().__init__()
+
+    def requires_client_login(self):
+        return True
+
+    @abstractmethod
+    def add_url_rules(self, selfapp):
+        """Add url rules to the app (like /login, /logout, etc)"""
+        pass
+
+    @abstractmethod
+    def get_login_url(self, data_adaptor):
+        """Return the url for the login route"""
+        pass
+
+    @abstractmethod
+    def get_logout_url(self, data_adaptor):
+        """Return the url for the logout route"""
+        pass
+
+
+class AuthTypeFactory:
+    """Factory class to create an authentication type"""
+
+    auth_types = {}
+
+    @staticmethod
+    def register(name, auth_type):
+        assert(issubclass(auth_type, AuthTypeBase))
+        AuthTypeFactory.auth_types[name] = auth_type
+
+    @staticmethod
+    def create(name):
+        auth_type = AuthTypeFactory.auth_types.get(name)
+        if auth_type is None:
+            return None
+        return auth_type()

server/auth/auth_none.py

@@ -0,0 +1,27 @@
+from server.auth.auth import AuthTypeBase, AuthTypeFactory
+from server.common.errors import ConfigurationError
+
+
+class AuthTypeNone(AuthTypeBase):
+
+    def __init__(self):
+        super().__init__()
+
+    def is_valid(self):
+        return False
+
+    def set_params(self, params):
+        if params:
+            raise ConfigurationError("not expecting authentication parameters")
+
+    def is_authenticated(self):
+        return True
+
+    def get_userid(self):
+        return None
+
+    def get_username(self):
+        return None
+
+
+AuthTypeFactory.register(None, AuthTypeNone)

server/auth/auth_session.py

@@ -0,0 +1,36 @@
+from server.auth.auth import AuthTypeBase, AuthTypeFactory
+from flask import session
+from uuid import uuid4
+
+
+class AuthTypeSession(AuthTypeBase):
+    """Session based authentication.  The user is always logged.  The user id is a random number
+    associated with the session.  This is a good choice for desktop servers."""
+
+    # key in the session token for userid
+    CXGUID = "cxguid"
+
+    def __init__(self):
+        super().__init__()
+
+    def is_valid(self):
+        return True
+
+    def set_params(self, params):
+        return
+
+    def is_authenticated(self):
+        # always authenticated
+        return True
+
+    def get_userid(self):
+        if self.CXGUID not in session:
+            session[self.CXGUID] = uuid4().hex
+            session.permanent = True
+        return session[self.CXGUID]
+
+    def get_username(self):
+        return "anonymous"
+
+
+AuthTypeFactory.register("session", AuthTypeSession)

server/auth/auth_test.py

@@ -0,0 +1,69 @@
+from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from flask import session, request, redirect, current_app
+
+
+class AuthTypeTest(AuthTypeClientBase):
+    """An authentication type for testing client based logins.  When the login route is accessed
+    the user is automatically logged in with a default or configured username"""
+
+    # key in session token with userid and username
+    CXGUID = "cxguid_test"
+    CXGUNAME = "cxguname_test"
+
+    def __init__(self):
+        super().__init__()
+        self.username = "test_account"
+        self.userid = "id0001"
+
+    def is_valid(self):
+        return True
+
+    def requires_client_login(self):
+        return True
+
+    def add_url_rules(self, app):
+        app.add_url_rule("/login", "login", self.login, methods=["GET"])
+        app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
+
+    def set_params(self, params):
+        if params:
+            self.username = params.get("username", self.username)
+            self.userid = params.get("userid", self.userid)
+
+    def is_authenticated(self):
+        return self.CXGUID in session
+
+    def get_userid(self):
+        return session.get(self.CXGUID)
+
+    def get_username(self):
+        return session.get(self.CXGUNAME)
+
+    def login(self):
+        args = request.args
+        return_to = args.get("dataset", "/")
+        session[self.CXGUID] = args.get("userid", self.userid)
+        session[self.CXGUNAME] = args.get("username", self.username)
+        return redirect(return_to)
+
+    def logout(self):
+        session.clear()
+        return_to = request.args.get("dataset", "/")
+        return redirect(return_to)
+
+    def get_login_url(self, data_adaptor):
+        """Return the url for the login route"""
+        if current_app.app_config.is_multi_dataset():
+            return f"/login?dataset={data_adaptor.uri_path}"
+        else:
+            return "/login"
+
+    def get_logout_url(self, data_adaptor):
+        """Return the url for the logout route"""
+        if current_app.app_config.is_multi_dataset():
+            return f"/logout?dataset={data_adaptor.uri_path}"
+        else:
+            return "/logout"
+
+
+AuthTypeFactory.register("test", AuthTypeTest)

server/common/annotations.py

@@ -1,6 +1,5 @@
 from datetime import datetime
 import re
-from uuid import uuid4
 import os
 import pandas as pd
 from hashlib import blake2b
@@ -11,7 +10,7 @@ from server.common.errors import AnnotationsError, OntologyLoadFailure
 from server.common.utils import series_to_schema
 import fsspec
 import fastobo
-from flask import session
+from flask import session, current_app
 from abc import ABCMeta, abstractmethod
 
 
@@ -80,7 +79,6 @@ class Annotations(metaclass=ABCMeta):
 
 class AnnotationsLocalFile(Annotations):
 
-    CXGUID = "cxguid"
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
     def __init__(self, output_dir, output_file):
@@ -159,18 +157,12 @@ class AnnotationsLocalFile(Annotations):
             self.last_fname = fname
             self.last_labels = df
 
-    def _get_userid(self):
-        if self.CXGUID not in session:
-            session[self.CXGUID] = uuid4().hex
-            session.permanent = True
-        return session[self.CXGUID]
-
     def _get_userdata_idhash(self, data_adaptor):
         """
         Return a short hash that weakly identifies the user and dataset.
         Used to create safe annotations output file names.
         """
-        uid = self._get_userid()
+        uid = current_app.auth.get_userid()
         id = (uid + data_adaptor.get_location()).encode()
         idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
         return idhash
@@ -257,8 +249,9 @@ class AnnotationsLocalFile(Annotations):
 
         elif session is not None:
             collection = self.get_collection()
-            params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
+            if current_app.auth.is_authenticated():
-            params["annotations-data-collection-is-read-only"] = False
+                params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-            params["annotations-data-collection-name"] = collection
+                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-name"] = collection
 
         parameters.update(params)

server/common/app_config.py

@@ -16,6 +16,7 @@ from server.common.annotations import AnnotationsLocalFile
 from server.common.utils import custom_format_warning
 import server.compute.diffexp_cxg as diffexp_tiledb
 from server.common.data_locator import discover_s3_region_name
+from server.auth.auth import AuthTypeFactory
 
 DEFAULT_SERVER_PORT = 5005
 # anything bigger than this will generate a special message
@@ -194,6 +195,7 @@ class AppConfig(object):
         server_config = self.server_config
         dataset_config = data_adaptor.dataset_config
         annotation = dataset_config.user_annotations
+        auth = server_config.auth
 
         # FIXME The current set of config is not consistently presented:
         # we have camalCase, hyphen-text, and underscore_text
@@ -257,6 +259,18 @@ class AppConfig(object):
             "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
         }
 
+        if dataset_config.app__authentication_enable and auth.is_valid():
+            config["authentication"] = {
+                "is_authenticated": auth.is_authenticated(),
+                "requires_client_login": auth.requires_client_login(),
+                "username": auth.get_username(),
+            }
+            if auth.requires_client_login():
+                config["authentication"].update({
+                    "login": auth.get_login_url(data_adaptor),
+                    "logout" : auth.get_logout_url(data_adaptor),
+                })
+
         return c
 
 
@@ -366,6 +380,7 @@ class ServerConfig(BaseConfig):
     def __init__(self, app_config, default_config):
         dictval_cases = [
             ("app", "csp_directives"),
+            ("authentication", "params"),
             ("adaptor", "cxg_adaptor", "tiledb_ctx"),
             ("multi_dataset", "dataroot"),
         ]
@@ -384,6 +399,9 @@ class ServerConfig(BaseConfig):
             self.app__server_timing_headers = dc["app"]["server_timing_headers"]
             self.app__csp_directives = dc["app"]["csp_directives"]
 
+            self.authentication__type = dc["authentication"]["type"]
+            self.authentication__params = dc["authentication"]["params"]
+
             self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
             self.multi_dataset__index = dc["multi_dataset"]["index"]
             self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
@@ -414,8 +432,12 @@ class ServerConfig(BaseConfig):
         # The matrix data cache manager is created during the complete_config and stored here.
         self.matrix_data_cache_manager = None
 
+        # The authentication object (BCM -- better name)
+        self.auth = None
+
     def complete_config(self, context):
         self.handle_app(context)
+        self.handle_authentication(context)
         self.handle_data_locator(context)
         self.handle_adaptor(context)  # may depend on data_locator
         self.handle_single_dataset(context)  # may depend on adaptor
@@ -484,6 +506,14 @@ class ServerConfig(BaseConfig):
                 elif not isinstance(v, str):
                     raise ConfigurationError("CSP directive value must be a string or list of strings.")
 
+    def handle_authentication(self, context):
+        self.check_attr("authentication__type", (type(None), str))
+        self.check_attr("authentication__params", (type(None), dict))
+        self.auth = AuthTypeFactory.create(self.authentication__type)
+        if self.auth is None:
+            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+        self.auth.set_params(self.authentication__params)
+
     def handle_data_locator(self, context):
         self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
         if self.data_locator__s3__region_name is True:
@@ -660,6 +690,7 @@ class DatasetConfig(BaseConfig):
             self.app__inline_scripts = dc["app"]["inline_scripts"]
             self.app__about_legal_tos = dc["app"]["about_legal_tos"]
             self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
+            self.app__authentication_enable = dc["app"]["authentication_enable"]
 
             self.presentation__max_categories = dc["presentation"]["max_categories"]
             self.presentation__custom_colors = dc["presentation"]["custom_colors"]
@@ -696,6 +727,7 @@ class DatasetConfig(BaseConfig):
         self.check_attr("app__inline_scripts", list)
         self.check_attr("app__about_legal_tos", (type(None), str))
         self.check_attr("app__about_legal_privacy", (type(None), str))
+        self.check_attr("app__authentication_enable", bool)
 
         # scripts can be string (filename) or dict (attributes).   Convert string to dict.
         scripts = []
@@ -721,6 +753,13 @@ class DatasetConfig(BaseConfig):
         self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
 
         if self.user_annotations__enable:
+            server_config = self.app_config.server_config
+            if not self.app__authentication_enable:
+                raise ConfigurationError("user annotations requires authentication to be enabled")
+            if not server_config.auth.is_valid():
+                auth_type = server_config.authentication__type
+                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
             # TODO, replace this with a factory pattern once we have more than one way
             # to do annotations.  currently only local_file_csv
             if self.user_annotations__type != "local_file_csv":

server/common/default_config.py

@@ -14,6 +14,15 @@ server:
     server_timing_headers: false
     csp_directives: null
 
+  authentication:
+    # The authentication types may be "none" or "session"
+    # none:  No authentication support, features like user_annotations must not be enabled.
+    # session:  A session based userid is automatically generated.
+    type: session
+
+    # a dictionary of parameters that may be required for an authentication type
+    params: null
+
   multi_dataset:
     # If dataroot is set, then cellxgene may serve multiple datasets.  This parameter is not
     # compatible with single_dataset/datapath.
@@ -132,6 +141,9 @@ dataset:
     about_legal_tos: null
     about_legal_privacy: null
 
+    # allow authentication support
+    authentication_enable: true
+
   presentation:
     max_categories: 1000
     custom_colors: true

server/data_common/data_adaptor.py

@@ -28,6 +28,11 @@ class DataAdaptor(metaclass=ABCMeta):
 
         # parameters set by this data adaptor based on the data.
         self.parameters = {}
+        self.uri_path = None
+
+    def set_uri_path(self, path):
+        # uri path to the dataset, e.g. /d/<datasetname>
+        self.uri_path = path
 
     @staticmethod
     @abstractmethod

server/test/__init__.py

@@ -86,15 +86,14 @@ def random_string(n):
     return "".join(random.choice(string.ascii_letters) for _ in range(n))
 
 
-@contextmanager
+def start_test_server(command_line_args=[], app_config=None):
-def test_server(command_line_args=[], app_config=None):
+    """
-    """A context to run the cellxgene server.
     Command line arguments can be passed in, as well as an app_config.
     This function is meant to be used like this, for example:
 
     with test_server(...) as server:
-       r = requests.get(f"{server}/...")
+        r = requests.get(f"{server}/...")
-       // check r
+        // check r
 
     where the server can be accessed within the context, and is terminated when
     the context is exited.
@@ -104,7 +103,8 @@ def test_server(command_line_args=[], app_config=None):
     yaml config file, which this server will read and parse.
     """
 
-    port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+    start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1)
+    port = int(os.environ.get("CXG_SERVER_PORT", start))
     port = find_available_port("localhost", port)
     command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args
 
@@ -128,10 +128,25 @@ def test_server(command_line_args=[], app_config=None):
     if tempdir:
         tempdir.cleanup()
 
+    return ps, server
+
+
+def stop_test_server(ps):
+    try:
+        ps.terminate()
+    except ProcessLookupError:
+        pass
+
+
+@contextmanager
+def test_server(command_line_args=[], app_config=None):
+    """A context to run the cellxgene server."""
+
+    ps, server = start_test_server(command_line_args, app_config)
     try:
         yield server
     finally:
         try:
-            ps.terminate()
+            stop_test_server(ps)
         except ProcessLookupError:
             pass

server/test/test_api.py

@@ -2,7 +2,6 @@ import shutil
 import time
 import unittest
 from http import HTTPStatus
-from subprocess import Popen
 
 import pandas as pd
 import requests
@@ -11,6 +10,7 @@ import server.test.decode_fbs as decode_fbs
 from server.data_common.matrix_loader import MatrixDataType
 from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT
 from server.test.test_datasets.fixtures import pbmc3k_colors
+from server.test import start_test_server, stop_test_server
 
 
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
@@ -21,9 +21,6 @@ BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
 class EndPoints(object):
     ANNOTATIONS_ENABLED = True
 
-    def setUp(self):
-        self.session = requests.Session()
-
     def test_initialize(self):
         endpoint = "schema"
         url = f"{self.URL_BASE}{endpoint}"
@@ -308,13 +305,13 @@ class EndPoints(object):
     def test_static(self):
         endpoint = "static"
         file = "assets/favicon.ico"
-        url = f"{self.LOCAL_URL}{endpoint}/{file}"
+        url = f"{self.server}/{endpoint}/{file}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
 
-    @staticmethod
+    def _setupClass(child_class, command_line):
-    def _setUpClass(child_class, start_command):
+        child_class.ps, child_class.server = start_test_server(command_line)
-        child_class.ps = Popen(start_command)
+        child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
         child_class.session = requests.Session()
         for i in range(90):
             try:
@@ -323,13 +320,6 @@ class EndPoints(object):
             except requests.exceptions.ConnectionError:
                 time.sleep(1)
 
-    @staticmethod
-    def _tearDownClass(child_class):
-        try:
-            child_class.ps.terminate()
-        except ProcessLookupError:
-            pass
-
 
 class EndPointsAnnotations(EndPoints):
     def test_get_schema_existing_writable(self):
@@ -385,32 +375,19 @@ class EndPointsAnnotations(EndPoints):
 class EndPointsAnndata(unittest.TestCase, EndPoints):
     """Test Case for endpoints"""
 
-    PORT = 5010
-    LOCAL_URL = f"http://127.0.0.1:{PORT}/"
-    VERSION = "v0.2"
-    URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
     ANNOTATIONS_ENABLED = False
 
     @classmethod
     def setUpClass(cls):
-        cls._setUpClass(
+        cls._setupClass(cls, [
-            cls,
+            f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
-            [
+            "--disable-annotations",
-                "cellxgene",
+            "--experimental-enable-reembedding",
-                "--no-upgrade-check",
+        ])
-                "launch",
-                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
-                "--disable-annotations",
-                "--verbose",
-                "--experimental-enable-reembedding",
-                "--port",
-                str(cls.PORT),
-            ],
-        )
 
     @classmethod
     def tearDownClass(cls):
-        cls._tearDownClass(cls)
+        stop_test_server(cls.ps)
 
     @property
     def annotations_enabled(self):
@@ -420,98 +397,57 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
 class EndPointsCxg(unittest.TestCase, EndPoints):
     """Test Case for endpoints"""
 
-    PORT = 5011
-    LOCAL_URL = f"http://127.0.0.1:{PORT}/"
-    VERSION = "v0.2"
-    URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
     ANNOTATIONS_ENABLED = False
 
     @classmethod
     def setUpClass(cls):
-        cls._setUpClass(
+        cls._setupClass(cls, [
-            cls,
+            f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg",
-            [
+            "--disable-annotations",
-                "cellxgene",
+        ])
-                "--no-upgrade-check",
-                "launch",
-                f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg",
-                "--disable-annotations",
-                "--verbose",
-                "--port",
-                str(cls.PORT),
-            ],
-        )
 
     @classmethod
     def tearDownClass(cls):
-        cls._tearDownClass(cls)
+        stop_test_server(cls.ps)
 
 
 class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
     """Test Case for endpoints"""
 
-    PORT = 5012
-    LOCAL_URL = f"http://127.0.0.1:{PORT}/"
-    VERSION = "v0.2"
-    URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
     ANNOTATIONS_ENABLED = True
-    MATRIX_DATA_TYPE = MatrixDataType.H5AD
 
     @classmethod
     def setUpClass(cls):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
             MatrixDataType.H5AD, annotations_fixture=True
         )
-        cls._setUpClass(
+        cls._setupClass(cls, [
-            cls,
+            "--annotations-file",
-            [
+            cls.annotations.output_file,
-                "cellxgene",
+            cls.data.get_location(),
-                "--no-upgrade-check",
+        ])
-                "launch",
-                "--annotations-file",
-                cls.annotations.output_file,
-                "--verbose",
-                "--port",
-                str(cls.PORT),
-                cls.data.get_location(),
-            ],
-        )
 
     @classmethod
     def tearDownClass(cls):
         shutil.rmtree(cls.tmp_dir)
-        cls._tearDownClass(cls)
+        stop_test_server(cls.ps)
 
 
 class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
     """Test Case for endpoints"""
 
-    PORT = 5013
-    LOCAL_URL = f"http://127.0.0.1:{PORT}/"
-    VERSION = "v0.2"
-    URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
     ANNOTATIONS_ENABLED = True
-    MATRIX_DATA_TYPE = MatrixDataType.CXG
 
     @classmethod
     def setUpClass(cls):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
-        cls._setUpClass(
+        cls._setupClass(cls, [
-            cls,
+            "--annotations-file",
-            [
+            cls.annotations.output_file,
-                "cellxgene",
+            cls.data.get_location(),
-                "--no-upgrade-check",
+        ])
-                "launch",
-                "--annotations-file",
-                cls.annotations.output_file,
-                "--verbose",
-                "--port",
-                str(cls.PORT),
-                cls.data.get_location(),
-            ],
-        )
 
     @classmethod
     def tearDownClass(cls):
         shutil.rmtree(cls.tmp_dir)
-        cls._tearDownClass(cls)
+        stop_test_server(cls.ps)

server/test/test_auth.py

@@ -0,0 +1,142 @@
+import unittest
+from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT, test_server
+import requests
+
+
+class AuthTest(unittest.TestCase):
+    def test_auth_none(self):
+        c = AppConfig()
+        c.update_server_config(
+            authentication__type=None, multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets"
+        )
+        c.update_default_dataset_config(user_annotations__enable=False)
+
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert "authentication" not in data_config["config"]
+
+    def test_auth_session(self):
+        c = AppConfig()
+        c.update_server_config(
+            authentication__type="session", multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets"
+        )
+        c.update_default_dataset_config(user_annotations__enable=True)
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["authentication"]["is_authenticated"]
+            assert not data_config["config"]["authentication"]["requires_client_login"]
+            assert data_config["config"]["authentication"]["username"] == "anonymous"
+
+    def test_auth_test(self):
+        c = AppConfig()
+        c.update_server_config(authentication__type="test")
+        c.update_server_config(
+            multi_dataset__dataroot=dict(
+                a1=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="auth"),
+                a2=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="no-auth"),
+            )
+        )
+
+        # specialize the configs
+        c.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
+        c.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
+
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+
+            # auth datasets
+            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["requires_client_login"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]
+
+            login_uri = data_config["config"]["authentication"]["login"]
+            logout_uri = data_config["config"]["authentication"]["logout"]
+
+            assert login_uri == "/login?dataset=auth/pbmc3k.cxg"
+            assert logout_uri == "/logout?dataset=auth/pbmc3k.cxg"
+
+            r = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/auth/pbmc3k.cxg/"
+
+            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] == "test_account"
+            assert data_config["config"]["parameters"]["annotations"]
+
+            r = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/auth/pbmc3k.cxg/"
+            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]
+
+            # no-auth datasets
+            r = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert "authentication" not in data_config["config"]
+            assert not data_config["config"]["parameters"]["annotations"]
+
+    def test_auth_test_single(self):
+        c = AppConfig()
+        c.update_server_config(
+            authentication__type="test",
+            single_dataset__datapath=f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
+
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+
+            r = session.get(f"{server}/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["requires_client_login"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]
+
+            login_uri = data_config["config"]["authentication"]["login"]
+            logout_uri = data_config["config"]["authentication"]["logout"]
+
+            assert login_uri == "/login"
+            assert logout_uri == "/logout"
+
+            r = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/"
+
+            r = session.get(f"{server}/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] == "test_account"
+            assert data_config["config"]["parameters"]["annotations"]
+
+            r = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/"
+            r = session.get(f"{server}/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]

server/test/test_nan_rest.py

@@ -1,17 +1,13 @@
 from http import HTTPStatus
-from subprocess import Popen
 import unittest
-import time
 import math
+from server.test import start_test_server, stop_test_server
 
 import server.test.decode_fbs as decode_fbs
 
 import requests
 
-LOCAL_URL = "http://127.0.0.1:5006/"
 VERSION = "v0.2"
-URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
-
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
 
 
@@ -20,33 +16,25 @@ class WithNaNs(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.ps = Popen(["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"])
+        cls.ps, cls.server = start_test_server(["test/test_datasets/nan.h5ad"])
-        session = requests.Session()
-        for i in range(90):
-            try:
-                session.get(f"{URL_BASE}schema")
-            except requests.exceptions.ConnectionError:
-                time.sleep(1)
 
     @classmethod
     def tearDownClass(cls):
-        try:
+        stop_test_server(cls.ps)
-            cls.ps.terminate()
-        except ProcessLookupError:
-            pass
 
     def setUp(self):
         self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
 
     def test_initialize(self):
         endpoint = "schema"
-        url = f"{URL_BASE}{endpoint}"
+        url = f"{self.url_base}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
 
     def test_data(self):
         endpoint = "data/var"
-        url = f"{URL_BASE}{endpoint}"
+        url = f"{self.url_base}{endpoint}"
         filter = {"filter": {"var": {"index": [[0, 20]]}}}
         result = self.session.put(url, json=filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -56,7 +44,7 @@ class WithNaNs(unittest.TestCase):
 
     def test_annotation_obs(self):
         endpoint = "annotations/obs"
-        url = f"{URL_BASE}{endpoint}"
+        url = f"{self.url_base}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
@@ -65,7 +53,7 @@ class WithNaNs(unittest.TestCase):
 
     def test_annotation_var(self):
         endpoint = "annotations/var"
-        url = f"{URL_BASE}{endpoint}"
+        url = f"{self.url_base}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")