add oauth authentication (#1681)

* add oauth authentication

Add support for OAuth2.

Change the interface to AuthTypeBase
  - better handling of config parameters
  - add a complete_setup function for additional setup steps

Added a function wrapper to enforce authentication for the
routes that require authenticaiton.

* change fsspec requirement

fsspec 0.8.0 breaks our tests
it imports a module that is does not require.

server/app/app.py

@@ -139,6 +139,18 @@ def get_data_adaptor(url_dataroot=None, dataset=None):
     return cache_manager.data_adaptor(dataset_key, datapath, config)
 
 
+def requires_authentication(func):
+    @wraps(func)
+    def wrapped_function(self, *args, **kwargs):
+        auth = current_app.auth
+        if auth.is_user_authenticated():
+            return func(self, *args, **kwargs)
+        else:
+            return make_response("not authenticated", HTTPStatus.UNAUTHORIZED)
+
+    return wrapped_function
+
+
 def rest_get_data_adaptor(func):
     @wraps(func)
     def wrapped_function(self, dataset=None):
@@ -164,11 +176,11 @@ def dataroot_test_index():
     server_config = config.server_config
 
     auth = server_config.auth
-    if auth.is_valid():
+    if auth.is_valid_authentication_type():
-        if server_config.auth.is_authenticated():
+        if server_config.auth.is_user_authenticated():
-            data += f"<p>Logged in as {auth.get_userid()} / {auth.get_username()}</p>"
+            data += f"<p>Logged in as {auth.get_user_id()} / {auth.get_user_name()} / {auth.get_user_email()}</p>"
         if auth.requires_client_login():
-            if server_config.auth.is_authenticated():
+            if server_config.auth.is_user_authenticated():
                 data += "<p><a href='/logout'>Logout</a></p>"
             else:
                 data += "<p><a href='/login'>Login</a></p>"
@@ -237,6 +249,7 @@ class AnnotationsObsAPI(DatasetResource):
     def get(self, data_adaptor):
         return common_rest.annotations_obs_get(request, data_adaptor)
 
+    @requires_authentication
     @cache_control(no_store=True)
     @rest_get_data_adaptor
     def put(self, data_adaptor):
@@ -357,9 +370,11 @@ class Server:
             resources = get_api_resources(bp_api)
             self.app.register_blueprint(resources.blueprint)
 
-        self.app.auth = server_config.auth
-        if self.app.auth.requires_client_login():
-            self.app.auth.add_url_rules(self.app)
-
         self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
         self.app.app_config = app_config
+
+        auth = server_config.auth
+        self.app.auth = auth
+        if auth.requires_client_login():
+            auth.add_url_rules(self.app)
+        auth.complete_setup(self.app)

server/auth/__init__.py

@@ -4,3 +4,4 @@
 import server.auth.auth_none  # noqa: F401
 import server.auth.auth_test  # noqa: F401
 import server.auth.auth_session  # noqa: F401
+import server.auth.auth_oauth  # noqa: F401

server/auth/auth.py

@@ -8,13 +8,9 @@ class AuthTypeBase(ABC):
         super().__init__()
 
     @abstractmethod
-    def set_params(self, params):
+    def is_valid_authentication_type(self):
-        """Set the parameters from app config.  raise ConfigurationError if any params are invalid"""
+        """Return True if the auth type is valid, e.g. it can return userinfo and username.
-        pass
+          (AuthTypeNone is the only one type that returns False)"""
-
-    @abstractmethod
-    def is_valid(self):
-        """Return True if the auth type can return user info (AuthTypeNone is the only one that cannot)"""
         pass
 
     def requires_client_login(self):
@@ -22,17 +18,28 @@ class AuthTypeBase(ABC):
         return False
 
     @abstractmethod
-    def is_authenticated(self):
+    def complete_setup(self, app):
+        """complete any setup that may be needed by this auth type.  The Flask app is passed in.
+        This is the last auth function called before the server starts to run."""
+        pass
+
+    @abstractmethod
+    def is_user_authenticated(self):
         """Return True if the user is authenticated"""
         pass
 
     @abstractmethod
-    def get_userid(self):
+    def get_user_id(self):
         """Return the id for this user (string)"""
         pass
 
     @abstractmethod
-    def get_username(self):
+    def get_user_name(self):
+        """Return the name of the user (string)"""
+        pass
+
+    @abstractmethod
+    def get_user_email(self):
         """Return the name of the user (string)"""
         pass
 
@@ -73,8 +80,8 @@ class AuthTypeFactory:
         AuthTypeFactory.auth_types[name] = auth_type
 
     @staticmethod
-    def create(name):
+    def create(name, app_config):
         auth_type = AuthTypeFactory.auth_types.get(name)
         if auth_type is None:
             return None
-        return auth_type()
+        return auth_type(app_config)

server/auth/auth_none.py

@@ -1,26 +1,27 @@
 from server.auth.auth import AuthTypeBase, AuthTypeFactory
-from server.common.errors import ConfigurationError
 
 
 class AuthTypeNone(AuthTypeBase):
 
-    def __init__(self):
+    def __init__(self, app_config):
         super().__init__()
 
-    def is_valid(self):
+    def is_valid_authentication_type(self):
         return False
 
-    def set_params(self, params):
+    def complete_setup(self, app):
-        if params:
+        pass
-            raise ConfigurationError("not expecting authentication parameters")
 
-    def is_authenticated(self):
+    def is_user_authenticated(self):
         return True
 
-    def get_userid(self):
+    def get_user_id(self):
         return None
 
-    def get_username(self):
+    def get_user_name(self):
+        return None
+
+    def get_user_email(self):
         return None
 
 

server/auth/auth_oauth.py

@@ -0,0 +1,188 @@
+from flask import session, request, redirect, current_app, has_request_context
+from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from server.common.errors import AuthenticationError, ConfigurationError
+from urllib.parse import urlencode
+from urllib.request import urlopen
+import json
+
+# It is not required to have authlib or jose.
+# However, it is a configuration error to use this auth type if they are not installed.
+missingimport = []
+try:
+    from authlib.integrations.flask_client import OAuth
+except ModuleNotFoundError:
+    missingimport.append("authlib")
+
+try:
+    from jose import jwt
+except ModuleNotFoundError:
+    missingimport.append("jose")
+
+
+class AuthTypeOAuth(AuthTypeClientBase):
+    """An authentication type for oauth2 logins."""
+
+    CXG_ID_TOKEN = "id_token"
+
+    def __init__(self, app_config):
+        super().__init__()
+        if missingimport:
+            raise ConfigurationError(f"oauth requires these modules: {', '.join(missingimport)}")
+        self.algorithms = ["RS256"]
+        self.api_base_url = app_config.authentication__params_oauth__api_base_url
+        self.client_id = app_config.authentication__params_oauth__client_id
+        self.client_secret = app_config.authentication__params_oauth__client_secret
+        self.callback_base_url = app_config.authentication__params_oauth__callback_base_url
+        self.audience = self.client_id
+
+        # load the jwks (JSON Web Key Set).
+        # The JSON Web Key Set (JWKS) is a set of keys which contains the public keys used to verify
+        # any JSON Web Token (JWT) issued by the authorization server and signed using the RS256
+        try:
+            jwksloc = f"{self.api_base_url}/.well-known/jwks.json"
+            jwksurl = urlopen(jwksloc)
+            self.jwks = json.loads(jwksurl.read())
+        except Exception:
+            raise ConfigurationError(f"error in oauth, api_url_base: {self.api_base_url}, cannot access {jwksloc}")
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def requires_client_login(self):
+        return True
+
+    def add_url_rules(self, app):
+        app.add_url_rule("/login", "login", self.login, methods=["GET"])
+        app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
+        app.add_url_rule("/oauth2/callback", "callback", self.callback, methods=["GET"])
+
+    def complete_setup(self, flask_app):
+        self.oauth = OAuth(flask_app)
+        if self.callback_base_url is None:
+            # In this case, assume the server is running on the same host as the client,
+            # and the oauth provider has been configured
+            # with a callback that understands a localhost callback (e.g. A http://localhost:5005).
+            server_config = flask_app.app_config.server_config
+            self.callback_base_url = f"http://{server_config.app__host}:{server_config.app__port}"
+
+        self.client = self.oauth.register(
+            "oauth",
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            api_base_url=self.api_base_url,
+            access_token_url=f"{self.api_base_url}/oauth/token",
+            authorize_url=f"{self.api_base_url}/authorize",
+            client_kwargs={
+                "scope" : "openid profile email",
+            }
+        )
+
+    def is_user_authenticated(self):
+        try:
+            payload = self.get_jwt_payload()
+            return payload is not None
+        except AuthenticationError:
+            return False
+
+    def get_user_id(self):
+        payload = self.get_jwt_payload()
+        if payload and payload.get("sub"):
+            return payload.get("sub")
+        return None
+
+    def get_user_name(self):
+        payload = self.get_jwt_payload()
+        if payload and payload.get("name"):
+            return payload.get("name")
+        return None
+
+    def get_user_email(self):
+        payload = self.get_jwt_payload()
+        if payload and payload.get("email"):
+            return payload.get("email")
+        return None
+
+    def login(self):
+        callbackurl = f'{self.callback_base_url}/oauth2/callback'
+        return_path = request.args.get("dataset", "")
+        return_to = f"{self.callback_base_url}/{return_path}"
+        # save the return path in the session cookie, accessed in the callback function
+        session["oauth_callback_redirect"] = return_to
+        return self.client.authorize_redirect(redirect_uri=callbackurl)
+
+    def logout(self):
+        if self.CXG_ID_TOKEN in session:
+            del session[self.CXG_ID_TOKEN]
+        return_path = request.args.get("dataset", "")
+        return_to = f"{self.callback_base_url}/{return_path}"
+        params = {'returnTo' : return_to, 'client_id' : self.client_id}
+        return redirect(self.client.api_base_url + '/v2/logout?' + urlencode(params))
+
+    def callback(self):
+        token = self.client.authorize_access_token()
+        id_token = token.get("id_token")
+        session[self.CXG_ID_TOKEN] = id_token
+        del session["oauth_callback_redirect"]
+        oauth_callback_redirect = session.get("oauth_callback_redirect", "/")
+        resp = redirect(oauth_callback_redirect)
+        return resp
+
+    def get_login_url(self, data_adaptor):
+        """Return the url for the login route"""
+        if current_app.app_config.is_multi_dataset():
+            return f"/login?dataset={data_adaptor.uri_path}"
+        else:
+            return "/login"
+
+    def get_logout_url(self, data_adaptor):
+        """Return the url for the logout route"""
+        if current_app.app_config.is_multi_dataset():
+            return f"/logout?dataset={data_adaptor.uri_path}"
+        else:
+            return "/logout"
+
+    def get_token(self):
+        """Function to return the token"""
+        return session.get(self.CXG_ID_TOKEN)
+
+    def get_jwt_payload(self):
+        if not has_request_context():
+            return None
+
+        token = self.get_token()
+        if token is None:
+            return None
+
+        unverified_header = jwt.get_unverified_header(token)
+        rsa_key = {}
+        for key in self.jwks['keys']:
+            if key['kid'] == unverified_header['kid']:
+                rsa_key = {
+                    'kty': key['kty'],
+                    'kid': key['kid'],
+                    'use': key['use'],
+                    'n': key['n'],
+                    'e': key['e']
+                }
+        if rsa_key:
+            try:
+                payload = jwt.decode(
+                    token,
+                    rsa_key,
+                    algorithms=self.algorithms,
+                    audience=self.audience,
+                    issuer=self.api_base_url + "/"
+                )
+                return payload
+
+            except jwt.JWTError as e:
+                raise AuthenticationError(f"invalid signature: {str(e)}")
+            except jwt.ExpiredSignatureError as e:
+                raise AuthenticationError(f"token expired: {str(e)}")
+            except jwt.JWTClaimsError as e:
+                raise AuthenticationError(f"invalid claims {str(e)}")
+
+        raise AuthenticationError("Unable to find the appropriate key")
+
+
+AuthTypeFactory.register("oauth", AuthTypeOAuth)

server/auth/auth_session.py

@@ -10,27 +10,30 @@ class AuthTypeSession(AuthTypeBase):
     # key in the session token for userid
     CXGUID = "cxguid"
 
-    def __init__(self):
+    def __init__(self, app_config):
         super().__init__()
 
-    def is_valid(self):
+    def is_valid_authentication_type(self):
         return True
 
-    def set_params(self, params):
+    def complete_setup(self, app):
-        return
+        pass
 
-    def is_authenticated(self):
+    def is_user_authenticated(self):
         # always authenticated
         return True
 
-    def get_userid(self):
+    def get_user_id(self):
         if self.CXGUID not in session:
             session[self.CXGUID] = uuid4().hex
             session.permanent = True
         return session[self.CXGUID]
 
-    def get_username(self):
+    def get_user_name(self):
         return "anonymous"
 
+    def get_user_email(self):
+        return None
+
 
 AuthTypeFactory.register("session", AuthTypeSession)

server/auth/auth_test.py

@@ -9,13 +9,15 @@ class AuthTypeTest(AuthTypeClientBase):
     # key in session token with userid and username
     CXGUID = "cxguid_test"
     CXGUNAME = "cxguname_test"
+    CXGUEMAIL = "cxguemail_test"
 
-    def __init__(self):
+    def __init__(self, app_config):
         super().__init__()
-        self.username = "test_account"
+        self.user_name = "test_account"
-        self.userid = "id0001"
+        self.user_id = "id0001"
+        self.user_email = "test_account@test.com"
 
-    def is_valid(self):
+    def is_valid_authentication_type(self):
         return True
 
     def requires_client_login(self):
@@ -25,25 +27,26 @@ class AuthTypeTest(AuthTypeClientBase):
         app.add_url_rule("/login", "login", self.login, methods=["GET"])
         app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
 
-    def set_params(self, params):
+    def complete_setup(self, app):
-        if params:
+        pass
-            self.username = params.get("username", self.username)
-            self.userid = params.get("userid", self.userid)
 
-    def is_authenticated(self):
+    def is_user_authenticated(self):
         return self.CXGUID in session
 
-    def get_userid(self):
+    def get_user_id(self):
         return session.get(self.CXGUID)
 
-    def get_username(self):
+    def get_user_name(self):
         return session.get(self.CXGUNAME)
 
+    def get_user_email(self):
+        return session.get(self.CXGUEMAIL)
+
     def login(self):
         args = request.args
         return_to = args.get("dataset", "/")
-        session[self.CXGUID] = args.get("userid", self.userid)
+        session[self.CXGUID] = args.get("userid", self.user_id)
-        session[self.CXGUNAME] = args.get("username", self.username)
+        session[self.CXGUNAME] = args.get("username", self.user_name)
         return redirect(return_to)
 
     def logout(self):

server/common/annotations.py

@@ -10,7 +10,7 @@ from server.common.errors import AnnotationsError, OntologyLoadFailure
 from server.common.utils import series_to_schema
 import fsspec
 import fastobo
-from flask import session, current_app
+from flask import session, current_app, has_request_context
 from abc import ABCMeta, abstractmethod
 
 
@@ -46,8 +46,8 @@ class Annotations(metaclass=ABCMeta):
             raise OntologyLoadFailure("Error loading OBO file") from e
 
     def get_schema(self, data_adaptor):
-        labels = self.read_labels(data_adaptor)
         schema = []
+        labels = self.read_labels(data_adaptor)
         if labels is not None and not labels.empty:
             for col in labels.columns:
                 col_schema = dict(name=col, writable=True)
@@ -113,6 +113,10 @@ class AnnotationsLocalFile(Annotations):
         return session.get(self.CXG_ANNO_COLLECTION)
 
     def read_labels(self, data_adaptor):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return pd.DataFrame()
+
         fname = self._get_filename(data_adaptor)
         with self.label_lock:
             if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
@@ -162,7 +166,7 @@ class AnnotationsLocalFile(Annotations):
         Return a short hash that weakly identifies the user and dataset.
         Used to create safe annotations output file names.
         """
-        uid = current_app.auth.get_userid()
+        uid = current_app.auth.get_user_id()
         id = (uid + data_adaptor.get_location()).encode()
         idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
         return idhash
@@ -249,7 +253,7 @@ class AnnotationsLocalFile(Annotations):
 
         elif session is not None:
             collection = self.get_collection()
-            if current_app.auth.is_authenticated():
+            if current_app.auth.is_user_authenticated():
                 params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
                 params["annotations-data-collection-is-read-only"] = False
                 params["annotations-data-collection-name"] = collection

server/common/app_config.py

@@ -272,11 +272,11 @@ class AppConfig(object):
             "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
         }
 
-        if dataset_config.app__authentication_enable and auth.is_valid():
+        if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
             config["authentication"] = {
-                "is_authenticated": auth.is_authenticated(),
+                "is_authenticated": auth.is_user_authenticated(),
                 "requires_client_login": auth.requires_client_login(),
-                "username": auth.get_username(),
+                "username": auth.get_user_name(),
             }
             if auth.requires_client_login():
                 config["authentication"].update({
@@ -393,7 +393,6 @@ class ServerConfig(BaseConfig):
     def __init__(self, app_config, default_config):
         dictval_cases = [
             ("app", "csp_directives"),
-            ("authentication", "params"),
             ("adaptor", "cxg_adaptor", "tiledb_ctx"),
             ("multi_dataset", "dataroot"),
         ]
@@ -413,7 +412,11 @@ class ServerConfig(BaseConfig):
             self.app__csp_directives = dc["app"]["csp_directives"]
 
             self.authentication__type = dc["authentication"]["type"]
-            self.authentication__params = dc["authentication"]["params"]
+            self.authentication__params_oauth__api_base_url = dc["authentication"]["params_oauth"]["api_base_url"]
+            self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
+            self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
+            self.authentication__params_oauth__callback_base_url = \
+                dc["authentication"]["params_oauth"]["callback_base_url"]
 
             self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
             self.multi_dataset__index = dc["multi_dataset"]["index"]
@@ -445,7 +448,7 @@ class ServerConfig(BaseConfig):
         # The matrix data cache manager is created during the complete_config and stored here.
         self.matrix_data_cache_manager = None
 
-        # The authentication object (BCM -- better name)
+        # The authentication object
         self.auth = None
 
     def complete_config(self, context):
@@ -521,11 +524,21 @@ class ServerConfig(BaseConfig):
 
     def handle_authentication(self, context):
         self.check_attr("authentication__type", (type(None), str))
-        self.check_attr("authentication__params", (type(None), dict))
+
-        self.auth = AuthTypeFactory.create(self.authentication__type)
+        # oauth
+        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
+        self.check_attr("authentication__params_oauth__api_base_url", ptypes)
+        self.check_attr("authentication__params_oauth__client_id", ptypes)
+        self.check_attr("authentication__params_oauth__client_secret", ptypes)
+        self.check_attr("authentication__params_oauth__callback_base_url", (type(None), str))
+        #   secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
+        #   second, from config file
+        self.authentication__params__oauth__client_secret = os.environ.get(
+            "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
+
+        self.auth = AuthTypeFactory.create(self.authentication__type, self)
         if self.auth is None:
             raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
-        self.auth.set_params(self.authentication__params)
 
     def handle_data_locator(self, context):
         self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
@@ -769,7 +782,7 @@ class DatasetConfig(BaseConfig):
             server_config = self.app_config.server_config
             if not self.app__authentication_enable:
                 raise ConfigurationError("user annotations requires authentication to be enabled")
-            if not server_config.auth.is_valid():
+            if not server_config.auth.is_valid_authentication_type():
                 auth_type = server_config.authentication__type
                 raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
 

server/common/default_config.py

@@ -5,7 +5,7 @@ server:
   app:
     verbose: false
     debug: false
-    host: "127.0.0.1"
+    host: localhost
     port : null
     open_browser: false
     force_https: false
@@ -15,13 +15,24 @@ server:
     csp_directives: null
 
   authentication:
-    # The authentication types may be "none" or "session"
+    # The authentication types may be "none", "session", "oauth"
     # none:  No authentication support, features like user_annotations must not be enabled.
-    # session:  A session based userid is automatically generated.
+    # session:  A session based userid is automatically generated. (no params needed)
+    # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
     type: session
 
-    # a dictionary of parameters that may be required for an authentication type
+    params_oauth:
-    params: null
+       # url to the auth server
+       api_base_url: null
+       # client_id of this app
+       client_id: null
+       # the client_secret known to the auth server and this app
+       client_secret: null
+       # cellxgene server location;
+       # the browser will be redirected to locations relative to this location during login and logout.
+       # A value of None, indicates the client and server are on the localhost.  http://localhost:<port> will be used.
+       callback_base_url: null
+
 
   multi_dataset:
     # If dataroot is set, then cellxgene may serve multiple datasets.  This parameter is not

server/common/errors.py

@@ -41,6 +41,10 @@ define_request_exception(
 )
 define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
 define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
+define_request_exception(
+    "AuthenticationError",
+    "Raised when there is an authentication error",
+    default_status_code=HTTPStatus.UNAUTHORIZED)
 
 define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")

server/requirements.txt

@@ -10,7 +10,7 @@ flask-server-timing>=0.1.2
 flask-talisman>=0.7.0
 flatbuffers>=1.10.0
 flatten-dict>=0.2.0
-fsspec>=0.4.4
+fsspec>=0.4.4,<0.8.0
 numba>=0.49.1
 numpy>=1.16.0
 packaging>=20.0