Various hardening to REST routes (#1293)

* URL reweriting for static

* request size limits

* improve quotas, make tests work

* remove debugging code

* pass limits to front-end

* fix renaming boggle

server/common/app_config.py

@@ -21,6 +21,18 @@ DEFAULT_SERVER_PORT = int(environ.get("CXG_SERVER_PORT", "5005"))
 # anything bigger than this will generate a special message
 BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
 
+""" Default limits for requests """
+Default_Limits = {
+    # Max number of columns that may be requested for /annotations or /data routes.
+    # This is a simplistic means of preventing excess resource consumption (eg,
+    # requesting the entire X matrix in one request) or other DoS style attacks/errors.
+    # Set to None to disable check.
+    "column_request_max": 32,
+    # Max number of cells that will be accepted for differential expression.
+    # Set to None to disable the check.
+    "diffexp_cellcount_max": None,  # None is disabled
+}
+
 
 class AppFeature(object):
     def __init__(self, path, available=False, method="POST", extra={}):
@@ -78,6 +90,9 @@ class AppConfig(object):
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
+        # Used for various limits, eg, size of requests.  Not currently configurable.
+        self.limits = Default_Limits
+
         # The annotation object is created during complete_config and stored here.
         self.user_annotations = None
 
@@ -425,5 +440,12 @@ class AppConfig(object):
         config["library_versions"] = library_versions
         config["links"] = links
         config["parameters"] = parameters
+        config["limits"] = self.limits
 
         return c
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = self.limits.get(limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value

server/common/errors.py

@@ -68,3 +68,11 @@ class ConfigurationError(Exception):
     """
 
     pass
+
+
+class ExceedsLimitError(Exception):
+    """
+    Raised when an HTTP request exceeds a limit/quota
+    """
+
+    pass

server/common/rest.py

@@ -9,6 +9,7 @@ from server.common.errors import (
     JSONEncodingValueError,
     PrepareError,
     DisabledFeatureError,
+    ExceedsLimitError,
 )
 
 import json
@@ -54,9 +55,13 @@ def config_get(app_config, data_adaptor, annotations):
 
 def annotations_obs_get(request, data_adaptor, annotations):
     fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
     preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
     if preferred_mimetype != "application/octet-stream":
         return abort(HTTPStatus.NOT_ACCEPTABLE)
+
     try:
         labels = None
         if annotations:
@@ -100,9 +105,13 @@ def annotations_obs_put(request, data_adaptor, annotations):
 
 def annotations_var_get(request, data_adaptor, annotations):
     fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
     preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
     if preferred_mimetype != "application/octet-stream":
         return abort(HTTPStatus.NOT_ACCEPTABLE)
+
     try:
         labels = None
         if annotations is not None:
@@ -129,7 +138,7 @@ def data_var_put(request, data_adaptor):
             HTTPStatus.OK,
             {"Content-Type": "application/octet-stream"},
         )
-    except FilterError as e:
+    except (FilterError, ValueError, ExceedsLimitError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
 
 
@@ -160,7 +169,7 @@ def diffexp_obs_post(request, data_adaptor):
     try:
         diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count)
         return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
-    except (ValueError, DisabledFeatureError, FilterError) as e:
+    except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
     except JSONEncodingValueError:
         # JSON encoding failure, usually due to bad data. Just let it ripple up
@@ -171,13 +180,13 @@ def diffexp_obs_post(request, data_adaptor):
 
 def layout_obs_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
     try:
-        if preferred_mimetype == "application/octet-stream":
-            return make_response(
-                data_adaptor.layout_to_fbs_matrix(), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
-            )
-        else:
-            return abort(HTTPStatus.NOT_ACCEPTABLE)
+        return make_response(
+            data_adaptor.layout_to_fbs_matrix(), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
+        )
     except PrepareError:
         return abort_and_log(
             HTTPStatus.NOT_IMPLEMENTED,