Auto format python, make sure it passes lint checks (#1401)

server/app/app.py

@@ -28,6 +28,7 @@ def _cache_control(always, **cache_kwargs):
     See Werkzeug for attributes that can be set, eg, no_cache, private, max_age, etc.
     https://werkzeug.palletsprojects.com/en/1.0.x/datastructures/#werkzeug.datastructures.ResponseCacheControl
     """
+
     def inner_cache_control(f):
         @wraps(f)
         def wrapper(*args, **kwargs):
@@ -39,7 +40,9 @@ def _cache_control(always, **cache_kwargs):
             for k, v in cache_kwargs.items():
                 setattr(response.cache_control, k, v)
             return response
+
         return wrapper
+
     return inner_cache_control
 
 
@@ -126,8 +129,7 @@ def dataroot_test_index():
     data += "<body><H1>Welcome to cellxgene</H1>"
 
     config = current_app.app_config
-    locator = DataLocator(config.multi_dataset__dataroot,
+    locator = DataLocator(config.multi_dataset__dataroot, region_name=config.data_locator__s3__region_name)
-                          region_name=config.data_locator__s3__region_name)
     datasets = []
     for fname in locator.ls():
         location = path_join(config.multi_dataset__dataroot, fname)

server/cli/launch.py

@@ -437,7 +437,7 @@ def launch(
             port=app_config.server__port,
             threaded=not app_config.server__debug,
             use_debugger=False,
-            use_reloader=False
+            use_reloader=False,
         )
     except OSError as e:
         if e.errno == errno.EADDRINUSE:

server/common/app_config.py

@@ -446,9 +446,7 @@ class AppConfig(object):
         cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
         cpu_count = os.cpu_count()
         max_workers = min(max_workers, cpu_multiplier * cpu_count)
-        diffexp_tiledb.set_config(
+        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
-            max_workers,
-            self.diffexp__alg_cxg__target_workunit)
 
     def handle_adaptor(self, context):
         # cxg
@@ -537,8 +535,8 @@ class AppConfig(object):
         config["links"] = links
         config["parameters"] = parameters
         config["limits"] = {
-            'column_request_max': self.limits__column_request_max,
+            "column_request_max": self.limits__column_request_max,
-            'diffexp_cellcount_max': self.limits__diffexp_cellcount_max,
+            "diffexp_cellcount_max": self.limits__diffexp_cellcount_max,
         }
 
         return c

server/common/data_locator.py

@@ -43,9 +43,7 @@ class DataLocator:
 
         # fsspec.filesystem will throw RuntimeError if the protocol is unsupported
         if self.protocol == "s3" and region_name:
-            self.fs = fsspec.filesystem(
+            self.fs = fsspec.filesystem(self.protocol, config_kwargs={"region_name": region_name})
-                self.protocol, config_kwargs={"region_name": region_name}
-            )
         else:
             self.fs = fsspec.filesystem(self.protocol)
 

server/common/immutable_kvcache.py

@@ -27,23 +27,23 @@ class ImmutableKVCache(MutableMapping):
                 return self.cache[key]
             if key not in self.factory_calls:
                 creation_thr = True
-                self.factory_calls[key] = {'cv': threading.Condition(), 'is_done': False, 'error': None}
+                self.factory_calls[key] = {"cv": threading.Condition(), "is_done": False, "error": None}
             factory_calls = self.factory_calls[key]
 
         # with the CV, create the value (or wait for it to be created)
-        cv = factory_calls['cv']
+        cv = factory_calls["cv"]
         with cv:
             if creation_thr:
                 try:
                     self.cache[key] = self.factory(key)
                 except Exception as e:
-                    factory_calls['error'] = e
+                    factory_calls["error"] = e
 
-                factory_calls['is_done'] = True
+                factory_calls["is_done"] = True
                 cv.notify_all()
             else:
                 """ wait for the value to be available """
-                while not factory_calls['is_done']:
+                while not factory_calls["is_done"]:
                     cv.wait()
 
         with self.lock:

server/common/rest.py

@@ -53,15 +53,15 @@ def _query_parameter_to_filter(args):
     # on name and value.
     try:
         for key, value in args.items(multi=True):
-            axis, name = key.split(':')
+            axis, name = key.split(":")
             if axis not in ("obs", "var"):
                 raise FilterError("unknown filter axis")
             name = url_unquote(name)
             current = filters[axis].setdefault(name, {"name": name})
 
-            val_split = value.split(',')
+            val_split = value.split(",")
             if len(val_split) == 1:
-                if 'min' in current or 'max' in current:
+                if "min" in current or "max" in current:
                     raise FilterError("do not mix range and value filters")
                 value = url_unquote(value)
                 values = current.setdefault("values", [])
@@ -72,7 +72,7 @@ def _query_parameter_to_filter(args):
                     raise FilterError("duplicate range specification")
                 min = url_unquote(val_split[0])
                 max = url_unquote(val_split[1])
-                if min != '*':
+                if min != "*":
                     current["min"] = float(min)
                 if max != "*":
                     current["max"] = float(max)
@@ -89,9 +89,7 @@ def _query_parameter_to_filter(args):
     for axis in ("obs", "var"):
         axis_filter = filters[axis]
         if len(axis_filter) > 0:
-            result[axis] = {
+            result[axis] = {"annotation_value": [val for val in axis_filter.values()]}
-                "annotation_value": [val for val in axis_filter.values()]
-            }
 
     return result
 

server/compute/diffexp_cxg.py

@@ -83,10 +83,10 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
         try:
             result = future.result()
             part_meanA, part_varA, part_meanB, part_varB, cols = result
-            meanA[cols[0]: cols[1]] += part_meanA
+            meanA[cols[0] : cols[1]] += part_meanA
-            varA[cols[0]: cols[1]] += part_varA
+            varA[cols[0] : cols[1]] += part_varA
-            meanB[cols[0]: cols[1]] += part_meanB
+            meanB[cols[0] : cols[1]] += part_meanB
-            varB[cols[0]: cols[1]] += part_varB
+            varB[cols[0] : cols[1]] += part_varB
         except Exception as e:
             for future in futures:
                 future.cancel()
@@ -98,7 +98,10 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
         nA,
         meanB.astype(dtype),
         varB.astype(dtype),
-        nB, top_n, diffexp_lfc_cutoff)
+        nB,
+        top_n,
+        diffexp_lfc_cutoff,
+    )
 
     return r
 

server/data_common/matrix_loader.py

@@ -167,7 +167,7 @@ class MatrixDataCacheManager(object):
                 self.datasets[location] = item
 
         try:
-            assert(cache_item)
+            assert cache_item
             if delete_adaptor:
                 delete_adaptor.delete()
             if data_adaptor is None:

server/data_cxg/cxg_adaptor.py

@@ -364,9 +364,7 @@ class CxgAdaptor(DataAdaptor):
         user_anno_keys = frozenset(labels.columns.tolist()) if need_labels else frozenset()
         return_keys = frozenset(fields)
 
-        label_join_index = (
+        label_join_index = frozenset([index_key]) if need_labels and (return_keys & user_anno_keys) else frozenset()
-            frozenset([index_key]) if need_labels and (return_keys & user_anno_keys) else frozenset()
-        )
 
         unknown_fields = return_keys - (cxg_keys | user_anno_keys)
         if unknown_fields:
@@ -376,7 +374,7 @@ class CxgAdaptor(DataAdaptor):
             list((return_keys & cxg_keys) | label_join_index),
             list(return_keys & user_anno_keys),
             list(return_keys),
-            index_key
+            index_key,
         )
 
     def annotation_to_fbs_matrix(self, axis, fields=None, labels=None):

server/eb/app.py

@@ -33,17 +33,12 @@ except Exception:
 
 def get_flask_secret_key(region_name, secret_name):
     session = boto3.session.Session()
-    client = session.client(
+    client = session.client(service_name="secretsmanager", region_name=region_name)
-        service_name='secretsmanager',
-        region_name=region_name
-    )
 
     try:
-        get_secret_value_response = client.get_secret_value(
+        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
-            SecretId=secret_name
+        if "SecretString" in get_secret_value_response:
-        )
+            var = get_secret_value_response["SecretString"]
-        if 'SecretString' in get_secret_value_response:
-            var = get_secret_value_response['SecretString']
             secret = json.loads(var)
             return secret.get("flask_secret_key")
     except Exception:

server/test/test_anndata_adaptor.py

@@ -42,7 +42,7 @@ class AdaptorTest(unittest.TestCase):
             "adaptor__anndata_adaptor__backed": self.backed,
             "single_dataset__datapath": self.data_locator,
             "limits__diffexp_cellcount_max": None,
-            "limits__column_request_max": None
+            "limits__column_request_max": None,
         }
         config = AppConfig()
         config.update(**args)

server/test/test_filter.py

@@ -15,50 +15,46 @@ class FilterParseTests(unittest.TestCase):
 
     def test_queryparam_to_filter_parse(self):
         # categories
-        self.assertEqual(_query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")), {
+        self.assertEqual(
-            "obs": {
+            _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
-                "annotation_value": [
+            {
-                    {"name": "foo", "values": ["bar"]},
+                "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
-                    {"name": "baz", "values": ["foo"]},
+                "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
-                ]
             },
-            "var": {
+        )
-                "annotation_value": [
-                    {"name": "baz", "values": ["133", "A"]}
-                ]
-            }
-        })
 
         # ranges
-        self.assertEqual(_query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")), {
+        self.assertEqual(
-            "obs": {
+            _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
-                "annotation_value": [
+            {
-                    {"name": "A", "min": 1, "max": 99.},
+                "obs": {
-                    {"name": "B", "max": 100.},
+                    "annotation_value": [
-                    {"name": "C", "min": 0.},
+                        {"name": "A", "min": 1, "max": 99.0},
-                ]
+                        {"name": "B", "max": 100.0},
+                        {"name": "C", "min": 0.0},
+                    ]
+                },
             },
-        })
+        )
 
         # combo
-        self.assertEqual(_query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")), {
+        self.assertEqual(
-            "var": {
+            _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
-                "annotation_value": [
+            {
-                    {"name": "B", "values": ["YES", "NO"]},
+                "var": {
-                    {"name": "A", "min": 1., "max": 99.},
+                    "annotation_value": [
-                ]
+                        {"name": "B", "values": ["YES", "NO"]},
+                        {"name": "A", "min": 1.0, "max": 99.0},
+                    ]
+                },
             },
-        })
+        )
 
     def test_queryparam_to_filter_escaping(self):
-        self.assertEqual(_query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")), {
+        self.assertEqual(
-            "obs": {
+            _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
-                "annotation_value": [
+            {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
-                    {"name": "var", "values": ["!,:OK="]},
+        )
-                    {"name": "A!", "values": ["YO"]},
-                ],
-            },
-        })
 
     def test_queryparam_to_filter_errors(self):