From 58e4c2489b82d5c148439b1e051eda9f4d9984e1 Mon Sep 17 00:00:00 2001 From: Matt Weiden <538456+mweiden@users.noreply.github.com> Date: Thu, 16 Apr 2020 08:07:10 -0700 Subject: [PATCH] Auto format python, make sure it passes lint checks (#1401) --- server/app/app.py | 6 ++- server/cli/launch.py | 2 +- server/common/app_config.py | 8 ++-- server/common/data_locator.py | 4 +- server/common/immutable_kvcache.py | 10 ++--- server/common/rest.py | 12 +++--- server/compute/diffexp_cxg.py | 13 +++--- server/data_common/matrix_loader.py | 2 +- server/data_cxg/cxg_adaptor.py | 6 +-- server/eb/app.py | 13 ++---- server/test/test_anndata_adaptor.py | 2 +- server/test/test_filter.py | 66 ++++++++++++++--------------- 12 files changed, 66 insertions(+), 78 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index 018a1097..66c0fcd3 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -28,6 +28,7 @@ def _cache_control(always, **cache_kwargs): See Werkzeug for attributes that can be set, eg, no_cache, private, max_age, etc. https://werkzeug.palletsprojects.com/en/1.0.x/datastructures/#werkzeug.datastructures.ResponseCacheControl """ + def inner_cache_control(f): @wraps(f) def wrapper(*args, **kwargs): @@ -39,7 +40,9 @@ def _cache_control(always, **cache_kwargs): for k, v in cache_kwargs.items(): setattr(response.cache_control, k, v) return response + return wrapper + return inner_cache_control @@ -126,8 +129,7 @@ def dataroot_test_index(): data += "

Welcome to cellxgene

" config = current_app.app_config - locator = DataLocator(config.multi_dataset__dataroot, - region_name=config.data_locator__s3__region_name) + locator = DataLocator(config.multi_dataset__dataroot, region_name=config.data_locator__s3__region_name) datasets = [] for fname in locator.ls(): location = path_join(config.multi_dataset__dataroot, fname) diff --git a/server/cli/launch.py b/server/cli/launch.py index 890ebad1..1be05b7d 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -437,7 +437,7 @@ def launch( port=app_config.server__port, threaded=not app_config.server__debug, use_debugger=False, - use_reloader=False + use_reloader=False, ) except OSError as e: if e.errno == errno.EADDRINUSE: diff --git a/server/common/app_config.py b/server/common/app_config.py index e71bd344..57dba4bd 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -446,9 +446,7 @@ class AppConfig(object): cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier cpu_count = os.cpu_count() max_workers = min(max_workers, cpu_multiplier * cpu_count) - diffexp_tiledb.set_config( - max_workers, - self.diffexp__alg_cxg__target_workunit) + diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit) def handle_adaptor(self, context): # cxg @@ -537,8 +535,8 @@ class AppConfig(object): config["links"] = links config["parameters"] = parameters config["limits"] = { - 'column_request_max': self.limits__column_request_max, - 'diffexp_cellcount_max': self.limits__diffexp_cellcount_max, + "column_request_max": self.limits__column_request_max, + "diffexp_cellcount_max": self.limits__diffexp_cellcount_max, } return c diff --git a/server/common/data_locator.py b/server/common/data_locator.py index 5b561302..50117682 100644 --- a/server/common/data_locator.py +++ b/server/common/data_locator.py @@ -43,9 +43,7 @@ class DataLocator: # fsspec.filesystem will throw RuntimeError if the protocol is unsupported if self.protocol == "s3" and region_name: - self.fs = fsspec.filesystem( - self.protocol, config_kwargs={"region_name": region_name} - ) + self.fs = fsspec.filesystem(self.protocol, config_kwargs={"region_name": region_name}) else: self.fs = fsspec.filesystem(self.protocol) diff --git a/server/common/immutable_kvcache.py b/server/common/immutable_kvcache.py index c5bc9f85..dddfe879 100644 --- a/server/common/immutable_kvcache.py +++ b/server/common/immutable_kvcache.py @@ -27,23 +27,23 @@ class ImmutableKVCache(MutableMapping): return self.cache[key] if key not in self.factory_calls: creation_thr = True - self.factory_calls[key] = {'cv': threading.Condition(), 'is_done': False, 'error': None} + self.factory_calls[key] = {"cv": threading.Condition(), "is_done": False, "error": None} factory_calls = self.factory_calls[key] # with the CV, create the value (or wait for it to be created) - cv = factory_calls['cv'] + cv = factory_calls["cv"] with cv: if creation_thr: try: self.cache[key] = self.factory(key) except Exception as e: - factory_calls['error'] = e + factory_calls["error"] = e - factory_calls['is_done'] = True + factory_calls["is_done"] = True cv.notify_all() else: """ wait for the value to be available """ - while not factory_calls['is_done']: + while not factory_calls["is_done"]: cv.wait() with self.lock: diff --git a/server/common/rest.py b/server/common/rest.py index 99df5b79..faa9f2bc 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -53,15 +53,15 @@ def _query_parameter_to_filter(args): # on name and value. try: for key, value in args.items(multi=True): - axis, name = key.split(':') + axis, name = key.split(":") if axis not in ("obs", "var"): raise FilterError("unknown filter axis") name = url_unquote(name) current = filters[axis].setdefault(name, {"name": name}) - val_split = value.split(',') + val_split = value.split(",") if len(val_split) == 1: - if 'min' in current or 'max' in current: + if "min" in current or "max" in current: raise FilterError("do not mix range and value filters") value = url_unquote(value) values = current.setdefault("values", []) @@ -72,7 +72,7 @@ def _query_parameter_to_filter(args): raise FilterError("duplicate range specification") min = url_unquote(val_split[0]) max = url_unquote(val_split[1]) - if min != '*': + if min != "*": current["min"] = float(min) if max != "*": current["max"] = float(max) @@ -89,9 +89,7 @@ def _query_parameter_to_filter(args): for axis in ("obs", "var"): axis_filter = filters[axis] if len(axis_filter) > 0: - result[axis] = { - "annotation_value": [val for val in axis_filter.values()] - } + result[axis] = {"annotation_value": [val for val in axis_filter.values()]} return result diff --git a/server/compute/diffexp_cxg.py b/server/compute/diffexp_cxg.py index 833946dd..b1268987 100644 --- a/server/compute/diffexp_cxg.py +++ b/server/compute/diffexp_cxg.py @@ -83,10 +83,10 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): try: result = future.result() part_meanA, part_varA, part_meanB, part_varB, cols = result - meanA[cols[0]: cols[1]] += part_meanA - varA[cols[0]: cols[1]] += part_varA - meanB[cols[0]: cols[1]] += part_meanB - varB[cols[0]: cols[1]] += part_varB + meanA[cols[0] : cols[1]] += part_meanA + varA[cols[0] : cols[1]] += part_varA + meanB[cols[0] : cols[1]] += part_meanB + varB[cols[0] : cols[1]] += part_varB except Exception as e: for future in futures: future.cancel() @@ -98,7 +98,10 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): nA, meanB.astype(dtype), varB.astype(dtype), - nB, top_n, diffexp_lfc_cutoff) + nB, + top_n, + diffexp_lfc_cutoff, + ) return r diff --git a/server/data_common/matrix_loader.py b/server/data_common/matrix_loader.py index fd386b14..7ee0184a 100644 --- a/server/data_common/matrix_loader.py +++ b/server/data_common/matrix_loader.py @@ -167,7 +167,7 @@ class MatrixDataCacheManager(object): self.datasets[location] = item try: - assert(cache_item) + assert cache_item if delete_adaptor: delete_adaptor.delete() if data_adaptor is None: diff --git a/server/data_cxg/cxg_adaptor.py b/server/data_cxg/cxg_adaptor.py index 857ace28..3de42eda 100644 --- a/server/data_cxg/cxg_adaptor.py +++ b/server/data_cxg/cxg_adaptor.py @@ -364,9 +364,7 @@ class CxgAdaptor(DataAdaptor): user_anno_keys = frozenset(labels.columns.tolist()) if need_labels else frozenset() return_keys = frozenset(fields) - label_join_index = ( - frozenset([index_key]) if need_labels and (return_keys & user_anno_keys) else frozenset() - ) + label_join_index = frozenset([index_key]) if need_labels and (return_keys & user_anno_keys) else frozenset() unknown_fields = return_keys - (cxg_keys | user_anno_keys) if unknown_fields: @@ -376,7 +374,7 @@ class CxgAdaptor(DataAdaptor): list((return_keys & cxg_keys) | label_join_index), list(return_keys & user_anno_keys), list(return_keys), - index_key + index_key, ) def annotation_to_fbs_matrix(self, axis, fields=None, labels=None): diff --git a/server/eb/app.py b/server/eb/app.py index d698a267..29725e0f 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -33,17 +33,12 @@ except Exception: def get_flask_secret_key(region_name, secret_name): session = boto3.session.Session() - client = session.client( - service_name='secretsmanager', - region_name=region_name - ) + client = session.client(service_name="secretsmanager", region_name=region_name) try: - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - if 'SecretString' in get_secret_value_response: - var = get_secret_value_response['SecretString'] + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + if "SecretString" in get_secret_value_response: + var = get_secret_value_response["SecretString"] secret = json.loads(var) return secret.get("flask_secret_key") except Exception: diff --git a/server/test/test_anndata_adaptor.py b/server/test/test_anndata_adaptor.py index cc069853..f13e8602 100644 --- a/server/test/test_anndata_adaptor.py +++ b/server/test/test_anndata_adaptor.py @@ -42,7 +42,7 @@ class AdaptorTest(unittest.TestCase): "adaptor__anndata_adaptor__backed": self.backed, "single_dataset__datapath": self.data_locator, "limits__diffexp_cellcount_max": None, - "limits__column_request_max": None + "limits__column_request_max": None, } config = AppConfig() config.update(**args) diff --git a/server/test/test_filter.py b/server/test/test_filter.py index 1639211f..91991a7b 100644 --- a/server/test/test_filter.py +++ b/server/test/test_filter.py @@ -15,50 +15,46 @@ class FilterParseTests(unittest.TestCase): def test_queryparam_to_filter_parse(self): # categories - self.assertEqual(_query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")), { - "obs": { - "annotation_value": [ - {"name": "foo", "values": ["bar"]}, - {"name": "baz", "values": ["foo"]}, - ] + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")), + { + "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]}, + "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]}, }, - "var": { - "annotation_value": [ - {"name": "baz", "values": ["133", "A"]} - ] - } - }) + ) # ranges - self.assertEqual(_query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")), { - "obs": { - "annotation_value": [ - {"name": "A", "min": 1, "max": 99.}, - {"name": "B", "max": 100.}, - {"name": "C", "min": 0.}, - ] + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")), + { + "obs": { + "annotation_value": [ + {"name": "A", "min": 1, "max": 99.0}, + {"name": "B", "max": 100.0}, + {"name": "C", "min": 0.0}, + ] + }, }, - }) + ) # combo - self.assertEqual(_query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")), { - "var": { - "annotation_value": [ - {"name": "B", "values": ["YES", "NO"]}, - {"name": "A", "min": 1., "max": 99.}, - ] + self.assertEqual( + _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")), + { + "var": { + "annotation_value": [ + {"name": "B", "values": ["YES", "NO"]}, + {"name": "A", "min": 1.0, "max": 99.0}, + ] + }, }, - }) + ) def test_queryparam_to_filter_escaping(self): - self.assertEqual(_query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")), { - "obs": { - "annotation_value": [ - {"name": "var", "values": ["!,:OK="]}, - {"name": "A!", "values": ["YO"]}, - ], - }, - }) + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")), + {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}}, + ) def test_queryparam_to_filter_errors(self):