gene set summary progress (#2127)

* revert removal of cache control headers

* checkpoint work on revising summary route

* add summary query support to annoMatrix

* summarize route cleanup

* add mising file

* clean up summarize route

* add summary histogram

* update deps

* lint

* more lint

* lint

* manage crossfiler during gene set state changes

* remove obsolete debugging code

* correctly perform async watch in histogram

* better error handling

backend/server/app/app.py

@@ -206,11 +206,16 @@ class GenesetsAPI(Resource):
         return common_rest.genesets_put(request, data_adaptor)
 
 
-class GenesetSummaryAPI(Resource):
+class SummarizeVarAPI(Resource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
     @rest_get_data_adaptor
     @cache_control(no_store=True)
-    def get(self, data_adaptor):
+    def post(self, data_adaptor):
-        return common_rest.geneset_summary_get(request, data_adaptor)
+        return common_rest.summarize_var_post(request, data_adaptor)
 
 
 def get_api_base_resources(bp_base):
@@ -239,7 +244,7 @@ def get_api_dataroot_resources(bp_dataroot):
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
     add_resource(GenesetsAPI, "/genesets")
-    add_resource(GenesetSummaryAPI, "/geneset_summary")
+    add_resource(SummarizeVarAPI, "/summarize/var")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes

backend/server/common/rest.py

@@ -3,6 +3,7 @@ import logging
 import sys
 from http import HTTPStatus
 import zlib
+import hashlib
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
@@ -383,31 +384,42 @@ def genesets_put(request, data_adaptor):
         return abort(HTTPStatus.NOT_FOUND, description=str(e))
 
 
-def geneset_summary_get(request, data_adaptor):
+def summarize_var_helper(request, data_adaptor, key, raw_query):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
     if preferred_mimetype != "application/octet-stream":
         return abort(HTTPStatus.NOT_ACCEPTABLE)
 
-    geneset_name = request.args.get("geneset_name", default=None)
+    summary_method = request.values.get("method", default="mean")
-    summary_method = request.args.get("method", default="mean")
+    query_hash = hashlib.sha1(raw_query).hexdigest()  # cache helper
-    request_tid = request.args.get("tid", default=None)
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
+
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
 
     try:    
-        annotations = data_adaptor.dataset_config.user_annotations
+        filter = _query_parameter_to_filter(args_filter_only)
-        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
-
-        if request_tid is not None and int(request_tid) != tid:
-            return abort(HTTPStatus.NOT_FOUND, "Obsolete TID")
-        if geneset_name is None or geneset_name not in genesets:
-            return abort(HTTPStatus.BAD_REQUEST, "Gene set name not found.")
-        genes = [g["gene_symbol"] for g in genesets.get(geneset_name)["genes"]]
-
         return make_response(
-            data_adaptor.get_gene_set_summary(geneset_name, genes, summary_method),
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
             HTTPStatus.OK,
             {"Content-Type": "application/octet-stream"},
         )
     except (ValueError) as e:
         return abort(HTTPStatus.NOT_FOUND, description=str(e))
-    except (UnsupportedSummaryMethod) as e:
+    except (UnsupportedSummaryMethod, FilterError) as e:
         return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/server/data_anndata/anndata_adaptor.py

@@ -5,7 +5,6 @@ import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
-import pandas as pd
 from scipy import sparse
 from server_timing import Timing as ServerTiming
 
@@ -13,7 +12,7 @@ import backend.server.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.server.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError, UnsupportedSummaryMethod
+from backend.common.errors import PrepareError, DatasetAccessError, FilterError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 from backend.server.compute.scanpy import scanpy_umap
 from backend.server.data_common.data_adaptor import DataAdaptor
@@ -69,11 +68,11 @@ class AnndataAdaptor(DataAdaptor):
 
     @staticmethod
     def _create_unique_column_name(df, col_name_prefix):
-        """ given the columns of a dataframe, and a name prefix, return a column name which
+        """given the columns of a dataframe, and a name prefix, return a column name which
-            does not exist in the dataframe, AND which is prefixed by `prefix`
+        does not exist in the dataframe, AND which is prefixed by `prefix`
 
-            The approach is to append a numeric suffix, starting at zero and increasing by
+        The approach is to append a numeric suffix, starting at zero and increasing by
-            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
+        one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
         suffix = 0
         while f"{col_name_prefix}{suffix}" in df:
@@ -200,10 +199,10 @@ class AnndataAdaptor(DataAdaptor):
             self.parameters.update({"diffexp_may_be_slow": True})
 
     def _is_valid_layout(self, arr):
-        """ return True if this layout data is a valid array for front-end presentation:
+        """return True if this layout data is a valid array for front-end presentation:
-            * ndarray, dtype float/int/uint
+        * ndarray, dtype float/int/uint
-            * with shape (n_obs, >= 2)
+        * with shape (n_obs, >= 2)
-            * with all values finite or NaN (no +Inf or -Inf)
+        * with all values finite or NaN (no +Inf or -Inf)
         """
         is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
         is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
@@ -368,29 +367,3 @@ class AnndataAdaptor(DataAdaptor):
     def get_var_keys(self):
         # return list of keys
         return self.data.var.keys().to_list()
-
-    def get_gene_set_summary(self, geneset_name, genes, method):
-        if method != "mean":
-            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
-
-        var_index = self.parameters.get("var_names")
-        obs_selector, var_selector = self._filter_to_mask(
-            {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": var_index,
-                            "values": genes,
-                        }
-                    ]
-                }
-            }
-        )
-
-        X = self.get_X_array(obs_selector, var_selector)
-        if sparse.issparse(X):
-            mean = X.mean(axis=1)
-        else:
-            mean = X.mean(axis=1, keepdims=True)
-        col_idx = pd.Index([geneset_name])
-        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/server/data_common/data_adaptor.py

@@ -1,14 +1,14 @@
 from abc import ABCMeta, abstractmethod
 from os.path import basename, splitext
 import re
-
 import numpy as np
 import pandas as pd
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
 from backend.server.common.config.app_config import AppConfig
 from backend.common.constants import Axis
-from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -482,6 +482,25 @@ class DataAdaptor(metaclass=ABCMeta):
             lastmod = None
         return lastmod
 
-    @abstractmethod
+    def summarize_var(self, method, filter, query_hash):
-    def get_gene_set_summary(self, geneset_name, genes, method):
+        if method != "mean":
-        pass
+            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
+
+        obs_selector, var_selector = self._filter_to_mask(filter)
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # if no filter, just return zeros.  We don't have a use case
+        # for summarizing the entire X without a filter, and it would
+        # potentially be quite compute / memory intensive.
+        if var_selector is None or np.count_nonzero(var_selector) == 0:
+            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
+        else:
+            X = self.get_X_array(obs_selector, var_selector)
+            if sparse.issparse(X):
+                mean = X.mean(axis=1)
+            else:
+                mean = X.mean(axis=1, keepdims=True)
+
+        col_idx = pd.Index([query_hash])
+        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/test/test_server/unit/common/test_api.py

@@ -5,6 +5,7 @@ import zlib
 from http import HTTPStatus
 import tempfile
 from os import path
+import hashlib
 
 import pandas as pd
 import requests
@@ -430,6 +431,75 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
         result_data = result.json()
         self.assertEqual(len(result_data), 10)
 
+    def test_get_summaryvar(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "summarize/var"
+
+        # single column
+        filter = f"var:{index_col_name}=F5"
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
+
+        # multi-column
+        col_names = ["F5", "BEB3", "SIK1"]
+        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
+
+    def test_post_summaryvar(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "summarize/var"
+        headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"}
+
+        # single column
+        filter = f"var:{index_col_name}=F5"
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
+        result = self.session.post(url, headers=headers, data=query)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
+
+        # multi-column
+        col_names = ["F5", "BEB3", "SIK1"]
+        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
+        result = self.session.post(url, headers=headers, data=query)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
+
 
 class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
     """Test Case for endpoints"""
@@ -738,86 +808,25 @@ summary test,,PIGU,\r
             original_data,
         )
 
-    def test_get_geneset_summary(self):
-        endpoint = "geneset_summary?geneset_name=summary%20test&method=mean"
-        url = f"{self.URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.session.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.content)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-        self.assertEqual(df["col_idx"], ["summary test"])
-        self.assertAlmostEqual(df["columns"][0][0], -0.19863907)
-
-    def test_get_geneset_summary_default_method(self):
-        endpoint = "geneset_summary?geneset_name=summary%20test"
-        url = f"{self.URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.session.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.content)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-        self.assertEqual(df["col_idx"], ["summary test"])
-        self.assertAlmostEqual(df["columns"][0][0], -0.19863907)
-
-    def test_get_geneset_summary_check_tid(self):
-        # get the TID
-        result = self.session.get(f"{self.URL_BASE}genesets", headers={"Accept": "application/json"})
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        tid = result.json()["tid"]
-
-        # current tid
-        endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid}"
-        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-
-        # future tid
-        endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid+1}"
-        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
-        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
-
-        # past tid
-        endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid-1}"
-        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
-        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
-
-        # No tid - ie, skip check
-        endpoint = "geneset_summary?geneset_name=summary%20test"
-        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-
     def test_get_geneset_summary_edge_cases(self):
         # attempt to summarize _all_ genesets, including edge cases with zero or one gene
         result = self.session.get(f"{self.URL_BASE}genesets", headers={"Accept": "application/json"})
         self.assertEqual(result.status_code, HTTPStatus.OK)
-        geneset_names = [gs["geneset_name"] for gs in result.json()["genesets"]]
+        genesets = result.json()["genesets"]
 
-        for gs in geneset_names:
+        endpoint = "summarize/var"
-            endpoint = f"geneset_summary?geneset_name={gs}"
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
-            result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        for gs in genesets:
+            genes = [g["gene_symbol"] for g in gs["genes"]]
+            filter = "&".join([f"var:{index_col_name}={gene}" for gene in genes])
+            query = f"method=mean&{filter}"
+            query_hash = hashlib.sha1(query.encode()).hexdigest()
+            url = f"{self.URL_BASE}{endpoint}?{query}"
+
+            result = self.session.get(url, headers={"Accept": "application/octet-stream"})
             self.assertEqual(result.status_code, HTTPStatus.OK)
             self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
             df = decode_fbs.decode_matrix_FBS(result.content)
             self.assertEqual(df["n_rows"], 2638)
             self.assertEqual(df["n_cols"], 1)
-            self.assertEqual(df["col_idx"], [gs])
+            self.assertEqual(df["col_idx"], [query_hash])
-
-    def test_get_geneset_error_handling(self):
-        # no geneset
-        endpoint = "geneset_summary"
-        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-        # unknown geneset
-        endpoint = "geneset_summary?geneset_name=NO_SUCH_GENE_SET"
-        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-        # unknown method
-        endpoint = "geneset_summary?geneset_name=summary%20test&method=NO_SUCH_METHOD"
-        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -85,9 +85,11 @@ describe("AnnoMatrix", () => {
       fetch.once(serverMocks.responder);
       await expect(
         annoMatrix.fetch("X", {
-          field: "var",
+          where: {
-          column: annoMatrix.schema.annotations.var.index,
+            field: "var",
-          value: "TYMP",
+            column: annoMatrix.schema.annotations.var.index,
+            value: "TYMP",
+          },
         })
       ).resolves.toBeInstanceOf(Dataframe);
 
@@ -103,14 +105,18 @@ describe("AnnoMatrix", () => {
       await expect(
         annoMatrix.fetch("X", [
           {
-            field: "var",
+            where: {
-            column: varIndex,
+              field: "var",
-            value: "SUMO3",
+              column: varIndex,
+              value: "SUMO3",
+            },
           },
           {
-            field: "var",
+            where: {
-            column: varIndex,
+              field: "var",
-            value: "TYMP",
+              column: varIndex,
+              value: "TYMP",
+            },
           },
         ])
       ).resolves.toBeInstanceOf(Dataframe);

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -170,9 +170,11 @@ describe("AnnoMatrixCrossfilter", () => {
       const xfltr = await crossfilter.select(
         "X",
         {
-          field: "var",
+          where: {
-          column: varIndex,
+            field: "var",
-          value: "TYMP",
+            column: varIndex,
+            value: "TYMP",
+          },
         },
         {
           mode: "range",
@@ -186,9 +188,11 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(xfltr.countSelected()).toEqual(501);
 
       const df = await annoMatrix.fetch("X", {
-        field: "var",
+        where: {
-        column: varIndex,
+          field: "var",
-        value: "TYMP",
+          column: varIndex,
+          value: "TYMP",
+        },
       });
       const values = df.icol(0).asArray();
       const selected = xfltr.allSelectedMask();

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -1,3 +1,4 @@
+import sha1 from "sha1";
 import {
   _whereCacheGet,
   _whereCacheCreate,
@@ -7,37 +8,82 @@ import {
 const schema = {};
 
 describe("whereCache", () => {
-  test("whereCacheGet - missing cache values", () => {
+  test("whereCacheGet - where query, missing cache values", () => {
     expect(
       _whereCacheGet({}, schema, "X", {
-        field: "var",
+        where: {
-        column: "foo",
+          field: "var",
-        value: "bar",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: {} }, schema, "X", {
+      _whereCacheGet({}, schema, "X", {
-        field: "var",
+        summarize: {
-        column: "foo",
+          field: "var",
-        value: "bar",
+          column: "foo",
+          values: ["bar"],
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
+      _whereCacheGet({ where: { X: {} } }, schema, "X", {
-        field: "var",
+        where: {
-        column: "foo",
+          field: "var",
-        value: "bar",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: { var: new Map() } } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(
-        { X: { var: new Map([["foo", new Map()]]) } },
+        { where: { X: { var: new Map([["foo", new Map()]]) } } },
         schema,
         "X",
         {
+          where: {
+            field: "var",
+            column: "foo",
+            value: "bar",
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - summarize query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          method: "mean",
           field: "var",
           column: "foo",
-          value: "bar",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { summarize: { X: { mean: { var: new Map() } } } },
+        schema,
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: "foo",
+            values: ["bar"],
+          },
         }
       )
     ).toEqual([undefined]);
@@ -45,140 +91,292 @@ describe("whereCache", () => {
 
   test("whereCacheGet - varied lookups", () => {
     const whereCache = {
-      X: {
+      where: {
-        var: new Map([
+        X: {
-          [
+          var: new Map([
-            "foo",
+            [
-            new Map([
+              "foo",
-              ["bar", [0]],
+              new Map([
-              ["baz", [1, 2]],
+                ["bar", [0]],
+                ["baz", [1, 2]],
+              ]),
+            ],
+          ]),
+        },
+      },
+      summarize: {
+        X: {
+          mean: {
+            var: new Map([
+              [
+                "foo",
+                new Map([
+                  [sha1("bar"), [0]],
+                  [sha1("baz"), [1, 2]],
+                ]),
+              ],
             ]),
-          ],
+          },
-        ]),
+        },
       },
     };
 
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
+        where: {
-        column: "foo",
+          field: "var",
-        value: "bar",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([0]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
+        summarize: {
-        column: "foo",
+          method: "mean",
-        value: "baz",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "baz",
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["baz"],
+        },
       })
     ).toEqual([1, 2]);
     expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "whoknows",
+        where: {
-        column: "whatever",
+          field: "whoknows",
-        value: "snork",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
+        where: {
-        column: "whatever",
+          field: "var",
-        value: "snork",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
+        where: {
-        column: "foo",
+          field: "var",
-        value: "snork",
+          column: "foo",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
   });
 
-  test("whereCacheCreate", () => {
+  test("whereCacheCreate, where query", () => {
     const query = {
-      field: "queryField",
+      where: {
-      column: "queryColumn",
+        field: "queryField",
-      value: "queryValue",
+        column: "queryColumn",
+        value: "queryValue",
+      },
     };
     const wc = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "queryValue" },
+      {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "queryValue",
+        },
+      },
       [0, 1, 2]
     );
     expect(wc).toBeDefined();
     expect(wc).toEqual(
       expect.objectContaining({
-        field: {
+        where: {
-          queryField: expect.any(Map),
+          field: {
+            queryField: expect.any(Map),
+          },
         },
       })
     );
-    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
-    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
-    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
+    expect(
-      true
+      wc.where.field.queryField.get("queryColumn").has("queryValue")
-    );
+    ).toEqual(true);
     expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
   });
 
-  test("whereCacheMerge", () => {
+  test("whereCacheCreate, summarize query", () => {
+    const query = {
+      summarize: {
+        method: "method",
+        field: "queryField",
+        column: "queryColumn",
+        values: ["queryValue"],
+      },
+    };
+    const wc = _whereCacheCreate("field", query, [0, 1, 2]);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, unknown query type", () => {
+    expect(_whereCacheCreate("field", { foobar: true }, [1])).toEqual({});
+  });
+
+  test("whereCacheMerge, where queries", () => {
     let wc;
 
     // remember, will mutate dst
     const src = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "foo" },
+      { where: { field: "queryField", column: "queryColumn", value: "foo" } },
       ["foo"]
     );
 
     const dst1 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst1"]
     );
     wc = _whereCacheMerge(dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
+        where: {
-        column: "queryColumn",
+          field: "queryField",
-        value: "foo",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
+        where: {
-        column: "queryColumn",
+          field: "queryField",
-        value: "bar",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     const dst2 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst2"]
     );
     wc = _whereCacheMerge(dst2, dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
+        where: {
-        column: "queryColumn",
+          field: "queryField",
-        value: "foo",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
+        where: {
-        column: "queryColumn",
+          field: "queryField",
-        value: "bar",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     wc = _whereCacheMerge({}, src);
     expect(wc).toEqual(src);
 
-    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
+    wc = _whereCacheMerge({ where: { field: { queryField: new Map() } } }, src);
     expect(wc).toEqual(src);
   });
+
+  test("whereCacheMerge, mixed queries", () => {
+    const wc = _whereCacheMerge(
+      _whereCacheCreate(
+        "field",
+        {
+          where: {
+            field: "queryField",
+            column: "queryColumn",
+            value: "foo",
+          },
+        },
+        ["a"]
+      ),
+      _whereCacheCreate(
+        "field",
+        {
+          summarize: {
+            method: "mean",
+            field: "queryField",
+            column: "queryColumn",
+            values: ["foo", "bar", "baz"],
+          },
+        },
+        ["b"]
+      )
+    );
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["a"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "mean",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["foo", "bar", "baz"],
+        },
+      })
+    ).toEqual(["b"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "does-not-exist",
+        },
+      })
+    ).toEqual([undefined]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "no-such-method",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["does-not-exist"],
+        },
+      })
+    ).toEqual([undefined]);
+  });
 });

client/package.json

@@ -70,6 +70,7 @@
     "regenerator-runtime": "^0.13.7",
     "regl": "^1.6.1",
     "script-ext-html-webpack-plugin": "^2.1.4",
+    "sha1": "^1.1.1",
     "tinyqueue": "^2.0.3",
     "webpack-merge": "^5.0.9",
     "whatwg-fetch": "^3.2.0"

client/src/actions/annotation.js

@@ -396,9 +396,9 @@ export const saveGenesetsAction = () => async (dispatch, getState) => {
   const { lastTid, genesets } = state.genesets;
 
   const genesetsAreAvailable =
-    config?.parameters?.["annotations_genesets"] ?? false;
+    config?.parameters?.annotations_genesets ?? false;
   const genesetsReadonly =
-    config?.parameters?.["annotations_genesets_readonly"] ?? true;
+    config?.parameters?.annotations_genesets_readonly ?? true;
   if (!genesetsAreAvailable || genesetsReadonly) {
     // our non-save was completed!
     return dispatch({

client/src/actions/geneset.js

@@ -0,0 +1,126 @@
+/*
+Action creators for gene sets
+
+Primarily used to keep the crossfilter and underlying data in sync with the UI.
+
+The behavior manifest in these action creators:
+
+    Delete a gene set, will
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state of each gene in the geneset
+
+    Delete a gene from a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state on the gene
+
+    Add a gene to a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * will NOT touch the selection state for the gene
+  
+Note that crossfilter indices are lazy created, as needed.
+*/
+
+export const genesetDelete = (genesetName) => (dispatch, getState) => {
+  const state = getState();
+  const { genesets } = state;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const geneSymbols = Array.from(gs.genes.keys());
+  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
+  dispatch({
+    type: "geneset: delete",
+    genesetName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+export const genesetAddGenes = (genesetName, genes) => (dispatch, getState) => {
+  const state = getState();
+  const { obsCrossfilter: prevObsCrossfilter } = state;
+  const obsCrossfilter = dropGenesetSummaryDimension(
+    prevObsCrossfilter,
+    state,
+    genesetName
+  );
+  dispatch({
+    type: "continuous metadata histogram cancel",
+    continuousNamespace: { isGeneSetSummary: true },
+    selection: genesetName,
+  });
+  return dispatch({
+    type: "geneset: add genes",
+    genesetName,
+    genes,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+export const genesetDeleteGenes = (genesetName, geneSymbols) => (
+  dispatch,
+  getState
+) => {
+  const state = getState();
+  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
+  return dispatch({
+    type: "geneset: delete genes",
+    genesetName,
+    geneSymbols,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+/*
+Private
+*/
+
+function dropGenesetSummaryDimension(obsCrossfilter, state, genesetName) {
+  const { annoMatrix, genesets } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const genes = Array.from(gs.genes.keys());
+  const query = {
+    summarize: {
+      method: "mean",
+      field: "var",
+      column: varIndex,
+      values: genes,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneDimension(obsCrossfilter, state, gene) {
+  const { annoMatrix } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const query = {
+    where: {
+      field: "var",
+      column: varIndex,
+      value: gene,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneset(dispatch, state, genesetName, geneSymbols) {
+  const { obsCrossfilter: prevObsCrossfilter } = state;
+  const obsCrossfilter = geneSymbols.reduce(
+    (crossfilter, gene) => dropGeneDimension(crossfilter, state, gene),
+    dropGenesetSummaryDimension(prevObsCrossfilter, state, genesetName)
+  );
+  dispatch({
+    type: "continuous metadata histogram cancel",
+    continuousNamespace: { isGeneSetSummary: true },
+    selection: genesetName,
+  });
+  geneSymbols.forEach((g) =>
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isUserDefined: true },
+      selection: g,
+    })
+  );
+  return obsCrossfilter;
+}

client/src/actions/index.js

@@ -13,6 +13,7 @@ import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
+import * as genesetActions from "./geneset";
 
 /*
 return promise fetching user-configured colors
@@ -58,7 +59,7 @@ async function genesetsFetch(dispatch, config) {
     genesets: [],
     tid: 0,
   };
-  if (config?.parameters?.["annotations_genesets"] ?? false) {
+  if (config?.parameters?.annotations_genesets ?? false) {
     fetchJson("genesets").then((response) => {
       dispatch({
         type: "geneset: initial load",
@@ -111,7 +112,7 @@ const doInitialDataLoad = () =>
       });
       dispatch({ type: "initial data load complete" });
 
-      const defaultEmbedding = config?.parameters?.["default_embedding"];
+      const defaultEmbedding = config?.parameters?.default_embedding;
       const layoutSchema = schema?.schema?.layout?.obs ?? [];
       if (
         defaultEmbedding &&
@@ -269,4 +270,7 @@ export default {
   needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
   layoutChoiceAction: embActions.layoutChoiceAction,
   setCellSetFromSelection: selnActions.setCellSetFromSelection,
+  genesetDelete: genesetActions.genesetDelete,
+  genesetAddGenes: genesetActions.genesetAddGenes,
+  genesetDeleteGenes: genesetActions.genesetDeleteGenes,
 };

client/src/annoMatrix/annoMatrix.js

@@ -12,6 +12,7 @@ import {
 import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
 import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
 import _shallowClone from "./clone";
+import { _queryValidate, _queryCacheKey } from "./query";
 
 const _dataframeCache = dataframeMemo(128);
 
@@ -181,7 +182,7 @@ export default class AnnoMatrix {
 		Returns a Promise for the query result, which will resolve to a dataframe.
 
 		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
-		represents the underlying object upon which the query is occuring.
+		represents the underlying object upon which the query is occurring.
 
 		Query is one of:
 			* a string, representing a single column name from the field, eg,
@@ -197,12 +198,6 @@ export default class AnnoMatrix {
 		field/column, similar to a join.  Currently only supported on the var
 		dimension, allowing query of X columns by var value (eg, gene name)
 
-    The query filter	is a single value filter:
-			{ "field name": [
-				{name: "column name", values: [ list of values ]}
-			]}
-		One and only one value filter is allowed in a value query.
-
 		Examples:
 
     1. Fetch the "n_genes" column the "obs":
@@ -220,7 +215,9 @@ export default class AnnoMatrix {
        value "TYMP" in the var index.
 
 			fetch("X", { 
-				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+				where: {
+          field: "var", column: this.schema.annotations.var.index, value: "TYMP"
+        }
 			})
 
       In AnnData & Pandas DataFrame API, this is equivalent to:
@@ -410,6 +407,14 @@ export default class AnnoMatrix {
     _subclassResponsibility();
   }
 
+  getCacheKeys(field, query) {
+    /*
+    Return cache keys for columns associated with this query.  May return
+    [unknown] if no keys are known (ie, nothing is or was cached).
+    */
+    return _whereCacheGet(this._whereCache, this.schema, field, query);
+  }
+
   /**
    ** Private interfaces below.
    **/
@@ -427,6 +432,7 @@ export default class AnnoMatrix {
   async _fetch(field, q) {
     if (!AnnoMatrix.fields().includes(field)) return undefined;
     const queries = Array.isArray(q) ? q : [q];
+    queries.forEach(_queryValidate);
 
     /* find cached columns we need, and GC the rest */
     const cachedColumns = this._resolveCachedQueries(field, queries);
@@ -507,7 +513,7 @@ export default class AnnoMatrix {
     * obs, var and emb do not grow without bounds, and are needed constantly 
       for rendering.  
       a) There is no upside to GC'ing these in the base (loader)
-      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+      b) The undo/redo cache can hold a large number in views, which is worth GC'ing
     * X is often much larger than memory, and the UI allows add/del from
       this.  Most of the GC potential is here in both the base and views.
 
@@ -522,7 +528,7 @@ export default class AnnoMatrix {
   as much of the cache is pinned by that data structure.
   */
   _gcField(field, isHot, pinnedColumns) {
-    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+    const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
 
     const cache = this._cache[field];
     if (cache.colIndex.size() < maxColumns) return; // trivial rejection
@@ -590,23 +596,18 @@ export default class AnnoMatrix {
     called each time a query is performed, allowing the gc to update any bookkeeping
     information.  Currently, this is just a simple last-fetched timestamp, stored
     in a Map.
-
-    Map objects preserve order of insertion. This is leveraged as a cheap way to
-    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
-    the map insertion order is least-recently-used first.
     */
     const cols = dataframe.colIndex.labels();
     const { _gcInfo } = this;
     const now = Date.now();
     cols.forEach((c) => {
-      // gcInfo.delete(c);
       _gcInfo.set(_columnCacheKey(field, c), now);
     });
   }
 
   /**
-  Cloning sublcass protocol - we rely in cloning to preserve immutable
+  Cloning subclass protocol - we rely in cloning to preserve immutable
-  symantics while not causing races or other side effects in internal
+  semantics while not causing races or other side effects in internal
   cache management.
 
   Subclasses must override _cloneDeeper() if they have state which requires 
@@ -639,15 +640,6 @@ export default class AnnoMatrix {
 /*
 private utility functions below
 */
-
-function _queryCacheKey(field, query) {
-  if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
-  }
-  return `${field}/${query}`;
-}
-
 function _columnCacheKey(field, column) {
   return `${field}/${column}`;
 }

client/src/annoMatrix/crossfilter.js

@@ -123,6 +123,24 @@ export default class AnnoMatrixObsCrossfilter {
     return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
   }
 
+  /**
+   * Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
+   * want to stop trackin the selection state, but aren't sure we want to blow the
+   * annomatrix cache.
+   */
+  dropDimension(field, query) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+    const keys = annoMatrix
+      .getCacheKeys(field, query)
+      .filter((k) => k !== undefined);
+    const dimName = _dimensionName(field, keys);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
   /**
   Selection state - API is identical to ImmutableTypedCrossfilter, as these
   are just wrappers to lazy create indices.

client/src/annoMatrix/fetchHelpers.js

@@ -1,4 +1,4 @@
-export { doBinaryRequest } from "../util/actionHelpers";
+export { doBinaryRequest, doFetch } from "../util/actionHelpers";
 
 /* double URI encode - needed for query-param filters */
 export function _dubEncURIComp(s) {

client/src/annoMatrix/loader.js

@@ -1,4 +1,4 @@
-import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { doBinaryRequest, doFetch } from "./fetchHelpers";
 import { matrixFBSToDataframe } from "../util/stateManager/matrix";
 import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
 import {
@@ -12,6 +12,13 @@ import { isArrayOrTypedArray } from "../util/typeHelpers";
 import { _whereCacheCreate } from "./whereCache";
 import AnnoMatrix from "./annoMatrix";
 import PromiseLimit from "../util/promiseLimit";
+import {
+  _expectSimpleQuery,
+  _expectComplexQuery,
+  _urlEncodeLabelQuery,
+  _urlEncodeComplexQuery,
+  _hashStringValues,
+} from "./query";
 
 const promiseThrottle = new PromiseLimit(5);
 
@@ -223,27 +230,23 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     /*
     _doLoad - evaluates the query against the field. Returns:
       * whereCache update: column query map mapping the query to the column labels
-      * Dataframe containing the new colums (one per dimension)
+      * Dataframe containing the new columns (one per dimension)
     */
-    let urlQuery;
+    let doRequest;
-    let urlBase;
     let priority = 10; // default fetch priority
 
     switch (field) {
       case "obs":
       case "var": {
-        urlBase = `${this.baseURL}annotations/${field}`;
+        doRequest = _obsOrVarLoader(this.baseURL, field, query);
-        urlQuery = _encodeQuery("annotation-name", query);
         break;
       }
       case "X": {
-        urlBase = `${this.baseURL}data/var`;
+        doRequest = _XLoader(this.baseURL, field, query);
-        urlQuery = _encodeQuery(undefined, query);
         break;
       }
       case "emb": {
-        urlBase = `${this.baseURL}layout/obs`;
+        doRequest = _embLoader(this.baseURL, field, query);
-        urlQuery = _encodeQuery("layout-name", query);
         priority = 0; // high prio load for embeddings
         break;
       }
@@ -251,12 +254,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
         throw new Error("Unknown field name");
     }
 
-    const url = `${urlBase}?${urlQuery}`;
+    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
-    const buffer = await promiseThrottle.priorityAdd(
-      priority,
-      doBinaryRequest,
-      url
-    );
     const result = matrixFBSToDataframe(buffer);
     if (!result || result.isEmpty()) throw Error("Unknown field/col");
 
@@ -267,7 +265,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     );
 
     if (field === "obs") {
-      /* cough, cough - see comment on method */
+      /* cough, cough - see comment on the function called */
       _normalizeCategoricalSchema(
         this.schema.annotations.obsByName[query],
         result.col(query)
@@ -282,17 +280,6 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
 Utility functions below
 */
 
-function _encodeQuery(colKey, q) {
-  if (typeof q === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = q;
-    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
-      queryColumn
-    )}=${_dubEncURIComp(queryValue)}`;
-  }
-  if (!colKey) throw new Error("Unsupported query by name");
-  return `${colKey}=${encodeURIComponent(q)}`;
-}
-
 function _writableCheck(colSchema) {
   if (!colSchema?.writable) {
     throw new Error("Unknown or readonly obs column");
@@ -305,3 +292,57 @@ function _writableCategoryTypeCheck(colSchema) {
     throw new Error("column must be categorical");
   }
 }
+
+function _embLoader(baseURL, _field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}layout/obs`;
+  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _obsOrVarLoader(baseURL, field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}annotations/${field}`;
+  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _XLoader(baseURL, field, query) {
+  _expectComplexQuery(query);
+
+  if (query.where) {
+    const urlBase = `${baseURL}data/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+    const url = `${urlBase}?${urlQuery}`;
+    return () => doBinaryRequest(url);
+  }
+
+  if (query.summarize) {
+    const urlBase = `${baseURL}summarize/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+
+    if (urlBase.length + urlQuery.length < 2000) {
+      const url = `${urlBase}?${urlQuery}`;
+      return () => doBinaryRequest(url);
+    }
+
+    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
+    return async () => {
+      const res = await doFetch(url, {
+        method: "POST",
+        body: urlQuery,
+        headers: new Headers({
+          Accept: "application/octet-stream",
+          "Content-Type": "application/x-www-form-urlencoded",
+        }),
+      });
+      return res.arrayBuffer();
+    };
+  }
+
+  throw new Error("Unknown query structure");
+}

client/src/annoMatrix/query.js

@@ -0,0 +1,126 @@
+import sha1 from "sha1";
+import { _dubEncURIComp } from "./fetchHelpers";
+
+/**
+ * Query utilities, mostly for debugging support and validation.
+ */
+
+/**
+ * Normalize & error check the query.
+ * @param {object | string} query - the query
+ * @returns {object | string} - the normalized query
+ */
+export function _queryValidate(query) {
+  if (typeof query !== "object") return query;
+
+  if (query.where && query.summarize)
+    throw new Error("query may not specify both where and summarize");
+  if (query.where) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
+    if (!queryField || !queryColumn || !queryValue)
+      throw new Error("Incomplete where query");
+    return query;
+  }
+  if (query.summarize) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    if (!queryField || !queryColumn || !queryValues)
+      throw new Error("Incomplete where query");
+    if (!Array.isArray(queryValues))
+      throw new Error("Summarize query values must be an array");
+    return query;
+  }
+  throw new Error("query must specify one of where or summarize");
+}
+
+export function _expectSimpleQuery(query) {
+  if (typeof query === "object") throw new Error("expected simple query");
+}
+
+export function _expectComplexQuery(query) {
+  if (typeof query !== "object") throw new Error("expected complex query");
+}
+
+/**
+ * Generate a unique key which can be used to reference this query.
+ *
+ * @param {string} field
+ * @param {string|object} query
+ * @returns the key
+ */
+export function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    // complex query
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      return `${field}/${method}/${queryField}/${queryColumn}/${queryValues.join(
+        ","
+      )}`;
+    }
+    throw new Error("Unrecognized complex query type");
+  }
+
+  // simple query
+  return `${field}/${query}`;
+}
+
+function _urlEncodeWhereQuery(q) {
+  const { field: queryField, column: queryColumn, value: queryValue } = q;
+  return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+    queryColumn
+  )}=${_dubEncURIComp(queryValue)}`;
+}
+
+function _urlEncodeSummarizeQuery(q) {
+  const { method, field, column, values } = q;
+  const filter = values
+    .map((value) => _urlEncodeWhereQuery({ field, column, value }))
+    .join("&");
+  return `method=${method}&${filter}`;
+}
+
+export function _urlEncodeComplexQuery(q) {
+  if (typeof q === "object") {
+    if (q.where) {
+      return _urlEncodeWhereQuery(q.where);
+    }
+    if (q.summarize) {
+      return _urlEncodeSummarizeQuery(q.summarize);
+    }
+  }
+  throw new Error("Unrecognized complex query type");
+}
+
+export function _urlEncodeLabelQuery(colKey, q) {
+  if (!colKey) throw new Error("Unsupported query by name");
+  if (typeof q !== "string") throw new Error("Query must be a simple label.");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+/**
+ * Generate the column key the server will send us for this query.
+ */
+export function _hashStringValues(arrayOfString) {
+  const hash = sha1(arrayOfString.join(""));
+  return hash;
+}

client/src/annoMatrix/schema.js

@@ -29,7 +29,7 @@ export function _getColumnSchema(schema, field, col) {
 export function _getColumnDimensionNames(schema, field, col) {
   /*
 		field/col may be an alias for multiple columns. Currently used to map ND 
-		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		values to 1D dataframe columns for embeddings/layout. Signified by the presence
 		of the "dims" value in the schema.
 		*/
   const colSchema = _getColumnSchema(schema, field, col);

client/src/annoMatrix/whereCache.js

@@ -1,27 +1,55 @@
 /*
 Private support functions.
 
-Support for a "where" query, eg,
+This implements a query resolver cache, mapping a query onto the column labels
+resolved by that query. These labels are then used to manage the acutal data cache,
+which stores data by the resolved label.
 
-	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+There are three query forms:
+  * primitive (string, number) - which is just reference the column label of same value
+  * where query (object) - eg, { where: { field: "var", column: "gene", value: "FOXP2" }}
+  * summary query (object) - eg, { summarize: { method: "mean", field: "var", column: "gene", values: ["FOXP2", "GNE", "F5"]}}
 
-These evaluate to a given column label.
+These queries all resolve to one or more column labels on a field.  This
-
+cache maintains a record of this, allowing direct access to the data caches
-The "where cache" is a map that saves evaluated queries and points 
+without a server round-trip.
-to the column label they resolve to.
-
-Data structure, using X as the example field being queried, and var as
-the index.
 
+The data structure for where queries, the following query against X as an example:
+  { where: { field: "var", column: "column_label_in_var", value: "value_in_var_column" } }
+results in the following cached entry:
 {
-	X: {
+  where: {
-		var: Map(
+    X: {
-			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
+      var: Map(
-		)
+        column_label_in_var => Map(
-	}
+          value_in_var_column => [column_label_in_X, ...]
+        )
+      )
+    }
+  },
+  summarize: {},
+}
+
+And for summarize queries, for the following summary on X:
+  { summarize: { method: "mean", field: "var", column: "gene", values: ["G1", "G2"]}}
+creates a cache entry of:
+{
+  where: {},
+  summarize: {
+    X: {
+      mean: {
+        var: Map(
+          "gene" => Map(
+            "G1,G2" => [summary_column_label, ...]
+          )
+        )
+      }
+    },
+  },
 }
 */
 import { _getColumnDimensionNames } from "./schema";
+import { _hashStringValues } from "./query";
 
 export function _whereCacheGet(whereCache, schema, field, query) {
   /* 
@@ -31,20 +59,30 @@ export function _whereCacheGet(whereCache, schema, field, query) {
 	*/
 
   if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
+    if (query.where) {
-
+      const {
-    const columnMap = whereCache?.[field]?.[queryField];
+        field: queryField,
-    if (columnMap === undefined) return [undefined];
+        column: queryColumn,
-
+        value: queryValue,
-    const valueMap = columnMap.get(queryColumn);
+      } = query.where;
-    if (valueMap === undefined) return [undefined];
+      const columnMap = whereCache?.where?.[field]?.[queryField];
-
+      return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
-    const columnLabels = valueMap.get(queryValue);
+    }
-    return columnLabels === undefined ? [undefined] : columnLabels;
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      const columnMap = whereCache?.summarize?.[field]?.[method]?.[queryField];
+      const queryValueHash = _hashStringValues(queryValues);
+      return columnMap?.get(queryColumn)?.get(queryValueHash) ?? [undefined];
+    }
+    return [undefined];
   }
 
-  const colDims = _getColumnDimensionNames(schema, field, query);
+  return _getColumnDimensionNames(schema, field, query) ?? [undefined];
-  return colDims === undefined ? [undefined] : colDims;
 }
 
 export function _whereCacheCreate(field, query, columnLabels) {
@@ -53,40 +91,86 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { field: queryField, column: queryColumn, value: queryValue } = query;
+  if (query.where) {
-  const whereCache = {
+    const {
-    [field]: {
+      field: queryField,
-      [queryField]: new Map([
+      column: queryColumn,
-        [queryColumn, new Map([[queryValue, columnLabels]])],
+      value: queryValue,
-      ]),
+    } = query.where;
-    },
+    return {
-  };
+      where: {
-  return whereCache;
+        [field]: {
+          [queryField]: new Map([
+            [queryColumn, new Map([[queryValue, columnLabels]])],
+          ]),
+        },
+      },
+    };
+  }
+  if (query.summarize) {
+    const {
+      method,
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    const queryValueHash = _hashStringValues(queryValues);
+    return {
+      summarize: {
+        [field]: {
+          [method]: {
+            [queryField]: new Map([
+              [queryColumn, new Map([[queryValueHash, columnLabels]])],
+            ]),
+          },
+        },
+      },
+    };
+  }
+  // oops, not sure what that query is!
+  return {};
+}
+
+function __mergeQueries(dst, src) {
+  for (const [queryField, columnMap] of Object.entries(src)) {
+    dst[queryField] = dst[queryField] || new Map();
+    for (const [queryColumn, valueMap] of columnMap) {
+      if (!dst[queryField].has(queryColumn))
+        dst[queryField].set(queryColumn, new Map());
+      for (const [queryValue, columnLabels] of valueMap) {
+        dst[queryField].get(queryColumn).set(queryValue, columnLabels);
+      }
+    }
+  }
 }
 
 function __whereCacheMerge(dst, src) {
   /*
 	merge src into dst (modifies dst)
 	*/
-  if (!dst) dst = {};
   if (!src || typeof src !== "object") return dst;
-  Object.entries(src).forEach(([field, query]) => {
+
-    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
+  if (src.where) {
-    Object.entries(query).forEach(([queryField, columnMap]) => {
+    dst.where = dst.where || {};
-      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
+    for (const [field, query] of Object.entries(src.where)) {
-        dst[field][queryField] = new Map();
+      dst.where[field] = dst.where[field] || {};
-      columnMap.forEach((valueMap, queryColumn) => {
+      __mergeQueries(dst.where[field], query);
-        if (!dst[field][queryField].has(queryColumn))
+    }
-          dst[field][queryField].set(queryColumn, new Map());
+  }
-        valueMap.forEach((columnLabels, queryValue) => {
+  if (src.summarize) {
-          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
+    dst.summarize = dst.summarize || {};
-        });
+    for (const [field, method] of Object.entries(src.summarize)) {
-      });
+      dst.summarize[field] = dst.summarize[field] || {};
-    });
+      for (const [methodName, query] of Object.entries(method)) {
-  });
+        dst.summarize[field][methodName] =
+          dst.summarize[field][methodName] || {};
+        __mergeQueries(dst.summarize[field][methodName], query);
+      }
+    }
+  }
   return dst;
 }
 
 export function _whereCacheMerge(...caches) {
-  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+  return caches.reduce(__whereCacheMerge, {});
 }

client/src/components/autosave/filenameDialog.js

@@ -18,7 +18,7 @@ import {
   userInfo: state.userInfo,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   writableGenesetsEnabled: !(
-    state.config?.parameters?.["annotations_genesets_readonly"] ?? true
+    state.config?.parameters?.annotations_genesets_readonly ?? true
   ),
 }))
 class FilenameDialog extends React.Component {

client/src/components/autosave/index.js

@@ -11,7 +11,7 @@ import FilenameDialog from "./filenameDialog";
   error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   writableGenesetsEnabled: !(
-    state.config?.parameters?.["annotations_genesets_readonly"] ?? true
+    state.config?.parameters?.annotations_genesets_readonly ?? true
   ),
   annoMatrix: state.annoMatrix,
   genesets: state.genesets,

client/src/components/brushableHistogram/index.js

@@ -1,5 +1,5 @@
 import React from "react";
-import { connect } from "react-redux";
+import { connect, shallowEqual } from "react-redux";
 import * as d3 from "d3";
 import Async from "react-async";
 import memoize from "memoize-one";
@@ -14,9 +14,9 @@ import StillLoading from "./loading";
 import ErrorLoading from "./error";
 
 @connect((state, ownProps) => {
-  const { isObs, isUserDefined, isDiffExp, field } = ownProps;
+  const { isObs, isUserDefined, isDiffExp, isGeneSetSummary, field } = ownProps;
   const myName = makeContinuousDimensionName(
-    { isObs, isUserDefined, isDiffExp },
+    { isObs, isUserDefined, isDiffExp, isGeneSetSummary },
     field
   );
   return {
@@ -28,6 +28,10 @@ import ErrorLoading from "./error";
   };
 })
 class HistogramBrush extends React.PureComponent {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
   /* memoized closure to prevent HistogramHeader unecessary repaint */
   handleColorAction = memoize((dispatch) => (field, isObs) => {
     if (isObs) {
@@ -71,7 +75,14 @@ class HistogramBrush extends React.PureComponent {
   onBrush = (selection, x, eventType) => {
     const type = `continuous metadata histogram ${eventType}`;
     return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const {
+        dispatch,
+        field,
+        isObs,
+        isUserDefined,
+        isDiffExp,
+        isGeneSetSummary,
+      } = this.props;
 
       // ignore programmatically generated events
       if (!d3.event.sourceEvent) return;
@@ -88,6 +99,7 @@ class HistogramBrush extends React.PureComponent {
           isObs,
           isUserDefined,
           isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
@@ -98,7 +110,14 @@ class HistogramBrush extends React.PureComponent {
 
   onBrushEnd = (selection, x) => {
     return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const {
+        dispatch,
+        field,
+        isObs,
+        isUserDefined,
+        isDiffExp,
+        isGeneSetSummary,
+      } = this.props;
       const minAllowedBrushSize = 10;
       const smallAmountToAvoidInfiniteLoop = 0.1;
 
@@ -138,6 +157,7 @@ class HistogramBrush extends React.PureComponent {
           isObs,
           isUserDefined,
           isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
@@ -300,19 +320,37 @@ class HistogramBrush extends React.PureComponent {
   }
 
   createQuery() {
-    const { isObs, field, annoMatrix } = this.props;
+    const { isObs, isGeneSetSummary, field, setGenes, annoMatrix } = this.props;
     const { schema } = annoMatrix;
     if (isObs) {
       return ["obs", field];
     }
     const varIndex = schema?.annotations?.var?.index;
     if (!varIndex) return null;
+
+    if (isGeneSetSummary) {
+      return [
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: varIndex,
+            values: setGenes,
+          },
+        },
+      ];
+    }
+
+    // else, we assume it is a gene expression
     return [
       "X",
       {
-        field: "var",
+        where: {
-        column: varIndex,
+          field: "var",
-        value: field,
+          column: varIndex,
+          value: field,
+        },
       },
     ];
   }
@@ -333,6 +371,7 @@ class HistogramBrush extends React.PureComponent {
       continuousSelectionRange,
       isObs,
       mini,
+      setGenes,
     } = this.props;
     const {
       margin,
@@ -346,7 +385,11 @@ class HistogramBrush extends React.PureComponent {
     const showScatterPlot = isDiffExp || isUserDefined;
 
     return (
-      <Async watch={annoMatrix} promiseFn={this.fetchAsyncProps}>
+      <Async
+        watchFn={HistogramBrush.watchAsync}
+        promiseFn={this.fetchAsyncProps}
+        watchProps={{ annoMatrix, setGenes }}
+      >
         <Async.Pending initial>
           <StillLoading displayName={field} zebra={zebra} />
         </Async.Pending>

client/src/components/geneExpression/gene.js

@@ -53,12 +53,7 @@ class Gene extends React.Component {
 
   handleDeleteGeneFromSet = () => {
     const { dispatch, gene, geneset } = this.props;
-
+    dispatch(actions.genesetDeleteGenes(geneset, [gene]));
-    dispatch({
-      type: "geneset: delete genes",
-      genesetName: geneset,
-      geneSymbols: [gene],
-    });
   };
 
   render() {

client/src/components/geneExpression/geneSet.js

@@ -9,6 +9,7 @@ import { memoize } from "../../util/dataframe/util";
 import Truncate from "../util/truncate";
 import * as globals from "../../globals";
 import GenesetMenus from "./menus/genesetMenus";
+import HistogramBrush from "../brushableHistogram";
 
 @connect((state, ownProps) => {
   return {
@@ -163,18 +164,25 @@ class GeneSet extends React.Component {
           ) : null}
         </div>
 
-        {isOpen && !toggleSummaryHisto
+        {isOpen &&
-          ? _.map(setGenes, (gene) => {
+          (!toggleSummaryHisto
-              return (
+            ? _.map(setGenes, (gene) => {
-                <Gene
+                return (
-                  key={gene}
+                  <Gene
-                  gene={gene}
+                    key={gene}
-                  geneset={setName}
+                    gene={gene}
-                  isDiffexp={isDiffexp}
+                    geneset={setName}
+                    isDiffexp={isDiffexp}
+                  />
+                );
+              })
+            : setGenes.length > 0 && (
+                <HistogramBrush
+                  isGeneSetSummary
+                  field={setName}
+                  setGenes={setGenes}
                 />
-              );
+              ))}
-            })
-          : null}
       </div>
     );
   }

client/src/components/geneExpression/menus/addGeneToGenesetDialogue.js

@@ -3,6 +3,7 @@ import { connect } from "react-redux";
 import AnnoDialog from "../../annoDialog";
 import LabelInput from "../../labelInput";
 import parseBulkGeneString from "../../../util/parseBulkGeneString";
+import actions from "../../../actions";
 
 @connect((state) => ({
   genesetsUI: state.genesetsUI,
@@ -36,11 +37,7 @@ class AddGeneToGenesetDialogue extends React.PureComponent {
       });
     });
 
-    dispatch({
+    dispatch(actions.genesetAddGenes(geneset, genesTmpHardcodedFormat));
-      type: "geneset: add genes",
-      genesetName: geneset,
-      genes: genesTmpHardcodedFormat,
-    });
     dispatch({
       type: "geneset: disable add new genes mode",
     });

client/src/components/geneExpression/menus/addGenes.js

@@ -77,6 +77,26 @@ class AddGenes extends React.Component {
     this.updateState(prevProps);
   }
 
+  handleClick(g) {
+    const { dispatch, userDefinedGenes } = this.props;
+    const { geneNames } = this.state;
+    if (!g) return;
+    const gene = g.target;
+    if (userDefinedGenes.indexOf(gene) !== -1) {
+      postUserErrorToast("That gene already exists");
+    } else if (userDefinedGenes.length > globals.maxUserDefinedGenes) {
+      postUserErrorToast(
+        `That's too many genes, you can have at most ${globals.maxUserDefinedGenes} user defined genes`
+      );
+    } else if (geneNames.indexOf(gene) === undefined) {
+      postUserErrorToast("That doesn't appear to be a valid gene name.");
+    } else {
+      dispatch({ type: "single user defined gene start" });
+      dispatch(actions.requestUserDefinedGene(gene));
+      dispatch({ type: "single user defined gene complete" });
+    }
+  }
+
   _genesToUpper = (listGenes) => {
     // Has to be a Map to preserve index
     const upperGenes = new Map();
@@ -190,26 +210,6 @@ class AddGenes extends React.Component {
     return "Apod, Cd74, ...";
   }
 
-  handleClick(g) {
-    const { dispatch, userDefinedGenes } = this.props;
-    const { geneNames } = this.state;
-    if (!g) return;
-    const gene = g.target;
-    if (userDefinedGenes.indexOf(gene) !== -1) {
-      postUserErrorToast("That gene already exists");
-    } else if (userDefinedGenes.length > globals.maxUserDefinedGenes) {
-      postUserErrorToast(
-        `That's too many genes, you can have at most ${globals.maxUserDefinedGenes} user defined genes`
-      );
-    } else if (geneNames.indexOf(gene) === undefined) {
-      postUserErrorToast("That doesn't appear to be a valid gene name.");
-    } else {
-      dispatch({ type: "single user defined gene start" });
-      dispatch(actions.requestUserDefinedGene(gene));
-      dispatch({ type: "single user defined gene complete" });
-    }
-  }
-
   render() {
     const { userDefinedGenesLoading } = this.props;
     const { tab, bulkAdd, activeItem, status, geneNames } = this.state;

client/src/components/geneExpression/menus/createGenesetDialogue.js

@@ -3,6 +3,7 @@ import React from "react";
 import { connect } from "react-redux";
 import AnnoDialog from "../../annoDialog";
 import LabelInput from "../../labelInput";
+import actions from "../../../actions";
 
 @connect((state) => ({
   annotations: state.annotations,
@@ -56,11 +57,7 @@ class CreateGenesetDialogue extends React.PureComponent {
         });
       });
 
-      dispatch({
+      dispatch(actions.genesetAddGenes(genesetName, genesTmpHardcodedFormat));
-        type: "geneset: add genes",
-        genesetName,
-        genes: genesTmpHardcodedFormat,
-      });
     }
     dispatch({
       type: "geneset: disable create geneset mode",

client/src/components/geneExpression/menus/genesetMenus.js

@@ -12,6 +12,7 @@ import {
 } from "@blueprintjs/core";
 
 import * as globals from "../../../globals";
+import actions from "../../../actions";
 import AddGeneToGenesetDialogue from "./addGeneToGenesetDialogue";
 
 @connect((state) => {
@@ -44,7 +45,7 @@ class GenesetMenus extends React.PureComponent {
 
   handleDeleteCategory = () => {
     const { dispatch, geneset } = this.props;
-    dispatch({ type: "geneset: delete", genesetName: geneset });
+    dispatch(actions.genesetDelete(geneset));
   };
 
   render() {

client/src/components/graph/graph.js

@@ -303,13 +303,6 @@ class Graph extends React.Component {
     window.removeEventListener("resize", this.handleResize);
   }
 
-  setReglCanvas = (canvas) => {
-    this.reglCanvas = canvas;
-    this.setState({
-      ...Graph.createReglState(canvas),
-    });
-  };
-
   handleResize = () => {
     const { state } = this.state;
     const viewport = this.getViewportDimensions();
@@ -321,14 +314,6 @@ class Graph extends React.Component {
     });
   };
 
-  getViewportDimensions = () => {
-    const { viewportRef } = this.props;
-    return {
-      height: viewportRef.clientHeight,
-      width: viewportRef.clientWidth,
-    };
-  };
-
   handleCanvasEvent = (e) => {
     const { camera, projectionTF } = this.state;
     if (e.type !== "wheel") e.preventDefault();
@@ -340,6 +325,143 @@ class Graph extends React.Component {
     }
   };
 
+  handleBrushDragAction() {
+    /*
+      event describing brush position:
+      @-------|
+      |       |
+      |       |
+      |-------@
+    */
+    // ignore programatically generated events
+    if (d3.event.sourceEvent === null || !d3.event.selection) return;
+
+    const { dispatch, layoutChoice } = this.props;
+    const s = d3.event.selection;
+    const northwest = this.mapScreenToPoint(s[0]);
+    const southeast = this.mapScreenToPoint(s[1]);
+    const [minX, maxY] = northwest;
+    const [maxX, minY] = southeast;
+    dispatch(
+      actions.graphBrushChangeAction(layoutChoice.current, {
+        minX,
+        minY,
+        maxX,
+        maxY,
+        northwest,
+        southeast,
+      })
+    );
+  }
+
+  handleBrushStartAction() {
+    // Ignore programatically generated events.
+    if (!d3.event.sourceEvent) return;
+
+    const { dispatch } = this.props;
+    dispatch(actions.graphBrushStartAction());
+  }
+
+  handleBrushEndAction() {
+    // Ignore programatically generated events.
+    if (!d3.event.sourceEvent) return;
+
+    /*
+    coordinates will be included if selection made, null
+    if selection cleared.
+    */
+    const { dispatch, layoutChoice } = this.props;
+    const s = d3.event.selection;
+    if (s) {
+      const northwest = this.mapScreenToPoint(s[0]);
+      const southeast = this.mapScreenToPoint(s[1]);
+      const [minX, maxY] = northwest;
+      const [maxX, minY] = southeast;
+      dispatch(
+        actions.graphBrushEndAction(layoutChoice.current, {
+          minX,
+          minY,
+          maxX,
+          maxY,
+          northwest,
+          southeast,
+        })
+      );
+    } else {
+      dispatch(actions.graphBrushDeselectAction(layoutChoice.current));
+    }
+  }
+
+  handleBrushDeselectAction() {
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphBrushDeselectAction(layoutChoice.current));
+  }
+
+  handleLassoStart() {
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphLassoStartAction(layoutChoice.current));
+  }
+
+  // when a lasso is completed, filter to the points within the lasso polygon
+  handleLassoEnd(polygon) {
+    const minimumPolygonArea = 10;
+    const { dispatch, layoutChoice } = this.props;
+
+    if (
+      polygon.length < 3 ||
+      Math.abs(d3.polygonArea(polygon)) < minimumPolygonArea
+    ) {
+      // if less than three points, or super small area, treat as a clear selection.
+      dispatch(actions.graphLassoDeselectAction(layoutChoice.current));
+    } else {
+      dispatch(
+        actions.graphLassoEndAction(
+          layoutChoice.current,
+          polygon.map((xy) => this.mapScreenToPoint(xy))
+        )
+      );
+    }
+  }
+
+  handleLassoCancel() {
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphLassoCancelAction(layoutChoice.current));
+  }
+
+  handleLassoDeselectAction() {
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphLassoDeselectAction(layoutChoice.current));
+  }
+
+  handleDeselectAction() {
+    const { selectionTool } = this.props;
+    if (selectionTool === "brush") this.handleBrushDeselectAction();
+    if (selectionTool === "lasso") this.handleLassoDeselectAction();
+  }
+
+  handleOpacityRangeChange(e) {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "change opacity deselected cells in 2d graph background",
+      data: e.target.value,
+    });
+  }
+
+  setReglCanvas = (canvas) => {
+    this.reglCanvas = canvas;
+    this.setState({
+      ...Graph.createReglState(canvas),
+    });
+  };
+
+  getViewportDimensions = () => {
+    const { viewportRef } = this.props;
+    return {
+      height: viewportRef.clientHeight,
+      width: viewportRef.clientWidth,
+    };
+  };
+
   createToolSVG = () => {
     /*
     Called from componentDidUpdate. Create the tool SVG, and return any
@@ -589,128 +711,6 @@ class Graph extends React.Component {
     ];
   }
 
-  handleBrushDragAction() {
-    /*
-      event describing brush position:
-      @-------|
-      |       |
-      |       |
-      |-------@
-    */
-    // ignore programatically generated events
-    if (d3.event.sourceEvent === null || !d3.event.selection) return;
-
-    const { dispatch, layoutChoice } = this.props;
-    const s = d3.event.selection;
-    const northwest = this.mapScreenToPoint(s[0]);
-    const southeast = this.mapScreenToPoint(s[1]);
-    const [minX, maxY] = northwest;
-    const [maxX, minY] = southeast;
-    dispatch(
-      actions.graphBrushChangeAction(layoutChoice.current, {
-        minX,
-        minY,
-        maxX,
-        maxY,
-        northwest,
-        southeast,
-      })
-    );
-  }
-
-  handleBrushStartAction() {
-    // Ignore programatically generated events.
-    if (!d3.event.sourceEvent) return;
-
-    const { dispatch } = this.props;
-    dispatch(actions.graphBrushStartAction());
-  }
-
-  handleBrushEndAction() {
-    // Ignore programatically generated events.
-    if (!d3.event.sourceEvent) return;
-
-    /*
-    coordinates will be included if selection made, null
-    if selection cleared.
-    */
-    const { dispatch, layoutChoice } = this.props;
-    const s = d3.event.selection;
-    if (s) {
-      const northwest = this.mapScreenToPoint(s[0]);
-      const southeast = this.mapScreenToPoint(s[1]);
-      const [minX, maxY] = northwest;
-      const [maxX, minY] = southeast;
-      dispatch(
-        actions.graphBrushEndAction(layoutChoice.current, {
-          minX,
-          minY,
-          maxX,
-          maxY,
-          northwest,
-          southeast,
-        })
-      );
-    } else {
-      dispatch(actions.graphBrushDeselectAction(layoutChoice.current));
-    }
-  }
-
-  handleBrushDeselectAction() {
-    const { dispatch, layoutChoice } = this.props;
-    dispatch(actions.graphBrushDeselectAction(layoutChoice.current));
-  }
-
-  handleLassoStart() {
-    const { dispatch, layoutChoice } = this.props;
-    dispatch(actions.graphLassoStartAction(layoutChoice.current));
-  }
-
-  // when a lasso is completed, filter to the points within the lasso polygon
-  handleLassoEnd(polygon) {
-    const minimumPolygonArea = 10;
-    const { dispatch, layoutChoice } = this.props;
-
-    if (
-      polygon.length < 3 ||
-      Math.abs(d3.polygonArea(polygon)) < minimumPolygonArea
-    ) {
-      // if less than three points, or super small area, treat as a clear selection.
-      dispatch(actions.graphLassoDeselectAction(layoutChoice.current));
-    } else {
-      dispatch(
-        actions.graphLassoEndAction(
-          layoutChoice.current,
-          polygon.map((xy) => this.mapScreenToPoint(xy))
-        )
-      );
-    }
-  }
-
-  handleLassoCancel() {
-    const { dispatch, layoutChoice } = this.props;
-    dispatch(actions.graphLassoCancelAction(layoutChoice.current));
-  }
-
-  handleLassoDeselectAction() {
-    const { dispatch, layoutChoice } = this.props;
-    dispatch(actions.graphLassoDeselectAction(layoutChoice.current));
-  }
-
-  handleDeselectAction() {
-    const { selectionTool } = this.props;
-    if (selectionTool === "brush") this.handleBrushDeselectAction();
-    if (selectionTool === "lasso") this.handleLassoDeselectAction();
-  }
-
-  handleOpacityRangeChange(e) {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "change opacity deselected cells in 2d graph background",
-      data: e.target.value,
-    });
-  }
-
   renderCanvas = renderThrottle(() => {
     const {
       regl,

client/src/components/infoDrawer/infoDrawer.js

@@ -11,7 +11,7 @@ import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers
     datasetTitle: state.config?.displayNames?.dataset ?? "",
     aboutURL: state.config?.links?.["about-dataset"],
     isOpen: state.controls.datasetDrawer,
-    dataPortalProps: state.config?.["corpora_props"],
+    dataPortalProps: state.config?.corpora_props,
   };
 })
 class InfoDrawer extends PureComponent {

client/src/components/infoDrawer/infoFormat.js

@@ -169,7 +169,7 @@ const InfoFormat = React.memo(
   ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
     if (
       ["1.0.0", "1.1.0"].indexOf(
-        dataPortalProps.version?.["corpora_schema_version"]
+        dataPortalProps.version?.corpora_schema_version
       ) === -1
     ) {
       dataPortalProps = {};

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -13,15 +13,14 @@ const DATASET_TITLE_FONT_SIZE = 14;
 @connect((state) => {
   const { corpora_props: corporaProps } = state.config;
   const correctVersion =
-    ["1.0.0", "1.1.0"].indexOf(
+    ["1.0.0", "1.1.0"].indexOf(corporaProps?.version?.corpora_schema_version) >
-      corporaProps?.version?.["corpora_schema_version"]
+    -1;
-    ) > -1;
   return {
     datasetTitle: state.config?.displayNames?.dataset ?? "",
-    libraryVersions: state.config?.["library_versions"],
+    libraryVersions: state.config?.library_versions,
     aboutLink: state.config?.links?.["about-dataset"],
-    tosURL: state.config?.parameters?.["about_legal_tos"],
+    tosURL: state.config?.parameters?.about_legal_tos,
-    privacyURL: state.config?.parameters?.["about_legal_privacy"],
+    privacyURL: state.config?.parameters?.about_legal_privacy,
     title: correctVersion ? corporaProps?.title : undefined,
   };
 })

client/src/components/menubar/diffexpButtons.js

@@ -11,7 +11,7 @@ import CellSetButton from "./cellSetButtons";
   celllist1: state.differential?.celllist1,
   celllist2: state.differential?.celllist2,
   diffexpMayBeSlow: state.config?.parameters?.["diffexp-may-be-slow"] ?? false,
-  diffexpCellcountMax: state.config?.limits?.["diffexp_cellcount_max"],
+  diffexpCellcountMax: state.config?.limits?.diffexp_cellcount_max,
 }))
 class DiffexpButtons extends React.PureComponent {
   computeDiffExp = () => {

client/src/components/menubar/index.js

@@ -40,7 +40,7 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
     celllist1: state.differential.celllist1,
     celllist2: state.differential.celllist2,
-    libraryVersions: state.config?.["library_versions"],
+    libraryVersions: state.config?.library_versions,
     auth: state.config?.authentication,
     userInfo: state.userInfo,
     undoDisabled: state["@@undoable/past"].length === 0,
@@ -50,8 +50,8 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     diffexpMayBeSlow:
       state.config?.parameters?.["diffexp-may-be-slow"] ?? false,
     showCentroidLabels: state.centroidLabels.showLabels,
-    tosURL: state.config?.parameters?.["about_legal_tos"],
+    tosURL: state.config?.parameters?.about_legal_tos,
-    privacyURL: state.config?.parameters?.["about_legal_privacy"],
+    privacyURL: state.config?.parameters?.about_legal_privacy,
     categoricalSelection: state.categoricalSelection,
     enableReembedding:
       state.config?.parameters?.["enable-reembedding"] ?? false,

client/src/components/scatterplot/scatterplot.js

@@ -303,9 +303,11 @@ class Scatterplot extends React.PureComponent {
     return [
       "X",
       {
-        field: "var",
+        where: {
-        column: varIndex,
+          field: "var",
-        value: geneName,
+          column: varIndex,
+          value: geneName,
+        },
       },
     ];
   }

client/src/components/termsPrompt/index.js

@@ -11,8 +11,8 @@ import {
 import { storageGet, storageSet, KEYS } from "../util/localStorage";
 
 @connect((state) => ({
-  tosURL: state.config?.parameters?.["about_legal_tos"],
+  tosURL: state.config?.parameters?.about_legal_tos,
-  privacyURL: state.config?.parameters?.["about_legal_privacy"],
+  privacyURL: state.config?.parameters?.about_legal_privacy,
 }))
 class TermsPrompt extends React.PureComponent {
   constructor(props) {

client/src/reducers/annotations.js

@@ -39,7 +39,7 @@ const Annotations = (
           "annotations-data-collection-name-is-read-only"
         ] ?? false;
       const promptForFilename =
-        action.config.parameters?.["user_annotation_collection_name_enabled"];
+        action.config.parameters?.user_annotation_collection_name_enabled;
       return {
         ...state,
         dataCollectionNameIsReadOnly,

client/src/reducers/autosave.js

@@ -76,7 +76,7 @@ const Autosave = (
       const { lastSavedGenesets } = action;
       return {
         ...state,
-        genesetSaveInProgess: false,
+        genesetSaveInProgress: false,
         error: false,
         lastSavedGenesets,
       };

client/src/reducers/genesets.js

@@ -57,12 +57,12 @@ const GeneSets = (
         for (const gene of gsData.genes) {
           genes.set(gene.gene_symbol, {
             geneSymbol: gene.gene_symbol,
-            geneDescription: gene?.["gene_description"] ?? "",
+            geneDescription: gene?.gene_description ?? "",
           });
         }
         const gs = {
           genesetName: gsData.geneset_name,
-          genesetDescription: gsData?.["geneset_description"] ?? "",
+          genesetDescription: gsData?.geneset_description ?? "",
           genes,
         };
         genesets.set(gsData.geneset_name, gs);

client/src/reducers/ontology.js

@@ -10,10 +10,8 @@ const Ontology = (
   switch (action.type) {
     case "configuration load complete": {
       const enabled =
-        action.config?.parameters?.["annotations_cell_ontology_enabled"] ??
+        action.config?.parameters?.annotations_cell_ontology_enabled ?? false;
-        false;
+      const terms = action.config?.parameters?.annotations_cell_ontology_terms;
-      const terms =
-        action.config?.parameters?.["annotations_cell_ontology_terms"];
 
       const termSet = new Set(terms);
       return {

client/src/reducers/undoableConfig.js

@@ -178,7 +178,7 @@ See undoable.js for description action filter interface description.
 Basic approach:
   * trivial handlers for skip, clear & save cases to keep config simple.
   * only implement complex state machines where absolutely required (eg,
-    multi-event seleciton and the like)
+    multi-event selection and the like)
 */
 const actionFilter = (debug) => (state, action, prevFilterState) => {
   const actionType = action.type;

client/src/reducers/undoableFsm.js

@@ -100,6 +100,12 @@ const createFsmTransitions = (
       to: "done",
       action: cancelPending,
     },
+    {
+      event: "continuous metadata histogram cancel",
+      from: "init",
+      to: "done",
+      action: save,
+    },
     {
       event: "continuous metadata histogram end",
       from: "continuous histo select in progress",

client/src/util/actionHelpers.js

@@ -30,22 +30,27 @@ export function catchErrorsWrap(fn, dispatchToUser = false) {
   };
 }
 
-/*
+/**
-Wrapper to perform async fetch with some modest error handling
+ * Wrapper to perform async fetch with some modest error handling
-and decoding.
+ * and decoding.  Arguments are identical to standard fetch.
-*/
+ */
-const doFetch = async (url, acceptType) => {
+export const doFetch = async (url, init = {}) => {
   try {
-    const res = await fetch(url, {
+    // add defaults to the fetch init param.
+    init = {
       method: "get",
-      headers: new Headers({
-        Accept: acceptType,
-      }),
       credentials: "include",
-    });
+      ...init,
-    if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
+    };
+    const acceptType = init.headers?.get("Accept");
+    const res = await fetch(url, init);
+    if (
+      res.ok &&
+      (!acceptType || res.headers.get("Content-Type").includes(acceptType))
+    ) {
       return res;
     }
+
     // else an error
     const msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
     dispatchNetworkErrorMessageToUser(msg);
@@ -61,16 +66,22 @@ const doFetch = async (url, acceptType) => {
 /*
 Wrapper to perform an async fetch and JSON decode response.
 */
-export const doJsonRequest = async (url) => {
+export const doJsonRequest = async (url, init = {}) => {
-  const res = await doFetch(url, "application/json");
+  const res = await doFetch(url, {
+    ...init,
+    headers: new Headers({ Accept: "application/json" }),
+  });
   return res.json();
 };
 
 /*
 Wrapper to perform an async fetch for binary data.
 */
-export const doBinaryRequest = async (url) => {
+export const doBinaryRequest = async (url, init = {}) => {
-  const res = await doFetch(url, "application/octet-stream");
+  const res = await doFetch(url, {
+    ...init,
+    headers: new Headers({ Accept: "application/octet-stream" }),
+  });
   return res.arrayBuffer();
 };
 

client/src/util/nameCreators.js

@@ -8,6 +8,9 @@ have a obsAnnotation named X, but we are using that for layout. So
 we namespace, and abstract to avoid proliferating strings throughout the
 codebase.
 
+It is _no longer_ used to remove collisions in the crossfilter or
+anno matrix namespaces. It is still used by the component tier.
+
 */
 
 const makeDimensionName = (namespace, key) => `${namespace}_${key}`;
@@ -18,12 +21,15 @@ export const diffexpDimensionName = (key) =>
   makeDimensionName("varData_diffexp", key);
 export const userDefinedDimensionName = (key) =>
   makeDimensionName("varData_userDefined", key);
+export const geneSetSummaryDimensionName = (key) =>
+  makeDimensionName("geneSetSummary", key);
 
 /*
     continuousNamespace = {
     isObs: true,
     isDiffExp: false,
-    isUserDefined: false
+    isUserDefined: false,
+    isGeneSet: false,
   }
 
   ie., makeContinuousDimensionName(continuousNamespace = {isObs: true}, "total_reads")
@@ -37,6 +43,8 @@ export const makeContinuousDimensionName = (continuousNamespace, key) => {
     name = diffexpDimensionName(key);
   } else if (continuousNamespace.isUserDefined) {
     name = userDefinedDimensionName(key);
+  } else if (continuousNamespace.isGeneSetSummary) {
+    name = geneSetSummaryDimensionName(key);
   } else {
     throw new Error("unknown continuous dimension");
   }

client/src/util/stateManager/colorHelpers.js

@@ -25,9 +25,11 @@ export function createColorQuery(colorMode, colorByAccessor, schema) {
       return [
         "X",
         {
-          field: "var",
+          where: {
-          column: varIndex,
+            field: "var",
-          value: colorByAccessor,
+            column: varIndex,
+            value: colorByAccessor,
+          },
         },
       ];
     }

client/src/util/stateManager/index.js

@@ -4,8 +4,16 @@ This is all VERY tightly integrated with reducers and actions, and
 exists to support those concepts.
 */
 
-export * as ColorHelpers from "./colorHelpers";
+import * as ColorHelpers from "./colorHelpers";
-export * as ControlsHelpers from "./controlsHelpers";
+import * as ControlsHelpers from "./controlsHelpers";
-export * as AnnotationsHelpers from "./annotationsHelpers";
+import * as AnnotationsHelpers from "./annotationsHelpers";
-export * as SchemaHelpers from "./schemaHelpers";
+import * as SchemaHelpers from "./schemaHelpers";
-export * as MatrixFBS from "./matrix";
+import * as MatrixFBS from "./matrix";
+
+export {
+  ColorHelpers,
+  ControlsHelpers,
+  AnnotationsHelpers,
+  SchemaHelpers,
+  MatrixFBS,
+};