gene set summary progress (#2127)

* revert removal of cache control headers

* checkpoint work on revising summary route

* add summary query support to annoMatrix

* summarize route cleanup

* add mising file

* clean up summarize route

* add summary histogram

* update deps

* lint

* more lint

* lint

* manage crossfiler during gene set state changes

* remove obsolete debugging code

* correctly perform async watch in histogram

* better error handling

client/src/annoMatrix/annoMatrix.js

@@ -12,6 +12,7 @@ import {
 import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
 import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
 import _shallowClone from "./clone";
+import { _queryValidate, _queryCacheKey } from "./query";
 
 const _dataframeCache = dataframeMemo(128);
 
@@ -181,7 +182,7 @@ export default class AnnoMatrix {
 		Returns a Promise for the query result, which will resolve to a dataframe.
 
 		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
-		represents the underlying object upon which the query is occuring.
+		represents the underlying object upon which the query is occurring.
 
 		Query is one of:
 			* a string, representing a single column name from the field, eg,
@@ -197,12 +198,6 @@ export default class AnnoMatrix {
 		field/column, similar to a join.  Currently only supported on the var
 		dimension, allowing query of X columns by var value (eg, gene name)
 
-    The query filter	is a single value filter:
-			{ "field name": [
-				{name: "column name", values: [ list of values ]}
-			]}
-		One and only one value filter is allowed in a value query.
-
 		Examples:
 
     1. Fetch the "n_genes" column the "obs":
@@ -220,7 +215,9 @@ export default class AnnoMatrix {
        value "TYMP" in the var index.
 
 			fetch("X", { 
-				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+				where: {
+          field: "var", column: this.schema.annotations.var.index, value: "TYMP"
+        }
 			})
 
       In AnnData & Pandas DataFrame API, this is equivalent to:
@@ -410,6 +407,14 @@ export default class AnnoMatrix {
     _subclassResponsibility();
   }
 
+  getCacheKeys(field, query) {
+    /*
+    Return cache keys for columns associated with this query.  May return
+    [unknown] if no keys are known (ie, nothing is or was cached).
+    */
+    return _whereCacheGet(this._whereCache, this.schema, field, query);
+  }
+
   /**
    ** Private interfaces below.
    **/
@@ -427,6 +432,7 @@ export default class AnnoMatrix {
   async _fetch(field, q) {
     if (!AnnoMatrix.fields().includes(field)) return undefined;
     const queries = Array.isArray(q) ? q : [q];
+    queries.forEach(_queryValidate);
 
     /* find cached columns we need, and GC the rest */
     const cachedColumns = this._resolveCachedQueries(field, queries);
@@ -507,7 +513,7 @@ export default class AnnoMatrix {
     * obs, var and emb do not grow without bounds, and are needed constantly 
       for rendering.  
       a) There is no upside to GC'ing these in the base (loader)
-      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+      b) The undo/redo cache can hold a large number in views, which is worth GC'ing
     * X is often much larger than memory, and the UI allows add/del from
       this.  Most of the GC potential is here in both the base and views.
 
@@ -522,7 +528,7 @@ export default class AnnoMatrix {
   as much of the cache is pinned by that data structure.
   */
   _gcField(field, isHot, pinnedColumns) {
-    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+    const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
 
     const cache = this._cache[field];
     if (cache.colIndex.size() < maxColumns) return; // trivial rejection
@@ -590,23 +596,18 @@ export default class AnnoMatrix {
     called each time a query is performed, allowing the gc to update any bookkeeping
     information.  Currently, this is just a simple last-fetched timestamp, stored
     in a Map.
-
-    Map objects preserve order of insertion. This is leveraged as a cheap way to
-    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
-    the map insertion order is least-recently-used first.
     */
     const cols = dataframe.colIndex.labels();
     const { _gcInfo } = this;
     const now = Date.now();
     cols.forEach((c) => {
-      // gcInfo.delete(c);
       _gcInfo.set(_columnCacheKey(field, c), now);
     });
   }
 
   /**
-  Cloning sublcass protocol - we rely in cloning to preserve immutable
-  symantics while not causing races or other side effects in internal
+  Cloning subclass protocol - we rely in cloning to preserve immutable
+  semantics while not causing races or other side effects in internal
   cache management.
 
   Subclasses must override _cloneDeeper() if they have state which requires 
@@ -639,15 +640,6 @@ export default class AnnoMatrix {
 /*
 private utility functions below
 */
-
-function _queryCacheKey(field, query) {
-  if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
-  }
-  return `${field}/${query}`;
-}
-
 function _columnCacheKey(field, column) {
   return `${field}/${column}`;
 }

client/src/annoMatrix/crossfilter.js

@@ -123,6 +123,24 @@ export default class AnnoMatrixObsCrossfilter {
     return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
   }
 
+  /**
+   * Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
+   * want to stop trackin the selection state, but aren't sure we want to blow the
+   * annomatrix cache.
+   */
+  dropDimension(field, query) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+    const keys = annoMatrix
+      .getCacheKeys(field, query)
+      .filter((k) => k !== undefined);
+    const dimName = _dimensionName(field, keys);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
   /**
   Selection state - API is identical to ImmutableTypedCrossfilter, as these
   are just wrappers to lazy create indices.

client/src/annoMatrix/fetchHelpers.js

@@ -1,4 +1,4 @@
-export { doBinaryRequest } from "../util/actionHelpers";
+export { doBinaryRequest, doFetch } from "../util/actionHelpers";
 
 /* double URI encode - needed for query-param filters */
 export function _dubEncURIComp(s) {

client/src/annoMatrix/loader.js

@@ -1,4 +1,4 @@
-import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { doBinaryRequest, doFetch } from "./fetchHelpers";
 import { matrixFBSToDataframe } from "../util/stateManager/matrix";
 import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
 import {
@@ -12,6 +12,13 @@ import { isArrayOrTypedArray } from "../util/typeHelpers";
 import { _whereCacheCreate } from "./whereCache";
 import AnnoMatrix from "./annoMatrix";
 import PromiseLimit from "../util/promiseLimit";
+import {
+  _expectSimpleQuery,
+  _expectComplexQuery,
+  _urlEncodeLabelQuery,
+  _urlEncodeComplexQuery,
+  _hashStringValues,
+} from "./query";
 
 const promiseThrottle = new PromiseLimit(5);
 
@@ -223,27 +230,23 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     /*
     _doLoad - evaluates the query against the field. Returns:
       * whereCache update: column query map mapping the query to the column labels
-      * Dataframe containing the new colums (one per dimension)
+      * Dataframe containing the new columns (one per dimension)
     */
-    let urlQuery;
-    let urlBase;
+    let doRequest;
     let priority = 10; // default fetch priority
 
     switch (field) {
       case "obs":
       case "var": {
-        urlBase = `${this.baseURL}annotations/${field}`;
-        urlQuery = _encodeQuery("annotation-name", query);
+        doRequest = _obsOrVarLoader(this.baseURL, field, query);
         break;
       }
       case "X": {
-        urlBase = `${this.baseURL}data/var`;
-        urlQuery = _encodeQuery(undefined, query);
+        doRequest = _XLoader(this.baseURL, field, query);
         break;
       }
       case "emb": {
-        urlBase = `${this.baseURL}layout/obs`;
-        urlQuery = _encodeQuery("layout-name", query);
+        doRequest = _embLoader(this.baseURL, field, query);
         priority = 0; // high prio load for embeddings
         break;
       }
@@ -251,12 +254,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
         throw new Error("Unknown field name");
     }
 
-    const url = `${urlBase}?${urlQuery}`;
-    const buffer = await promiseThrottle.priorityAdd(
-      priority,
-      doBinaryRequest,
-      url
-    );
+    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
     const result = matrixFBSToDataframe(buffer);
     if (!result || result.isEmpty()) throw Error("Unknown field/col");
 
@@ -267,7 +265,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     );
 
     if (field === "obs") {
-      /* cough, cough - see comment on method */
+      /* cough, cough - see comment on the function called */
       _normalizeCategoricalSchema(
         this.schema.annotations.obsByName[query],
         result.col(query)
@@ -282,17 +280,6 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
 Utility functions below
 */
 
-function _encodeQuery(colKey, q) {
-  if (typeof q === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = q;
-    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
-      queryColumn
-    )}=${_dubEncURIComp(queryValue)}`;
-  }
-  if (!colKey) throw new Error("Unsupported query by name");
-  return `${colKey}=${encodeURIComponent(q)}`;
-}
-
 function _writableCheck(colSchema) {
   if (!colSchema?.writable) {
     throw new Error("Unknown or readonly obs column");
@@ -305,3 +292,57 @@ function _writableCategoryTypeCheck(colSchema) {
     throw new Error("column must be categorical");
   }
 }
+
+function _embLoader(baseURL, _field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}layout/obs`;
+  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _obsOrVarLoader(baseURL, field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}annotations/${field}`;
+  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _XLoader(baseURL, field, query) {
+  _expectComplexQuery(query);
+
+  if (query.where) {
+    const urlBase = `${baseURL}data/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+    const url = `${urlBase}?${urlQuery}`;
+    return () => doBinaryRequest(url);
+  }
+
+  if (query.summarize) {
+    const urlBase = `${baseURL}summarize/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+
+    if (urlBase.length + urlQuery.length < 2000) {
+      const url = `${urlBase}?${urlQuery}`;
+      return () => doBinaryRequest(url);
+    }
+
+    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
+    return async () => {
+      const res = await doFetch(url, {
+        method: "POST",
+        body: urlQuery,
+        headers: new Headers({
+          Accept: "application/octet-stream",
+          "Content-Type": "application/x-www-form-urlencoded",
+        }),
+      });
+      return res.arrayBuffer();
+    };
+  }
+
+  throw new Error("Unknown query structure");
+}

client/src/annoMatrix/query.js

@@ -0,0 +1,126 @@
+import sha1 from "sha1";
+import { _dubEncURIComp } from "./fetchHelpers";
+
+/**
+ * Query utilities, mostly for debugging support and validation.
+ */
+
+/**
+ * Normalize & error check the query.
+ * @param {object | string} query - the query
+ * @returns {object | string} - the normalized query
+ */
+export function _queryValidate(query) {
+  if (typeof query !== "object") return query;
+
+  if (query.where && query.summarize)
+    throw new Error("query may not specify both where and summarize");
+  if (query.where) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
+    if (!queryField || !queryColumn || !queryValue)
+      throw new Error("Incomplete where query");
+    return query;
+  }
+  if (query.summarize) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    if (!queryField || !queryColumn || !queryValues)
+      throw new Error("Incomplete where query");
+    if (!Array.isArray(queryValues))
+      throw new Error("Summarize query values must be an array");
+    return query;
+  }
+  throw new Error("query must specify one of where or summarize");
+}
+
+export function _expectSimpleQuery(query) {
+  if (typeof query === "object") throw new Error("expected simple query");
+}
+
+export function _expectComplexQuery(query) {
+  if (typeof query !== "object") throw new Error("expected complex query");
+}
+
+/**
+ * Generate a unique key which can be used to reference this query.
+ *
+ * @param {string} field
+ * @param {string|object} query
+ * @returns the key
+ */
+export function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    // complex query
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      return `${field}/${method}/${queryField}/${queryColumn}/${queryValues.join(
+        ","
+      )}`;
+    }
+    throw new Error("Unrecognized complex query type");
+  }
+
+  // simple query
+  return `${field}/${query}`;
+}
+
+function _urlEncodeWhereQuery(q) {
+  const { field: queryField, column: queryColumn, value: queryValue } = q;
+  return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+    queryColumn
+  )}=${_dubEncURIComp(queryValue)}`;
+}
+
+function _urlEncodeSummarizeQuery(q) {
+  const { method, field, column, values } = q;
+  const filter = values
+    .map((value) => _urlEncodeWhereQuery({ field, column, value }))
+    .join("&");
+  return `method=${method}&${filter}`;
+}
+
+export function _urlEncodeComplexQuery(q) {
+  if (typeof q === "object") {
+    if (q.where) {
+      return _urlEncodeWhereQuery(q.where);
+    }
+    if (q.summarize) {
+      return _urlEncodeSummarizeQuery(q.summarize);
+    }
+  }
+  throw new Error("Unrecognized complex query type");
+}
+
+export function _urlEncodeLabelQuery(colKey, q) {
+  if (!colKey) throw new Error("Unsupported query by name");
+  if (typeof q !== "string") throw new Error("Query must be a simple label.");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+/**
+ * Generate the column key the server will send us for this query.
+ */
+export function _hashStringValues(arrayOfString) {
+  const hash = sha1(arrayOfString.join(""));
+  return hash;
+}

client/src/annoMatrix/schema.js

@@ -29,7 +29,7 @@ export function _getColumnSchema(schema, field, col) {
 export function _getColumnDimensionNames(schema, field, col) {
   /*
 		field/col may be an alias for multiple columns. Currently used to map ND 
-		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		values to 1D dataframe columns for embeddings/layout. Signified by the presence
 		of the "dims" value in the schema.
 		*/
   const colSchema = _getColumnSchema(schema, field, col);

client/src/annoMatrix/whereCache.js

@@ -1,27 +1,55 @@
 /*
 Private support functions.
 
-Support for a "where" query, eg,
+This implements a query resolver cache, mapping a query onto the column labels
+resolved by that query. These labels are then used to manage the acutal data cache,
+which stores data by the resolved label.
 
-	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+There are three query forms:
+  * primitive (string, number) - which is just reference the column label of same value
+  * where query (object) - eg, { where: { field: "var", column: "gene", value: "FOXP2" }}
+  * summary query (object) - eg, { summarize: { method: "mean", field: "var", column: "gene", values: ["FOXP2", "GNE", "F5"]}}
 
-These evaluate to a given column label.
+These queries all resolve to one or more column labels on a field.  This
+cache maintains a record of this, allowing direct access to the data caches
+without a server round-trip.
 
-The "where cache" is a map that saves evaluated queries and points 
-to the column label they resolve to.
+The data structure for where queries, the following query against X as an example:
+  { where: { field: "var", column: "column_label_in_var", value: "value_in_var_column" } }
+results in the following cached entry:
+{
+  where: {
+    X: {
+      var: Map(
+        column_label_in_var => Map(
+          value_in_var_column => [column_label_in_X, ...]
+        )
+      )
+    }
+  },
+  summarize: {},
+}
 
-Data structure, using X as the example field being queried, and var as
-the index.
-
-{ 
-	X: {
-		var: Map(
-			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
-		)
-	}
+And for summarize queries, for the following summary on X:
+  { summarize: { method: "mean", field: "var", column: "gene", values: ["G1", "G2"]}}
+creates a cache entry of:
+{
+  where: {},
+  summarize: {
+    X: {
+      mean: {
+        var: Map(
+          "gene" => Map(
+            "G1,G2" => [summary_column_label, ...]
+          )
+        )
+      }
+    },
+  },
 }
 */
 import { _getColumnDimensionNames } from "./schema";
+import { _hashStringValues } from "./query";
 
 export function _whereCacheGet(whereCache, schema, field, query) {
   /* 
@@ -31,20 +59,30 @@ export function _whereCacheGet(whereCache, schema, field, query) {
 	*/
 
   if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-
-    const columnMap = whereCache?.[field]?.[queryField];
-    if (columnMap === undefined) return [undefined];
-
-    const valueMap = columnMap.get(queryColumn);
-    if (valueMap === undefined) return [undefined];
-
-    const columnLabels = valueMap.get(queryValue);
-    return columnLabels === undefined ? [undefined] : columnLabels;
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      const columnMap = whereCache?.where?.[field]?.[queryField];
+      return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      const columnMap = whereCache?.summarize?.[field]?.[method]?.[queryField];
+      const queryValueHash = _hashStringValues(queryValues);
+      return columnMap?.get(queryColumn)?.get(queryValueHash) ?? [undefined];
+    }
+    return [undefined];
   }
 
-  const colDims = _getColumnDimensionNames(schema, field, query);
-  return colDims === undefined ? [undefined] : colDims;
+  return _getColumnDimensionNames(schema, field, query) ?? [undefined];
 }
 
 export function _whereCacheCreate(field, query, columnLabels) {
@@ -53,40 +91,86 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { field: queryField, column: queryColumn, value: queryValue } = query;
-  const whereCache = {
-    [field]: {
-      [queryField]: new Map([
-        [queryColumn, new Map([[queryValue, columnLabels]])],
-      ]),
-    },
-  };
-  return whereCache;
+  if (query.where) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
+    return {
+      where: {
+        [field]: {
+          [queryField]: new Map([
+            [queryColumn, new Map([[queryValue, columnLabels]])],
+          ]),
+        },
+      },
+    };
+  }
+  if (query.summarize) {
+    const {
+      method,
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    const queryValueHash = _hashStringValues(queryValues);
+    return {
+      summarize: {
+        [field]: {
+          [method]: {
+            [queryField]: new Map([
+              [queryColumn, new Map([[queryValueHash, columnLabels]])],
+            ]),
+          },
+        },
+      },
+    };
+  }
+  // oops, not sure what that query is!
+  return {};
+}
+
+function __mergeQueries(dst, src) {
+  for (const [queryField, columnMap] of Object.entries(src)) {
+    dst[queryField] = dst[queryField] || new Map();
+    for (const [queryColumn, valueMap] of columnMap) {
+      if (!dst[queryField].has(queryColumn))
+        dst[queryField].set(queryColumn, new Map());
+      for (const [queryValue, columnLabels] of valueMap) {
+        dst[queryField].get(queryColumn).set(queryValue, columnLabels);
+      }
+    }
+  }
 }
 
 function __whereCacheMerge(dst, src) {
   /*
 	merge src into dst (modifies dst)
 	*/
-  if (!dst) dst = {};
   if (!src || typeof src !== "object") return dst;
-  Object.entries(src).forEach(([field, query]) => {
-    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
-    Object.entries(query).forEach(([queryField, columnMap]) => {
-      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
-        dst[field][queryField] = new Map();
-      columnMap.forEach((valueMap, queryColumn) => {
-        if (!dst[field][queryField].has(queryColumn))
-          dst[field][queryField].set(queryColumn, new Map());
-        valueMap.forEach((columnLabels, queryValue) => {
-          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
-        });
-      });
-    });
-  });
+
+  if (src.where) {
+    dst.where = dst.where || {};
+    for (const [field, query] of Object.entries(src.where)) {
+      dst.where[field] = dst.where[field] || {};
+      __mergeQueries(dst.where[field], query);
+    }
+  }
+  if (src.summarize) {
+    dst.summarize = dst.summarize || {};
+    for (const [field, method] of Object.entries(src.summarize)) {
+      dst.summarize[field] = dst.summarize[field] || {};
+      for (const [methodName, query] of Object.entries(method)) {
+        dst.summarize[field][methodName] =
+          dst.summarize[field][methodName] || {};
+        __mergeQueries(dst.summarize[field][methodName], query);
+      }
+    }
+  }
   return dst;
 }
 
 export function _whereCacheMerge(...caches) {
-  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+  return caches.reduce(__whereCacheMerge, {});
 }