bumped version (#602)

0.6.0 was burned on pypi

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.3
+current_version = 0.6.1
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.dockerignore

@@ -0,0 +1,5 @@
+bin
+client
+dist
+docs
+server

.gitignore

@@ -35,3 +35,7 @@ npm-debug.log
 __pycache__
 *.DS_Store*
 data
+
+# Jekyll
+docs/_site/
+docs/Gemfile.lock

.travis.yml

@@ -8,13 +8,14 @@ cache:
 install:
   - set -eo pipefail
   - pip install flake8
-  - ./bin/build-client
+  - make build
-  - pip install -e .
+  - make install
   - pip install -r server/requirements-dev.txt
+  - docker build .
 script:
   - set -eo pipefail
-  - flake8 server/app/
+  - flake8 server
-  - flake8 server/cli/
+  - black --check
   - npm run --prefix client/ build
   - npm run --prefix client/ test
   - pytest -s server/test

Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu:bionic
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev && \
+    pip3 install cellxgene
+
+ENTRYPOINT ["cellxgene"]

README.md

@@ -8,7 +8,8 @@
 
 ## getting started
 
-You'll need **python 3.6** and **Google Chrome**. The web UI is tested on OSX and Windows using Chrome, and the python CLI is tested on OSX and Ubuntu (via WSL/Windows). It should work on other platforms, but if you run into trouble let us know (see [help](#help-and-contact) below).
+You'll need **python 3.6** and **Google Chrome**. (_Warning_: Python 3.7 is **not** supported at this time)
+The web UI is tested on OSX and Windows using Chrome, and the python CLI is tested on OSX and Ubuntu (via WSL/Windows). It should work on other platforms, but if you run into trouble let us know (see [help](#help-and-contact) below).
 
 To install run
 
@@ -27,6 +28,7 @@ If you want an example dataset download [this file](https://github.com/chanzucke
 ```
 cellxgene launch pbmc3k.h5ad --open
 ```
+
 You should see your web browser open with the following
 
 <img width="450" src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-opening-screenshot.png" pad="50px">
@@ -35,7 +37,7 @@ You should see your web browser open with the following
 
 There are several options available, such as:
 
-- `--layout` to specify the layout as  `tsne` or `umap`
+- `--layout` to specify the layout as `tsne` or `umap`
 - `--title` to show a title on the explorer
 - `--open` to automatically open the web browser after launching (OS X only)
 
@@ -56,11 +58,11 @@ The `launch` command assumes that the data is stored in the `.h5ad` format from
 - an `obs` field has a unique identifier for every cell (you can specify which field to use with the `--obs-names` option, by default it will use the value of `data.obs_names`)
 - a `var` field has a unique identifier for every gene (you can specify which field to use with the `--var-names` option, by default it will use the value of `data.var_names`)
 - an `obsm` field contains the two-dimensional coordinates for the layout that you want to render (e.g. `X_tsne` for the `tsne` layout or `X_umap` for the `umap` layout)
-- any additional  `obs` fields will be rendered as per-cell continuous or categorical metadata by the app (e.g. `louvain` cluster assignments) 
+- any additional `obs` fields will be rendered as per-cell continuous or categorical metadata by the app (e.g. `louvain` cluster assignments)
 
 ### prepare
 
-The `prepare` command is included to help you format your data. It uses `scanpy` under the hood. This is especially useful if you are starting with raw unanalyzed data and are unfamiliar with `scanpy`. 
+The `prepare` command is included to help you format your data. It uses `scanpy` under the hood. This is especially useful if you are starting with raw unanalyzed data and are unfamiliar with `scanpy`.
 
 To prepare from an existing `.h5ad` file use
 
@@ -68,7 +70,7 @@ To prepare from an existing `.h5ad` file use
 cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad
 ```
 
-This will load the input data, perform PCA and nearest neighbor calculations, compute `umap` and `tsne` layouts and `louvain` cluster assignments, and save the results in a new file called `dataset-processed.h5ad` that can be loaded using `cellxgene launch`. Data can be loaded from several formats, including `.h5ad` `.loom` and a `10-Genomics-formatted` `mtx` directory. Several options are available, including running one of the preprocessing `recipes` included with `scanpy`, which include steps like cell filtering and gene selection. 
+This will load the input data, perform PCA and nearest neighbor calculations, compute `umap` and `tsne` layouts and `louvain` cluster assignments, and save the results in a new file called `dataset-processed.h5ad` that can be loaded using `cellxgene launch`. Data can be loaded from several formats, including `.h5ad` `.loom` and a `10-Genomics-formatted` `mtx` directory. Several options are available, including running one of the preprocessing `recipes` included with `scanpy`, which include steps like cell filtering and gene selection. To learn more about the `recipes` please see the `scanpy` [documentation](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes).
 
 Depending on the options chosen, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option and only computing the layout for `umap`, using a call like this
 
@@ -88,6 +90,12 @@ cellxgene prepare --help
 pip install cellxgene[louvain]
 ```
 
+If the aforementioned optional package installation fails, you can also install these packages directly:
+
+```
+pip install python-igraph louvain>=0.6
+```
+
 ## conda and virtual environments
 
 If you use conda and want to create a conda environment for `cellxgene` you can use the following commands
@@ -102,13 +110,27 @@ Or you can create a virtual environment by using
 
 ```
 ENV_NAME=cellxgene
-python3 -m venv ${ENV_NAME}
+python3.6 -m venv ${ENV_NAME}
 source ${ENV_NAME}/bin/activate
 pip install cellxgene
 ```
 
+## docker
+
+We have included a dockerfile to conveniently run cellxgene from docker.
+
+1. Build the image `docker build . -t cellxgene`
+2. Run the container and mount data `docker run -v "$PWD/example-dataset/:/data/" -p 5005:5005 cellxgene launch --host 0.0.0.0 data/pbmc3k.h5ad`
+   - You will need to use --host 0.0.0.0 to have the container listen to incoming requests from the browser
+
 ## FAQ
 
+<details>
+
+<summary> questions about data formatting </summary>
+  
+<hr>
+  
 > Someone sent me a directory of `10X-Genomics` data with a `mtx` file and I've never used `scanpy`, can I use `cellxgene`?
 
 Yep! This should only take a couple steps. We'll assume your data is in a folder called `data/` and you've successfully installed `cellxgene` with the `louvain` packages as described above. Just run
@@ -125,14 +147,18 @@ cellxgene launch data-processed.h5ad --layout=umap --open
 
 And your web browser should open with an interactive view of your data.
 
+<hr>
+
 > In my `prepare` command I received the following error `Warning: louvain module is not installed, no clusters will be calculated. To fix this please install cellxgene with the optional feature louvain enabled`
 
-Louvain clustering requires additional dependencies that are somewhat complex, so we don't include them by default. For now, you need to specify that you want these packages by using 
+Louvain clustering requires additional dependencies that are somewhat complex, so we don't include them by default. For now, you need to specify that you want these packages by using
 
 ```
 pip install cellxgene[louvain]
 ```
 
+<hr>
+
 > I ran `prepare` and I'm getting results that look unexpected
 
 You might want to try running one of the preprocessing recipes included with `scanpy` (read more about them [here](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes)). You can specify this with the `--recipe` option, such as
@@ -143,22 +169,56 @@ cellxgene prepare data/ --output=data-processed.h5ad --recipe=zheng17
 
 It should be easy to run `prepare` then call `cellxgene launch` a few times with different settings to explore different behaviors. We may explore adding other preprocessing options in the future.
 
+<hr>
+
 > I have extra metadata that I want to add to my dataset
 
 Currently this is not supported directly, but you should be able to do this manually using `scanpy`. For example, this [notebook](https://github.com/falexwolf/fun-analyses/blob/master/tabula_muris/tabula_muris.ipynb) shows adding the contents of a `csv` file with metadata to an `anndata` object. For now, you could do this manually on your data in the same way and then save out the result before loading into `cellxgene`.
 
-> I tried to `pip install cellxgene` and got a weird error I don't understand
+<hr>
 
-This may happen, especially as we work out bugs in our installation process! Please create a new [Github issue](https://github.com/chanzuckerberg/cellxgene/issues), explain what you did, and include all the error messages you saw. It'd also be super helpful if you call `pip freeze` and include the full output alongside your issue.
+> What part of the anndata objects does cellxgene pull in for visualization?
 
-> How are you computing and sorting differential expression results?
+- `.obs` and `.var` annotations are use to extract metadata for filtering
+- `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression
+- `.obsm` is used for layout
 
-Currently we use a [Welch's *t*-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) implementation including the same variance overestimation correction as used in `scanpy`. We sort the `tscore` to identify the top N genes, and then filter to remove any that fall below a cutoff log fold change value, which can help remove spurious test results. The default threshold is `0.01` and can be changed using the option `--diffexp-lfc-cutoff`. We can explore adding support for other test types in the future.
+</details>
+
+<details>
+  
+<summary> questions about installing and building </summary>
+
+<hr>
+
+> I tried to `pip install cellxgene` and got a weird error about missing paths to an HDF5 library?  
+
+You probably just need to install HDF5 first. If you're on a mac, you can simply `brew install hdf5` and then try `pip install cellxgene` again.  
+
+> I tried to `pip install cellxgene` and got another weird error I don't understand  
+
+This may happen, especially as we work out bugs in our installation process! Please create a new [Github issue](https://github.com/chanzuckerberg/cellxgene/issues), explain what you did, and include all the error messages you saw. It'd also be super helpful if you call `pip freeze` and include the full output alongside your issue.  
+
+<hr>
 
 > I'm following the developer instructions and get an error about "missing files and directories” when trying to build the client
 
 This is likely because you do not have node and npm installed, we recommend using [nvm](https://github.com/creationix/nvm) if you're new to using these tools.
 
+</details>
+
+<details>
+
+<summary> questions about algorithms </summary>
+
+<hr>
+
+> How are you computing and sorting differential expression results?
+
+Currently we use a [Welch's _t_-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) implementation including the same variance overestimation correction as used in `scanpy`. We sort the `tscore` to identify the top N genes, and then filter to remove any that fall below a cutoff log fold change value, which can help remove spurious test results. The default threshold is `0.01` and can be changed using the option `--diffexp-lfc-cutoff`. We can explore adding support for other test types in the future.
+
+</details>
+
 ## developer guide
 
 This project has made a few key design choices
@@ -194,15 +254,15 @@ pip install -e .
 
 You can start the app while developing either by calling `cellxgene` or by calling `python -m server`. We recommend using the `--debug` flag to see more output, which you can include when reporting bugs.
 
-If you have any questions about developing or contributing, come hang out with us by joining the [CZI Science Slack](https://cziscience.slack.com/messages/CCTA8DF1T) and posting in the `#cellxgene-dev` channel.
+If you have any questions about developing or contributing, come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-dev` channel.
 
 ## development roadmap
 
-`cellxgene` is still very much in development, and we've love to include the community as we plan new features to work on. We are thinking about working on the following features over the next 3-12 months. If you are interested in updates, want to give feedback, want to contribute, or have ideas about other features we should work on, please [contact us](#help-and-contact) 
+`cellxgene` is still very much in development, and we've love to include the community as we plan new features to work on. We are thinking about working on the following features over the next 3-12 months. If you are interested in updates, want to give feedback, want to contribute, or have ideas about other features we should work on, please [contact us](#help-and-contact)
 
 - **Visualizaling spatial metadata** Image-based transcriptomics methods also generate large cell by gene matrices, alongside rich metadata about spatial location; we would like to render this information in `cellxgene`
-- **Visualizing trajectories** Trajectory analyses infer progression along some ordering or pseudotime; we would like `cellxgene ` to render the results of these analyses when they have been performed
+- **Visualizing trajectories** Trajectory analyses infer progression along some ordering or pseudotime; we would like `cellxgene` to render the results of these analyses when they have been performed
-- **Deploy to web**  Many projects release public data browser websites  alongside their publicatons; we would like to make it easy for anyone to deploy `cellxgene` to a custom URL with their own dataset that they own and operate
+- **Deploy to web** Many projects release public data browser websites alongside their publicatons; we would like to make it easy for anyone to deploy `cellxgene` to a custom URL with their own dataset that they own and operate
 - **HCA Integration** The [Human Cell Atlas](https://humancellatlas.org) is generating a large corpus of single-cell expression data and will make it available through the Data Coordination Platform; we would like `cellxgene` to be one of several different portals for browsing these data
 
 ## contributing
@@ -215,13 +275,13 @@ We've been heavily inspired by several other related single-cell visualization p
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 
-We have been working closely with the [`scanpy`](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset. 
+We have been working closely with the [`scanpy`](https://github.com/theislab/scanpy) team to integrate with their awesome analysis tools. Special thanks to Alex Wolf, Fabian Theis, and the rest of the team for their help during development and for providing an example dataset.
 
 We are eager to explore integrations with other computational backends such as [`Seurat`](https://github.com/satijalab/seurat) or [`Bioconductor`](https://github.com/Bioconductor)
 
 ## help and contact
 
-Have questions, suggestions, or comments? You can come hang out with us by joining the [CZI Science Slack](https://cziscience.slack.com/messages/CCTA8DF1T) and posting in the `#cellxgene-users` channel. As mentioned above, please submit any feature requests or bugs as [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). We'd love to hear from you!
+Have questions, suggestions, or comments? You can come hang out with us by joining the [CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and posting in the `#cellxgene-users` channel. As mentioned above, please submit any feature requests or bugs as [Github issues](https://github.com/chanzuckerberg/cellxgene/issues). We'd love to hear from you!
 
 ## reuse
 

bin/build-client

@@ -1,15 +0,0 @@
-#!/bin/bash
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-CELLXGENE_DIR=$(dirname $DIR)
-
-cd $CELLXGENE_DIR
-
-npm install --prefix client/ client
-npm run  --prefix client build
-rm -rf server/app/web/static
-mkdir -p server/app/web/static/img
-cp client/build/index.html server/app/web/templates/
-cp -r client/build/static server/app/web/
-
-cp client/build/favicon.png server/app/web/static/img
-cp client/build/service-worker.js server/app/web/static/js/

bin/clean

@@ -1,18 +0,0 @@
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-CELLXGENE_DIR=$(dirname $DIR)
-
-echo "Uninstalling cellxgene"
-yes | pip uninstall cellxgene
-echo "removing node_modules"
-rm -rf $CELLXGENE_DIR/client/node_modules
-echo "removing client_build"
-rm -rf $CELLXGENE_DIR/client/build
-echo "removing dist"
-rm -rf $CELLXGENE_DIR/dist
-echo "removing egg-info"
-rm -rf $CELLXGENE_DIR/cellxgene.egg-info
-echo "removing static files"
-rm -f $CELLXGENE_DIR/server/app/web/templates/index.html
-rm -rf $CELLXGENE_DIR/server/app/web/static
-echo "cellxgene cleanup complete"
-

client/__tests__/setupMissingGlobals.js

@@ -0,0 +1,9 @@
+/*
+Define globals which are present in the client, but not in node (and therefore not in
+the jest test environment).
+*/
+
+import { TextDecoder, TextEncoder } from "util";
+
+global.TextDecoder = TextDecoder;
+global.TextEncoder = TextEncoder;

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,5 +1,7 @@
 /* eslint no-bitwise: "off" */
 import _ from "lodash";
+import { flatbuffers } from "flatbuffers";
+import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
 
 /*
 test data mocking REST 0.2 API responses.  Used in several tests.
@@ -58,7 +60,7 @@ const aSchemaResponse = {
   }
 };
 
-const anAnnotationsObsResponse = {
+const anAnnotationsObsJSONResponse = {
   names: ["name", "field1", "field2", "field3", "field4"],
   data: _()
     .range(nObs)
@@ -73,7 +75,7 @@ const anAnnotationsObsResponse = {
     .value()
 };
 
-const anAnnotationsVarResponse = {
+const anAnnotationsVarJSONResponse = {
   names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
   data: _()
     .range(nVar)
@@ -88,7 +90,74 @@ const anAnnotationsVarResponse = {
     .value()
 };
 
-const aLayoutResponse = {
+function encodeTypedArray(builder, uType, uData) {
+  const uTypeName = NetEncoding.TypedArray[uType];
+  const ArrayType = NetEncoding[uTypeName];
+  const dv = ArrayType.createDataVector(builder, uData);
+  builder.startObject(1);
+  builder.addFieldOffset(0, dv, 0);
+  return builder.endObject();
+}
+
+function encodeMatrix(columns, colIndex = undefined) {
+  const utf8Encoder = new TextEncoder("utf-8");
+  const builder = new flatbuffers.Builder(1024);
+  const cols = _.map(columns, carr => {
+    let uType;
+    let tarr;
+    if (_.every(carr, _.isNumber)) {
+      uType = NetEncoding.TypedArray.Float32Array;
+      tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
+    } else {
+      uType = NetEncoding.TypedArray.JSONEncodedArray;
+      const json = JSON.stringify(carr);
+      const jsonUTF8 = utf8Encoder.encode(json);
+      tarr = encodeTypedArray(builder, uType, jsonUTF8);
+    }
+    NetEncoding.Column.startColumn(builder);
+    NetEncoding.Column.addUType(builder, uType);
+    NetEncoding.Column.addU(builder, tarr);
+    return NetEncoding.Column.endColumn(builder);
+  });
+
+  const encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
+
+  let encColIndex;
+  if (colIndex) {
+    encColIndex = encodeTypedArray(
+      builder,
+      NetEncoding.TypedArray.JSONEncodedArray,
+      utf8Encoder.encode(JSON.stringify(colIndex))
+    );
+  }
+
+  NetEncoding.Matrix.startMatrix(builder);
+  NetEncoding.Matrix.addNRows(builder, columns[0].length);
+  NetEncoding.Matrix.addNCols(builder, columns.length);
+  NetEncoding.Matrix.addColumns(builder, encColumns);
+  if (colIndex) {
+    NetEncoding.Matrix.addColIndexType(
+      builder,
+      NetEncoding.TypedArray.JSONEncodedArray
+    );
+    NetEncoding.Matrix.addColIndex(builder, encColIndex);
+  }
+  const root = NetEncoding.Matrix.endMatrix(builder);
+  builder.finish(root);
+  return builder.asUint8Array();
+}
+
+const anAnnotationsObsFBSResponse = (() => {
+  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
+})();
+
+const anAnnotationsVarFBSResponse = (() => {
+  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
+})();
+
+const aLayoutJSONResponse = {
   layout: {
     ndims: 2,
     coordinates: _()
@@ -98,6 +167,36 @@ const aLayoutResponse = {
   }
 };
 
+const aLayoutFBSResponse = (() => {
+  const coords = [
+    new Float32Array(nObs).fill(Math.random()),
+    new Float32Array(nObs).fill(Math.random())
+  ];
+  const builder = new flatbuffers.Builder(1024);
+
+  const cols = _.map(coords, carr => {
+    const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
+    NetEncoding.Float32Array.startFloat32Array(builder);
+    NetEncoding.Float32Array.addData(builder, cdv);
+    const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
+
+    NetEncoding.Column.startColumn(builder);
+    NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
+    NetEncoding.Column.addU(builder, floatArr);
+    return NetEncoding.Column.endColumn(builder);
+  });
+
+  const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
+
+  NetEncoding.Matrix.startMatrix(builder);
+  NetEncoding.Matrix.addNRows(builder, nObs);
+  NetEncoding.Matrix.addNCols(builder, nVar);
+  NetEncoding.Matrix.addColumns(builder, columns);
+  const matrix = NetEncoding.Matrix.endMatrix(builder);
+  builder.finish(matrix);
+  return builder.asUint8Array();
+})();
+
 const aDataObsResponse = {
   var: [2, 4, 29],
   obs: _()
@@ -107,10 +206,10 @@ const aDataObsResponse = {
 };
 
 export {
-  aLayoutResponse as layoutObs,
+  aLayoutFBSResponse as layoutObs,
   aDataObsResponse as dataObs,
-  anAnnotationsVarResponse as annotationsVar,
+  anAnnotationsVarFBSResponse as annotationsVar,
-  anAnnotationsObsResponse as annotationsObs,
+  anAnnotationsObsFBSResponse as annotationsObs,
   aSchemaResponse as schema,
   aConfigResponse as config
 };

client/__tests__/util/stateManager/summarizeAnnotations.test.js

@@ -0,0 +1,280 @@
+import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations";
+
+describe("summarizeAnnotations", () => {
+  const schema = {
+    annotations: {
+      obs: [
+        { name: "name", type: "string" },
+        { name: "nameString", type: "string" },
+        { name: "nameBoolean", type: "boolean" },
+        { name: "nameFloat32", type: "float32" },
+        { name: "nameInt32", type: "int32" },
+        {
+          name: "nameCategorical",
+          type: "categorical",
+          categories: [true, false, 1, 0, 0.00001, 4383.4833, "test", "", "0"]
+        }
+      ],
+      var: [{ name: "name", type: "string" }]
+    }
+  };
+
+  test("empty test", () => {
+    const summary = summarizeAnnotations(schema, [], []);
+    expect(summary).toEqual(
+      expect.objectContaining({
+        obs: {
+          nameString: {
+            categorical: true,
+            categories: [],
+            categoryCounts: new Map(),
+            numCategories: 0
+          },
+          nameBoolean: {
+            categorical: true,
+            categories: [],
+            categoryCounts: new Map(),
+            numCategories: 0
+          },
+          nameFloat32: {
+            categorical: false,
+            range: {
+              max: undefined,
+              min: undefined,
+              nan: 0,
+              ninf: 0,
+              pinf: 0
+            }
+          },
+          nameInt32: {
+            categorical: false,
+            range: {
+              max: undefined,
+              min: undefined,
+              nan: 0,
+              ninf: 0,
+              pinf: 0
+            }
+          },
+          nameCategorical: {
+            categorical: true,
+            categories: [],
+            categoryCounts: new Map(),
+            numCategories: 0
+          }
+        },
+        var: {}
+      })
+    );
+  });
+
+  test("simple test", () => {
+    const obsAnnotations = [
+      {
+        __index__: 0,
+        name: "n1",
+        nameString: "hi",
+        nameBoolean: true,
+        nameFloat32: 39.3,
+        nameInt32: 99,
+        nameCategorical: 1
+      }
+    ];
+    const varAnnotations = [];
+
+    const summary = summarizeAnnotations(
+      schema,
+      obsAnnotations,
+      varAnnotations
+    );
+
+    expect(summary).toEqual(
+      expect.objectContaining({
+        obs: {
+          nameString: {
+            categorical: true,
+            categories: ["hi"],
+            categoryCounts: new Map([["hi", 1]]),
+            numCategories: 1
+          },
+          nameBoolean: {
+            categorical: true,
+            categories: [true],
+            categoryCounts: new Map([[true, 1]]),
+            numCategories: 1
+          },
+          nameFloat32: {
+            categorical: false,
+            range: { min: 39.3, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
+          },
+          nameInt32: {
+            categorical: false,
+            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
+          },
+          nameCategorical: {
+            categorical: true,
+            categories: [1],
+            categoryCounts: new Map([[1, 1]]),
+            numCategories: 1
+          }
+        },
+        var: {}
+      })
+    );
+  });
+
+  test("multi test", () => {
+    const obsAnnotations = [
+      {
+        __index__: 0,
+        name: "n0",
+        nameString: "hi",
+        nameBoolean: false,
+        nameFloat32: 39.3,
+        nameInt32: 99,
+        nameCategorical: 1
+      },
+      {
+        __index__: 1,
+        name: "n1",
+        nameString: "hi",
+        nameBoolean: true,
+        nameFloat32: 39.3,
+        nameInt32: 99,
+        nameCategorical: false
+      },
+      {
+        __index__: 2,
+        name: "n2",
+        nameString: "bye",
+        nameBoolean: true,
+        nameFloat32: 0,
+        nameInt32: 99,
+        nameCategorical: "0"
+      }
+    ];
+    const varAnnotations = [];
+
+    const summary = summarizeAnnotations(
+      schema,
+      obsAnnotations,
+      varAnnotations
+    );
+
+    expect(summary).toMatchObject(
+      expect.objectContaining({
+        obs: {
+          nameString: {
+            categorical: true,
+            categories: expect.arrayContaining(["hi", "bye"]),
+            categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+            numCategories: 2
+          },
+          nameBoolean: {
+            categorical: true,
+            categories: expect.arrayContaining([true, false]),
+            categoryCounts: new Map([[true, 2], [false, 1]]),
+            numCategories: 2
+          },
+          nameFloat32: {
+            categorical: false,
+            range: { min: 0, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
+          },
+          nameInt32: {
+            categorical: false,
+            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
+          },
+          nameCategorical: {
+            categorical: true,
+            categories: expect.arrayContaining([1, false, "0"]),
+            categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+            numCategories: 3
+          }
+        },
+        var: {}
+      })
+    );
+  });
+
+  test("non-finite numbers", () => {
+    const obsAnnotations = [
+      {
+        __index__: 0,
+        name: "n0",
+        nameString: "hi",
+        nameBoolean: false,
+        nameFloat32: 39.3,
+        nameInt32: 99,
+        nameCategorical: 1
+      },
+      {
+        __index__: 1,
+        name: "n1",
+        nameString: "hi",
+        nameBoolean: true,
+        nameFloat32: Number.NEGATIVE_INFINITY,
+        nameInt32: 99,
+        nameCategorical: false
+      },
+      {
+        __index__: 2,
+        name: "n2",
+        nameString: "bye",
+        nameBoolean: true,
+        nameFloat32: Number.NaN,
+        nameInt32: 99,
+        nameCategorical: "0"
+      },
+      {
+        __index__: 3,
+        name: "n2",
+        nameString: "bye",
+        nameBoolean: true,
+        nameFloat32: Number.POSITIVE_INFINITY,
+        nameInt32: 99,
+        nameCategorical: "0"
+      }
+    ];
+    const varAnnotations = [];
+
+    const summary = summarizeAnnotations(
+      schema,
+      obsAnnotations,
+      varAnnotations
+    );
+
+    expect(summary).toMatchObject(
+      expect.objectContaining({
+        obs: {
+          nameString: {
+            categorical: true,
+            categories: expect.arrayContaining(["hi", "bye"]),
+            categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+            numCategories: 2
+          },
+          nameBoolean: {
+            categorical: true,
+            categories: expect.arrayContaining([true, false]),
+            categoryCounts: new Map([[true, 2], [false, 1]]),
+            numCategories: 2
+          },
+          nameFloat32: {
+            categorical: false,
+            range: { min: 39.3, max: 39.3, nan: 1, ninf: 1, pinf: 1 }
+          },
+          nameInt32: {
+            categorical: false,
+            range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
+          },
+          nameCategorical: {
+            categorical: true,
+            categories: expect.arrayContaining([1, false, "0"]),
+            categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+            numCategories: 3
+          }
+        },
+        var: {}
+      })
+    );
+  });
+});

client/__tests__/util/stateManager/universe.test.js

@@ -30,7 +30,6 @@ describe("createUniverseFromRestV02Response", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-
     const universe = Universe.createUniverseFromRestV02Response(
       REST.config,
       REST.schema,
@@ -66,56 +65,3 @@ describe("createUniverseFromRestV02Response", () => {
     expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
   });
 });
-
-describe("convertExpressionRESTv02ToObject", () => {
-  /*
-  test convertExpressionRESTv02ToObject
-
-  convertExpressionRESTv02ToObject(
-    universe,
-    response) --> { geneName: Float32Array, geneName: Float32Array, ... }
-
-  reponse is a /data/obs response:
-  {
-    var: [ varIndices fetched ],
-    obs: [
-      [ obsIndex, evalue, ... ],
-      ...
-    ]
-  }
-  */
-  test("create from response data", () => {
-    const universe = Universe.createUniverseFromRestV02Response(
-      REST.config,
-      REST.schema,
-      REST.annotationsObs,
-      REST.annotationsVar,
-      REST.layoutObs
-    );
-    const expression = Universe.convertExpressionRESTv02ToObject(
-      universe,
-      REST.dataObs
-    );
-
-    /* Check that the expected keys are present */
-    const expectedGeneNames = _.map(
-      REST.dataObs.var,
-      v => REST.annotationsVar.data[v][5]
-    );
-    expect(Object.keys(expression)).toEqual(
-      expect.arrayContaining(expectedGeneNames)
-    );
-
-    const expectedExpressionValues = _.map(
-      _.unzip(REST.dataObs.obs),
-      a => new Float32Array(a)
-    );
-
-    _.forEach(REST.dataObs.var, (varIdx, idx) => {
-      const varName = universe.varAnnotations[varIdx].name;
-      expect(varName).toBeDefined();
-      expect(varIdx).toBe(universe.varNameToIndexMap[varName]);
-      expect(expression[varName]).toEqual(expectedExpressionValues[idx + 1]);
-    });
-  });
-});

client/__tests__/util/stateManager/world.test.js

@@ -168,10 +168,13 @@ describe("createObsDimensionMap", () => {
     */
 
     const { dimensionMap } = defaultBigBang();
-
+    const annotationNames = _.map(
+      REST.schema.schema.annotations.obs,
+      c => c.name
+    );
     const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
     expect(dimensionMap).toBeDefined();
-    REST.annotationsObs.names.forEach(name => {
+    annotationNames.forEach(name => {
       const dim = dimensionMap[obsAnnoDimensionName(name)];
       if (name === "name") {
         expect(dim).toBeUndefined();
@@ -184,11 +187,8 @@ describe("createObsDimensionMap", () => {
         }
       }
     });
-    expect(dimensionMap[layoutDimensionName("X")]).toBeInstanceOf(
+    expect(dimensionMap[layoutDimensionName("XY")]).toBeInstanceOf(
-      Crossfilter.ScalarDimension
+      Crossfilter.SpatialDimension
-    );
-    expect(dimensionMap[layoutDimensionName("Y")]).toBeInstanceOf(
-      Crossfilter.ScalarDimension
     );
   });
 });

client/__tests__/util/stateManager/worldUtil.test.js

@@ -0,0 +1,45 @@
+import {
+  countCategoryValues2D,
+  clearCaches
+} from "../../../src/util/stateManager/worldUtil";
+
+describe("WorldUtil cache management", () => {
+  test("empty", () => {
+    const count = countCategoryValues2D("a", "b", []);
+    expect(count).toMatchObject(new Map());
+  });
+
+  test("simple couts", () => {
+    const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+    const count = countCategoryValues2D("a", "b", rows);
+    expect(count).toMatchObject(
+      new Map([
+        [0, new Map([[true, 1], [false, 1]])],
+        [1, new Map([[false, 1]])]
+      ])
+    );
+  });
+
+  test("memo cache clear", () => {
+    clearCaches();
+    const row1 = [];
+    const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+    const count1 = countCategoryValues2D("a", "b", row1);
+    const count2 = countCategoryValues2D("a", "b", row1);
+    const count3 = countCategoryValues2D("a", "b", []);
+    const count4 = countCategoryValues2D("a", "b", row2);
+
+    clearCaches();
+    const count10 = countCategoryValues2D("a", "b", row1);
+    const count11 = countCategoryValues2D("a", "b", row2);
+
+    expect(count1).toEqual(count2);
+    expect(count1).toEqual(count3);
+    expect(count1).toEqual(count10);
+    expect(count1).not.toBe(count3);
+    expect(count1).not.toBe(count10);
+
+    expect(count4).toEqual(count11);
+    expect(count4).not.toBe(count11);
+  });
+});

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -102,22 +102,20 @@ const someData = [
 ];
 
 function groupReduce(data, valueMap, valueReduce, valueInit) {
-  return _
+  return _.reduce(
-    .reduce(
+    data,
-      data,
+    (acc, value) => {
-      (acc, value) => {
+      const k = valueMap(value);
-        const k = valueMap(value);
+      let r = _.find(acc, o => o.key === k);
-        let r = _.find(acc, o => o.key === k);
+      if (!r) {
-        if (!r) {
+        r = { key: k, value: valueInit() };
-          r = { key: k, value: valueInit() };
+        acc.push(r);
-          acc.push(r);
+      }
-        }
+      r.value = valueReduce(r.value, value);
-        r.value = valueReduce(r.value, value);
+      return acc;
-        return acc;
+    },
-      },
+    []
-      []
+  ).sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-    )
-    .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
 }
 
 function groupCount(data, map) {
@@ -139,7 +137,11 @@ describe("typedCrossfilter", () => {
     expect(payments.size()).toEqual(someData.length);
     expect(payments.all()).toEqual(someData);
 
-    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const quantity = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.quantity,
+      Int32Array
+    );
     expect(quantity).toBeDefined();
     expect(quantity.id()).toBeDefined();
 
@@ -150,10 +152,22 @@ describe("typedCrossfilter", () => {
 
   test("filterAll and filterNone", () => {
     expect(payments).toBeDefined();
-    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const quantity = payments.dimension(
-    const tip = payments.dimension(r => r.tip, Float32Array);
+      crossfilter.ScalarDimension,
-    const total = payments.dimension(r => r.total, Float32Array);
+      r => r.quantity,
-    const type = payments.dimension(r => r.type, "enum");
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Float32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     expect(quantity).toBeDefined();
     expect(tip).toBeDefined();
@@ -198,10 +212,22 @@ describe("typedCrossfilter", () => {
 
   test("filterExact", () => {
     expect(payments).toBeDefined();
-    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const quantity = payments.dimension(
-    const tip = payments.dimension(r => r.tip, Float32Array);
+      crossfilter.ScalarDimension,
-    const total = payments.dimension(r => r.total, Float32Array);
+      r => r.quantity,
-    const type = payments.dimension(r => r.type, "enum");
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Float32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     quantity.filterExact(1);
     expect(payments.countFiltered()).toEqual(
@@ -222,10 +248,22 @@ describe("typedCrossfilter", () => {
 
   test("filterRange", () => {
     expect(payments).toBeDefined();
-    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const quantity = payments.dimension(
-    const tip = payments.dimension(r => r.tip, Float32Array);
+      crossfilter.ScalarDimension,
-    const total = payments.dimension(r => r.total, Float32Array);
+      r => r.quantity,
-    const type = payments.dimension(r => r.type, "enum");
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Float32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     tip.filterRange([0, 91]);
     expect(payments.allFiltered()).toEqual(
@@ -251,10 +289,22 @@ describe("typedCrossfilter", () => {
 
   test("filterEnum", () => {
     expect(payments).toBeDefined();
-    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const quantity = payments.dimension(
-    const tip = payments.dimension(r => r.tip, Float32Array);
+      crossfilter.ScalarDimension,
-    const total = payments.dimension(r => r.total, Float32Array);
+      r => r.quantity,
-    const type = payments.dimension(r => r.type, "enum");
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Float32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     type.filterEnum(["tab", "cash"]);
     expect(payments.allFiltered()).toEqual(
@@ -274,15 +324,31 @@ describe("typedCrossfilter", () => {
 
   test("more than 32 dimensions", () => {
     expect(payments).toBeDefined();
-    const quantity = payments.dimension(r => r.quantity, Int32Array);
+    const quantity = payments.dimension(
-    const tip = payments.dimension(r => r.tip, Float32Array);
+      crossfilter.ScalarDimension,
-    const total = payments.dimension(r => r.total, Float32Array);
+      r => r.quantity,
-    const type = payments.dimension(r => r.type, "enum");
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.tip,
+      Float32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Float32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     // Create a bunch of fake dimensions to ensure we can handle > 32
     let dimMap = {};
     for (let i = 0; i < 65; i++) {
-      dimMap[i] = payments.dimension(r => Math.random(), Float32Array);
+      dimMap[i] = payments.dimension(
+        crossfilter.ScalarDimension,
+        r => Math.random(),
+        Float32Array
+      );
       expect(dimMap[i]).toBeDefined();
       expect(dimMap[i].id()).toBeDefined();
     }
@@ -304,10 +370,22 @@ describe("typedCrossfilter", () => {
   test("group, default mapping, default reducer, no filter", () => {
     expect(payments).toBeDefined();
 
-    var quantity = payments.dimension(r => r.quantity, Int32Array);
+    const quantity = payments.dimension(
-    var tip = payments.dimension(r => r.tip, Int32Array);
+      crossfilter.ScalarDimension,
-    var type = payments.dimension(r => r.type, "enum");
+      r => r.quantity,
-    var total = payments.dimension(r => r.total, Int32Array);
+      Int32Array
+    );
+    const tip = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.tip,
+      Int32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Int32Array
+    );
 
     _.each(
       {
@@ -331,9 +409,17 @@ describe("typedCrossfilter", () => {
     // custom mapping in groups only works for scalar types.  Enums do not
     // currently implement it.
 
-    const tip = payments.dimension(r => r.tip, Int32Array);
+    const tip = payments.dimension(
-    const totalX10 = payments.dimension(r => r.total * 10, Int32Array);
+      crossfilter.ScalarDimension,
-    const type = payments.dimension(r => r.type, "enum");
+      r => r.tip,
+      Int32Array
+    );
+    const totalX10 = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total * 10,
+      Int32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     const paymentsByTip_A = tip.group();
     const paymentsByTip_B = tip.group(r => 10 * r);
@@ -370,8 +456,12 @@ describe("typedCrossfilter", () => {
   test("group, default map, custom reducer, no filters", () => {
     expect(payments).toBeDefined();
 
-    const total = payments.dimension(r => r.total, Float32Array);
+    const total = payments.dimension(
-    const type = payments.dimension(r => r.type, "enum");
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Float32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     const paymentsByTotal = total.group();
     const paymentsByType = type.group();
@@ -407,9 +497,17 @@ describe("typedCrossfilter", () => {
 
     expect(payments).toBeDefined();
 
-    const tip = payments.dimension(r => r.tip, Int32Array);
+    const tip = payments.dimension(
-    const total = payments.dimension(r => r.total, Int32Array);
+      crossfilter.ScalarDimension,
-    const type = payments.dimension(r => r.type, "enum");
+      r => r.tip,
+      Int32Array
+    );
+    const total = payments.dimension(
+      crossfilter.ScalarDimension,
+      r => r.total,
+      Int32Array
+    );
+    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     const paymentsByTip = tip.group();
     const paymentsByTotal = total.group();

client/configuration/webpack/webpack.config.prod.js

@@ -10,7 +10,7 @@ const nodeModules = path.resolve("node_modules");
 
 const babelOptions = require("../babel/babel.prod");
 
-const publicPath = "/";
+const publicPath = "";
 
 module.exports = {
   mode: "production",

client/index_template.html

@@ -24,7 +24,7 @@
 <script type="text/javascript">
 window.CELLXGENE = {};
 window.CELLXGENE.API = {
-  prefix: "{{ prefix | safe }}",
+  prefix: window.location.href + "api/",
   version: "v0.2/"
 };
 </script>

client/package.json

@@ -1,10 +1,11 @@
 {
   "name": "cellxgene",
-  "version": "0.2.3",
+  "version": "0.6.1",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
+    "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
     "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
     "clean": "rimraf build",
@@ -19,10 +20,9 @@
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
   },
-  "nyc": {
+  "eslintIgnore": [
-    "sourceMap": false,
+    "src/util/stateManager/matrix_generated.js"
-    "instrument": false
+  ],
-  },
   "resolutions": {
     "eslint-scope": "3.7.1"
   },
@@ -33,9 +33,12 @@
     "canvas-fit": "^1.5.0",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.3.0",
+    "flatbuffers": "^1.10.2",
     "font-color-contrast": "^1.0.3",
+    "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.1.4",
     "gl-matrix": "^2.7.1",
+    "is-number": "^7.0.0",
     "key-pressed": "0.0.1",
     "lodash": "^4.17.4",
     "memoize-one": "^4.0.0",
@@ -73,30 +76,28 @@
     "babel-eslint": "^10.0.1",
     "babel-jest": "^23.6.0",
     "babel-loader": "^8.0.0",
-    "babel-plugin-istanbul": "^5.1.0",
     "babel-preset-modern-browsers": "^12.0.0",
-    "chalk": "^2.4.1",
+    "chalk": "^2.4.2",
-    "connect-history-api-fallback": "^1.3.0",
+    "connect-history-api-fallback": "^1.6.0",
     "copy-webpack-plugin": "^4.6.0",
     "css-loader": "^1.0.1",
-    "eslint": "^5.8.0",
+    "eslint": "^5.13.0",
     "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-prettier": "^3.1.0",
+    "eslint-config-prettier": "^4.0.0",
-    "eslint-loader": "^2.1.1",
+    "eslint-loader": "^2.1.2",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.14.0",
+    "eslint-plugin-import": "^2.16.0",
-    "eslint-plugin-jest": "^21.27.2",
+    "eslint-plugin-jest": "^22.2.2",
-    "eslint-plugin-jsx-a11y": "^6.1.1",
+    "eslint-plugin-jsx-a11y": "^6.2.1",
-    "eslint-plugin-react": "^7.11.1",
+    "eslint-plugin-react": "^7.12.4",
     "express": "^4.14.0",
     "file-loader": "^2.0.0",
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
-    "jest": "^23.5.0",
+    "jest": "^24.1.0",
     "json-loader": "^0.5.4",
     "mini-css-extract-plugin": "^0.4.1",
-    "nyc": "^13.0.1",
+    "rimraf": "^2.6.3",
-    "rimraf": "^2.5.4",
     "serve-favicon": "^2.3.0",
     "style-loader": "^0.23.1",
     "sw-precache-webpack-plugin": "^0.11.5",
@@ -109,7 +110,10 @@
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
     ],
-    "testURL": "http://localhost/"
+    "testURL": "http://localhost/",
+    "setupFiles": [
+      "./__tests__/setupMissingGlobals.js"
+    ]
   },
   "babel": {
     "env": {

client/src/actions/index.js

@@ -5,6 +5,7 @@ import { Universe, kvCache } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
+  doBinaryRequest,
   rangeEncodeIndices,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
@@ -13,24 +14,28 @@ import {
 Bootstrap application with the initial data loading.
   * /config - application configuration
   * /schema - schema of dataframe
-  * /annotations/obs - all metadata annotation
+  * /annotations - all metadata annotation
+  * /layout - all default layout
 */
 const doInitialDataLoad = () =>
   catchErrorsWrap(async dispatch => {
     dispatch({ type: "initial data load start" });
 
     try {
-      const requests = _([
+      const requestJson = _(["config", "schema"])
-        "config",
-        "schema",
-        "annotations/obs",
-        "annotations/var",
-        "layout/obs"
-      ])
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
         .map(url => doJsonRequest(url))
         .value();
-      const results = await Promise.all(requests);
+      const requestBinary = _([
+        "annotations/obs",
+        "annotations/var?annotation-name=name",
+        "layout/obs"
+      ])
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url))
+        .value();
+
+      const results = await Promise.all(_.concat(requestJson, requestBinary));
 
       /* set config defaults */
       const config = { ...globals.configDefaults, ...results[0].config };
@@ -87,6 +92,38 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  /* helper for this function only */
+  const fetchData = async geneNames => {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}data/var`,
+      {
+        method: "PUT",
+        body: JSON.stringify({
+          filter: {
+            var: {
+              annotation_value: [{ name: "name", values: geneNames }]
+            }
+          }
+        }),
+        headers: new Headers({
+          accept: "application/octet-stream",
+          "Content-Type": "application/json"
+        })
+      }
+    );
+
+    if (
+      !res.ok ||
+      res.headers.get("Content-Type") !== "application/octet-stream"
+    ) {
+      // WILL throw
+      return dispatchExpressionErrors(dispatch, res);
+    }
+
+    const data = await res.arrayBuffer();
+    return Universe.convertDataFBStoObject(universe, data);
+  };
+
   const state = getState();
   const { universe } = state.controls;
   /* preload data already in cache */
@@ -108,35 +145,10 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   /* Fetch data for any genes not in cache */
   if (genesToFetch.length) {
     try {
-      // XXX: TODO - this could be using /data/var rather than /data/obs,
+      const newExpressionData = await fetchData(genesToFetch);
-      // as that would simplify the transformation in convertExpressionRESTv02ToObject
-      const res = await fetch(
-        `${globals.API.prefix}${globals.API.version}data/obs`,
-        {
-          method: "PUT",
-          body: JSON.stringify({
-            filter: {
-              var: {
-                annotation_value: [{ name: "name", values: genesToFetch }]
-              }
-            }
-          }),
-          headers: new Headers({
-            accept: "application/json",
-            "Content-Type": "application/json"
-          })
-        }
-      );
-
-      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-        // WILL throw
-        return dispatchExpressionErrors(dispatch, res);
-      }
-
-      const data = await res.json();
       expressionData = {
         ...expressionData,
-        ...Universe.convertExpressionRESTv02ToObject(universe, data)
+        ...newExpressionData
       };
     } catch (error) {
       dispatch({ type: "expression load error", error });

client/src/components/brushableHistogram/index.js

@@ -13,6 +13,7 @@ import memoize from "memoize-one";
 import { kvCache } from "../../util/stateManager";
 import * as globals from "../../globals";
 import actions from "../../actions";
+import finiteExtent from "../../util/finiteExtent";
 
 @connect(state => ({
   world: state.controls.world,
@@ -26,7 +27,7 @@ import actions from "../../actions";
   obsAnnotations: _.get(state.controls.world, "obsAnnotations", null)
 }))
 class HistogramBrush extends React.Component {
-  calcHistogramCache = memoize((obsAnnotations, field, ranges) => {
+  calcHistogramCache = memoize((obsAnnotations, field, rangeMin, rangeMax) => {
     const { world } = this.props;
     const histogramCache = {};
 
@@ -40,7 +41,7 @@ class HistogramBrush extends React.Component {
 
       histogramCache.x = d3
         .scaleLinear()
-        .domain([ranges.min, ranges.max])
+        .domain([rangeMin, rangeMax])
         .range([0, this.width]);
 
       histogramCache.bins = d3
@@ -56,7 +57,7 @@ class HistogramBrush extends React.Component {
       histogramCache.x = d3
         .scaleLinear()
         .domain(
-          d3.extent(varValues)
+          finiteExtent(varValues)
         ) /* replace this if we have ranges for genes back from server like we do for annotations on cells */
         .range([0, this.width]);
 
@@ -130,7 +131,8 @@ class HistogramBrush extends React.Component {
     const histogramCache = this.calcHistogramCache(
       obsAnnotations,
       field,
-      ranges
+      ranges.min,
+      ranges.max
     );
 
     const { x, y, bins, numValues } = histogramCache;
@@ -159,7 +161,13 @@ class HistogramBrush extends React.Component {
   }
 
   removeHistogram() {
-    const { dispatch, field, colorAccessor } = this.props;
+    const {
+      dispatch,
+      field,
+      colorAccessor,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor
+    } = this.props;
     dispatch({
       type: "clear user defined gene",
       data: field
@@ -169,6 +177,18 @@ class HistogramBrush extends React.Component {
         type: "reset colorscale"
       });
     }
+    if (field === scatterplotXXaccessor) {
+      dispatch({
+        type: "set scatterplot x",
+        data: null
+      });
+    }
+    if (field === scatterplotYYaccessor) {
+      dispatch({
+        type: "set scatterplot y",
+        data: null
+      });
+    }
   }
 
   handleSetGeneAsScatterplotX() {
@@ -265,7 +285,7 @@ class HistogramBrush extends React.Component {
         }}
       >
         <div style={{ display: "flex", justifyContent: "flex-end" }}>
-          {isDiffExp ? (
+          {isDiffExp || isUserDefined ? (
             <span>
               <span
                 style={{ marginRight: 7 }}

client/src/components/categorical/category.js

@@ -6,7 +6,7 @@ import { Button, Tooltip } from "@blueprintjs/core";
 
 import * as globals from "../../globals";
 import Value from "./value";
-import alphabeticallySortedValues from "./util";
+import sortedCategoryValues from "./util";
 
 @connect(state => ({
   colorAccessor: state.controls.colorAccessor,
@@ -25,24 +25,22 @@ class Category extends React.Component {
     const { categoricalSelectionState, metadataField } = this.props;
     const cat = categoricalSelectionState[metadataField];
     const categoryCount = {
-      // total number of options in this category
+      // total number of categories in this dimension
-      totalOptionCount: cat.numOptions,
+      totalCatCount: cat.numCategories,
       // number of selected options in this category
-      selectedOptionCount: _.reduce(
+      selectedCatCount: _.reduce(
-        cat.optionSelected,
+        cat.categorySelected,
         (res, cond) => (cond ? res + 1 : res),
         0
       )
     };
-    if (categoryCount.selectedOptionCount === categoryCount.totalOptionCount) {
+    if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
       /* everything is on, so not indeterminate */
       this.checkbox.indeterminate = false;
-    } else if (categoryCount.selectedOptionCount === 0) {
+    } else if (categoryCount.selectedCatCount === 0) {
       /* nothing is on, so no */
       this.checkbox.indeterminate = false;
-    } else if (
+    } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
-      categoryCount.selectedOptionCount < categoryCount.totalOptionCount
-    ) {
       /* to be explicit... */
       this.checkbox.indeterminate = true;
     }
@@ -88,12 +86,13 @@ class Category extends React.Component {
     const { categoricalSelectionState, metadataField } = this.props;
 
     const cat = categoricalSelectionState[metadataField];
-    const optTuples = alphabeticallySortedValues([...cat.optionIndex]);
+    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
+        optTuples={optTuples}
         key={tuple[1]}
         metadataField={metadataField}
-        optionIndex={tuple[1]}
+        categoryIndex={tuple[1]}
         i={i}
       />
     ));

client/src/components/categorical/occupancy.js

@@ -0,0 +1,73 @@
+// jshint esversion: 6
+import React from "react";
+import _ from "lodash";
+import { connect } from "react-redux";
+import * as d3 from "d3";
+
+@connect()
+class Occupancy extends React.Component {
+  render() {
+    const {
+      occupancy,
+      colorScale,
+      categoricalSelectionState,
+      colorAccessor,
+      schema
+    } = this.props;
+    const width = 100;
+    const height = 11;
+
+    const categories = _.filter(schema.annotations.obs, {
+      name: colorAccessor
+    })[0].categories;
+
+    const x = d3
+      .scaleLinear()
+      /* get all the keys d[1] as an array, then find the sum */
+      .domain([0, d3.sum(Array.from(occupancy, d => d[1]))])
+      .range([0, width]);
+
+    let currentOffset = 0;
+
+    const stacks = categoricalSelectionState[colorAccessor].categoryValues.map(
+      d => {
+        const o = occupancy.get(d);
+
+        const scaledValue = x(o);
+
+        const stackItem = {
+          key: d,
+          value: o || 0,
+          rectWidth: o ? scaledValue : 0,
+          offset: currentOffset,
+          fill: o ? colorScale(categories.indexOf(d)) : "rgb(255,255,255)"
+        };
+        currentOffset += o ? scaledValue : 0;
+        return stackItem;
+      }
+    );
+
+    return (
+      <svg
+        style={{
+          marginRight: 5,
+          width,
+          height
+        }}
+      >
+        {stacks.map(d => (
+          <rect
+            key={d.key}
+            width={d.rectWidth}
+            height={height}
+            x={d.offset}
+            title={d.metadataField}
+            fill={d.fill}
+          />
+        ))}
+      </svg>
+    );
+  }
+}
+
+export default Occupancy;

client/src/components/categorical/util.js

@@ -3,9 +3,33 @@
 // values is [ [optVal, optIdx], ...]
 // index is range array
 // return sorted index
-export default values =>
+
-  values.sort((a, b) => {
+import isNumber from "is-number";
+import _ from "lodash";
+
+const sortedCategoryValues = values => {
+  /* this sort could be memoized for perf */
+
+  const strings = [];
+  const ints = [];
+
+  _.forEach(values, v => {
+    if (isNumber(v[0])) {
+      ints.push(v);
+    } else {
+      strings.push(v);
+    }
+  });
+
+  strings.sort((a, b) => {
     const textA = String(a[0]).toUpperCase();
     const textB = String(b[0]).toUpperCase();
     return textA < textB ? -1 : textA > textB ? 1 : 0;
   });
+
+  ints.sort((a, b) => +a[0] - +b[0]);
+
+  return ints.concat(strings);
+};
+
+export default sortedCategoryValues;

client/src/components/categorical/value.js

@@ -2,29 +2,33 @@
 import { connect } from "react-redux";
 import React from "react";
 import _ from "lodash";
+import Occupancy from "./occupancy";
+import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
+import * as globals from "../../globals";
 
 @connect(state => ({
   categoricalSelectionState: state.controls.categoricalSelectionState,
   colorScale: state.controls.colorScale,
   colorAccessor: state.controls.colorAccessor,
-  schema: _.get(state.controls.world, "schema", null)
+  schema: _.get(state.controls.world, "schema", null),
+  world: state.controls.world
 }))
 class CategoryValue extends React.Component {
   toggleOff() {
-    const { dispatch, metadataField, optionIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex } = this.props;
     dispatch({
       type: "categorical metadata filter deselect",
       metadataField,
-      optionIndex
+      categoryIndex
     });
   }
 
   toggleOn() {
-    const { dispatch, metadataField, optionIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex } = this.props;
     dispatch({
       type: "categorical metadata filter select",
       metadataField,
-      optionIndex
+      categoryIndex
     });
   }
 
@@ -32,31 +36,43 @@ class CategoryValue extends React.Component {
     const {
       categoricalSelectionState,
       metadataField,
-      optionIndex,
+      categoryIndex,
       colorAccessor,
       colorScale,
       i,
-      schema
+      schema,
+      world
     } = this.props;
 
     if (!categoricalSelectionState) return null;
 
     const category = categoricalSelectionState[metadataField];
-    const selected = category.optionSelected[optionIndex];
+    const selected = category.categorySelected[categoryIndex];
-    const count = category.optionCount[optionIndex];
+    const count = category.categoryCounts[categoryIndex];
-    const value = category.optionValue[optionIndex];
+    const value = category.categoryValues[categoryIndex];
-    const displayString = String(category.optionValue[optionIndex]).valueOf();
+    const displayString = String(
+      category.categoryValues[categoryIndex]
+    ).valueOf();
 
     /* this is the color scale, so add swatches below */
-    const c = metadataField === colorAccessor;
+    const isColorBy = metadataField === colorAccessor;
     let categories = null;
+    let occupancy = null;
 
-    if (c && schema) {
+    if (isColorBy && schema) {
       categories = _.filter(schema.annotations.obs, {
         name: colorAccessor
       })[0].categories;
     }
 
+    if (colorAccessor && !isColorBy) {
+      occupancy = countCategoryValues2D(
+        metadataField,
+        colorAccessor,
+        world.obsAnnotations
+      );
+    }
+
     return (
       <div
         key={i}
@@ -70,7 +86,10 @@ class CategoryValue extends React.Component {
           style={{
             margin: 0,
             padding: 0,
-            userSelect: "none"
+            userSelect: "none",
+            width: globals.leftSidebarWidth - 130,
+            display: "flex",
+            justifyContent: "space-between"
           }}
         >
           <label className="bp3-control bp3-checkbox">
@@ -84,6 +103,18 @@ class CategoryValue extends React.Component {
             <span className="bp3-control-indicator" />
             {displayString}
           </label>
+          <span style={{ flexShrink: 0 }}>
+            {colorAccessor &&
+            !isColorBy &&
+            categoricalSelectionState[colorAccessor] ? (
+              <Occupancy
+                occupancy={occupancy.get(
+                  category.categoryValues[categoryIndex]
+                )}
+                {...this.props}
+              />
+            ) : null}
+          </span>
         </div>
         <span>
           <span>{count}</span>
@@ -93,7 +124,7 @@ class CategoryValue extends React.Component {
               width: 11,
               height: 11,
               backgroundColor:
-                c && categories
+                isColorBy && categories
                   ? colorScale(categories.indexOf(value))
                   : "inherit"
             }}

client/src/components/framework/toasters.js

@@ -13,6 +13,13 @@ A "user" error - eg, bad input
 export const postUserErrorToast = message =>
   ErrorToastTopCenter.show({ message, intent: Intent.WARNING });
 
+/*
+A toast the user must dismiss manually, because they need to act on its information,
+ie., 8 bulk add genes out of 40 were bad. Manually see which ones and fix.
+*/
+export const keepAroundErrorToast = message =>
+  ErrorToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
+
 /*
 a hard network error
 */

client/src/components/geneExpression/cellSetButtons.js

@@ -1,7 +1,7 @@
 // jshint esversion: 6
 import React from "react";
 import _ from "lodash";
-import { Button, Tooltip } from "@blueprintjs/core";
+import { AnchorButton, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 
 @connect()
@@ -33,9 +33,10 @@ class CellSetButton extends React.Component {
         content="Save current selection for differential expression computation"
         position="top"
       >
-        <Button
+        <AnchorButton
           style={{ marginRight: 10 }}
           type="button"
+          disabled={differential.diffExp}
           onClick={this.set.bind(this)}
         >
           {eitherCellSetOneOrTwo}
@@ -43,7 +44,7 @@ class CellSetButton extends React.Component {
           {differential[cellListName]
             ? `${differential[cellListName].length} cells`
             : "0 cells"}
-        </Button>
+        </AnchorButton>
       </Tooltip>
     );
   }

client/src/components/geneExpression/expressionButtons.js

@@ -68,6 +68,7 @@ class Expression extends React.Component {
               style={{ marginTop: 10 }}
               disabled={!haveBothCellSets}
               intent="primary"
+              loading={differential.loading}
               fill
               type="button"
               onClick={this.computeDiffExp.bind(this)}

client/src/components/geneExpression/index.js

@@ -3,14 +3,60 @@
 
 import React from "react";
 import _ from "lodash";
-import * as d3 from "d3";
+import fuzzysort from "fuzzysort";
+
 import { connect } from "react-redux";
-import { Button, Tooltip } from "@blueprintjs/core";
+import {
+  MenuItem,
+  Button,
+  FormGroup,
+  InputGroup,
+  ControlGroup
+} from "@blueprintjs/core";
+import { Suggest } from "@blueprintjs/select";
 import HistogramBrush from "../brushableHistogram";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import { postUserErrorToast } from "../framework/toasters";
+import {
+  postUserErrorToast,
+  keepAroundErrorToast
+} from "../framework/toasters";
 import ExpressionButtons from "./expressionButtons";
+import finiteExtent from "../../util/finiteExtent";
+
+const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
+  if (!modifiers.matchesPredicate) {
+    return null;
+  }
+  /* the fuzzysort wraps the object with other properties, like a score */
+  const gene = fuzzySortResult.obj;
+  const text = gene.name;
+
+  return (
+    <MenuItem
+      active={modifiers.active}
+      disabled={modifiers.disabled}
+      // Use of annotations in this way is incorrect and dataset specific.
+      // See https://github.com/chanzuckerberg/cellxgene/issues/483
+      // label={gene.n_counts}
+      key={gene.name}
+      onClick={g => {
+        /* this fires when user clicks a menu item */
+        handleClick(g);
+      }}
+      text={text}
+    />
+  );
+};
+
+const filterGenes = (query, genes) => {
+  /* fires on load, once, and then for each character typed into the input */
+  return fuzzysort.go(query, genes, {
+    key: "name",
+    limit: 5,
+    threshold: -10000 // don't return bad results
+  });
+};
 
 @connect(state => {
   const metadata = _.get(state.controls.world, "obsAnnotations", null);
@@ -22,6 +68,7 @@ import ExpressionButtons from "./expressionButtons";
     metadata,
     initializeRanges,
     userDefinedGenes: state.controls.userDefinedGenes,
+    userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
     world: state.controls.world,
     colorAccessor: state.controls.colorAccessor,
     allGeneNames: state.controls.allGeneNames,
@@ -32,20 +79,14 @@ class GeneExpression extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      gene: ""
+      bulkAdd: "",
+      tab: "autosuggest"
     };
   }
 
-  keyPress(e) {
+  handleClick(g) {
-    if (e.keyCode === 13) {
-      this.handleClick();
-    }
-  }
-
-  handleClick() {
     const { world, dispatch, userDefinedGenes } = this.props;
-    const { gene } = this.state;
+    const gene = g.target;
-
     if (userDefinedGenes.indexOf(gene) !== -1) {
       postUserErrorToast("That gene already exists");
     } else if (userDefinedGenes.length > 15) {
@@ -60,13 +101,49 @@ class GeneExpression extends React.Component {
         type: "user defined gene",
         data: gene
       });
-      this.setState({ gene: "" });
     }
   }
 
+  handleBulkAddClick() {
+    const { world, dispatch, userDefinedGenes } = this.props;
+    const { bulkAdd } = this.state;
+
+    /*
+      test:
+      Apod,,, Cd74,,    ,,,    Foo,    Bar-2,,
+    */
+    if (bulkAdd !== "") {
+      const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
+
+      genes.forEach(gene => {
+        if (userDefinedGenes.indexOf(gene) !== -1) {
+          keepAroundErrorToast("That gene already exists");
+        } else if (!_.find(world.varAnnotations, { name: gene })) {
+          keepAroundErrorToast(
+            `${gene} doesn't appear to be a valid gene name.`
+          );
+        } else {
+          dispatch(actions.requestUserDefinedGene(gene));
+          dispatch({
+            type: "user defined gene",
+            data: gene
+          });
+        }
+      });
+    }
+
+    this.setState({ bulkAdd: "" });
+  }
+
   render() {
-    const { world, userDefinedGenes, differential } = this.props;
+    const {
-    const { gene } = this.state;
+      world,
+      userDefinedGenes,
+      userDefinedGenesLoading,
+      differential
+    } = this.props;
+
+    const { tab, bulkAdd } = this.state;
 
     return (
       <div>
@@ -84,31 +161,105 @@ class GeneExpression extends React.Component {
             Selected Genes
           </p>
           <div
-            style={{ padding: globals.leftSidebarSectionPadding }}
+            style={{
-            className="bp3-control-group"
+              padding: globals.leftSidebarSectionPadding
+            }}
           >
-            <div className="bp3-input-group bp3-fill">
+            <Button
-              <input
+              active={tab === "autosuggest"}
-                onKeyDown={this.keyPress.bind(this)}
+              style={{ marginRight: 5 }}
-                onChange={e => {
+              minimal
-                  this.setState({ gene: e.target.value });
+              small
-                }}
+              onClick={() => {
-                value={gene}
+                this.setState({ tab: "autosuggest" });
-                type="text"
+              }}
-                className="bp3-input"
-                placeholder="Enter a gene name"
-                style={{ paddingRight: 94 }}
-              />
-            </div>
-            <Tooltip
-              content="Add a gene to see its expression levels"
-              position="bottom"
             >
-              <Button intent="primary" onClick={this.handleClick.bind(this)}>
+              Autosuggest
+            </Button>
+            <Button
+              active={tab === "bulkadd"}
+              minimal
+              small
+              onClick={() => {
+                this.setState({ tab: "bulkadd" });
+              }}
+            >
+              Bulk add genes
+            </Button>
+          </div>
+
+          {tab === "autosuggest" ? (
+            <ControlGroup
+              style={{
+                paddingLeft: globals.leftSidebarSectionPadding,
+                paddingBottom: globals.leftSidebarSectionPadding
+              }}
+            >
+              <Suggest
+                closeOnSelect
+                openOnKeyDown
+                resetOnSelect
+                itemDisabled={
+                  userDefinedGenesLoading ? () => true : () => false
+                }
+                noResults={<MenuItem disabled text="No matching genes." />}
+                onItemSelect={g => {
+                  /* this happens on 'enter' */
+                  this.handleClick(g);
+                }}
+                inputValueRenderer={g => {
+                  return "";
+                }}
+                itemListPredicate={filterGenes}
+                itemRenderer={renderGene.bind(this)}
+                items={
+                  world && world.varAnnotations
+                    ? world.varAnnotations
+                    : [{ name: "No genes" }]
+                }
+                popoverProps={{ minimal: true }}
+              />
+              <Button
+                className="bp3-button bp3-intent-primary"
+                loading={userDefinedGenesLoading}
+              >
                 Add
               </Button>
-            </Tooltip>
+            </ControlGroup>
-          </div>
+          ) : null}
+          {tab === "bulkadd" ? (
+            <div style={{ paddingLeft: globals.leftSidebarSectionPadding }}>
+              <form
+                onSubmit={e => {
+                  e.preventDefault();
+                  this.handleBulkAddClick();
+                }}
+              >
+                <FormGroup
+                  helperText="Add a list of genes (comma delimited)"
+                  labelFor="text-input-bulk-add"
+                >
+                  <ControlGroup>
+                    <InputGroup
+                      onChange={e => {
+                        this.setState({ bulkAdd: e.target.value });
+                      }}
+                      id="text-input-bulk-add"
+                      placeholder="Apod, Cd74, ..."
+                      value={bulkAdd}
+                    />
+                    <Button
+                      intent="primary"
+                      onClick={this.handleBulkAddClick.bind(this)}
+                      loading={userDefinedGenesLoading}
+                    >
+                      Add
+                    </Button>
+                  </ControlGroup>
+                </FormGroup>
+              </form>
+            </div>
+          ) : null}
           {world && userDefinedGenes.length > 0
             ? _.map(userDefinedGenes, (geneName, index) => {
                 const values = world.varDataCache[geneName];
@@ -120,7 +271,7 @@ class GeneExpression extends React.Component {
                     key={geneName}
                     field={geneName}
                     zebra={index % 2 === 0}
-                    ranges={d3.extent(values)}
+                    ranges={finiteExtent(values)}
                     isUserDefined
                   />
                 );
@@ -150,7 +301,7 @@ class GeneExpression extends React.Component {
                     key={name}
                     field={name}
                     zebra={index % 2 === 0}
-                    ranges={d3.extent(values)}
+                    ranges={finiteExtent(values)}
                     isDiffExp
                     logFoldChange={value[1]}
                     pval={value[2]}

client/src/components/graph/graph.js

@@ -22,7 +22,8 @@ import scaleLinear from "../../util/scaleLinear";
   responsive: state.responsive,
   colorRGB: _.get(state.controls, "colorRGB", null),
   opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
+  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
+  resettingInterface: state.controls.resettingInterface
 }))
 class Graph extends React.Component {
   constructor(props) {
@@ -39,7 +40,7 @@ class Graph extends React.Component {
     this.state = {
       svg: null,
       brush: null,
-      mode: "brush"
+      mode: "lasso"
     };
   }
 
@@ -201,7 +202,9 @@ class Graph extends React.Component {
         this.handleBrushSelectAction.bind(this),
         this.handleBrushDeselectAction.bind(this),
         responsive,
-        this.graphPaddingRight
+        this.graphPaddingRight,
+        this.handleLassoStart.bind(this),
+        this.handleLassoEnd.bind(this)
       );
       this.setState({ svg: newSvg, brush });
     }
@@ -250,54 +253,54 @@ class Graph extends React.Component {
     });
   }
 
+  invertPoint(pin) {
+    const { responsive } = this.props;
+    const { regl, camera, offset } = this.state;
+
+    const gl = regl._gl;
+
+    // get aspect ratio
+    const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+
+    // compute inverse view matrix
+    const inverse = mat4.invert([], camera.view());
+
+    // transform screen coordinates -> cell coordinates
+    const x = (2 * pin[0]) / (responsive.width - this.graphPaddingRight) - 1;
+    const y = 2 * (1 - pin[1] / (responsive.height - this.graphPaddingTop)) - 1;
+    const pout = [
+      x * inverse[14] * aspect + inverse[12],
+      y * inverse[14] + inverse[13]
+    ];
+    return [(pout[0] + 1) / 2 + offset[0], (pout[1] + 1) / 2 + offset[1]];
+  }
+
   handleBrushSelectAction() {
     /*
-    This conditional handles procedural brush deselect. Brush emits
+      This conditional handles procedural brush deselect. Brush emits
-    an event on procedural deselect because it is move: null
+      an event on procedural deselect because it is move: null
     */
-
+    /*
-    const { camera, offset } = this.state;
-    const { dispatch, responsive } = this.props;
-
-    if (d3.event.sourceEvent !== null) {
-      /*
-      No idea why d3 event scope works like this
-      but apparently
-      it does
-      https://bl.ocks.org/EfratVil/0e542f5fc426065dd1d4b6daaa345a9f
-    */
-      const s = d3.event.selection;
-      const gl = this.state.regl._gl;
-      /*
       event describing brush position:
       @-------|
       |       |
       |       |
       |-------@
     */
+    /*
+      No idea why d3 event scope works like this
+      but apparently
+      it does
+      https://bl.ocks.org/EfratVil/0e542f5fc426065dd1d4b6daaa345a9f
+    */
+    const { dispatch } = this.props;
 
-      // get aspect ratio
+    if (d3.event.sourceEvent !== null) {
-      const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+      const s = d3.event.selection;
-
-      // compute inverse view matrix
-      const inverse = mat4.invert([], camera.view());
-
-      // transform screen coordinates -> cell coordinates
-      const invert = pin => {
-        const x =
-          (2 * pin[0]) / (responsive.width - this.graphPaddingRight) - 1;
-        const y =
-          2 * (1 - pin[1] / (responsive.height - this.graphPaddingTop)) - 1;
-        const pout = [
-          x * inverse[14] * aspect + inverse[12],
-          y * inverse[14] + inverse[13]
-        ];
-        return [(pout[0] + 1) / 2 + offset[0], (pout[1] + 1) / 2 + offset[1]];
-      };
 
       const brushCoords = {
-        northwest: invert([s[0][0], s[0][1]]),
+        northwest: this.invertPoint([s[0][0], s[0][1]]),
-        southeast: invert([s[1][0], s[1][1]])
+        southeast: this.invertPoint([s[1][0], s[1][1]])
       };
 
       dispatch({
@@ -329,6 +332,25 @@ class Graph extends React.Component {
     }
   }
 
+  handleLassoStart() {
+    const { dispatch } = this.props;
+    // reset selected points when starting a new polygon
+    // making it easier for the user to make the next selection
+    dispatch({
+      type: "lasso started"
+    });
+  }
+
+  // when a lasso is completed, filter to the points within the lasso polygon
+  handleLassoEnd(polygon) {
+    const { dispatch } = this.props;
+
+    dispatch({
+      type: "lasso selection",
+      polygon: polygon.map(xy => this.invertPoint(xy)) // transform the polygon
+    });
+  }
+
   handleOpacityRangeChange(e) {
     const { dispatch } = this.props;
     dispatch({
@@ -339,11 +361,20 @@ class Graph extends React.Component {
 
   resetInterface() {
     const { dispatch } = this.props;
+    dispatch({
+      type: "interface reset started"
+    });
     dispatch(actions.resetInterface());
   }
 
   render() {
-    const { dispatch, responsive, crossfilter } = this.props;
+    const {
+      dispatch,
+      responsive,
+      crossfilter,
+      resettingInterface
+    } = this.props;
+
     const { mode } = this.state;
     return (
       <div id="graphWrapper">
@@ -392,6 +423,7 @@ class Graph extends React.Component {
                   /* world && universe ? worldEqUniverse(world, universe) : false */
                 }
                 type="button"
+                loading={resettingInterface}
                 intent="warning"
                 style={{ marginRight: 10 }}
                 onClick={this.resetInterface.bind(this)}
@@ -401,13 +433,18 @@ class Graph extends React.Component {
             </Tooltip>
             <div>
               <div className="bp3-button-group">
-                <Tooltip content="Lasso cells" position="left">
+                <Tooltip content="Lasso selection" position="left">
                   <Button
-                    className="bp3-button bp3-icon-select"
                     type="button"
-                    active={mode === "brush"}
+                    className="bp3-button bp3-icon-polygon-filter"
+                    active={mode === "lasso"}
                     onClick={() => {
-                      this.setState({ mode: "brush" });
+                      this.handleBrushDeselectAction();
+                      // this.restartReglLoop();
+                      this.setState({ mode: "lasso" });
+                    }}
+                    style={{
+                      cursor: "pointer"
                     }}
                   />
                 </Tooltip>
@@ -440,7 +477,7 @@ class Graph extends React.Component {
         >
           <div
             style={{
-              display: mode === "brush" ? "inherit" : "none"
+              display: mode === "lasso" ? "inherit" : "none"
             }}
             id="graphAttachPoint"
           />

client/src/components/graph/setupLasso.js

@@ -0,0 +1,127 @@
+// https://bl.ocks.org/pbeshai/8008075f9ce771ee8be39e8c38907570
+
+import * as d3 from "d3";
+
+const Lasso = () => {
+  const dispatch = d3.dispatch("start", "end");
+
+  const polygonToPath = polygon =>
+    `M${polygon.map(d => d.join(",")).join("L")}`;
+
+  const distance = (pt1, pt2) =>
+    Math.sqrt((pt2[0] - pt1[0]) ** 2 + (pt2[1] - pt1[1]) ** 2);
+
+  // distance last point has to be to first point before it auto closes when mouse is released
+  const closeDistance = 75;
+
+  const lasso = svg => {
+    let lassoPolygon;
+    let lassoPath;
+    let closePath;
+
+    const handleDragStart = () => {
+      lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
+
+      if (lassoPath) {
+        lassoPath.remove();
+      }
+
+      lassoPath = g
+        .append("path")
+        .attr("fill", "#0bb")
+        .attr("fill-opacity", 0.1)
+        .attr("stroke", "#0bb")
+        .attr("stroke-dasharray", "3, 3");
+
+      closePath = g
+        .append("line")
+        .attr("x2", lassoPolygon[0][0])
+        .attr("y2", lassoPolygon[0][1])
+        .attr("stroke", "#0bb")
+        .attr("stroke-dasharray", "3, 3")
+        .attr("opacity", 0);
+
+      dispatch.call("start", lasso, lassoPolygon);
+    };
+
+    const handleDrag = () => {
+      const point = d3.mouse(svg.node());
+      lassoPolygon.push(point);
+      lassoPath.attr("d", polygonToPath(lassoPolygon));
+
+      // indicate if we are within closing distance
+      if (
+        distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
+        closeDistance
+      ) {
+        closePath
+          .attr("x1", point[0])
+          .attr("y1", point[1])
+          .attr("opacity", 1);
+      } else {
+        closePath.attr("opacity", 0);
+      }
+    };
+
+    const handleDragEnd = () => {
+      // remove the close path
+      closePath.remove();
+      closePath = null;
+
+      // succesfully closed
+      if (
+        distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
+        closeDistance
+      ) {
+        lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
+        dispatch.call("end", lasso, lassoPolygon);
+
+        // otherwise cancel
+      } else {
+        lassoPath.remove();
+        lassoPath = null;
+        lassoPolygon = null;
+      }
+    };
+
+    // append a <g> with a rect
+    const g = svg.append("g").attr("class", "lasso-group");
+    const bbox = svg.node().getBoundingClientRect();
+    const area = g
+      .append("rect")
+      .attr("width", bbox.width)
+      .attr("height", bbox.height)
+      .attr("fill", "tomato")
+      .attr("opacity", 0);
+
+    const drag = d3
+      .drag()
+      .on("start", handleDragStart)
+      .on("drag", handleDrag)
+      .on("end", handleDragEnd);
+
+    area.call(drag);
+
+    lasso.reset = () => {
+      if (lassoPath) {
+        lassoPath.remove();
+        lassoPath = null;
+      }
+
+      lassoPolygon = null;
+      if (closePath) {
+        closePath.remove();
+        closePath = null;
+      }
+    };
+  };
+
+  lasso.on = (type, callback) => {
+    dispatch.on(type, callback);
+    return lasso;
+  };
+
+  return lasso;
+};
+
+export default Lasso;

client/src/components/graph/setupSVGandBrush.js

@@ -1,6 +1,7 @@
 // jshint esversion: 6
 import * as d3 from "d3";
 import styles from "./graph.css";
+import Lasso from "./setupLasso";
 
 /******************************************
 *******************************************
@@ -12,7 +13,9 @@ export default (
   handleBrushSelectAction,
   handleBrushDeselectAction,
   responsive,
-  graphPaddingRight
+  graphPaddingRight,
+  handleLassoStart,
+  handleLassoEnd
 ) => {
   const svg = d3
     .select("#graphAttachPoint")
@@ -32,9 +35,16 @@ export default (
     .attr("class", "graph_brush")
     .call(brush);
 
+  const lassoInstance = Lasso()
+    .on("end", handleLassoEnd)
+    .on("start", handleLassoStart);
+
+  const lasso = svg.call(lassoInstance);
+
   return {
     svg,
     brushContainer,
-    brush
+    brush,
+    lasso
   };
 };

client/src/components/scatterplot/scatterplot.js

@@ -20,6 +20,7 @@ import scaleLinear from "../../util/scaleLinear";
 
 import { margin, width, height } from "./util";
 import { kvCache } from "../../util/stateManager";
+import finiteExtent from "../../util/finiteExtent";
 
 @connect(state => {
   const {
@@ -227,11 +228,11 @@ class Scatterplot extends React.Component {
   static setupScales(expressionX, expressionY) {
     const xScale = d3
       .scaleLinear()
-      .domain(d3.extent(expressionX))
+      .domain(finiteExtent(expressionX))
       .range([0, width]);
     const yScale = d3
       .scaleLinear()
-      .domain(d3.extent(expressionY))
+      .domain(finiteExtent(expressionY))
       .range([height, 0]);
 
     return {

client/src/globals.js

@@ -1,3 +1,5 @@
+import { Colors } from "@blueprintjs/core";
+
 // jshint esversion: 6
 /* these will be either (preferably) specified or inferred */
 export const categories = [
@@ -44,8 +46,8 @@ export const configDefaults = {
 };
 
 /* colors */
-export const blue = "#4a90e2";
+export const blue = Colors.BLUE3;
-export const hcaBlue = "#1c7cc7";
+export const linkBlue = Colors.BLUE5;
 export const lightestGrey = "rgb(249,249,249)";
 export const lighterGrey = "rgb(245,245,245)";
 export const lightGrey = "rgb(211,211,211)";
@@ -57,6 +59,7 @@ export const brightBlue = "#4a90e2";
 export const brightGreen = "#A2D729";
 export const darkGreen = "#448C4D";
 
+export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
 
 /* typography constants */

client/src/middleware/updateCellColors.js

@@ -1,15 +1,10 @@
 // jshint esversion: 6
 import _ from "lodash";
 import * as d3 from "d3";
-import {
+import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
-  interpolateViridis,
-  interpolateSpectral,
-  interpolateRainbow,
-  interpolateBlues,
-  interpolateCool
-} from "d3-scale-chromatic";
 import * as globals from "../globals";
 import parseRGB from "../util/parseRGB";
+import finiteExtent from "../util/finiteExtent";
 
 /*
   https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
@@ -54,62 +49,89 @@ const updateCellColorsMiddleware = store => next => action => {
 
   const { obsAnnotations } = s.controls.world;
   let colorScale;
-  const colorsByName = new Array(obsAnnotations.length);
   const colorsByRGB = new Array(obsAnnotations.length);
 
   /*
   in plain language...
    (a) once the cells have loaded.
    (b) each time a user changes a color control we need to update cellsMetadata colors
-  This is available to all the draw functions as world.colorName[index] or world.colorRGB[index]
+  This is available to all the draw functions as controls.colorRGB[index]
   */
 
   if (action.type === "color by categorical metadata") {
-    const categories = _.filter(s.controls.world.schema.annotations.obs, {
+    const { categories } = _.filter(s.controls.world.schema.annotations.obs, {
       name: action.colorAccessor
-    })[0].categories;
+    })[0];
 
     colorScale = d3
       .scaleSequential(interpolateRainbow)
       .domain([0, categories.length]);
 
-    for (let i = 0; i < obsAnnotations.length; i += 1) {
+    /* pre-create colors - much faster than doing it for each obs */
+    const colors = _.transform(categories, (acc, cat, idx) => {
+      acc[cat] = parseRGB(colorScale(idx));
+    });
+
+    const key = action.colorAccessor;
+    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
       const obs = obsAnnotations[i];
-      const c = colorScale(categories.indexOf(obs[action.colorAccessor]));
+      const cat = obs[key];
-      colorsByName[i] = c;
+      colorsByRGB[i] = colors[cat];
-      colorsByRGB[i] = parseRGB(c);
     }
   }
 
   if (action.type === "color by continuous metadata") {
+    const colorBins = 100;
+    const [min, max] = [0, action.rangeMaxForColorAccessor];
     colorScale = d3
-      .scaleLinear()
+      .scaleQuantile()
-      .domain([0, action.rangeMaxForColorAccessor])
+      .domain([min, max])
-      .range([1, 0]);
+      .range(_.range(colorBins - 1, -1, -1));
 
-    for (let i = 0; i < obsAnnotations.length; i += 1) {
+    /* pre-create colors - much faster than doing it for each obs */
-      const obs = obsAnnotations[i];
+    const colors = new Array(colorBins);
-      const c = interpolateCool(colorScale(obs[action.colorAccessor]));
+    for (let i = 0; i < colorBins; i += 1) {
-      colorsByName[i] = c;
+      colors[i] = parseRGB(interpolateCool(i / colorBins));
-      colorsByRGB[i] = parseRGB(c);
+    }
+
+    const key = action.colorAccessor;
+    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
+    for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
+      const val = obsAnnotations[i][key];
+      if (Number.isFinite(val)) {
+        const c = colorScale(val);
+        colorsByRGB[i] = colors[c];
+      } else {
+        colorsByRGB[i] = nonFiniteColor;
+      }
     }
   }
 
   if (action.type === "color by expression") {
     const { gene, data } = action;
     const expression = data[gene]; // Float32Array
+    const colorBins = 100;
+    const [min, max] = finiteExtent(expression);
     colorScale = d3
-      .scaleLinear()
+      .scaleQuantile()
-      .domain([_.min(expression), _.max(expression)])
+      .domain([min, max])
-      .range([
+      .range(_.range(colorBins - 1, -1, -1));
-        1,
+
-        0
+    /* pre-create colors - much faster than doing it for each obs */
-      ]); /* invert viridis... probably pass this scale through to others */
+    const colors = new Array(colorBins);
+    for (let i = 0; i < colorBins; i += 1) {
+      colors[i] = parseRGB(interpolateCool(i / colorBins));
+    }
+    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
 
     for (let i = 0, len = expression.length; i < len; i += 1) {
-      const c = interpolateCool(colorScale(expression[i]));
+      const e = expression[i];
-      colorsByName[i] = c;
+      if (Number.isFinite(e)) {
-      colorsByRGB[i] = parseRGB(c);
+        const c = colorScale(e);
+        colorsByRGB[i] = colors[c];
+      } else {
+        colorsByRGB[i] = nonFiniteColor;
+      }
     }
   }
 
@@ -117,7 +139,7 @@ const updateCellColorsMiddleware = store => next => action => {
   append the result of all the filters to the action the user just triggered
   */
   const modifiedAction = Object.assign({}, action, {
-    colors: { name: colorsByName, rgb: colorsByRGB },
+    colors: { rgb: colorsByRGB },
     colorScale
   });
 

client/src/reducers/controls.js

@@ -1,7 +1,9 @@
 // jshint esversion: 6
 
 import _ from "lodash";
-import { World, kvCache } from "../util/stateManager";
+import { polygonContains } from "d3";
+
+import { World, kvCache, WorldUtil } from "../util/stateManager";
 import parseRGB from "../util/parseRGB";
 import Crossfilter from "../util/typedCrossfilter";
 import * as globals from "../globals";
@@ -24,25 +26,27 @@ Remember that option values can be ANY js type, except undefined/null.
   {
     _category_name_1: {
       // map of option value to index
-      optionIndex: Map([
+      categoryIndices: Map([
-        optval1: index,
+        catval1: index,
         ...
       ])
 
       // index->selection true/false state
-      optionSelected: [ true/false, true/false, ... ]
+      categorySelected: [ true/false, true/false, ... ]
 
       // number of options
-      numOptions: number,
+      numCategories: number,
 
       // isTruncated - true if the options for selection has
       // been truncated (ie, was too large to implement)
     }
   }
 */
-function topNoptions(summary) {
+function topNCategories(summary) {
-  const counts = _.map(summary.categories, cat => summary.options[cat]);
+  const counts = _.map(summary.categories, cat =>
-  const sortIndex = fillRange(new Array(summary.numOptions)).sort(
+    summary.categoryCounts.get(cat)
+  );
+  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
     (a, b) => counts[b] - counts[a]
   );
   const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
@@ -65,20 +69,18 @@ function createCategoricalSelectionState(state, world) {
         key !== "name" &&
         value.categories.length < state.maxCategoryItems;
       if (isSelectableCategory) {
-        const [optionValue, optionCount] = topNoptions(value);
+        const [categoryValues, categoryCounts] = topNCategories(value);
-        // const optionCount = Object.values(value.options);
+        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-
+        const numCategories = categoryIndices.size;
-        const optionIndex = new Map(optionValue.map((v, i) => [v, i]));
+        const categorySelected = new Array(numCategories).fill(true);
-        const numOptions = optionIndex.size;
+        const isTruncated = categoryValues.length < value.numCategories;
-        const optionSelected = new Array(numOptions).fill(true);
-        const isTruncated = optionValue.length < value.numOptions;
         res[key] = {
-          optionValue, // array: of natively typed option values
+          categoryValues, // array: of natively typed category values
-          optionIndex, // map: option value (native type) -> option index
+          categoryIndices, // map: category value (native type) -> category index
-          optionSelected, // array: t/f selection state
+          categorySelected, // array: t/f selection state
-          numOptions, // number: of options
+          numCategories, // number: of categories
           isTruncated, // bool: true if list was truncated
-          optionCount // array: cardinality of each option
+          categoryCounts // array: cardinality of each category
         };
       }
     }
@@ -87,12 +89,12 @@ function createCategoricalSelectionState(state, world) {
 }
 
 /*
-given a categoricalSelectionState, return the list of all option values
+given a categoricalSelectionState, return the list of all category values
 where selection state is true (ie, they are selected).
 */
 function selectedValuesForCategory(categorySelectionState) {
-  const selectedValues = _([...categorySelectionState.optionIndex])
+  const selectedValues = _([...categorySelectionState.categoryIndices])
-    .filter(tuple => categorySelectionState.optionSelected[tuple[1]])
+    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
     .map(tuple => tuple[0])
     .value();
   return selectedValues;
@@ -112,16 +114,17 @@ const Controls = (
 
     // all of the data + selection state
     world: null,
-    colorName: null,
     colorRGB: null,
     categoricalSelectionState: null,
     crossfilter: null,
     dimensionMap: null,
     userDefinedGenes: [],
+    userDefinedGenesLoading: false,
     diffexpGenes: [],
 
     colorAccessor: null,
     colorScale: null,
+    resettingInterface: false,
 
     opacityForDeselectedCells: 0.2,
     graphBrushSelection: null,
@@ -167,14 +170,16 @@ const Controls = (
       /* first light - create world & other data-driven defaults */
       const { universe } = action;
       const world = World.createWorldFromEntireUniverse(universe);
-      const colorName = new Array(universe.nObs).fill(globals.defaultCellColor);
+      const colorRGB = new Array(universe.nObs).fill(
-      const colorRGB = _.map(colorName, c => parseRGB(c));
+        parseRGB(globals.defaultCellColor)
+      );
       const categoricalSelectionState = createCategoricalSelectionState(
         state,
         world
       );
       const crossfilter = Crossfilter(world.obsAnnotations);
       const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+      WorldUtil.clearCaches();
 
       const worldVarDataCache = world.varDataCache;
 
@@ -222,12 +227,12 @@ const Controls = (
         error: null,
         universe,
         world,
-        colorName,
         colorRGB,
         categoricalSelectionState,
         crossfilter,
         dimensionMap,
-        colorAccessor: null
+        colorAccessor: null,
+        resettingInterface: false
       };
     }
     case "set World to current selection": {
@@ -239,14 +244,16 @@ const Controls = (
         action.world,
         action.crossfilter
       );
-      const colorName = new Array(world.nObs).fill(globals.defaultCellColor);
+      const colorRGB = new Array(world.nObs).fill(
-      const colorRGB = _.map(colorName, c => parseRGB(c));
+        parseRGB(globals.defaultCellColor)
+      );
       const categoricalSelectionState = createCategoricalSelectionState(
         state,
         world
       );
       const crossfilter = Crossfilter(world.obsAnnotations);
       const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+      WorldUtil.clearCaches();
 
       const worldVarDataCache = world.varDataCache;
       /* var dimensions */
@@ -287,7 +294,6 @@ const Controls = (
         loading: false,
         error: null,
         world,
-        colorName,
         colorRGB,
         categoricalSelectionState,
         crossfilter,
@@ -322,6 +328,18 @@ const Controls = (
         }
       };
     }
+    case "request user defined gene started": {
+      return {
+        ...state,
+        userDefinedGenesLoading: true
+      };
+    }
+    case "request user defined gene error": {
+      return {
+        ...state,
+        userDefinedGenesLoading: false
+      };
+    }
     case "request user defined gene success": {
       const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
       const worldVarDataCache = world.varDataCache;
@@ -339,7 +357,8 @@ const Controls = (
       return {
         ...state,
         dimensionMap,
-        userDefinedGenes: _userDefinedGenes
+        userDefinedGenes: _userDefinedGenes,
+        userDefinedGenesLoading: false
       };
     }
     case "request differential expression success": {
@@ -440,11 +459,11 @@ const Controls = (
     }
     case "reset colorscale": {
       const { world } = state;
-      const colorName = new Array(world.nObs).fill(globals.defaultCellColor);
+      const colorRGB = new Array(world.nObs).fill(
-      const colorRGB = _.map(colorName, c => parseRGB(c));
+        parseRGB(globals.defaultCellColor)
+      );
       return {
         ...state,
-        colorName,
         colorRGB,
         colorAccessor: null
       };
@@ -462,27 +481,36 @@ const Controls = (
              User Events
      *******************************/
     case "graph brush selection change": {
-      state.dimensionMap[layoutDimensionName("X")].filterRange([
+      state.dimensionMap[layoutDimensionName("XY")].filterWithinRect(
-        action.brushCoords.northwest[0],
+        action.brushCoords.northwest,
-        action.brushCoords.southeast[0]
+        action.brushCoords.southeast
-      ]);
+      );
-      state.dimensionMap[layoutDimensionName("Y")].filterRange([
-        action.brushCoords.southeast[1],
-        action.brushCoords.northwest[1]
-      ]);
       return {
         ...state,
         graphBrushSelection: action.brushCoords
       };
     }
+    case "lasso deselect":
     case "graph brush deselect": {
-      state.dimensionMap[layoutDimensionName("X")].filterAll();
+      state.dimensionMap[layoutDimensionName("XY")].filterAll();
-      state.dimensionMap[layoutDimensionName("Y")].filterAll();
       return {
         ...state,
         graphBrushSelection: null
       };
     }
+    case "lasso selection": {
+      const { polygon } = action;
+      const dXY = state.dimensionMap[layoutDimensionName("XY")];
+      if (polygon.length < 3) {
+        // single point or a line is not a polygon, and is therefore a deselect
+        dXY.filterAll();
+      } else {
+        dXY.filterWithinPolygon(polygon);
+      }
+      return {
+        ...state
+      };
+    }
     case "continuous metadata histogram brush": {
       const name = makeContinuousDimensionName(
         action.continuousNamespace,
@@ -510,19 +538,25 @@ const Controls = (
         graphRenderCounter: c
       };
     }
+    case "interface reset started": {
+      return {
+        ...state,
+        resettingInterface: true
+      };
+    }
     /*******************************
           Categorical metadata
     *******************************/
     case "categorical metadata filter select": {
-      const newOptionSelected = Array.from(
+      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].optionSelected
+        state.categoricalSelectionState[action.metadataField].categorySelected
       );
-      newOptionSelected[action.optionIndex] = true;
+      newCategorySelected[action.categoryIndex] = true;
       const newCategoricalSelectionState = {
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: newOptionSelected
+          categorySelected: newCategorySelected
         }
       };
 
@@ -538,15 +572,15 @@ const Controls = (
       };
     }
     case "categorical metadata filter deselect": {
-      const newOptionSelected = Array.from(
+      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].optionSelected
+        state.categoricalSelectionState[action.metadataField].categorySelected
       );
-      newOptionSelected[action.optionIndex] = false;
+      newCategorySelected[action.categoryIndex] = false;
       const newCategoricalSelectionState = {
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: newOptionSelected
+          categorySelected: newCategorySelected
         }
       };
 
@@ -566,8 +600,9 @@ const Controls = (
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: Array.from(
+          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField].optionSelected
+            state.categoricalSelectionState[action.metadataField]
+              .categorySelected
           ).fill(false)
         }
       };
@@ -584,8 +619,9 @@ const Controls = (
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: Array.from(
+          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField].optionSelected
+            state.categoricalSelectionState[action.metadataField]
+              .categorySelected
           ).fill(true)
         }
       };
@@ -605,7 +641,6 @@ const Controls = (
     case "color by continuous metadata": {
       return {
         ...state,
-        colorName: action.colors.name,
         colorRGB: action.colors.rgb,
         colorAccessor: action.colorAccessor,
         colorScale: action.colorScale
@@ -614,7 +649,6 @@ const Controls = (
     case "color by expression": {
       return {
         ...state,
-        colorName: action.colors.name,
         colorRGB: action.colors.rgb,
         colorAccessor: action.gene,
         colorScale: action.colorScale

client/src/reducers/index.js

@@ -1,9 +1,9 @@
 // jshint esversion: 6
 import { combineReducers, createStore, applyMiddleware } from "redux";
 import thunk from "redux-thunk";
+import { composeWithDevTools } from "redux-devtools-extension";
 import updateURLMiddleware from "../middleware/updateURLMiddleware";
 import updateCellColors from "../middleware/updateCellColors";
-import { composeWithDevTools } from "redux-devtools-extension";
 
 import config from "./config";
 import differential from "./differential";

client/src/util/actionHelpers.js

@@ -25,24 +25,45 @@ export function catchErrorsWrap(fn, dispatchToUser = false) {
 }
 
 /*
-Wrapper to perform an async fetch and JSON decode response.
+Wrapper to perform async fetch with some modest error handling
+and decoding.
 */
-export const doJsonRequest = async url => {
+const doFetch = async (url, acceptType) => {
   const res = await fetch(url, {
     method: "get",
     headers: new Headers({
-      "Content-Type": "application/json"
+      Accept: acceptType
     })
   });
-  if (res.ok && res.headers.get("Content-Type") === "application/json") {
+  if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
-    return res.json();
+    return res;
   }
   // else an error
-  const msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
   dispatchNetworkErrorMessageToUser(msg);
   throw new Error(msg);
 };
 
+/*
+Wrapper to perform an async fetch and JSON decode response.
+*/
+export const doJsonRequest = async url => {
+  const res = await doFetch(url, "application/json");
+  return res.json();
+};
+
+/*
+Wrapper to perform an async fetch for binary data.
+*/
+export const doBinaryRequest = async url => {
+  const res = await doFetch(url, "application/octet-stream");
+  return res.arrayBuffer();
+};
+
 /*
 This function "packs" filter index lists into the more efficient
 "range" form specified in the REST 0.2 spec.

client/src/util/finiteExtent.js

@@ -0,0 +1,33 @@
+/*
+Return the [minimum, maximum] extent, of the given typed array, ignoring
+non-finite values (ie, +Infinity, -Infinity).
+
+If undefined or empty array, or array contains only non-finite numbers,
+will return [undefined, undefined]
+*/
+
+function finiteExtent(tarr) {
+  let min;
+  let max;
+  let i;
+
+  for (i = 0; i < tarr.length; i += 1) {
+    const val = tarr[i];
+    if (Number.isFinite(val)) {
+      min = val;
+      max = val;
+      i += 1;
+      break;
+    }
+  }
+  for (; i < tarr.length; i += 1) {
+    const val = tarr[i];
+    if (Number.isFinite(val)) {
+      if (min > val) min = val;
+      if (max < val) max = val;
+    }
+  }
+  return [min, max];
+}
+
+export default finiteExtent;

client/src/util/stateManager/index.js

@@ -3,7 +3,7 @@
 /*
 Model manager providing an abstraction for the use of the reducer code.
 This module provides several buckets of functionality:
-  - schema and config driven tranformation of the dataframe wire protocol
+  - schema and config driven tranformation of the wire protocol
     into a format that is easy for the UI code to use.
   - manage the universe/world abstraction:
     + universe: all of the server-provided, read-only data
@@ -17,3 +17,4 @@ exists to support those concepts.
 export * as Universe from "./universe";
 export * as World from "./world";
 export * as kvCache from "./keyvalcache";
+export * as WorldUtil from "./worldUtil";

client/src/util/stateManager/matrix.js

@@ -0,0 +1,75 @@
+import { flatbuffers } from "flatbuffers";
+import { NetEncoding } from "./matrix_generated";
+
+const utf8Decoder = new TextDecoder("utf-8");
+
+/*
+Matrix flatbuffer decoding support.   See fbs/matrix.fbs
+*/
+
+/*
+Decode NetEncoding.TypedArray
+*/
+function decodeTypedArray(uType, uValF, inplace = false) {
+  if (uType === NetEncoding.TypedArray.NONE) {
+    return null;
+  }
+
+  // Convert to a JS class that supports this type
+  const TypeClass = NetEncoding[NetEncoding.TypedArray[uType]];
+  // Create a TypedArray that references the underlying buffer
+  let arr = uValF(new TypeClass()).dataArray();
+  if (uType === NetEncoding.TypedArray.JSONEncodedArray) {
+    const json = utf8Decoder.decode(arr);
+    arr = JSON.parse(json);
+  } else if (!inplace) {
+    /* force copy to release underlying FBS buffer */
+    arr = new arr.constructor(arr);
+  }
+  return arr;
+}
+
+/*
+Parameter: Uint8Array or ArrayBuffer containing raw flatbuffer Matrix
+Returns: object containing decoded Matrix:
+{
+  nRows: num,
+  nCols: num,
+  columns: [
+    each column, which will be a TypedArray or Array
+  ]
+  colIdx: []|null
+}
+*/
+function decodeMatrixFBS(arrayBuffer, inplace = false) {
+  const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
+  const df = NetEncoding.Matrix.getRootAsMatrix(bb);
+
+  const nRows = df.nRows();
+  const nCols = df.nCols();
+
+  /* decode columns */
+  const columnsLength = df.columnsLength();
+  const columns = Array(columnsLength).fill(null);
+  for (let c = 0; c < columnsLength; c += 1) {
+    const col = df.columns(c);
+    columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
+  }
+
+  /* decode col_idx */
+  const colIdx = decodeTypedArray(
+    df.colIndexType(),
+    df.colIndex.bind(df),
+    inplace
+  );
+
+  return {
+    nRows,
+    nCols,
+    columns,
+    colIdx,
+    rowIdx: null
+  };
+}
+
+export default decodeMatrixFBS;

client/src/util/stateManager/matrix_generated.js

@@ -0,0 +1,835 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/**
+ * @const
+ * @namespace
+ */
+var NetEncoding = NetEncoding || {};
+
+/**
+ * @enum
+ */
+NetEncoding.TypedArray = {
+  NONE: 0, 0: 'NONE',
+  Float32Array: 1, 1: 'Float32Array',
+  Int32Array: 2, 2: 'Int32Array',
+  Uint32Array: 3, 3: 'Uint32Array',
+  Float64Array: 4, 4: 'Float64Array',
+  JSONEncodedArray: 5, 5: 'JSONEncodedArray'
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Float32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Float32Array}
+ */
+NetEncoding.Float32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Float32Array=} obj
+ * @returns {NetEncoding.Float32Array}
+ */
+NetEncoding.Float32Array.getRootAsFloat32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Float32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Float32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readFloat32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Float32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Float32Array}
+ */
+NetEncoding.Float32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Float32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Float32Array.startFloat32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Float32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addFloat32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Float32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float32Array.endFloat32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Uint32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Uint32Array}
+ */
+NetEncoding.Uint32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Uint32Array=} obj
+ * @returns {NetEncoding.Uint32Array}
+ */
+NetEncoding.Uint32Array.getRootAsUint32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Uint32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Uint32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Uint32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Uint32Array}
+ */
+NetEncoding.Uint32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Uint32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Uint32Array.startUint32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Uint32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Uint32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Uint32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Uint32Array.endUint32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Int32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Int32Array}
+ */
+NetEncoding.Int32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Int32Array=} obj
+ * @returns {NetEncoding.Int32Array}
+ */
+NetEncoding.Int32Array.getRootAsInt32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Int32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Int32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Int32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Int32Array}
+ */
+NetEncoding.Int32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Int32Array.startInt32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Int32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Int32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Int32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Int32Array.endInt32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Float64Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Float64Array}
+ */
+NetEncoding.Float64Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Float64Array=} obj
+ * @returns {NetEncoding.Float64Array}
+ */
+NetEncoding.Float64Array.getRootAsFloat64Array = function(bb, obj) {
+  return (obj || new NetEncoding.Float64Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Float64Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readFloat64(this.bb.__vector(this.bb_pos + offset) + index * 8) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Float64Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Float64Array}
+ */
+NetEncoding.Float64Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Float64Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Float64Array.startFloat64Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Float64Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float64Array.createDataVector = function(builder, data) {
+  builder.startVector(8, data.length, 8);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addFloat64(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Float64Array.startDataVector = function(builder, numElems) {
+  builder.startVector(8, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float64Array.endFloat64Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.JSONEncodedArray = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.JSONEncodedArray}
+ */
+NetEncoding.JSONEncodedArray.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.JSONEncodedArray=} obj
+ * @returns {NetEncoding.JSONEncodedArray}
+ */
+NetEncoding.JSONEncodedArray.getRootAsJSONEncodedArray = function(bb, obj) {
+  return (obj || new NetEncoding.JSONEncodedArray).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.JSONEncodedArray.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.JSONEncodedArray.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Uint8Array}
+ */
+NetEncoding.JSONEncodedArray.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Uint8Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.JSONEncodedArray.startJSONEncodedArray = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.JSONEncodedArray.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.JSONEncodedArray.createDataVector = function(builder, data) {
+  builder.startVector(1, data.length, 1);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.JSONEncodedArray.startDataVector = function(builder, numElems) {
+  builder.startVector(1, numElems, 1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.JSONEncodedArray.endJSONEncodedArray = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Column = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Column.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Column=} obj
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Column.getRootAsColumn = function(bb, obj) {
+  return (obj || new NetEncoding.Column).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Column.prototype.uType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Column.prototype.u = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Column.startColumn = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} uType
+ */
+NetEncoding.Column.addUType = function(builder, uType) {
+  builder.addFieldInt8(0, uType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} uOffset
+ */
+NetEncoding.Column.addU = function(builder, uOffset) {
+  builder.addFieldOffset(1, uOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Column.endColumn = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Matrix = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Matrix}
+ */
+NetEncoding.Matrix.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Matrix=} obj
+ * @returns {NetEncoding.Matrix}
+ */
+NetEncoding.Matrix.getRootAsMatrix = function(bb, obj) {
+  return (obj || new NetEncoding.Matrix).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.nRows = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.nCols = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @param {NetEncoding.Column=} obj
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Matrix.prototype.columns = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? (obj || new NetEncoding.Column).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.columnsLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Matrix.prototype.colIndexType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 10);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Matrix.prototype.colIndex = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 12);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Matrix.prototype.rowIndexType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 14);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Matrix.prototype.rowIndex = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 16);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Matrix.startMatrix = function(builder) {
+  builder.startObject(7);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} nRows
+ */
+NetEncoding.Matrix.addNRows = function(builder, nRows) {
+  builder.addFieldInt32(0, nRows, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} nCols
+ */
+NetEncoding.Matrix.addNCols = function(builder, nCols) {
+  builder.addFieldInt32(1, nCols, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} columnsOffset
+ */
+NetEncoding.Matrix.addColumns = function(builder, columnsOffset) {
+  builder.addFieldOffset(2, columnsOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Matrix.createColumnsVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Matrix.startColumnsVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} colIndexType
+ */
+NetEncoding.Matrix.addColIndexType = function(builder, colIndexType) {
+  builder.addFieldInt8(3, colIndexType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} colIndexOffset
+ */
+NetEncoding.Matrix.addColIndex = function(builder, colIndexOffset) {
+  builder.addFieldOffset(4, colIndexOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} rowIndexType
+ */
+NetEncoding.Matrix.addRowIndexType = function(builder, rowIndexType) {
+  builder.addFieldInt8(5, rowIndexType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} rowIndexOffset
+ */
+NetEncoding.Matrix.addRowIndex = function(builder, rowIndexOffset) {
+  builder.addFieldOffset(6, rowIndexOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Matrix.endMatrix = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+NetEncoding.Matrix.finishMatrixBuffer = function(builder, offset) {
+  builder.finish(offset);
+};
+
+// Exports for ECMAScript6 Modules
+export {NetEncoding};

client/src/util/stateManager/summarizeAnnotations.js

@@ -1,4 +1,5 @@
 import _ from "lodash";
+import finiteExtent from "../finiteExtent";
 
 /*
 Build and return obs/var summary using any annotation in the schema
@@ -8,6 +9,7 @@ Value will be an object, containing summary information.
 
 For continuous annotations (int, float, etc):
   <annotation_name>: {
+    categorical: false,
     range {
       min: <number>,
       max: <number>
@@ -15,12 +17,14 @@ For continuous annotations (int, float, etc):
   }
 
 For categorical annotations (boolean, string, category):
-  <annotatoin_name>: {
+  <annotation_name>: {
-    options: {
+    categorical: true,
-      <option1>: <number>,
+    categories: [ <category1>, <category2>, ... ]
+    categoryCounts: Map {
+      <category1>: <number>,
       ...
     },
-    numOptions: <number>
+    numCategories: <number>
   }
 
 Summarize will be returned for BOTH obs and var annotations.
@@ -28,19 +32,19 @@ Summarize will be returned for BOTH obs and var annotations.
 Example:
   {
     "Splice_sites_Annotated": {
-      "range": {
+      categorical: false,
+      range: {
         "min": 26,
         "max": 1075869
       }
     },
     "Selection": {
-      numOptions, 6,
+      categorical: true,
-      "options": {
+      numCategories, 3,
+      categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ],
+      categoryCounts: Map {
         "Astrocytes(HEPACAM)": 714,
         "Endothelial(BSC)": 123,
-        "Oligodendrocytes(GC)": 294,
-        "Neurons(Thy1)": 685,
-        "Microglia(CD45)": 1108,
         "Unpanned": 665
       }
     }
@@ -48,46 +52,62 @@ Example:
 
 NOTE: will not summarize the required 'name' annotation, as that is
 specified as unique per element.
-
-TODO: XXX - this data structure coerces all metadata categories into a string
-(ie, stores values as an Object property in the `options` field).   This looses
-information (eg, type) for category types which are not strings.  Consider an
-alterative data structure that does not use the object property for non-string
-data types (and does not use _.countBy to summarize).
 */
-function summarizeDimension(schema, annotations) {
+function _summarizeAnnotations(_schema, annotations) {
-  return _(schema)
+  const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
     .filter(v => v.name !== "name")
     .keyBy("name")
     .mapValues(anno => {
       const { name, type } = anno;
       const continuous = type === "int32" || type === "float32";
 
-      if (!continuous) {
+      if (continuous) {
-        const categories = _.uniq(_.flatMap(annotations, name));
+        let min;
-        const options = _.countBy(annotations, name);
+        let max;
-        const numOptions = _.size(options);
+        let nan = 0;
+        let pinf = 0;
+        let ninf = 0;
+        for (let r = 0; r < annotations.length; r += 1) {
+          const val = Number(annotations[r][name]);
+          if (Number.isFinite(val)) {
+            if (min === undefined) {
+              min = val;
+              max = val;
+            } else {
+              min = val < min ? val : min;
+              max = val > max ? val : max;
+            }
+          } else if (Number.isNaN(val)) {
+            nan += 1;
+          } else if (val > 0) {
+            pinf += 1;
+          } else {
+            ninf += 1;
+          }
+        }
         return {
-          numOptions,
+          categorical: false,
-          options,
+          range: { min, max, nan, pinf, ninf }
-          categories
         };
       }
 
-      if (continuous) {
+      /* else categorical */
-        let min = Number.POSITIVE_INFINITY;
+      const categoryCounts = new Map();
-        let max = Number.NEGATIVE_INFINITY;
+      for (let r = 0; r < annotations.length; r += 1) {
-        _.forEach(annotations, obs => {
+        const val = annotations[r][name];
-          const val = Number(obs[name]);
+        let curCount = categoryCounts.get(val);
-          min = val < min ? val : min;
+        if (curCount === undefined) curCount = 0;
-          max = val > max ? val : max;
+        categoryCounts.set(val, curCount + 1);
-        });
-        return { range: { min, max } };
       }
-
+      return {
-      throw new Error("incomprehensible schema");
+        categorical: true,
+        categories: [...categoryCounts.keys()],
+        categoryCounts,
+        numCategories: categoryCounts.size
+      };
     })
     .value();
+  return summary;
 }
 
 export default function summarizeAnnotations(
@@ -96,7 +116,7 @@ export default function summarizeAnnotations(
   varAnnotations
 ) {
   return {
-    obs: summarizeDimension(schema.annotations.obs, obsAnnotations),
+    obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations),
-    var: summarizeDimension(schema.annotations.var, varAnnotations)
+    var: _summarizeAnnotations(schema.annotations.var, varAnnotations)
   };
 }

client/src/util/stateManager/universe.js

@@ -1,8 +1,10 @@
 // jshint esversion: 6
 
 import _ from "lodash";
+
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
+import decodeMatrixFBS from "./matrix";
 
 /*
 Private helper function - create and return a template Universe
@@ -92,73 +94,40 @@ function finalize(universe) {
   return universe;
 }
 
-function RESTv02AnnotationsResponseToInternal(response) {
+function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
   /*
-  Source per the spec:
+  Convert a Matrix FBS to our internal format -- row-major array of
-  {
+  observations/cells, stored as an object.  Each obs has a key for each
-    names: [
+  annotation, plus __index__ containing its obsIndex.
-      'tissue_type', 'sex', 'num_reads', 'clusters'
-    ],
-    data: [
-      [ 0, 'lung', 'F', 39844, 99 ],
-      [ 1, 'heart', 'M', 83, 1 ],
-      [ 49, 'spleen', null, 2, "unknown cluster" ],
-      // [ obsOrVarIndex, value, value, value, value ],
-      // ...
-    ]
-  }
 
-  Internal (target) format:
+  Example:
   [
     { __index__: 0, tissue_type: "lung", sex: "F", ... },
     ...
   ]
+
+  XXX TODO: we could make use of the columns in building crossfilter
+  dimensions (they have to be recreated).  Future optimization.
   */
-  const { names, data } = response;
+  const fbs = decodeMatrixFBS(arrayBuffer);
-  const keys = ["__index__", ...names];
+  const keys = fbs.colIdx;
-  return _(data)
+  const result = Array(fbs.nRows);
-    .map(obs => _.zipObject(keys, obs))
+  for (let row = 0; row < fbs.nRows; row += 1) {
-    .value();
+    const rec = { __index__: row };
+    for (let col = 0; col < fbs.nCols; col += 1) {
+      rec[keys[col]] = fbs.columns[col][row];
+    }
+    result[row] = rec;
+  }
+  return result;
 }
 
-function RESTv02LayoutResponseToInternal(response) {
+function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
-  /*
+  const fbs = decodeMatrixFBS(arrayBuffer, true);
-  Source per the spec:
+  return {
-  {
+    X: fbs.columns[0],
-    layout: {
+    Y: fbs.columns[1]
-      ndims: 2,
-      coordinates: [
-        [ 0, 0.284483, 0.983744 ],
-        [ 1, 0.038844, 0.739444 ],
-        // [ obsOrVarIndex, X_coord, Y_coord ],
-        // ...
-      ]
-    }
-  }
-
-  Target (internal) format:
-  {
-    X: Float32Array(numObs),
-    Y: Float32Array(numObs)
-  }
-  In the same order as obsAnnotations
-  */
-  const { ndims, coordinates } = response.layout;
-  if (ndims !== 2) {
-    throw new Error("Unsupported layout dimensionality");
-  }
-
-  const layout = {
-    X: new Float32Array(coordinates.length),
-    Y: new Float32Array(coordinates.length)
   };
-
-  for (let i = 0; i < coordinates.length; i += 1) {
-    const [idx, x, y] = coordinates[i];
-    layout.X[idx] = x;
-    layout.Y[idx] = y;
-  }
-  return layout;
 }
 
 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -192,7 +161,7 @@ export function createUniverseFromRestV02Response(
   schemaResponse,
   annotationsObsResponse,
   annotationsVarResponse,
-  layoutObsResponse
+  layoutFBSResponse
 ) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -209,15 +178,15 @@ export function createUniverseFromRestV02Response(
   universe.nVar = schema.dataframe.nVar;
 
   /* annotations */
-  universe.obsAnnotations = RESTv02AnnotationsResponseToInternal(
+  universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
     annotationsObsResponse
   );
-  universe.varAnnotations = RESTv02AnnotationsResponseToInternal(
+  universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
     annotationsVarResponse
   );
 
   /* layout */
-  universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse);
+  universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
 
   universe.summary = summarizeAnnotations(
     universe.schema,
@@ -229,32 +198,24 @@ export function createUniverseFromRestV02Response(
   return finalize(universe);
 }
 
-export function convertExpressionRESTv02ToObject(universe, response) {
+export function convertDataFBStoObject(universe, arrayBuffer) {
   /*
-  /data/obs response looks like:
+  /data/var returns a flatbuffer (FBS) as described by cellxgene/fbs/matrix.fbs
-  {
-    var: [ varIndices fetched ],
-    obs: [
-      [ obsIndex, evalue, ... ],
-      ...
-    ]
-  }
 
-  convert expression toa simple Float32Array, and return
+  This routine converts the binary wire encoding into a JS object:
-  { geneName: array, geneName: array, ... }
+
-  NOTE: geneName, not varIndex
+  {
+    gene: Float32Array,
+    ...
+  }
   */
-  const vars = response.var;
+  const fbs = decodeMatrixFBS(arrayBuffer);
-  const { obs } = response;
+  const { colIdx, columns } = fbs;
   const result = {};
-  // XXX TODO: could this use _.unzip and have less code?
+
-  for (let varIdx = 0; varIdx < vars.length; varIdx += 1) {
+  for (let c = 0; c < colIdx.length; c += 1) {
-    const gene = universe.varAnnotations[vars[varIdx]].name;
+    const gene = universe.varAnnotations[colIdx[c]].name;
-    const data = new Float32Array(universe.nObs);
+    result[gene] = columns[c];
-    for (let obsIdx = 0; obsIdx < obs.length; obsIdx += 1) {
-      data[obsIdx] = obs[obsIdx][varIdx + 1];
-    }
-    result[gene] = data;
   }
   return result;
 }

client/src/util/stateManager/world.js

@@ -4,6 +4,7 @@ import _ from "lodash";
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
 import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
+import Crossfilter from "../typedCrossfilter";
 import { sliceByIndex } from "../typedCrossfilter/util";
 
 /*
@@ -231,7 +232,12 @@ export function createVarDimension(
   crossfilter,
   geneName
 ) {
-  return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
+  // return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
+  return crossfilter.dimension(
+    Crossfilter.ScalarDimension,
+    _worldVarDataCache[geneName],
+    Float32Array
+  );
 }
 
 export function createObsDimensionMap(crossfilter, world) {
@@ -246,9 +252,14 @@ export function createObsDimensionMap(crossfilter, world) {
     .filter(anno => anno.name !== "name")
     .transform((result, anno) => {
       const dimType = deduceDimensionType(anno, anno.name);
-      // XXX if dimtype is a scalar, we may be able to do better?
+      if (dimType === "enum") {
-      if (dimType) {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
+          Crossfilter.EnumDimension,
+          r => r[anno.name]
+        );
+      } else {
+        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
+          Crossfilter.ScalarDimension,
           r => r[anno.name],
           dimType
         );
@@ -259,13 +270,10 @@ export function createObsDimensionMap(crossfilter, world) {
   /*
   Add crossfilter dimensions allowing filtering on layout
   */
-  dimensionMap[layoutDimensionName("X")] = crossfilter.dimension(
+  dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
+    Crossfilter.SpatialDimension,
     obsLayout.X,
-    Float32Array
+    obsLayout.Y
-  );
-  dimensionMap[layoutDimensionName("Y")] = crossfilter.dimension(
-    obsLayout.Y,
-    Float32Array
   );
 
   return dimensionMap;

client/src/util/stateManager/worldUtil.js

@@ -0,0 +1,65 @@
+/* eslint-disable import/prefer-default-export */
+import _ from "lodash";
+
+/*
+Various utility functions operating on World/Universe
+*/
+
+/*
+Count unique category values, binning first by dim1 then by dim2
+Return:
+
+Map {
+   dim1_val1: Map {
+      dim2_val1: number,
+      dim2_val2: number,
+      ...
+   },
+   ...
+}
+
+*/
+function _countCategoryValues2D(dim1, dim2, rows) {
+  const dimMap = new Map();
+  for (let r = 0; r < rows.length; r += 1) {
+    const row = rows[r];
+    const val1 = row[dim1];
+    const val2 = row[dim2];
+    let d2Map = dimMap.get(val1);
+    if (d2Map === undefined) {
+      d2Map = new Map();
+      dimMap.set(val1, d2Map);
+    }
+    let curCount = d2Map.get(val2);
+    if (curCount === undefined) {
+      curCount = 0;
+    }
+    d2Map.set(val2, curCount + 1);
+  }
+  return dimMap;
+}
+
+let __worldUtilMemoId__ = 0;
+function _memoizedId(x) {
+  if (!x.__worldUtilMemoId__) {
+    __worldUtilMemoId__ += 1;
+    x.__worldUtilMemoId__ = __worldUtilMemoId__;
+  }
+  return x.__worldUtilMemoId__;
+}
+function _countCategoryValues2DResolver(...args) {
+  const id = args[0] + args[1] + _memoizedId(args[2]);
+  return id;
+}
+
+export const countCategoryValues2D = _.memoize(
+  _countCategoryValues2D,
+  _countCategoryValues2DResolver
+);
+
+/*
+Clear any cached data within WorldUtil caches, eg, memoized functions
+*/
+export function clearCaches() {
+  countCategoryValues2D.cache.clear();
+}

client/src/util/typedCrossfilter/index.js

@@ -27,6 +27,8 @@ more complex API.  In a few cases, elements of that API were incorporated.
     https://github.com/square/crossfilter/
 
 */
+// XXX replace
+import { polygonContains } from "d3";
 
 import PositiveIntervals from "./positiveIntervals";
 import BitArray from "./bitArray";
@@ -66,14 +68,22 @@ class TypedCrossfilter {
     return this.data;
   }
 
-  dimension(value, valueArrayType) {
+  /*
+  Create a crossfilter dimension, upon which filtering (subselection) can
+  be done.  Each dimension is typed, and has a particular set of filtering
+  semantics.
+    * ScalarDimension - backed by TypedArray values, supporting filtering
+      by value (within a value range, or one or more exact values)
+    * EnumDimension - backed by an enumeration (eg, strings, bools), filtering
+      by one or more enum categories.
+    * SpatialDimension - backed by 2D points, filter by containment within
+      various shapes (currently supports within Rectangle and within Polygon).
+  Call this method to create a dimension, passing arguments appropriate for
+  the dimension constructor.
+  */
+  dimension(DimensionType, ...rest) {
     const id = this.selection.allocDimension();
-    let dim;
+    const dim = new DimensionType(this, id, ...rest);
-    if (valueArrayType === "enum") {
-      dim = new EnumDimension(value, this, id);
-    } else {
-      dim = new ScalarDimension(value, valueArrayType, this, id);
-    }
     this.filters.push({ id, dim });
     dim.filterAll();
     return dim;
@@ -115,13 +125,34 @@ class TypedCrossfilter {
   }
 }
 
-// Base dimension type - value must be a scalar type (eg, int, float),
+// Base dimension type - not exported.
-// and value array must be a TypedArray.
+class _Dimension {
-//
+  constructor(xfltr, id) {
-class ScalarDimension {
-  constructor(value, ValueArrayType, xfltr, id) {
     this.crossfilter = xfltr;
     this._id = id;
+    this.groups = [];
+  }
+
+  dispose() {
+    this.crossfilter._freeDimension(this._id);
+    return this;
+  }
+
+  id() {
+    return this._id;
+  }
+
+  _filterUpdate() {
+    this.crossfilter.updateTime += 1;
+  }
+}
+
+// Scalar dimension type - value must be a scalar type (eg, int, float),
+// and value array must be a TypedArray.
+//
+class ScalarDimension extends _Dimension {
+  constructor(xfltr, id, value, ValueArrayType) {
+    super(xfltr, id);
 
     // current selection filter, expressed as PostiveIntervals.
     this.currentFilter = [];
@@ -151,9 +182,6 @@ class ScalarDimension {
 
     // create sort index
     this.index = makeSortIndex(array);
-
-    // groups, if any
-    this.groups = [];
   }
 
   _createValueArray(value, array) {
@@ -167,15 +195,6 @@ class ScalarDimension {
     return larray;
   }
 
-  dispose() {
-    this.crossfilter._freeDimension(this._id);
-    return this;
-  }
-
-  id() {
-    return this._id;
-  }
-
   // Argument is an array of intervals indicating records newly selected/filtered
   //
   _updateFilters(newFilter) {
@@ -209,7 +228,7 @@ class ScalarDimension {
     );
 
     this.currentFilter = cNewFilter;
-    this.crossfilter.updateTime += 1;
+    this._filterUpdate();
   }
 
   // filter by value - exact match
@@ -355,8 +374,8 @@ class ScalarDimension {
 // strings, which can be mapped into an fixed numeric range [0..n).
 //
 class EnumDimension extends ScalarDimension {
-  constructor(value, xfltr, id) {
+  constructor(xfltr, id, value) {
-    super(value, Uint32Array, xfltr, id);
+    super(xfltr, id, value, Uint32Array);
   }
 
   _createValueArray(value, array) {
@@ -408,6 +427,127 @@ class EnumDimension extends ScalarDimension {
   }
 }
 
+/*
+Super simple 2D spatial dimension, supporting basic "filter within"
+operations.
+*/
+class SpatialDimension extends _Dimension {
+  constructor(xfltr, id, X, Y) {
+    super(xfltr, id);
+
+    if (X.length !== Y.length && X.length !== this.crossfilter.data.length) {
+      throw new RangeError(
+        "SpatialDimension values must have same dimensionality as crossfilter"
+      );
+    }
+    this.X = X;
+    this.Y = Y;
+
+    this.Xindex = makeSortIndex(X);
+    this.Yindex = makeSortIndex(Y);
+  }
+
+  filterAll() {
+    this.crossfilter.selection.selectAll(this._id);
+    this._filterUpdate();
+  }
+
+  filterNone() {
+    this.crossfilter.selection.deselectAll(this._id);
+    this._filterUpdate();
+  }
+
+  /*
+  this could be smarter, but we don't currently use it...
+  */
+  filterWithinRect(northwest, southeast) {
+    const [x0, y0] = northwest;
+    const [x1, y1] = southeast;
+    const { X, Y } = this;
+    const seln = this.crossfilter.selection;
+    const { _id } = this;
+    seln.deselectAll(_id);
+    for (let i = 0, l = this.X.length; i < l; i += 1) {
+      const x = X[i];
+      const y = Y[i];
+      if (x0 <= x && x < x1 && y0 <= y && y < y1) {
+        seln.selectOne(_id, i);
+      }
+    }
+    this._filterUpdate();
+  }
+
+  /*
+  Relatively brute force filter by polygon.   Polygon is array of points, where
+  each point is [x,y].   Eg, [[x0,y0], [x1,y1], ...].
+
+  Currently uses d3.polygonContains() to test for polygon inclusion, which itself
+  uses a ray casting (crossing number) algorithm.  There are a series of optimizations
+  to make this faster:
+    * first sliced by X or Y, using an index on the axis
+    * then the polygon bounding box is used for trivial rejection
+    * then the polygon test is applied
+  */
+  filterWithinPolygon(polygon) {
+    /* return bounding box of the polygon */
+    function polygonBoundingBox(pg) {
+      let minX = Number.MAX_VALUE;
+      let minY = Number.MAX_VALUE;
+      let maxX = Number.MIN_VALUE;
+      let maxY = Number.MIN_VALUE;
+      for (let i = 0, l = pg.length; i < l; i += 1) {
+        const p = pg[i];
+        const x = p[0];
+        const y = p[1];
+        if (x < minX) minX = x;
+        if (y < minY) minY = y;
+        if (x > maxX) maxX = x;
+        if (y > maxY) maxY = y;
+      }
+      return [minX, minY, maxX, maxY];
+    }
+
+    const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
+    const { X, Y } = this;
+    let slice;
+    let index;
+    if (maxY - minY > maxX - minX) {
+      slice = [
+        lowerBoundIndirect(X, this.Xindex, minX, 0, X.length),
+        upperBoundIndirect(X, this.Xindex, maxX, 0, X.length)
+      ];
+      index = this.Xindex;
+    } else {
+      slice = [
+        lowerBoundIndirect(Y, this.Yindex, minY, 0, Y.length),
+        upperBoundIndirect(Y, this.Yindex, maxY, 0, Y.length)
+      ];
+      index = this.Yindex;
+    }
+
+    const seln = this.crossfilter.selection;
+    const { _id } = this;
+    const testWithin = polygonContains; // d3.polygonContains()
+    seln.deselectAll(_id);
+
+    for (let i = slice[0], e = slice[1]; i < e; i += 1) {
+      const rid = index[i];
+      const x = X[rid];
+      const y = Y[rid];
+      if (
+        minX <= x &&
+        x < maxX &&
+        minY <= y &&
+        y < maxY &&
+        testWithin(polygon, [x, y])
+      ) {
+        seln.selectOne(_id, rid);
+      }
+    }
+    this._filterUpdate();
+  }
+}
+
 // Groups!  Map/reduce
 //
 class ScalarGroup {
@@ -611,5 +751,6 @@ crossfilter.BitArray = BitArray;
 crossfilter.TypedCrossfilter = TypedCrossfilter;
 crossfilter.ScalarDimension = ScalarDimension;
 crossfilter.EnumDimension = EnumDimension;
+crossfilter.SpatialDimension = SpatialDimension;
 
 export default crossfilter;

dev_docs/REST_API.md

@@ -1,5 +1,11 @@
 # cellxgene REST API 0.2 specification
 
+_Note:_ this document is increasingly divergent from the code base and should be read with great suspicion. For example, it lacks any information
+about the binary encoding used by various routes, and has incorrect information about "required" routes and features. We may update it at a
+later date when the protocol stabilizes a bit.
+
+---
+
 Items marked as (_future_) are intended for future implementation, and are included in the design to round out the concept, and highlight what we would do when/if we needed more functionality. The (_future_) items are not currently used by the cellxgene web application, and may be omitted from any backend - see [Current Front-End Dependencies](#current-front-end-dependencies) for more details.
 
 _Caveat emptor, partial spec_: this is a sketch for a spec, not a full spec, and some shortcuts have been taken in the authorship. Best practices for a REST API are assumed but not documented here, such as API versioning, reasonable choices for HTTP response codes, etc. In addition, for clarity the JSON examples will not always have all required quoting (eg, on keys) - the actual implementation should use legal JSON/CSV.

dev_docs/pull_request_guidelines.md

@@ -0,0 +1,35 @@
+## Creating PR
+1. Name [username]/branchname 
+    1. Branch name should be all lowercase
+    2. Words separated by “-”
+2. Code should address only one issue ideally, make a separate PR for each task
+3. Description 
+    1. Clear explanation of issues solved
+    2. Describe why and how, when appropriate
+    3. Call out specific areas you want extra attention in review (optional)
+    4. If your PR requires more than one reviewer tag those people in the description or comments and let them know that you specifically require them
+4. Ensure that the PR updates tests and documentation and adds tests where appropriate
+5. Use github’s issue keywords when PR is addressing an issue https://help.github.com/articles/closing-issues-using-keywords/
+6. Tags (add at beginning of title)
+    1. [EASY] - small non-controversial change, easy to review
+    2. [DO NOT MERGE] - PR is in progress, do not merge changes
+
+## Review
+1. Assign at least one reviewer to submitted PRs.   Reviewers should be selected based on expertise in areas affected by the PR (eg, web UI: Colin), and should include Comp Bio and PM as needed.
+2. Reviewers should approve or request changes (not just comment) and put general and line level comments where appropriate
+3. As a PR submitter respond to all comments (eg, comment, commit a change, etc)
+4. External PRs
+    1. For external PRs or PRs not from our core team, core team should assign a reviewer and make initial contact within 1 business day
+    2. Build code on local environment and run smoke tests 
+
+## Required to Merge
+1. Travis CI Build passing
+2. At least one reviewer approved
+    1. Exceptions: 
+        1. Release PRs where version is just bumped should not need review
+        2. Complex PRs which touch multiple parts of the codebase should have reviews from all relevant parties
+3. License and Security checks (SNYK) passing. If their server is down and you didn’t add any new external npm or python packages, merge is OK
+
+## Merging
+1. Use "squash and merge" option when merging
+2. If you resolved conflicts, wait until the build passes to merge

dev_docs/release_process.md

@@ -0,0 +1,66 @@
+# cellxgene release process
+
+_This document defines the release process for cellxgene_
+
+## Overview
+
+The goal of the release process is to publish an installable package
+to PyPi, with a matching tagged release on github.
+
+The release process should result in the following side-effects:
+
+- Version number bump, using semantic versioning
+- JS assets built & packaged, committed to the repo
+- Tagged github release
+- Publication to PyPi
+
+## Recipe
+
+Follow these steps to create a release.
+
+1.  Preparation:
+    - python3.6 environment, and a cellxgene clone
+    - Define the release version number, using [semantic versioning](https://semver.org/),
+      and specifying all three digits (eg, 0.3.0)
+    - Write the release title and release notes and add to
+      [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
+2.  Create a release branch, eg, `release-version`
+3.  In the release branch:
+    - Run `make release-stage-1 PART=[major | minor | patch]` where you choose major/minor/patch depending on which part of the version
+      is being bumped (eg, 0.2.9->0.3 is minor).
+4.  Commit and push the new branch
+5.  Create a PR for the release.
+    - [optional] As needed, conduct PR review.
+6.  Merge to master
+7.  Create Github release using the version number and release notes
+    ([instructions](https://help.github.com/articles/creating-releases/)).
+    - Draft new release
+    - Type version name matching release version number from (1)
+    - Select `master` as release branch (ensure you merged the release PR)
+    - Type title `Release {version num}`
+    - [optional] Check pre-release if this release is not ready for production
+    - Publish Release
+8. Publish to pypi by performing the following steps (assumes you that you have registered for pypi, 
+and that you have write access to the cellxgene pypi package):
+    - Build the distribution and upload to test pypi `make release-stage-2`
+    - [optional] Test the test installation in a fresh virtual environment using `make install-release-test`
+    - Upload the package to real pypi using `make release-stage-final`
+    - [optional] Test the installation in a fresh virtual environment using
+      `pip install cellxgene`
+    - **Troubleshooting**:
+        - Fails to upload to test.pypi: pypi doesn't allow you to reupload a release with the same version number, 
+        if you accidentally burned a release number you want to use on prod, you have a couple options. 
+            1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
+             then upload to prod `make release-stage-final`. 
+            2) OPTION 2: (DANGER) release directly to prod: `make release-burned`.  
+            3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option
+            PART=patch until you get to an unburned version.
+        - The release doesn't install or fails your tests when you install it: Delete it from pypi - Go to pypi.org, sign in, 
+        go to the cellxgene package, click manage, then in the options drop down, click delete and 
+        follow the instructions. You will not be able to use that release number again. If it is a minor bug 
+        and not a major regression, you can just release a patch. 
+    
+
+The optional steps are for testing purposes, and are recommended
+for publishing any major releases, and any releases that significantly
+change the packaging (e.g. new bundled files, new dependencies, etc.)

dev_docs/testing_environment_for_web_hosting.md

@@ -0,0 +1,69 @@
+### How to set up a testing environment for changes related to web hosting.
+
+We often get PRs related to someone using a server to host cellxgene externally or on a local network (ex. https://github.com/chanzuckerberg/cellxgene/pull/568 ). Here is how you can test these changes locally.
+
+We are going to run docker containers for cellxgene and an apache server running a reverse proxy on a local docker network. We run the cellxgene container without exposing any ports so that we cannot access it directly, only through the apache server. We can also update our cellxgene Dockerfile so that we can install a local build instead of having to deploy to pypi.
+
+1 Create Docker network, this allows the containers to communicate with each other.
+
+```
+docker network create cxg
+```
+
+2 Create and run cellxgene container
+
+(optional) To install cellxgene from the local codebase
+
+a Create sdist file
+`make pydist`
+
+b Update Dockerfile to install from dist
+
+```
+FROM ubuntu:bionic
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+COPY [ "dist/",    "/cellxgene/dist/" ]
+
+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev && \
+    pip3 install /cellxgene/dist/cellxgene-0.5.1.tar.gz
+
+ENTRYPOINT ["cellxgene"]
+```
+
+(required) Build container
+`docker build . -t cellxgene`
+
+3 Create the proxy container
+
+In a separate directory create these two files
+
+Dockerfile
+
+```
+FROM rgoyard/apache-proxy:latest
+ADD proxy.conf /conf/
+```
+
+proxy.conf
+
+```
+ProxyPass "/data/" http://cellxgene:5005/
+ProxyPassReverse "/data/" http://cellxgene:5005/
+```
+
+Build the container
+`docker build -t proxy .`
+
+4 Run containers and attach to network
+
+```
+docker run -d -p 80:80 --network cxg --name proxy proxy
+docker run -v "$PWD/example-dataset/:/data/" --name cellxgene --network cxg cellxgene launch --host 0.0.0.0 data/pbmc3k.h5ad
+```
+
+5 Go to served site
+
+http://localhost/data/

docs/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem "github-pages", group: :jekyll_plugins

docs/_config.yml

@@ -0,0 +1,9 @@
+theme: jekyll-theme-cayman
+show_downloads: false
+baseurl: /cellxgene
+
+nav:
+  - title: Data
+    url: data.html
+  - title: FAQ
+    url: faq.html

docs/_layouts/default.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html lang="{{ site.lang | default: "en-US" }}">
+  <head>
+
+    {% if site.google_analytics %}
+      <script async src="https://www.googletagmanager.com/gtag/js?id={{ site.google_analytics }}"></script>
+      <script>
+        window.dataLayer = window.dataLayer || [];
+        function gtag(){dataLayer.push(arguments);}
+        gtag('js', new Date());
+        gtag('config', '{{ site.google_analytics }}');
+      </script>
+    {% endif %}
+    <meta charset="UTF-8">
+
+{% seo %}
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="theme-color" content="#157878">
+    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
+    <link rel="stylesheet" href="{{ '/assets/css/style.css?v=' | append: site.github.build_revision | relative_url }}">
+  </head>
+  <body>
+
+    <header class="page-header" role="banner">
+      <h1 class="project-name">{{ site.title | default: site.github.repository_name }}</h1>
+      <h2 class="project-tagline">{{ site.description | default: site.github.project_tagline }}</h2>
+      {% if site.nav %}
+        <a href="{{ site.baseurl }}/" class="btn">Home</a>
+        {% for item in site.nav %}
+          <a href="{{ item.url }}" class="btn">{{ item.title }}</a>
+        {% endfor %}
+      {% endif %}
+      {% if site.github.is_project_page %}
+        <a href="{{ site.github.repository_url }}" class="btn" target="_blank">Code</a>
+      {% endif %}
+    </header>
+
+    <main id="content" class="main-content" role="main">
+      {{ content }}
+
+      <footer class="site-footer">
+        {% if site.github.is_project_page %}
+          <span class="site-footer-owner"><a href="{{ site.github.repository_url }}">{{ site.github.repository_name }}</a> is maintained by <a href="{{ site.github.owner_url }}">{{ site.github.owner_name }}</a>.</span>
+        {% endif %}
+        <span class="site-footer-credits">This page was generated by <a href="https://pages.github.com">GitHub Pages</a>.</span>
+      </footer>
+    </main>
+  </body>
+</html>

docs/assets/css/style.scss

@@ -0,0 +1,4 @@
+---
+---
+
+@import "{{ site.theme }}";

docs/data.md

@@ -0,0 +1,29 @@
+---
+layout: default
+title: data
+description: Data
+---
+
+
+# data vignette: how to use cellxgene prepare
+
+#### coming soon!
+
+# example datasets to use with cellxgene
+
+### Examination of single cells from primary human pancreas tissue
+cells: 2,544  
+tissue(s): pancreas  
+data: [Human Cell Atlas Data Portal](https://prod.data.humancellatlas.org/explore/projects?filter=%5B%7B%22facetName%22%3A%22organ%22%2C%22terms%22%3A%5B%22pancreas%22%5D%7D%2C%7B%22facetName%22%3A%22project%22%2C%22terms%22%3A%5B%22Single+cell+transcriptome+analysis+of+human+pancreas%22%5D%7D%5D)  
+paper: [Enge, Martin, et al.](https://www.cell.com/cell/fulltext/S0092-8674(17)31053-X?_returnURL=https%3A%2F%2Flinkinghub.elsevier.com%2Fretrieve%2Fpii%2FS009286741731053X%3Fshowall%3Dtrue)
+
+### Tabula Muris
+cells: 53,800  
+tissue(s): muscle, pancreas, bone, large intestine, heart, brain, fat, mammary gland, tongue , diaphragm, bladder, spleen, thymus, lung , skin, liver, trachea, kidney  
+data: [Tabula Muris Data](https://github.com/czbiohub/tabula-muris-vignettes/tree/master/data)  
+paper: [Tabula Muris Consortium](https://www.nature.com/articles/s41586-018-0590-4)  
+
+### Transcriptional profiling of 1.3 million brain cells
+cells: 1,330,000  
+tissue(s): brain  
+data: [10x Genomics](https://community.10xgenomics.com/t5/10x-Blog/Our-1-3-million-single-cell-dataset-is-ready-to-download/ba-p/276)  

docs/faq.md

@@ -0,0 +1,94 @@
+---
+layout: default
+title: FAQ
+description: Frequently Asked Questions
+---
+
+# Data formatting
+
+#### What file formats can I use with _cellxgene_?
+
+Currently, you can go straight into `cellxgene launch` with your own analyzed data in h5ad format, after you have performed dimenstionality reduction (tsne, umap) and clustering (louvain).
+
+If your data is in a different format, and/or you still need to perform dimensionality reduction and clustering, `cellxgene` can do that for you with the `prepare` command. `cellxgene prepare` runs `scanpy` under the hood and can read in any format that is currently supported by `scanpy` (including mtx, loom, and more listed [here](https://scanpy.readthedocs.io/en/latest/api/index.html#reading)).
+
+The output of `cellxgene prepare` is a h5ad file with your computed clusters and tsne/umap projections that can be used in `cellxgene launch`.
+
+#### I have a directory of 10X-Genomics data with _mtx_ files and I've never used _scanpy_, can I use _cellxgene_?
+
+Yep! This should only take a couple steps. We'll assume your data is in a folder called `data/` and you've successfully installed `cellxgene` with the `louvain` packages as described above. Just run
+
+```
+cellxgene prepare data/ --output=data-processed.h5ad --layout=umap
+```
+
+Depending on the size of the dataset, this may take some time. Once it's done, call
+
+```
+cellxgene launch data-processed.h5ad --layout=umap --open
+```
+
+And your web browser should open with an interactive view of your data.
+
+#### I have extra metadata that I want to add to my dataset
+
+Currently this is not supported directly, but you should be able to do this yourself using `scanpy`. For example, this [notebook](https://github.com/falexwolf/fun-analyses/blob/master/tabula_muris/tabula_muris.ipynb) shows adding the contents of a `csv` file with metadata to an `anndata` object. For now, you could do this manually on your data in the same way and then save out the result before loading into `cellxgene`.
+
+#### What part of the _anndata_ objects does cellxgene pull in for visualization?
+
+- `.obs` and `.var` annotations are use to extract metadata for filtering
+- `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression
+- `.obsm` is used for layout
+
+# Algorithms
+
+#### How are you computing and sorting differential expression results?
+
+We use a [Welch's _t_-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) implementation including the same variance overestimation correction as used in `scanpy`. We sort the `tscore` to identify the top N genes, and then filter to remove any that fall below a cutoff log fold change value, which can help remove spurious test results. The default threshold is `0.01` and can be changed using the option `--diffexp-lfc-cutoff`.
+
+# Problems, errors, & bugs
+
+#### How do I create a Python 3.6 environment for _cellxgene_?
+
+If you use conda and want to create a [conda environment](https://conda.io/docs/user-guide/tasks/manage-environments.html) for _cellxgene_ you can use the following commands
+
+```
+conda create --yes -n cellxgene python=3.6
+conda activate cellxgene
+pip install cellxgene
+```
+
+Or you can create a virtual environment by using
+
+```
+ENV_NAME=cellxgene
+python3.6 -m venv ${ENV_NAME}
+source ${ENV_NAME}/bin/activate
+pip install cellxgene
+```
+
+#### In my _prepare_ command I received the following error `Warning: louvain module is not installed, no clusters will be calculated. To fix this please install cellxgene with the optional feature louvain enabled`
+
+Louvain clustering requires additional dependencies, so we don't include them by default. For now, you need to specify that you want these packages by using
+
+```
+pip install cellxgene[louvain]
+```
+
+#### I ran _prepare_ and I'm getting results that look unexpected
+
+You might want to try running one of the preprocessing recipes included with `scanpy` (read more about them [here](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes)). You can specify this with the `--recipe` option, such as
+
+```
+cellxgene prepare data/ --output=data-processed.h5ad --recipe=zheng17
+```
+
+It should be easy to run `prepare` then call `cellxgene launch` a few times with different settings to explore different behaviors. We may explore adding other preprocessing options in the future.
+
+#### I tried to `pip install cellxgene` and got a weird error I don't understand
+
+This may happen, especially as we work out bugs in our installation process! Please create a new [Github issue](https://github.com/chanzuckerberg/cellxgene/issues), explain what you did, and include all the error messages you saw. It'd also be super helpful if you call `pip freeze` and include the full output alongside your issue.
+
+#### I'm following the developer instructions and get an error about "missing files and directories” when trying to build the client
+
+This is likely because you do not have node and npm installed, we recommend using [nvm](https://github.com/creationix/nvm) if you're new to using these tools.

docs/index.md

@@ -0,0 +1,37 @@
+_cellxgene_ is an interactive data explorer for single-cell transcriptomics data. Whether you need to visualize one thousand cells or one million, _cellxgene_ helps you gain insight into your single-cell data.
+
+## features
+
+#### Flexible selections, coloring, and differential expression of your selected sets of cells
+<img src="diffexp.gif" width="600"/>
+
+#### Single-gene analyses (e.g. expression analysis)
+<img src="customGene.gif" width="600" />
+
+## getting started
+
+_cellxgene_ **only** supports Python 3.6. We recommend [installing _cellxgene_ into a conda or virtual environment.](/faq.html#how-do-i-create-a-python-36-environment-for-cellxgene)
+
+Install the package.
+``` bash
+pip install cellxgene
+```
+
+Download an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+
+``` bash
+curl -o pbmc3k.h5ad https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad
+```
+
+Launch _cellxgene_
+``` bash
+cellxgene launch pbmc3k.h5ad
+```
+
+## getting help
+
+We'd love to hear from you!
+
+For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cziscience-slack.herokuapp.com/) and say "hi!".
+
+For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).

docs/release_process.md

@@ -1,57 +0,0 @@
-# cellxgene release process
-
-_This document defines the release process for cellxgene_
-
-## Overview
-
-The goal of the release process is to publish an installable package
-to PyPi, with a matching tagged release on github.
-
-The release process should result in the following side-effects:
-
-- Version number bump, using semantic versioning
-- JS assets built & packaged, committed to the repo
-- Tagged github release
-- Publication to PyPi
-
-## Process
-
-Follow these steps to create a release.
-
-1.  Preparation:
-    - Define the release version number, using [semantic versioning](https://semver.org/)
-    - Write the release title and release notes and add to 
-    [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
-2.  Create a release branch, eg, `release-version`
-3.  In the release branch:
-    - Run `bumpversion --config-file .bumpversion.cfg [major | minor | patch]`
-    - Clean up existing environment using `bin/clean`
-    - Build the JS asserts using `bin/build-client`
-4.  Commit and push the new branch
-5.  Create a PR for the release.
-    - [optional] As needed, conduct PR review.
-6.  Merge to master
-7.  Create Github release using the version number and release notes 
-([instructions](https://help.github.com/articles/creating-releases/)).
-    - Draft new release
-    - Type version name matching release version number from (1)
-    - Select `master` as release branch (ensure you merged the release PR)
-    - Type title `Release {version num}`
-    - [optional] Check pre-release if this release is not ready for production
-    - Publish Release
-8.  Publish to pypi by performing the following steps (assumes you have `setuptools` and `twine` installed and that you 
-have registered for pypi and have write access to the cellxgene pypi package)
-    - Build the distribution by calling
-    `python setup.py sdist`
-      inside the top-level directory
-    - [optional] Upload the package to test pypi
-      `twine upload --repository-url https://test.pypi.org/legacy/ dist/*`
-    - [optional] Test the test installation in a fresh virtual environment using
-      `pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene`
-    - Upload the package to real pypi using `twine upload dist/*`
-    - [optional] Test the installation in a fresh virtual environment using
-      `pip install cellxgene`
-
-The optional steps are for testing purposes, and are recommended
-for publishing any major releases, and any releases that significantly
-change the packaging (e.g. new bundled files, new dependencies, etc.)

fbs/matrix.fbs

@@ -0,0 +1,73 @@
+
+/*
+
+  Flatbuffers schema for use in cellxgene wire-format.
+
+  Schema defines a general purpose, polymorphic, 2D matrix.  Data is
+  organized in a columnar layout.   Each column is homomorphic, and
+  several column types are supported:
+    - IEEE 32 and 64 bit floats
+    - signed and unsigned 32 bit integers
+    - JSON/UTF8 encoded array (for other types)
+
+  https://github.com/google/flatbuffers
+  http://google.github.io/flatbuffers/
+
+  NOTE: IF YOU MODIFY THIS FILE, YOU MUST RECOMPILE AND COMMIT
+  RESULTING FILES TO THE REPO:
+  * server/app/util/fbs/NetEncoding/*
+  * client/src/util/stateManager/matrix_generated.js
+
+*/
+
+namespace NetEncoding;
+
+table Float32Array {
+  data: [float32];
+}
+
+table Uint32Array {
+  data: [uint32];
+}
+
+table Int32Array {
+  data: [int32];
+}
+
+table Float64Array {
+  data: [float64];
+}
+
+table JSONEncodedArray {
+  // contains a UTF-8/JSON encoded array.  Used to store other
+  // types (or polymorphic arrays)
+  data: [uint8];
+}
+
+union TypedArray {
+  Float32Array,
+  Int32Array,
+  Uint32Array,
+  Float64Array,
+  JSONEncodedArray
+}
+
+// Extra level of indirection required because vector of union not yet supported
+table Column {
+  u: TypedArray;
+}
+
+// 2D matrix stored in columnar layout
+//
+table Matrix {
+  n_rows: uint32;     // all columns have this length
+  n_cols: uint32;     // same as columns.length
+  columns: [Column];  // length n_cols
+
+  // optional row and column index, with same length as corresponding dimension.
+  // If null, defaults to numeric index, ie, [0, n_rows) or [0, n_cols)
+  col_index: TypedArray;
+  row_index: TypedArray;
+}
+
+root_type Matrix;

makefile

@@ -0,0 +1,122 @@
+BUILDDIR := build
+CLIENTBUILD := $(BUILDDIR)/client
+SERVERBUILD := $(BUILDDIR)/server
+CLEANFILES :=  $(BUILDDIR)/ client/build dist cellxgene.egg-info
+
+PART ?= patch
+
+# BUILDING PACKAGE
+
+build : clean build-server
+	@echo "done"
+
+build-server : build-client
+	mkdir -p $(SERVERBUILD)
+	cp -r server/* $(SERVERBUILD)
+	cp -r client/build/  $(CLIENTBUILD)
+	mkdir -p $(SERVERBUILD)/app/web/static/img
+	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/
+	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/
+	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img
+	cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/
+	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
+
+build-client :
+	npm install --prefix client/ client
+	npm run  --prefix client build
+
+# If you are actively developing in the server folder use this, dirties the source tree
+build-for-server-dev : clean-server build-client
+	mkdir -p server/app/web/static/img
+	cp client/build/index.html server/app/web/templates/
+	cp -r client/build/static server/app/web/
+	cp client/build/favicon.png server/app/web/static/img
+	cp client/build/service-worker.js server/app/web/static/js/
+
+clean : clean-lite clean-server
+	rm -rf client/node_modules
+
+# cleaning node_modules is the longest one, so we avoid that if possible
+clean-lite :
+	rm -rf $(CLEANFILES)
+
+clean-server :
+	rm -f server/app/web/templates/index.html
+	rm -rf server/app/web/static
+
+.PHONY : build build-server build-client build-for-server-dev clean clean-lite clean-server
+
+# CREATING DISTRIBUTION RELEASE
+
+pydist : build
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
+.PHONY : pydist
+
+# RELEASE HELPERS
+
+# create new version to commit to master
+release-stage-1 : dev-env bump clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# build dist and release to dev pypi
+release-stage-2 : dev-env pydist twine
+	@echo "Dist built and uploaded to test.pypi.org"
+	@echo "Test the install `make install-release-test` and then upload to Pypi prod"
+	@echo "`make twine-prod`"
+
+release-stage-final: twine-prod
+	@echo "Release uploaded to pypi.org"
+
+# DANGER: releases directly to prod
+# use this if you accidently burned a test release version number,
+release-burned : dev-env pydist twine-prod
+	@echo "Dist built and uploaded to pypi.org"
+	@echo "Test the install `make install-release`"
+
+dev-env :
+	pip install -r server/requirements-dev.txt
+
+# give PART=[major, minor, part] as param to make bump
+bump :
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
+twine :
+	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+
+twine-prod :
+	twine upload dist/*
+
+# quicker than re-building client
+gen-package-lock :
+	npm install --prefix client/ client
+
+.PHONY : release-stage-1 release-stage-2 release-stage-final release-burned dev-env bump twine twine-prod gen-package-lock
+
+# INSTALL
+
+# setup.py sucks when you have your library in a separate folder, adding these in to help setup envs
+
+# install from build directory
+install : uninstall
+	cd $(BUILDDIR); pip install -e .
+
+# install from source tree for development
+install-dev : uninstall
+	pip install -e .
+
+# install from test.pypi to test your release
+install-release-test : uninstall
+	pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
+
+# install from pypi to test your release
+install-release : uninstall
+	pip install cellxgene
+	@echo "Installed cellxgene from pypi.org"
+
+uninstall :
+	yes | pip uninstall cellxgene || true
+
+.PHONY : install install-dev install-release-test install-release uninstall

server/__main__.py

@@ -2,11 +2,14 @@
 if __package__ is None:
     import sys
     from pathlib import Path
+
     PKG_PATH = Path(__file__).parent
     sys.path.insert(0, str(PKG_PATH.parent))
-    import server
+    import server # noqa F401
+
     __package__ = PKG_PATH.name
 
 # Main thing
-from .cli.cli import cli
+from .cli.cli import cli # noqa F402
+
 cli()

server/app/app.py

@@ -14,16 +14,14 @@ REACTIVE_LIMIT = 1_000_000
 
 app = Flask(__name__, static_folder="web/static")
 app.json_encoder = Float32JSONEncoder
-cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860000})
+cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000})
 Compress(app)
 CORS(app)
 
 # Config
 SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
 
-app.config.update(
+app.config.update(SECRET_KEY=SECRET_KEY)
-    SECRET_KEY=SECRET_KEY,
-)
 
 # Application Data
 data = None
@@ -36,7 +34,13 @@ docs.append(resources.get_swagger_doc())
 app.register_blueprint(webapp.bp)
 app.register_blueprint(resources.blueprint)
 app.register_blueprint(
-    get_swagger_blueprint(docs, "/api/swagger", produces=["application/json"], title="cellxgene rest api",
+    get_swagger_blueprint(
-                          description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene"))
+        docs,
+        "/api/swagger",
+        produces=["application/json"],
+        title="cellxgene rest api",
+        description="An API connecting ExpressionMatrix2 clustering algorithm to cellxgene",
+    )
+)
 
 app.add_url_rule("/", endpoint="index")

server/app/driver/driver.py

@@ -11,7 +11,6 @@ Sort order for methods
 
 
 class CXGDriver(metaclass=ABCMeta):
-
     def __init__(self, data, args):
         self.data = self._load_data(data)
         self.layout_method = args["layout"]
@@ -24,11 +23,8 @@ class CXGDriver(metaclass=ABCMeta):
     def features(self):
         features = {
             "cluster": {"available": False},
-            "layout": {
+            "layout": {"obs": {"available": False}, "var": {"available": False}},
-                "obs": {"available": False},
+            "diffexp": {"available": False},
-                "var": {"available": False},
-            },
-            "diffexp": {"available": False}
         }
         # TODO - Interactive limit should be generated from the actual available methods see GH issue #94
         if self.layout_method:
@@ -70,6 +66,11 @@ class CXGDriver(metaclass=ABCMeta):
         """
         pass
 
+    @abstractmethod
+    def annotation_to_fbs_matrix(self, axis, field=None):
+        """ Same as annotation(), except returns a flatbuffer, and does not support filtering. """
+        pass
+
     @abstractmethod
     def data_frame(self, filter, axis):
         """
@@ -83,6 +84,10 @@ class CXGDriver(metaclass=ABCMeta):
         """
         pass
 
+    @abstractmethod
+    def data_frame_to_fbs_matrix(self, filter, axis):
+        pass
+
     @abstractmethod
     def diffexp_topN(self, obsFilter1, obsFilter2, top_n=None, interactive_limit=None):
         """
@@ -108,3 +113,8 @@ class CXGDriver(metaclass=ABCMeta):
         :return:  [cellid, x, y, ...]
         """
         pass
+
+    @abstractmethod
+    def layout_to_fbs_matrix(self, filter):
+        """ same as layout, except returns a flatbuffer """
+        pass

server/app/scanpy_engine/diffexp.py

@@ -1,4 +1,3 @@
-
 import numpy as np
 from scipy import sparse, stats
 
@@ -10,18 +9,31 @@ def _mean_var_n(X):
     than naive methods (and same method used by numpy.var())
     https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass
     """
-    n = X.shape[0]
+    # fp_err_occurred is a flag indicating that a floating point error
-    if sparse.issparse(X):
+    # occured somewhere in our compute.  Used to trigger non-finite
-        mean = X.mean(axis=0).A1
+    # number handling.
-        dfm = X - mean
+    fp_err_occurred = False
-        sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
-        v = sumsq / (n - 1)
-    else:
-        mean = X.mean(axis=0)
-        dfm = X - mean
-        sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
-        v = sumsq / (n - 1)
 
+    def fp_err_set(err, flag):
+        nonlocal fp_err_occurred
+        fp_err_occurred = True
+
+    with np.errstate(divide="call", invalid="call", call=fp_err_set):
+        n = X.shape[0]
+        if sparse.issparse(X):
+            mean = X.mean(axis=0).A1
+            dfm = X - mean
+            sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
+            v = sumsq / (n - 1)
+        else:
+            mean = X.mean(axis=0)
+            dfm = X - mean
+            sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
+            v = sumsq / (n - 1)
+
+    if fp_err_occurred:
+        mean[np.isfinite(mean) == False] = 0    # noqa: E712
+        v[np.isfinite(v) == False] = 0          # noqa: E712
     return mean, v, n
 
 
@@ -64,19 +76,19 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     sum_vn = vnA + vnB
 
     # degrees of freedom for Welch's t-test
-    with np.errstate(divide='ignore', invalid='ignore'):
+    with np.errstate(divide="ignore", invalid="ignore"):
-        dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
+        dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
     dof[np.isnan(dof)] = 1
 
     # Welch's t-test score calculation
-    with np.errstate(divide='ignore', invalid='ignore'):
+    with np.errstate(divide="ignore", invalid="ignore"):
         tscores = (meanA - meanB) / np.sqrt(sum_vn)
     tscores[np.isnan(tscores)] = 0
 
     # p-value
     pvals = stats.t.sf(np.abs(tscores), dof) * 2
     pvals_adj = pvals * adata._X.shape[1]
-    pvals_adj[pvals_adj > 1] = 1        # cap adjusted p-value at 1
+    pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
 
     # logfoldchanges: log2(meanA / meanB)
     logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
@@ -106,8 +118,5 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = [[sort_order[i],
+    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
-               logfoldchanges_top_n[i],
-               pvals_top_n[i],
-               pvals_adj_top_n[i]] for i in range(top_n)]
     return result

server/app/scanpy_engine/scanpy_engine.py

@@ -8,8 +8,16 @@ from scipy import sparse
 
 from server.app.driver.driver import CXGDriver
 from server.app.util.constants import Axis, DEFAULT_TOP_N
-from server.app.util.errors import FilterError, InteractiveError, PrepareError, ScanpyFileError
+from server.app.util.errors import (
+    FilterError,
+    InteractiveError,
+    JSONEncodingValueError,
+    PrepareError,
+    ScanpyFileError,
+)
+from server.app.util.utils import jsonify_scanpy
 from server.app.scanpy_engine.diffexp import diffexp_ttest
+from server.app.util.fbs.matrix import encode_matrix_fbs
 
 """
 Sort order for methods
@@ -22,7 +30,6 @@ Sort order for methods
 
 
 class ScanpyEngine(CXGDriver):
-
     def __init__(self, data, args):
         super().__init__(data, args)
         self._alias_annotation_names(Axis.OBS, args["obs_names"])
@@ -55,21 +62,29 @@ class ScanpyEngine(CXGDriver):
             df_axis.rename(inplace=True, columns={"index": "name"})
         elif name in df_axis.columns:
             if name not in df_axis.columns:
-                raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
+                raise KeyError(
+                    f"Annotation name {name}, specified in --{ax_name}-name does not exist."
+                )
             if not df_axis[name].is_unique:
-                raise KeyError(f"Values in -{ax_name}-name must be unique. "
+                raise KeyError(
-                               "Please prepare data to contain unique values.")
+                    f"Values in -{ax_name}-name must be unique. "
+                    "Please prepare data to contain unique values."
+                )
             # reset index to simple range; alias user-specified annotation to "name"
             df_axis.reset_index(drop=True, inplace=True)
             df_axis.rename(inplace=True, columns={name: "name"})
         else:
-            raise KeyError(f"Annotation name {name}, specified in --{ax_name}_name does not exist.")
+            raise KeyError(
+                f"Annotation name {name}, specified in --{ax_name}_name does not exist."
+            )
 
     @staticmethod
     def _can_cast_to_float32(ann):
         if ann.dtype.kind == "f":
             if not np.can_cast(ann.dtype, np.float32):
-                warnings.warn(f"Annotation {ann.name} will be converted to 32 bit float and may loose precision.")
+                warnings.warn(
+                    f"Annotation {ann.name} will be converted to 32 bit float and may lose precision."
+                )
             return True
         return False
 
@@ -88,12 +103,9 @@ class ScanpyEngine(CXGDriver):
             "dataframe": {
                 "nObs": self.cell_count,
                 "nVar": self.gene_count,
-                "type": str(self.data.X.dtype)
+                "type": str(self.data.X.dtype),
             },
-            "annotations": {
+            "annotations": {"obs": [], "var": []},
-                "obs": [],
-                "var": []
-            }
         }
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
@@ -114,7 +126,9 @@ class ScanpyEngine(CXGDriver):
                     ann_schema["type"] = "categorical"
                     ann_schema["categories"] = curr_axis[ann].dtype.categories.tolist()
                 else:
-                    raise TypeError(f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene.")
+                    raise TypeError(
+                        f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
+                    )
                 self.schema["annotations"][ax].append(ann_schema)
 
     @staticmethod
@@ -125,32 +139,41 @@ class ScanpyEngine(CXGDriver):
         try:
             result = sc.read(data, cache=True)
         except ValueError:
-            raise ScanpyFileError("File must be in the .h5ad format. Please read "
+            raise ScanpyFileError(
-                                  "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
+                "File must be in the .h5ad format. Please read "
-                                  "learn more about this format. You may be able to convert your file into this format "
+                "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
-                                  "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
+                "learn more about this format. You may be able to convert your file into this format "
-                                  "information.")
+                "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
+                "information."
+            )
         except Exception as e:
-            raise ScanpyFileError(f"Error while loading file: {e}, File must be in the .h5ad format, please check "
+            raise ScanpyFileError(
-                                  f"that your input and try again.")
+                f"Error while loading file: {e}, File must be in the .h5ad format, please check "
+                f"that your input and try again."
+            )
         return result
 
     def _validate_data_types(self):
         if self.data.X.dtype != "float32":
-            warnings.warn(f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
+            warnings.warn(
-                          f"Precision may be truncated.")
+                f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
+                f"Precision may be truncated."
+            )
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
             for ann in curr_axis:
                 datatype = curr_axis[ann].dtype
-                downcast_map = {"int64": "int32",
+                downcast_map = {
-                                "uint32": "int32",
+                    "int64": "int32",
-                                "uint64": "int32",
+                    "uint32": "int32",
-                                "float64": "float32",
+                    "uint64": "int32",
-                                }
+                    "float64": "float32",
+                }
                 if datatype in downcast_map:
-                    warnings.warn(f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
+                    warnings.warn(
-                                  f"Data will be downcast to {downcast_map[datatype]}.")
+                        f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
+                        f"Data will be downcast to {downcast_map[datatype]}."
+                    )
                 if isinstance(datatype, CategoricalDtype):
                     category_num = len(curr_axis[ann].dtype.categories)
                     if category_num > 500 and category_num > self.max_category_items:
@@ -158,7 +181,8 @@ class ScanpyEngine(CXGDriver):
                             f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
                             f"cumbersome or slow to display. We recommend setting the "
                             f"--max-category-items option to 500, this will hide categorical "
-                            f"annotations with more than 500 categories in the UI")
+                            f"annotations with more than 500 categories in the UI"
+                        )
 
     def _validate_data_calculations(self):
         layout_key = f"X_{self.layout_method}"
@@ -170,7 +194,8 @@ class ScanpyEngine(CXGDriver):
                 f" layout may have been computed. The requested layout must be pre-calculated and saved "
                 f"back in the h5ad file. You can run "
                 f"`cellxgene prepare --layout {self.layout_method} <datafile>` "
-                f"to solve this problem. ")
+                f"to solve this problem. "
+            )
 
     def filter_dataframe(self, filter):
         """
@@ -191,7 +216,7 @@ class ScanpyEngine(CXGDriver):
 
     @staticmethod
     def _annotation_filter_to_mask(filter, d_axis, count):
-        mask = np.ones((count, ), dtype=bool)
+        mask = np.ones((count,), dtype=bool)
         for v in filter:
             if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]:
                 key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"])
@@ -209,24 +234,28 @@ class ScanpyEngine(CXGDriver):
 
     @staticmethod
     def _index_filter_to_mask(filter, count):
-        mask = np.zeros((count, ), dtype=bool)
+        mask = np.zeros((count,), dtype=bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0]:i[1]] = True
+                mask[i[0] : i[1]] = True
             else:
                 mask[i] = True
         return mask
 
     @staticmethod
     def _axis_filter_to_mask(filter, d_axis, count):
-        mask = np.ones((count, ), dtype=bool)
+        mask = np.ones((count,), dtype=bool)
         if "index" in filter:
-            mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count))
+            mask = np.logical_and(
+                mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)
+            )
         if "annotation_value" in filter:
-            mask = np.logical_and(mask,
+            mask = np.logical_and(
-                                  ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"],
+                mask,
-                                                                          d_axis,
+                ScanpyEngine._annotation_filter_to_mask(
-                                                                          count))
+                    filter["annotation_value"], d_axis, count
+                ),
+            )
         return mask
 
     def _filter_to_mask(self, filter, use_slices=True):
@@ -239,9 +268,13 @@ class ScanpyEngine(CXGDriver):
 
         if filter is not None:
             if Axis.OBS in filter:
-                obs_selector = self._axis_filter_to_mask(filter["obs"], self.data.obs, self.data.n_obs)
+                obs_selector = self._axis_filter_to_mask(
+                    filter["obs"], self.data.obs, self.data.n_obs
+                )
             if Axis.VAR in filter:
-                var_selector = self._axis_filter_to_mask(filter["var"], self.data.var, self.data.n_vars)
+                var_selector = self._axis_filter_to_mask(
+                    filter["var"], self.data.var, self.data.n_vars
+                )
         return obs_selector, var_selector
 
     @staticmethod
@@ -256,8 +289,11 @@ class ScanpyEngine(CXGDriver):
 
         https://docs.scipy.org/doc/scipy/reference/sparse.html
         """
-        prefer_row_access = sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) \
+        prefer_row_access = (
+            sparse.isspmatrix_csr(data._X)
+            or sparse.isspmatrix_lil(data._X)
             or sparse.isspmatrix_bsr(data._X)
+        )
         if prefer_row_access:
             # Row-major slicing
             if obs_selector is not None:
@@ -292,7 +328,7 @@ class ScanpyEngine(CXGDriver):
                 fields = obs.columns.tolist()
             result = {
                 "names": fields,
-                "data": DataFrame(obs[fields]).to_records(index=True).tolist()
+                "data": DataFrame(obs[fields]).to_records(index=True).tolist(),
             }
         else:
             var = self.data.var[var_selector]
@@ -300,9 +336,21 @@ class ScanpyEngine(CXGDriver):
                 fields = var.columns.tolist()
             result = {
                 "names": fields,
-                "data": DataFrame(var[fields]).to_records(index=True).tolist()
+                "data": DataFrame(var[fields]).to_records(index=True).tolist(),
             }
-        return result
+        try:
+            return jsonify_scanpy(result)
+        except ValueError:
+            raise JSONEncodingValueError("Error encoding annotations to JSON")
+
+    def annotation_to_fbs_matrix(self, axis, fields=None):
+        if axis == Axis.OBS:
+            df = self.data.obs
+        else:
+            df = self.data.var
+        if fields is not None and len(fields) > 0:
+            df = df[fields]
+        return encode_matrix_fbs(df, col_idx=df.columns)
 
     def data_frame(self, filter, axis):
         """
@@ -326,27 +374,71 @@ class ScanpyEngine(CXGDriver):
         if axis == Axis.OBS:
             result = {
                 "var": var_index_sliced.tolist(),
-                "obs": DataFrame(_X, index=obs_index_sliced).to_records(index=True).tolist()
+                "obs": DataFrame(_X, index=obs_index_sliced)
+                .to_records(index=True)
+                .tolist(),
             }
         else:
             result = {
                 "obs": obs_index_sliced.tolist(),
-                "var": DataFrame(_X.T, index=var_index_sliced).to_records(index=True).tolist()
+                "var": DataFrame(_X.T, index=var_index_sliced)
+                .to_records(index=True)
+                .tolist(),
             }
-        return result
+        try:
+            return jsonify_scanpy(result)
+        except ValueError:
+            raise JSONEncodingValueError("Error encoding dataframe to JSON")
+
+    def data_frame_to_fbs_matrix(self, filter, axis):
+        """
+        Retrieves data 'X' and returns in a flatbuffer Matrix.
+        :param filter: filter: dictionary with filter params
+        :param axis: string obs or var
+        :return: flatbuffer Matrix
+
+        Caveats:
+        * currently only supports access on VAR axis
+        * currently only supports filtering on VAR axis
+        """
+        if axis != Axis.VAR:
+            raise ValueError("Only VAR dimension access is supported")
+        try:
+            obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
+        except (KeyError, IndexError) as e:
+            raise FilterError(f"Error parsing filter: {e}") from e
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # Currently only handles VAR dimension
+        X = self.data._X
+        if var_selector is not None:
+            X = X[:, var_selector]
+        return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None)
 
     def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None):
         if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
             raise FilterError("Observation filters may not contain vaiable conditions")
         try:
-            obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"], self.data.obs, self.data.n_obs)
+            obs_mask_A = self._axis_filter_to_mask(
-            obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"], self.data.obs, self.data.n_obs)
+                obsFilterA["obs"], self.data.obs, self.data.n_obs
+            )
+            obs_mask_B = self._axis_filter_to_mask(
+                obsFilterB["obs"], self.data.obs, self.data.n_obs
+            )
         except (KeyError, IndexError) as e:
             raise FilterError(f"Error parsing filter: {e}") from e
         if top_n is None:
             top_n = DEFAULT_TOP_N
-        result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n, self.diffexp_lfc_cutoff)
+        result = diffexp_ttest(
-        return result
+            self.data, obs_mask_A, obs_mask_B, top_n, self.diffexp_lfc_cutoff
+        )
+        try:
+            return jsonify_scanpy(result)
+        except ValueError:
+            raise JSONEncodingValueError(
+                "Error encoding differential expression to JSON"
+            )
 
     def layout(self, filter, interactive_limit=None):
         """
@@ -370,11 +462,41 @@ class ScanpyEngine(CXGDriver):
         try:
             df_layout = df.obsm[f"X_{self.layout_method}"]
         except ValueError as e:
-            raise PrepareError(f"Layout has not been calculated using {self.layout_method}, "
+            raise PrepareError(
-                               f"please prepare your datafile and relaunch cellxgene") from e
+                f"Layout has not been calculated using {self.layout_method}, "
-        normalized_layout = DataFrame((df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
+                f"please prepare your datafile and relaunch cellxgene"
-                                      index=df.obs.index)
+            ) from e
-        return {
+        normalized_layout = DataFrame(
-            "ndims": normalized_layout.shape[1],
+            (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min()),
-            "coordinates": normalized_layout.to_records(index=True).tolist()
+            index=df.obs.index,
-        }
+        )
+        try:
+            return jsonify_scanpy(
+                {
+                    "layout": {
+                        "ndims": normalized_layout.shape[1],
+                        "coordinates": normalized_layout.to_records(
+                            index=True
+                        ).tolist(),
+                    }
+                }
+            )
+        except ValueError:
+            raise JSONEncodingValueError("Error encoding layout to JSON")
+
+    def layout_to_fbs_matrix(self):
+        """
+        Return the default 2-D layout for cells as a FBS Matrix.
+
+        Caveats:
+        * does not support filtering
+        * only returns Matrix in columnar layout
+        """
+        try:
+            df_layout = self.data.obsm[f"X_{self.layout_method}"]
+        except ValueError as e:
+            raise PrepareError(
+                f"Layout has not been calculated using {self.layout_method}, "
+                f"please prepare your datafile and relaunch cellxgene") from e
+        normalized_layout = (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min())
+        return encode_matrix_fbs(normalized_layout.astype(dtype=np.float32), col_idx=None, row_idx=None)

server/app/util/constants.py

@@ -25,3 +25,8 @@ class Axis(AugmentedEnum):
 class DiffExpMode(AugmentedEnum):
     TOP_N = "topN"
     VAR_FILTER = "varFilter"
+
+
+JSON_NaN_to_num_warning_msg = (
+    "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
+)

server/app/util/errors.py

@@ -16,6 +16,15 @@ class InteractiveError(Exception):
         self.message = message
 
 
+class JSONEncodingValueError(Exception):
+    """
+    Raised when file loaded into scanpy is misformatted
+    """
+
+    def __init__(self, message):
+        self.message = message
+
+
 class MimeTypeError(Exception):
     """
     Raised when incompatible MIME type selected

server/app/util/fbs/NetEncoding/Column.py

@@ -0,0 +1,41 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Column(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsColumn(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Column()
+        x.Init(buf, n + offset)
+        return x
+
+    # Column
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Column
+    def UType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Column
+    def U(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+def ColumnStart(builder): builder.StartObject(2)
+def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
+def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
+def ColumnEnd(builder): return builder.EndObject()

server/app/util/fbs/NetEncoding/Float32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Float32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFloat32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Float32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Float32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Float32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Float32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
+        return 0
+
+    # Float32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Float32ArrayStart(builder): builder.StartObject(1)
+def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Float32ArrayEnd(builder): return builder.EndObject()

server/app/util/fbs/NetEncoding/Float64Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Float64Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFloat64Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Float64Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Float64Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Float64Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
+        return 0
+
+    # Float64Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
+        return 0
+
+    # Float64Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Float64ArrayStart(builder): builder.StartObject(1)
+def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
+def Float64ArrayEnd(builder): return builder.EndObject()

server/app/util/fbs/NetEncoding/Int32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Int32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsInt32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Int32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Int32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Int32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Int32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
+        return 0
+
+    # Int32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Int32ArrayStart(builder): builder.StartObject(1)
+def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Int32ArrayEnd(builder): return builder.EndObject()

server/app/util/fbs/NetEncoding/JSONEncodedArray.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class JSONEncodedArray(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsJSONEncodedArray(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = JSONEncodedArray()
+        x.Init(buf, n + offset)
+        return x
+
+    # JSONEncodedArray
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # JSONEncodedArray
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
+        return 0
+
+    # JSONEncodedArray
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
+    # JSONEncodedArray
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def JSONEncodedArrayStart(builder): builder.StartObject(1)
+def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
+def JSONEncodedArrayEnd(builder): return builder.EndObject()

server/app/util/fbs/NetEncoding/Matrix.py

@@ -0,0 +1,98 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Matrix(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsMatrix(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Matrix()
+        x.Init(buf, n + offset)
+        return x
+
+    # Matrix
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Matrix
+    def NRows(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def NCols(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def Columns(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            x = self._tab.Vector(o)
+            x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
+            x = self._tab.Indirect(x)
+            from .Column import Column
+            obj = Column()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+    # Matrix
+    def ColumnsLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Matrix
+    def ColIndexType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def ColIndex(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+    # Matrix
+    def RowIndexType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def RowIndex(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+def MatrixStart(builder): builder.StartObject(7)
+def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
+def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
+def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
+def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
+def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
+def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
+def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
+def MatrixEnd(builder): return builder.EndObject()

server/app/util/fbs/NetEncoding/TypedArray.py

@@ -0,0 +1,12 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+class TypedArray(object):
+    NONE = 0
+    Float32Array = 1
+    Int32Array = 2
+    Uint32Array = 3
+    Float64Array = 4
+    JSONEncodedArray = 5
+

server/app/util/fbs/NetEncoding/Uint32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Uint32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsUint32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Uint32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Uint32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Uint32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Uint32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
+        return 0
+
+    # Uint32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Uint32ArrayStart(builder): builder.StartObject(1)
+def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Uint32ArrayEnd(builder): return builder.EndObject()

server/app/util/fbs/matrix.py

@@ -0,0 +1,207 @@
+import flatbuffers
+import numpy as np
+from scipy import sparse
+import pandas as pd
+
+import server.app.util.fbs.NetEncoding.Column as Column
+import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
+import server.app.util.fbs.NetEncoding.Matrix as Matrix
+
+
+# Placeholder until recent enhancements to flatbuffers Python
+# runtime are released, at which point we can use the default
+# version.  This code is a port of the head.  See:
+#
+#       https://github.com/google/flatbuffers/pull/4829
+#
+def CreateNumpyVector(builder, x):
+    """CreateNumpyVector writes a numpy array into the buffer."""
+
+    if not isinstance(x, np.ndarray):
+        raise TypeError("non-numpy-ndarray passed to CreateNumpyVector")
+
+    if x.dtype.kind not in ['b', 'i', 'u', 'f']:
+        raise TypeError("numpy-ndarray holds elements of unsupported datatype")
+
+    if x.ndim > 1:
+        raise TypeError("multidimensional-ndarray passed to CreateNumpyVector")
+
+    builder.StartVector(x.itemsize, x.size, x.dtype.alignment)
+
+    # Ensure little endian byte ordering
+    if x.dtype.str[0] == "<":
+        x_little_endian = x
+    else:
+        x_little_endian = x.byteswap(inplace=False)
+
+    # Calculate total length
+    len = int(x_little_endian.itemsize * x_little_endian.size)
+    builder.head = int(builder.Head() - len)
+
+    # tobytes ensures c_contiguous ordering
+    builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C')
+
+    return builder.EndVector(x.size)
+
+
+# Serialization helper
+def serialize_column(builder, typed_arr):
+    """ Serialize NetEncoding.Column """
+    (u_type, u_value) = typed_arr
+    Column.ColumnStart(builder)
+    Column.ColumnAddUType(builder, u_type)
+    Column.ColumnAddU(builder, u_value)
+    return Column.ColumnEnd(builder)
+
+
+# Serialization helper
+def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
+    """ Serialize NetEncoding.Matrix """
+    Matrix.MatrixStart(builder)
+    Matrix.MatrixAddNRows(builder, n_rows)
+    Matrix.MatrixAddNCols(builder, n_cols)
+    Matrix.MatrixAddColumns(builder, columns)
+    if col_idx is not None:
+        (u_type, u_val) = col_idx
+        Matrix.MatrixAddColIndexType(builder, u_type)
+        Matrix.MatrixAddColIndex(builder, u_val)
+    return Matrix.MatrixEnd(builder)
+
+
+# Serialization helper
+def serialize_typed_array(builder, source_array, encoding_info):
+    """
+    Serialize any of the various typed arrays, eg, Float32Array.   Specific
+    means of serialization and type conversion are provided by type_info.
+    """
+    arr = source_array
+    (array_type, as_type) = encoding_info(source_array)
+
+    if isinstance(arr, pd.Index):
+        arr = arr.to_series()
+
+    # convert to a simple ndarray
+    if as_type == 'json':
+        as_json = arr.to_json(orient='records')
+        arr = np.array(bytearray(as_json, 'utf-8'))
+    else:
+        if sparse.issparse(arr):
+            arr = arr.toarray()
+        elif isinstance(arr, pd.Series):
+            arr = arr.get_values()
+        if arr.dtype != as_type:
+            arr = arr.astype(as_type)
+
+    # serialize the ndarray into a vector
+    if arr.ndim == 2 and arr.shape[0] == 1:
+        arr = arr[0]
+    vec = CreateNumpyVector(builder, arr)
+
+    # serialize the typed array table
+    builder.StartObject(1)
+    builder.PrependUOffsetTRelativeSlot(0, vec, 0)
+    array_value = builder.EndObject()
+    return (array_type, array_value)
+
+
+def column_encoding(arr):
+    type_map = {
+        # dtype:  ( array_type, as_type )
+        np.float64: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.float32: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.float16: (TypedArray.TypedArray.Float32Array, np.float32),
+
+        np.int8: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.int16: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
+
+        np.uint8: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.uint16: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
+    }
+    type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
+    return type_map.get(arr.dtype.type, type_map_default)
+
+
+def index_encoding(arr):
+    type_map = {
+        # dtype:  ( array_type, as_type )
+        np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
+
+        np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
+    }
+    type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
+    return type_map.get(arr.dtype.type, type_map_default)
+
+
+def guess_at_mem_needed(matrix):
+    (n_rows, n_cols) = matrix.shape
+    if isinstance(matrix, np.ndarray) or sparse.issparse(matrix):
+        guess = (n_rows * n_cols * matrix.dtype.itemsize) + 1024
+    elif isinstance(matrix, pd.DataFrame):
+        # XXX TODO - DataFrame type estimate
+        guess = 1
+    else:
+        guess = 1
+
+    # round up to nearest 1024 bytes
+    guess = (guess + 0x400) & (~0x3ff)
+    return guess
+
+
+def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
+    """
+    Given a 2D DataFrame, ndarray or sparse equivalent, create and return a
+    Matrix flatbuffer.
+
+    :param matrix: 2D DataFrame, ndarray or sparse equivalent
+    :param row_idx: index for row dimension, Index or ndarray
+    :param col_idx: index for col dimension, Index or ndarray
+
+    NOTE: row indices are (currently) unsupported and must be None
+    """
+
+    if row_idx is not None:
+        raise ValueError("row indexing not supported for FBS Matrix")
+    if matrix.ndim != 2:
+        raise ValueError("FBS Matrix must be 2D")
+
+    (n_rows, n_cols) = matrix.shape
+
+    # estimate size needed, so we don't unnecessarily realloc.
+    builder = flatbuffers.Builder(guess_at_mem_needed(matrix))
+
+    if isinstance(matrix, pd.DataFrame):
+        matrix_columns = reversed(tuple(matrix[name] for name in matrix))
+    else:
+        matrix_columns = reversed(tuple(c for c in matrix.T))
+
+    columns = []
+    # for idx in reversed(np.arange(n_cols)):
+    for c in matrix_columns:
+        # serialize the typed array
+        typed_arr = serialize_typed_array(builder, c, column_encoding)
+
+        # serialize the Column union
+        columns.append(serialize_column(builder, typed_arr))
+
+    # Serialize Matrix.columns[]
+    Matrix.MatrixStartColumnsVector(builder, n_cols)
+    for c in columns:
+        builder.PrependUOffsetTRelative(c)
+    matrix_column_vec = builder.EndVector(n_cols)
+
+    # serialize the colIndex if provided
+    cidx = None
+    if col_idx is not None:
+        cidx = serialize_typed_array(builder, col_idx, index_encoding)
+
+    # Serialize Matrix
+    matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx)
+
+    builder.Finish(matrix)
+    return builder.Output()

server/app/util/filter.py

@@ -7,7 +7,6 @@ from server.app.util.constants import Axis
 
 
 class QueryStringError(Exception):
-
     def __init__(self, key, message):
         self.key = key
         self.message = message

server/app/util/models.py

@@ -5,24 +5,13 @@ class AnnotationModel(Schema):
     type = "object"
     description = "Filter by annotation key: value"
     properties = {
-        "name": {
+        "name": {"type": "string"},
-            "type": "string"
-        },
         # TODO update to OpenAPI v3.0 when a library is available that supports it
         # Unfortunately 2.0 doesn't have a way to have a schema that accepts multiple types
         # Overloading the type key with a list seems to work ok and makes it to the page
-        "values": {
+        "values": {"type": "array", "items": {"type": ["float32", "string", "int32", "bool"]}},
-            "type": "array",
+        "min": {"type": ["int32", "float32"]},
-            "items": {
+        "max": {"type": ["int32", "float32"]},
-                "type": ["float32", "string", "int32", "bool"]
-            }
-        },
-        "min": {
-            "type": ["int32", "float32"],
-        },
-        "max": {
-            "type": ["int32", "float32"],
-        }
     }
     required = ["name"]
 
@@ -30,36 +19,16 @@ class AnnotationModel(Schema):
 class IndexModel(Schema):
     type = "object"
     description = "Filter by index of observation/variable ex. [0, 5, 15]"
-    properties = {
+    properties = {"index": {"type": "array", "items": {"format": "int32", "type": "integer"}}}
-        "index": {
-            "type": "array",
-            "items": {
-                "format": "int32",
-                "type": "integer"
-            }
-
-        }
-    }
 
 
 class AxisModel(Schema):
     type = "object"
     description = "Axis of data -- obs or var"
-    properties = {
+    properties = {"index": IndexModel, "annotation_value": AnnotationModel.array()}
-        "index": IndexModel,
-        "annotation_value": AnnotationModel.array()
-    }
 
 
 class FilterModel(Schema):
     type = "object"
     description = "Complex filter"
-    properties = {
+    properties = {"filter": {"type": "object", "properties": {"obs": AxisModel, "var": AxisModel}}}
-        "filter": {
-            "type": "object",
-            "properties": {
-                "obs": AxisModel,
-                "var": AxisModel
-            }
-        }
-    }

server/app/util/utils.py

@@ -7,6 +7,17 @@ from server.app.util.errors import MimeTypeError
 
 
 class Float32JSONEncoder(json.JSONEncoder):
+    def __init__(self, *args, **kwargs):
+        """
+        NaN/Infinities are illegal in standard JSON.  Python extends JSON with
+        non-standard symbols that most JavaScript JSON parsers do not understand.
+        The `allow_nan` parameter will force Python simplejson to throw an ValueError
+        if it runs into non-finite floating point values which are unsupported by
+        standard JSON.
+        """
+        kwargs["allow_nan"] = False
+        super().__init__(*args, **kwargs)
+
     def default(self, obj):
         if isinstance(obj, float32):
             return float(obj)
@@ -15,8 +26,13 @@ class Float32JSONEncoder(json.JSONEncoder):
         return json.JSONEncoder.default(self, obj)
 
 
-def get_mime_type(default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None,
+def custom_format_warning(msg, *args, **kwargs):
-                  header=None):
+    return f"[cellxgene] Warning: {msg} \n"
+
+
+def get_mime_type(
+    default="application/json", acceptable_types=["application/json", "text/csv"], query_param=None, header=None
+):
     mime_type = default
     if query_param:
         if query_param in acceptable_types:
@@ -38,3 +54,7 @@ def whole_number(value):
     if value < 0:
         raise ArgumentTypeError(f"{value} is not >= 0")
     return value
+
+
+def jsonify_scanpy(data):
+    return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)

server/app/web/templates/swagger.html

@@ -73,7 +73,7 @@
 <script>
 	window.onload = function () {
 		const ui = SwaggerUIBundle({
-			url: window.location.origin + "/api/swagger.json",
+			url: window.location.href.replace(/\/swagger$/, "") + "/api/swagger.json",
 			dom_id: '#swagger-ui',
 			deepLinking: true,
 			presets: [

server/app/web/webapp.py

@@ -1,7 +1,5 @@
 import os
-from flask import (
+from flask import Blueprint, render_template, send_from_directory, current_app
-    Blueprint, render_template, send_from_directory, current_app
-)
 
 
 bp = Blueprint("webapp", __name__, template_folder="templates")
@@ -9,9 +7,8 @@ bp = Blueprint("webapp", __name__, template_folder="templates")
 
 @bp.route("/")
 def index():
-    url_base = current_app.config["CXG_API_BASE"]
     dataset_title = current_app.config["DATASET_TITLE"]
-    return render_template("index.html", prefix=url_base, datasetTitle=dataset_title)
+    return render_template("index.html", datasetTitle=dataset_title)
 
 
 # renders swagger documentation

server/cli/cli.py

@@ -5,7 +5,7 @@ from .prepare import prepare
 
 
 @click.group(name="cellxgene", context_settings=dict(max_content_width=85))
-@click.version_option(version="0.2.3", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+@click.version_option(version="0.6.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
 def cli():
     pass
 

server/cli/launch.py

@@ -1,37 +1,80 @@
-import sys
-import click
 import logging
 from os import devnull
 from os.path import splitext, basename
+import sys
+import warnings
 import webbrowser
 
+import click
+
 from server.app.util.errors import ScanpyFileError
+from server.app.util.utils import custom_format_warning
 
 
 @click.command()
 @click.argument("data", metavar="<data file>", type=click.Path(exists=True, file_okay=True, dir_okay=False))
-@click.option("--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True,
+@click.option(
-              help="Method for layout.")
+    "--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True, help="Method for layout."
-@click.option("--diffexp", "-d", type=click.Choice(["ttest"]), default="ttest", show_default=True,
+)
-              help="Method for differential expression.")
+@click.option(
+    "--diffexp",
+    "-d",
+    type=click.Choice(["ttest"]),
+    default="ttest",
+    show_default=True,
+    help="Method for differential expression.",
+)
 @click.option("--title", "-t", help="Title to display (if omitted will use file name).", metavar="")
-@click.option("--verbose", "-v", is_flag=True, default=False, show_default=True,
+@click.option(
-              help="Provide verbose output, including warnings and all server requests.")
+    "--verbose",
-@click.option("--debug", "-d", is_flag=True, default=False, show_default=True,
+    "-v",
-              help="Run in debug mode.")
+    is_flag=True,
-@click.option("--open", "-o", "open_browser", is_flag=True, default=False, show_default=True,
+    default=False,
-              help="Open the web browser after launch.")
+    show_default=True,
+    help="Provide verbose output, including warnings and all server requests.",
+)
+@click.option("--debug", is_flag=True, default=False, show_default=True, help="Run in debug mode.")
+@click.option(
+    "--open",
+    "-o",
+    "open_browser",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Open the web browser after launch.",
+)
 @click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True)
 @click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
 @click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
-@click.option("--listen-all", is_flag=True, default=False, show_default=True,
+@click.option("--host", default="127.0.0.1", help="Host IP address")
-              help="Bind to all interfaces (this makes the server accessible beyond this computer).")
+@click.option(
-@click.option("--max-category-items", default=100, metavar="", show_default=True,
+    "--max-category-items",
-              help="Limits the number of categorical annotation items displayed.")
+    default=100,
-@click.option("--diffexp-lfc-cutoff", default=0.01, show_default=True,
+    metavar="",
-              help="Relative expression cutoff used when selecting top N differentially expressed genes")
+    show_default=True,
-def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
+    help="Limits the number of categorical annotation items displayed.",
-           open_browser, port, listen_all, max_category_items, diffexp_lfc_cutoff):
+)
+@click.option(
+    "--diffexp-lfc-cutoff",
+    default=0.01,
+    show_default=True,
+    help="Relative expression cutoff used when selecting top N differentially expressed genes",
+)
+def launch(
+    data,
+    layout,
+    diffexp,
+    title,
+    verbose,
+    debug,
+    obs_names,
+    var_names,
+    open_browser,
+    port,
+    host,
+    max_category_items,
+    diffexp_lfc_cutoff,
+):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
     Data must be in a format that cellxgene expects, read the
@@ -46,9 +89,6 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
     # Startup message
     click.echo("[cellxgene] Starting the CLI...")
 
-    # Import Flask app
-    from server.app.app import app
-
     # Argument checking
     name, extension = splitext(data)
     if extension != ".h5ad":
@@ -57,6 +97,8 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
     if debug:
         verbose = True
         open_browser = False
+    else:
+        warnings.formatwarning = custom_format_warning
 
     if not verbose:
         sys.tracebacklimit = 0
@@ -65,19 +107,13 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
         file_parts = splitext(basename(data))
         title = file_parts[0]
 
-    if listen_all:
-        host = "0.0.0.0"
-    else:
-        host = "127.0.0.1"
-
     # Setup app
     cellxgene_url = f"http://{host}:{port}"
-    api_base = f"{cellxgene_url}/api/"
 
-    app.config.update(
+    # Import Flask app
-        DATASET_TITLE=title,
+    from server.app.app import app
-        CXG_API_BASE=api_base
+
-    )
+    app.config.update(DATASET_TITLE=title)
 
     if not verbose:
         log = logging.getLogger("werkzeug")
@@ -88,7 +124,8 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
     # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
     # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
     import matplotlib as mpl
-    mpl.use('TkAgg')
+
+    mpl.use("TkAgg")
     from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
 
     args = {
@@ -97,7 +134,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
         "max_category_items": max_category_items,
         "diffexp_lfc_cutoff": diffexp_lfc_cutoff,
         "obs_names": obs_names,
-        "var_names": var_names
+        "var_names": var_names,
     }
 
     try:
@@ -114,7 +151,7 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
     click.echo("[cellxgene] Type CTRL-C at any time to exit.")
 
     if not verbose:
-        f = open(devnull, 'w')
+        f = open(devnull, "w")
         sys.stdout = f
 
     app.run(host=host, debug=debug, port=port, threaded=True)

server/cli/prepare.py

@@ -7,22 +7,48 @@ from scipy.sparse.csc import csc_matrix
 
 @click.command()
 @click.argument("data", nargs=1, metavar="<dataset: file or path to data>", required=True)
-@click.option("--layout", "-l", default=["umap", "tsne"], multiple=True, type=click.Choice(["umap", "tsne"]),
+@click.option(
-              help="Layout algorithm", show_default=True)
+    "--layout",
-@click.option("--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]),
+    "-l",
-              help="Preprocessing to run.", show_default=True)
+    default=["umap", "tsne"],
+    multiple=True,
+    type=click.Choice(["umap", "tsne"]),
+    help="Layout algorithm",
+    show_default=True,
+)
+@click.option(
+    "--recipe",
+    "-r",
+    default="none",
+    type=click.Choice(["none", "seurat", "zheng17"]),
+    help="Preprocessing to run.",
+    show_default=True,
+)
 @click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
 @click.option("--plotting", "-p", default=False, is_flag=True, help="Whether to generate plots.", show_default=True)
 @click.option("--sparse", default=False, is_flag=True, help="Whether to force sparsity.", show_default=True)
 @click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
 @click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
 @click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
-@click.option("--make-obs-names-unique", default=True, is_flag=True,
+@click.option(
-              help="Ensure obs index is unique.", show_default=True)
+    "--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True
-@click.option("--make-var-names-unique", default=True, is_flag=True,
+)
-              help="Ensure var index is unique.", show_default=True)
+@click.option(
-def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
+    "--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True
-            set_obs_names, set_var_names, make_obs_names_unique, make_var_names_unique):
+)
+def prepare(
+    data,
+    layout,
+    recipe,
+    output,
+    plotting,
+    sparse,
+    overwrite,
+    set_obs_names,
+    set_var_names,
+    make_obs_names_unique,
+    make_var_names_unique,
+):
     """Preprocesses data for use with cellxgene.
 
     This tool runs a series of scanpy routines for preparing a dataset
@@ -35,6 +61,7 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
     # collect slow imports here to make CLI startup more responsive
     click.echo("[cellxgene] Starting CLI...")
     import matplotlib
+
     matplotlib.use("Agg")
     import scanpy.api as sc
 
@@ -49,8 +76,10 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
     output = expanduser(output)
 
     if not output:
-        click.echo("Warning: No file will be saved, to save the results of cellxgene prepare include "
+        click.echo(
-                   "--output <filename> to save output to a new file")
+            "Warning: No file will be saved, to save the results of cellxgene prepare include "
+            "--output <filename> to save output to a new file"
+        )
     if isfile(output) and not overwrite:
         raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
 
@@ -119,9 +148,11 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
         try:
             sc.tl.louvain(adata)
         except ModuleNotFoundError:
-            click.echo("\nWarning: louvain module is not installed, no clusters will be calculated. "
+            click.echo(
-                       "To fix this please install cellxgene with the optional feature louvain enabled: "
+                "\nWarning: louvain module is not installed, no clusters will be calculated. "
-                       "`pip install cellxgene[louvain]`")
+                "To fix this please install cellxgene with the optional feature louvain enabled: "
+                "`pip install cellxgene[louvain]`"
+            )
 
     def run_layout(adata):
         if len(unique(adata.obs["louvain"].values)) < 10:
@@ -142,11 +173,11 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
     def show_step(item):
         names = {
             "make_sparse": "Ensuring sparsity",
-            "run_recipe": f"Running preprocessing recipe \"{recipe}\"",
+            "run_recipe": f'Running preprocessing recipe "{recipe}"',
             "run_pca": "Running PCA",
             "run_neighbors": "Calculating neighbors",
             "run_louvain": "Calculating clusters",
-            "run_layout": "Computing layout"
+            "run_layout": "Computing layout",
         }
         if item is not None:
             return names[item.__name__]

server/requirements-dev.txt

@@ -1,5 +1,6 @@
+black
+bumpversion>=0.5
 pytest>=3.6.3
 requests>=2.18.4
 twine>=1.12.1
-bumpversion>=0.5
 -r requirements.txt

server/requirements.txt

@@ -6,9 +6,10 @@ Flask-Compress>=1.4.0
 Flask-Cors>=3.0.6
 Flask-RESTful>=0.3.6
 flask-restful-swagger-2>=0.35
+flatbuffers>=1.10.0
 matplotlib>=2.2
 numpy>=1.14.5
 pandas>=0.23.1
 scanpy>=1.3.2
 scipy>=1.1.0
-scikit-learn==0.19.1
+scikit-learn>=0.19.1,!=0.20.0

server/test/decode_fbs.py

@@ -0,0 +1,69 @@
+
+"""
+Code to decode, for testing purposes, the flatbuffer encoded blobs.
+This code will need to be updated if fbs/matrix.fbs changes.
+
+For more information, see fbs/matrix.fbs and server/app/util/fbs/
+"""
+import json
+
+import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
+import server.app.util.fbs.NetEncoding.Matrix as Matrix
+import server.app.util.fbs.NetEncoding.Int32Array as Int32Array
+import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
+import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
+import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
+import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+
+
+def decode_typed_array(tarr):
+    type_map = {
+        TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
+        TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
+        TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
+        TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
+        TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
+    }
+    (u_type, u) = tarr
+    if u_type == TypedArray.TypedArray.NONE:
+        return None
+
+    TarType = type_map.get(u_type, None)
+    assert(TarType is not None)
+
+    arr = TarType()
+    arr.Init(u.Bytes, u.Pos)
+    narr = arr.DataAsNumpy()
+    if u_type == TypedArray.TypedArray.JSONEncodedArray:
+        narr = json.loads(narr.tostring().decode('utf-8'))
+    return narr
+
+
+def decode_matrix_FBS(buf):
+    """
+    Given a FBS Matrix, return an decoded Python dict containing
+    same info in native format.
+
+    NOTE / TODO: row_idx not currently implemented
+    """
+    df = Matrix.Matrix.GetRootAsMatrix(buf, 0)
+    n_rows = df.NRows()
+    n_cols = df.NCols()
+
+    columns_length = df.ColumnsLength()
+
+    decoded_columns = []
+    for col_idx in range(0, columns_length):
+        col = df.Columns(col_idx)
+        tarr = (col.UType(), col.U())
+        decoded_columns.append(decode_typed_array(tarr))
+
+    cidx = decode_typed_array((df.ColIndexType(), df.ColIndex()))
+
+    return {
+        "n_rows": n_rows,
+        "n_cols": n_cols,
+        "columns": decoded_columns,
+        "col_idx": cidx,
+        "row_idx": None
+    }

server/test/test_api.py

@@ -5,19 +5,13 @@ import time
 
 import requests
 
+import decode_fbs
+
 LOCAL_URL = "http://127.0.0.1:5005/"
 VERSION = "v0.2"
 URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
 
-BAD_FILTER = {
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
-            "filter": {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "xyz"},
-                    ],
-                }
-            }
-        }
 
 
 class EndPoints(unittest.TestCase):
@@ -48,6 +42,7 @@ class EndPoints(unittest.TestCase):
         url = f"{URL_BASE}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
         self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
@@ -57,6 +52,7 @@ class EndPoints(unittest.TestCase):
         url = f"{URL_BASE}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
         self.assertEqual(len(result_data["config"]["features"]), 4)
@@ -66,10 +62,26 @@ class EndPoints(unittest.TestCase):
         url = f"{URL_BASE}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["layout"]["ndims"], 2)
         self.assertEqual(len(result_data["layout"]["coordinates"]), 2638)
 
+    def test_get_layout_fbs(self):
+        endpoint = "layout/obs"
+        url = f"{URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 2638)
+        self.assertEqual(df['n_cols'], 2)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+
     # def test_put_layout(self):
     #     endpoint = "layout/obs"
     #     url = f"{URL_BASE}{endpoint}"
@@ -101,6 +113,7 @@ class EndPoints(unittest.TestCase):
         url = f"{URL_BASE}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
         self.assertEqual(len(result_data["data"]), 2638)
@@ -111,11 +124,28 @@ class EndPoints(unittest.TestCase):
         query = "annotation-name=n_genes&annotation-name=percent_mito"
         url = f"{URL_BASE}{endpoint}?{query}"
         result = self.session.get(url)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         self.assertEqual(result.status_code, HTTPStatus.OK)
         result_data = result.json()
         self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
         self.assertEqual(len(result_data["data"][0]), 3)
 
+    def test_get_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        url = f"{URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 2638)
+        self.assertEqual(df['n_cols'], 5)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNotNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+        self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
+
     def test_get_annotations_obs_error(self):
         endpoint = "annotations/obs"
         query = "annotation-name=notakey"
@@ -133,12 +163,13 @@ class EndPoints(unittest.TestCase):
                         {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
                         {"name": "n_counts", "min": 3000},
                     ],
-                    "index": [1, 99, [1000, 2000]]
+                    "index": [1, 99, [1000, 2000]],
                 }
             }
         }
         result = self.session.put(url, json=obs_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
         self.assertEqual(len(result_data["data"]), 15)
@@ -154,12 +185,13 @@ class EndPoints(unittest.TestCase):
                         {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
                         {"name": "n_counts", "min": 3000},
                     ],
-                    "index": [1, 99, [1000, 2000]]
+                    "index": [1, 99, [1000, 2000]],
                 }
             }
         }
         result = self.session.put(url, json=obs_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
         self.assertEqual(len(result_data["data"][0]), 3)
@@ -170,26 +202,13 @@ class EndPoints(unittest.TestCase):
         url = f"{URL_BASE}{endpoint}"
         params = {
             "mode": "topN",
-            "set1": {
+            "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
-                "filter": {
+            "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
-                    "obs": {"annotation_value": [
+            "count": 7,
-                        {"name": "louvain", "values": ["NK cells"]}
-                    ]
-                    }
-                }
-            },
-            "set2": {
-                "filter": {
-                    "obs": {"annotation_value": [
-                        {"name": "louvain", "values": ["CD8 T cells"]}
-                    ]
-                    }
-                }
-            },
-            "count": 7
         }
         result = self.session.post(url, json=params)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(len(result_data), 7)
 
@@ -199,23 +218,12 @@ class EndPoints(unittest.TestCase):
         params = {
             "mode": "topN",
             "count": 10,
-            "set1": {
+            "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
-                "filter": {
+            "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
-                    "obs": {
-                        "index": [[0, 500]]
-                    }
-                }
-            },
-            "set2": {
-                "filter": {
-                    "obs": {
-                        "index": [[500, 1000]]
-                    }
-                }
-            }
         }
         result = self.session.post(url, json=params)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(len(result_data), 10)
 
@@ -224,6 +232,7 @@ class EndPoints(unittest.TestCase):
         url = f"{URL_BASE}{endpoint}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["names"], ["name", "n_cells"])
         self.assertEqual(len(result_data["data"]), 1838)
@@ -235,10 +244,27 @@ class EndPoints(unittest.TestCase):
         url = f"{URL_BASE}{endpoint}?{query}"
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["names"], ["n_cells"])
         self.assertEqual(len(result_data["data"][0]), 2)
 
+    def test_get_annotations_var_fbs(self):
+        endpoint = "annotations/var"
+        url = f"{URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 1838)
+        self.assertEqual(df['n_cols'], 2)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNotNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+        self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
+
     def test_get_annotations_var_error(self):
         endpoint = "annotations/var"
         query = "annotation-name=notakey"
@@ -249,17 +275,10 @@ class EndPoints(unittest.TestCase):
     def test_put_annotations_var(self):
         endpoint = "annotations/var"
         url = f"{URL_BASE}{endpoint}"
-        var_filter = {
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {"name": "name", "values": ["ATAD3C", "RER1"]},
-                    ]
-                }
-            }
-        }
         result = self.session.put(url, json=var_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["names"], ["name", "n_cells"])
         self.assertEqual(len(result_data["data"]), 2)
@@ -268,17 +287,10 @@ class EndPoints(unittest.TestCase):
         endpoint = "annotations/var"
         query = "annotation-name=n_cells"
         url = f"{URL_BASE}{endpoint}?{query}"
-        var_filter = {
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
-            "filter": {
-                "var": {
-                    "annotation_value": [
-                        {"name": "name", "values": ["ATAD3C", "RER1"]},
-                    ]
-                }
-            }
-        }
         result = self.session.put(url, json=var_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["names"], ["n_cells"])
         self.assertEqual(len(result_data["data"][0]), 2)
@@ -291,6 +303,7 @@ class EndPoints(unittest.TestCase):
             url = f"{URL_BASE}{endpoint}?{query}"
             result = self.session.get(url)
             self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.headers["Content-Type"], "application/json")
             result_data = result.json()
             self.assertEqual(len(result_data["obs"]), 2638)
 
@@ -312,6 +325,7 @@ class EndPoints(unittest.TestCase):
             url = f"{URL_BASE}{endpoint}"
             result = self.session.get(url)
             self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.headers["Content-Type"], "application/json")
 
     def test_data_filter(self):
         for axis in ["obs", "var"]:
@@ -320,10 +334,11 @@ class EndPoints(unittest.TestCase):
             url = f"{URL_BASE}{endpoint}?{query}"
             result = self.session.get(url)
             self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.headers["Content-Type"], "application/json")
             result_data = result.json()
             self.assertEqual(len(result_data["obs"]), 38)
 
-    def test_data_put(self):
+    def test_data_json_put(self):
         for axis in ["obs", "var"]:
             endpoint = f"data/{axis}"
             url = f"{URL_BASE}{endpoint}"
@@ -335,31 +350,48 @@ class EndPoints(unittest.TestCase):
                             {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
                             {"name": "n_counts", "min": 3000},
                         ],
-                        "index": [1, 99, [1000, 2000]]
+                        "index": [1, 99, [1000, 2000]],
                     }
                 }
             }
             result = self.session.put(url, headers=header, json=obs_filter)
             self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.headers["Content-Type"], "application/json")
             result_data = result.json()
             self.assertEqual(len(result_data["obs"]), 15)
 
+    def test_data_put_fbs(self):
+        endpoint = f"data/var"
+        url = f"{URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        filter = {
+            "filter": {
+                "var": {
+                    "index": [0, 1, 4]
+                }
+            }
+        }
+        result = self.session.put(url, headers=header, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df['n_rows'], 2638)
+        self.assertEqual(df['n_cols'], 3)
+        self.assertIsNotNone(df['columns'])
+        self.assertIsNotNone(df['col_idx'])
+        self.assertIsNone(df['row_idx'])
+        self.assertEqual(len(df['columns']), df['n_cols'])
+        self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
+
     def test_data_put_single_var(self):
         for axis in ["obs", "var"]:
             endpoint = f"data/{axis}"
             url = f"{URL_BASE}{endpoint}"
             header = {"Accept": "application/json"}
-            var_filter = {
+            var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
-                "filter": {
-                    "var": {
-                        "annotation_value": [
-                            {"name": "name", "values": ["RER1"]},
-                        ]
-                    }
-                }
-            }
             result = self.session.put(url, headers=header, json=var_filter)
             self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.headers["Content-Type"], "application/json")
             result_data = result.json()
             if axis == "obs":
                 self.assertEqual(len(result_data["obs"][0]), 2)
@@ -371,18 +403,48 @@ class EndPoints(unittest.TestCase):
     def test_cache(self):
         endpoint = "annotations/var"
         url = f"{URL_BASE}{endpoint}"
-        f1 = {"filter": {"var": {"annotation_value": [{"name": "name",
+        f1 = {
-                                                       "values": ["HLA-DRB1", "HLA-DQA1", "HLA-DQB1", "HLA-DPA1",
+            "filter": {
-                                                                  "HLA-DPB1", "MS4A1", "IL32", "CCL5", "CD79B",
+                "var": {
-                                                                  "CD79A"]}]}}}
+                    "annotation_value": [
+                        {
+                            "name": "name",
+                            "values": [
+                                "HLA-DRB1",
+                                "HLA-DQA1",
+                                "HLA-DQB1",
+                                "HLA-DPA1",
+                                "HLA-DPB1",
+                                "MS4A1",
+                                "IL32",
+                                "CCL5",
+                                "CD79B",
+                                "CD79A",
+                            ],
+                        }
+                    ]
+                }
+            }
+        }
         result = self.session.put(url, json=f1)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data1 = result.json()
-        f2 = {"filter": {"var": {"annotation_value": [{"name": "name",
+        f2 = {
-                                                       "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH",
+            "filter": {
-                                                                  "CCL5", "CCL4", "CST7", "NKG7"]}]}}}
+                "var": {
+                    "annotation_value": [
+                        {
+                            "name": "name",
+                            "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
+                        }
+                    ]
+                }
+            }
+        }
         result = self.session.put(url, json=f2)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data2 = result.json()
         self.assertNotEqual(result_data1, result_data2)
 
@@ -393,11 +455,21 @@ class EndPoints(unittest.TestCase):
         result = self.session.put(url, json=f1)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         result_data1 = result.json()
-        f2 = {"filter": {"var": {"annotation_value": [{"name": "name",
+        f2 = {
-                                                       "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH",
+            "filter": {
-                                                                  "CCL5", "CCL4", "CST7", "NKG7"]}]}}}
+                "var": {
+                    "annotation_value": [
+                        {
+                            "name": "name",
+                            "values": ["FGFBP2", "GZMA", "LTB", "PRF1", "CTSW", "GZMH", "CCL5", "CCL4", "CST7", "NKG7"],
+                        }
+                    ]
+                }
+            }
+        }
         result = self.session.put(url, json=f2)
         self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data2 = result.json()
         self.assertNotEqual(result_data1, result_data2)
 

server/test/test_filter.py

@@ -54,13 +54,17 @@ class UtilTest(unittest.TestCase):
 
     def test_complex_filter(self):
         filter_dict = ImmutableMultiDict(
-            [("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")])
+            [("obs:louvain", "NK cells"), ("obs:louvain", "CD8 T cells"), ("obs:n_counts", "3000,*")]
+        )
         filter_ = parse_filter(filter_dict, self.schema)
         self.assertIn("obs", filter_)
-        self.assertEqual(filter_["obs"]["annotation_value"], [{"name": "louvain",
+        self.assertEqual(
-                                                               "values": ["NK cells", "CD8 T cells"]},
+            filter_["obs"]["annotation_value"],
-                                                              {"name": "n_counts",
+            [
-                                                               "max": None, "min": 3000.0}])
+                {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
+                {"name": "n_counts", "max": None, "min": 3000.0},
+            ],
+        )
 
     def test_bad_filter(self):
         bad_annotation_type = ImmutableMultiDict([("obs:tissue", "lung")])
@@ -71,9 +75,7 @@ class UtilTest(unittest.TestCase):
             parse_filter(bad_axis, self.schema)
 
     def test_boolean_filter(self):
-        schema = {
+        schema = {"obs": [{"name": "bool_filter", "type": "boolean"}]}
-            "obs": [{"name": "bool_filter", "type": "boolean"}]
-        }
         filter_dict = ImmutableMultiDict([("obs:bool_filter", "false")])
         filter_ = parse_filter(filter_dict, schema)
         self.assertIn("obs", filter_)

server/test/test_nan_rest.py

@@ -0,0 +1,51 @@
+from http import HTTPStatus
+from subprocess import Popen
+import unittest
+import time
+
+import requests
+
+LOCAL_URL = "http://127.0.0.1:5005/"
+VERSION = "v0.2"
+URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
+
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+class WithNaNs(unittest.TestCase):
+    """Test Case for endpoints"""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ps = Popen(
+            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug"]
+        )
+        session = requests.Session()
+        for i in range(90):
+            try:
+                session.get(f"{URL_BASE}schema")
+            except requests.exceptions.ConnectionError:
+                time.sleep(1)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cls.ps.terminate()
+        except ProcessLookupError:
+            pass
+
+    def setUp(self):
+        self.session = requests.Session()
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_errors(self):
+        endpoints = ["annotations/obs", "annotations/var", "data/obs", "data/var"]
+        for endpoint in endpoints:
+            url = f"{URL_BASE}{endpoint}"
+            result = self.session.get(url)
+            self.assertEqual(result.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)

server/test/test_nan_scanpy_engine.py

@@ -0,0 +1,72 @@
+import json
+import pytest
+import unittest
+import warnings
+import math
+
+import decode_fbs
+
+from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
+from server.app.util.errors import JSONEncodingValueError
+
+
+class NaNTest(unittest.TestCase):
+    def setUp(self):
+        self.args = {
+            "layout": "umap",
+            "diffexp": "ttest",
+            "max_category_items": 100,
+            "obs_names": None,
+            "var_names": None,
+            "diffexp_lfc_cutoff": 0.01,
+        }
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=UserWarning)
+            self.data = ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args)
+            self.data._create_schema()
+
+    def test_load(self):
+        with self.assertWarns(UserWarning):
+            ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 100)
+        self.assertEqual(self.data.gene_count, 100)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_dataframe(self):
+        data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
+        self.assertIsNotNone(data_frame_var)
+        self.assertEqual(data_frame_var["n_rows"], 100)
+        self.assertEqual(data_frame_var["n_cols"], 100)
+        self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
+
+        with pytest.raises(JSONEncodingValueError):
+            json.loads(self.data.data_frame(None, "obs"))
+        with pytest.raises(JSONEncodingValueError):
+            json.loads(self.data.data_frame(None, "var"))
+
+    def test_dataframe_obs_not_implemented(self):
+        with self.assertRaises(ValueError) as cm:
+            decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
+        self.assertIsNotNone(cm.exception)
+
+    def test_annotation(self):
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        self.assertEqual(
+            annotations["col_idx"],
+            ["name", "n_genes", "percent_mito", "n_counts", "louvain"]
+        )
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))
+
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
+        self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))
+
+        with pytest.raises(JSONEncodingValueError):
+            json.loads(self.data.annotation(None, "obs"))
+        with pytest.raises(JSONEncodingValueError):
+            json.loads(self.data.annotation(None, "var"))

server/test/test_scanpy_engine.py

@@ -3,7 +3,6 @@ from os import path
 import pytest
 import time
 import unittest
-import argparse
 
 import numpy as np
 from pandas import Series
@@ -13,8 +12,14 @@ from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
 
 class UtilTest(unittest.TestCase):
     def setUp(self):
-        args = {'layout': 'umap', 'diffexp': 'ttest', 'max_category_items': 100,
+        args = {
-                'obs_names': None, 'var_names': None, 'diffexp_lfc_cutoff': 0.01}
+            "layout": "umap",
+            "diffexp": "ttest",
+            "max_category_items": 100,
+            "obs_names": None,
+            "var_names": None,
+            "diffexp_lfc_cutoff": 0.01,
+        }
 
         self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args)
         self.data._create_schema()
@@ -22,8 +27,8 @@ class UtilTest(unittest.TestCase):
     def test_init(self):
         self.assertEqual(self.data.cell_count, 2638)
         self.assertEqual(self.data.gene_count, 1838)
-        epsilon = 0.000005
+        epsilon = 0.000_005
-        self.assertTrue(self.data.data.X[0, 0] - -0.17146951 < epsilon)
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
 
     def test_mandatory_annotations(self):
         self.assertIn("name", self.data.data.obs)
@@ -40,12 +45,8 @@ class UtilTest(unittest.TestCase):
     def test_filter_idx(self):
         filter_ = {
             "filter": {
-                "var": {
+                "var": {"index": [1, 99, [200, 300]]},
-                    "index": [1, 99, [200, 300]]
+                "obs": {"index": [1, 99, [1000, 2000]]},
-                },
-                "obs": {
-                    "index": [1, 99, [1000, 2000]]
-                }
             }
         }
         data = self.data.filter_dataframe(filter_["filter"])
@@ -56,7 +57,7 @@ class UtilTest(unittest.TestCase):
             "filter": {
                 "obs": {
                     "annotation_value": [
-                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
+                        {"name": "louvain", "values": ["NK cells", "CD8 T cells"]}
                     ]
                 }
             }
@@ -64,13 +65,7 @@ class UtilTest(unittest.TestCase):
         data = self.data.filter_dataframe(filter_["filter"])
         self.assertEqual(data.shape, (470, 1838))
         filter_ = {
-            "filter": {
+            "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
-                "obs": {
-                    "annotation_value": [
-                        {"name": "n_counts", "min": 3000},
-                    ]
-                }
-            }
         }
         data = self.data.filter_dataframe(filter_["filter"])
         self.assertEqual(data.shape, (497, 1838))
@@ -78,11 +73,7 @@ class UtilTest(unittest.TestCase):
     def test_filter_annotation_no_uns(self):
         filter_ = {
             "filter": {
-                "var": {
+                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
-                    "annotation_value": [
-                        {"name": "name", "values": ["RER1"]},
-                    ]
-                }
             }
         }
         data = self.data.filter_dataframe(filter_["filter"])
@@ -91,16 +82,14 @@ class UtilTest(unittest.TestCase):
     def test_filter_complex(self):
         filter_ = {
             "filter": {
-                "var": {
+                "var": {"index": [1, 99, [200, 300]]},
-                    "index": [1, 99, [200, 300]]
-                },
                 "obs": {
                     "annotation_value": [
                         {"name": "louvain", "values": ["NK cells", "CD8 T cells"]},
                         {"name": "n_counts", "min": 3000},
                     ],
-                    "index": [1, 99, [1000, 2000]]
+                    "index": [1, 99, [1000, 2000]],
-                }
+                },
             }
         }
         data = self.data.filter_dataframe(filter_["filter"])
@@ -116,118 +105,101 @@ class UtilTest(unittest.TestCase):
             self.assertEqual(self.data.schema, schema)
 
     def test_schema_produces_error(self):
-        self.data.data.obs["time"] = Series(list([time.time() for i in range(self.data.cell_count)]),
+        self.data.data.obs["time"] = Series(
-                                            dtype="datetime64[ns]")
+            list([time.time() for i in range(self.data.cell_count)]),
+            dtype="datetime64[ns]",
+        )
         with pytest.raises(TypeError):
             self.data._create_schema()
 
     def test_config(self):
-        self.assertEqual(self.data.features["layout"]["obs"], {'available': True, 'interactiveLimit': 50000})
+        self.assertEqual(
+            self.data.features["layout"]["obs"],
+            {"available": True, "interactiveLimit": 50000},
+        )
 
     def test_layout(self):
-        layout = self.data.layout(None)
+        layout = json.loads(self.data.layout(None))
-        self.assertEqual(layout["ndims"], 2)
+        self.assertEqual(layout["layout"]["ndims"], 2)
-        self.assertEqual(len(layout["coordinates"]), 2638)
+        self.assertEqual(len(layout["layout"]["coordinates"]), 2638)
-        self.assertEqual(layout["coordinates"][0][0], 0)
+        self.assertEqual(layout["layout"]["coordinates"][0][0], 0)
-        for idx, val in enumerate(layout["coordinates"]):
+        for idx, val in enumerate(layout["layout"]["coordinates"]):
             self.assertLessEqual(val[1], 1)
             self.assertLessEqual(val[2], 1)
 
     def test_annotations(self):
-        annotations = self.data.annotation(None, "obs")
+        annotations = json.loads(self.data.annotation(None, "obs"))
-        self.assertEqual(annotations["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
+        self.assertEqual(
+            annotations["names"],
+            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
+        )
         self.assertEqual(len(annotations["data"]), 2638)
-        annotations = self.data.annotation(None, "var")
+        annotations = json.loads(self.data.annotation(None, "var"))
         self.assertEqual(annotations["names"], ["name", "n_cells"])
         self.assertEqual(len(annotations["data"]), 1838)
 
     def test_annotation_fields(self):
-        annotations = self.data.annotation(None, "obs", ["n_genes", "n_counts"])
+        annotations = json.loads(
+            self.data.annotation(None, "obs", ["n_genes", "n_counts"])
+        )
         self.assertEqual(annotations["names"], ["n_genes", "n_counts"])
         self.assertEqual(len(annotations["data"]), 2638)
-        annotations = self.data.annotation(None, "var", ["name"])
+        annotations = json.loads(self.data.annotation(None, "var", ["name"]))
         self.assertEqual(annotations["names"], ["name"])
         self.assertEqual(len(annotations["data"]), 1838)
 
     def test_filtered_annotation(self):
         filter_ = {
             "filter": {
-                "obs": {
+                "obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]},
-                    "annotation_value": [
-                        {"name": "n_counts", "min": 3000},
-                    ]
-                },
                 "var": {
-                    "annotation_value": [
+                    "annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]
-                        {"name": "name", "values": ["ATAD3C", "RER1"]},
+                },
-                    ]
-                }
             }
         }
-        annotations = self.data.annotation(filter_["filter"], "obs")
+        annotations = json.loads(self.data.annotation(filter_["filter"], "obs"))
-        self.assertEqual(annotations["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
+        self.assertEqual(
+            annotations["names"],
+            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
+        )
         self.assertEqual(len(annotations["data"]), 497)
-        annotations = self.data.annotation(filter_["filter"], "var")
+        annotations = json.loads(self.data.annotation(filter_["filter"], "var"))
         self.assertEqual(annotations["names"], ["name", "n_cells"])
         self.assertEqual(len(annotations["data"]), 2)
 
     def test_filtered_layout(self):
         filter_ = {
-            "filter": {
+            "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
-                "obs": {
-                    "annotation_value": [
-                        {"name": "n_counts", "min": 3000},
-                    ]
-                }
-            }
         }
-        layout = self.data.layout(filter_["filter"])
+        layout = json.loads(self.data.layout(filter_["filter"]))
-        self.assertEqual(len(layout["coordinates"]), 497)
+        self.assertEqual(len(layout["layout"]["coordinates"]), 497)
 
     def test_diffexp_topN(self):
-        f1 = {
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
-            "filter": {
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
-                "obs": {
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-                    "index": [[0, 500]]
-                }
-            }
-        }
-        f2 = {
-            "filter": {
-                "obs": {
-                    "index": [[500, 1000]]
-                }
-            }
-        }
-        result = self.data.diffexp_topN(f1["filter"], f2["filter"])
         self.assertEqual(len(result), 10)
-        result = self.data.diffexp_topN(f1["filter"], f2["filter"], 20)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
         self.assertEqual(len(result), 20)
 
     def test_data_frame(self):
-        data_frame_obs = self.data.data_frame(None, "obs")
+        data_frame_obs = json.loads(self.data.data_frame(None, "obs"))
         self.assertEqual(len(data_frame_obs["var"]), 1838)
         self.assertEqual(len(data_frame_obs["obs"]), 2638)
-        data_frame_var = self.data.data_frame(None, "var")
+        data_frame_var = json.loads(self.data.data_frame(None, "var"))
         self.assertEqual(len(data_frame_var["var"]), 1838)
         self.assertEqual(len(data_frame_var["obs"]), 2638)
 
     def test_filtered_data_frame(self):
         filter_ = {
-            "filter": {
+            "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
-                "obs": {
-                    "annotation_value": [
-                        {"name": "n_counts", "min": 3000},
-                    ]
-                }
-            }
         }
-        data_frame_obs = self.data.data_frame(filter_["filter"], "obs")
+        data_frame_obs = json.loads(self.data.data_frame(filter_["filter"], "obs"))
         self.assertEqual(len(data_frame_obs["var"]), 1838)
         self.assertEqual(len(data_frame_obs["obs"]), 497)
         self.assertIsInstance(data_frame_obs["obs"][0], (list, tuple))
         self.assertEqual(type(data_frame_obs["var"][0]), int)
-        data_frame_var = self.data.data_frame(filter_["filter"], "var")
+        data_frame_var = json.loads(self.data.data_frame(filter_["filter"], "var"))
         self.assertEqual(len(data_frame_var["var"]), 1838)
         self.assertEqual(len(data_frame_var["obs"]), 497)
         self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
@@ -237,14 +209,10 @@ class UtilTest(unittest.TestCase):
         for axis in ["obs", "var"]:
             filter_ = {
                 "filter": {
-                    "var": {
+                    "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
-                        "annotation_value": [
-                            {"name": "name", "values": ["RER1"]},
-                        ]
-                    }
                 }
             }
-            data_frame_var = self.data.data_frame(filter_["filter"], axis)
+            data_frame_var = json.loads(self.data.data_frame(filter_["filter"], axis))
             if axis == "obs":
                 self.assertEqual(type(data_frame_var["var"][0]), int)
                 self.assertIsInstance(data_frame_var["obs"][0], (list, tuple))
@@ -252,5 +220,5 @@ class UtilTest(unittest.TestCase):
                 self.assertEqual(type(data_frame_var["obs"][0]), int)
                 self.assertIsInstance(data_frame_var["var"][0], (list, tuple))
 
-    if __name__ == '__main__':
+    if __name__ == "__main__":
         unittest.main()