Merge remote-tracking branch 'origin/master' into sidneymbell/docs-overhaul

client/package.json

@@ -32,9 +32,9 @@
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.18.1",
+    "@blueprintjs/core": "^3.20.0",
-    "@blueprintjs/icons": "^3.10.0",
+    "@blueprintjs/icons": "^3.12.0",
-    "@blueprintjs/select": "^3.10.0",
+    "@blueprintjs/select": "^3.11.2",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
     "flatbuffers": "^1.11.0",
@@ -45,48 +45,48 @@
     "is-number": "^7.0.0",
     "lodash": "^4.17.15",
     "memoize-one": "^5.1.1",
-    "react": "^16.9.0",
+    "react": "^16.11.0",
     "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.9.0",
+    "react-dom": "^16.11.0",
     "react-flip-toolkit": "7.0.6",
     "react-helmet": "^5.2.1",
-    "react-icons": "^3.7.0",
+    "react-icons": "^3.8.0",
-    "react-redux": "^7.1.1",
+    "react-redux": "^7.1.3",
     "redux": "^4.0.4",
     "redux-thunk": "^2.2.0",
     "regl": "^1.3.13"
   },
   "devDependencies": {
-    "@babel/core": "^7.6.0",
+    "@babel/core": "^7.7.2",
-    "@babel/plugin-proposal-class-properties": "^7.5.5",
+    "@babel/plugin-proposal-class-properties": "^7.7.0",
-    "@babel/plugin-proposal-decorators": "^7.6.0",
+    "@babel/plugin-proposal-decorators": "^7.7.0",
     "@babel/plugin-proposal-export-namespace-from": "^7.5.2",
     "@babel/plugin-proposal-function-bind": "^7.2.0",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
     "@babel/plugin-proposal-optional-chaining": "^7.6.0",
-    "@babel/plugin-transform-react-constant-elements": "^7.6.0",
+    "@babel/plugin-transform-react-constant-elements": "^7.6.3",
-    "@babel/plugin-transform-runtime": "^7.6.0",
+    "@babel/plugin-transform-runtime": "^7.6.2",
-    "@babel/preset-env": "^7.6.0",
+    "@babel/preset-env": "^7.7.1",
-    "@babel/preset-react": "^7.0.0",
+    "@babel/preset-react": "^7.7.0",
-    "@babel/register": "^7.6.0",
+    "@babel/register": "^7.7.0",
-    "@babel/runtime": "^7.6.0",
+    "@babel/runtime": "^7.7.2",
     "babel-eslint": "^10.0.3",
     "babel-jest": "^24.9.0",
     "babel-loader": "^8.0.6",
     "babel-preset-modern-browsers": "^14.0.0",
-    "chalk": "^2.4.2",
+    "chalk": "^3.0.0",
     "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^5.0.4",
+    "copy-webpack-plugin": "^5.0.5",
     "css-loader": "^3.2.0",
-    "eslint": "^6.4.0",
+    "eslint": "^6.6.0",
     "eslint-config-airbnb": "^18.0.1",
-    "eslint-config-prettier": "^6.3.0",
+    "eslint-config-prettier": "^6.5.0",
-    "eslint-loader": "^3.0.0",
+    "eslint-loader": "^3.0.2",
     "eslint-plugin-filenames": "^1.3.2",
     "eslint-plugin-import": "^2.18.2",
-    "eslint-plugin-jest": "^22.17.0",
+    "eslint-plugin-jest": "^23.0.4",
     "eslint-plugin-jsx-a11y": "^6.2.3",
-    "eslint-plugin-react": "^7.14.3",
+    "eslint-plugin-react": "^7.16.0",
     "express": "^4.17.1",
     "file-loader": "^4.2.0",
     "html-webpack-inline-source-plugin": "0.0.10",
@@ -95,16 +95,16 @@
     "jest-puppeteer": "^4.3.0",
     "json-loader": "^0.5.4",
     "mini-css-extract-plugin": "^0.8.0",
-    "puppeteer": "^1.20.0",
+    "puppeteer": "^2.0.0",
     "rimraf": "^3.0.0",
     "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.10.2",
+    "start-server-and-test": "^1.10.6",
     "style-loader": "^1.0.0",
     "sw-precache-webpack-plugin": "^0.11.5",
-    "url-loader": "^2.1.0",
+    "url-loader": "^2.2.0",
-    "webpack": "^4.40.2",
+    "webpack": "^4.41.2",
-    "webpack-cli": "^3.3.8",
+    "webpack-cli": "^3.3.10",
-    "webpack-dev-middleware": "^3.7.1"
+    "webpack-dev-middleware": "^3.7.2"
   },
   "jest": {
     "testMatch": [

client/src/components/brushableHistogram/index.js

@@ -44,7 +44,7 @@ class HistogramBrush extends React.PureComponent {
     return varData.col(field);
   }
 
-  calcHistogramCache = memoize((col, field) => {
+  calcHistogramCache = memoize(col => {
     /*
      recalculate expensive stuff, notably bins, summaries, etc.
     */
@@ -240,15 +240,23 @@ class HistogramBrush extends React.PureComponent {
     };
   }
 
-  drawHistogram(svgRef) {
+  handleSetGeneAsScatterplotX = () => {
-    const { field, world } = this.props;
+    const { dispatch, field } = this.props;
-    const col = HistogramBrush.getColumn(world, field);
+    dispatch({
-    const histogramCache = this.calcHistogramCache(col, field);
+      type: "set scatterplot x",
-    const { x, y, bins } = histogramCache;
+      data: field
-    this._histogram = { x, y, bins, svgRef };
+    });
-  }
+  };
 
-  handleColorAction() {
+  handleSetGeneAsScatterplotY = () => {
+    const { dispatch, field } = this.props;
+    dispatch({
+      type: "set scatterplot y",
+      data: field
+    });
+  };
+
+  handleColorAction = () => {
     const { dispatch, field, world, ranges } = this.props;
 
     if (world.obsAnnotations.hasCol(field)) {
@@ -260,9 +268,9 @@ class HistogramBrush extends React.PureComponent {
     } else if (world.varData.hasCol(field)) {
       dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field));
     }
-  }
+  };
 
-  removeHistogram() {
+  removeHistogram = () => {
     const {
       dispatch,
       field,
@@ -291,26 +299,14 @@ class HistogramBrush extends React.PureComponent {
         data: null
       });
     }
-  }
+  };
 
-  handleSetGeneAsScatterplotX() {
+  drawHistogram(svgRef) {
-    return () => {
+    const { field, world } = this.props;
-      const { dispatch, field } = this.props;
+    const col = HistogramBrush.getColumn(world, field);
-      dispatch({
+    const histogramCache = this.calcHistogramCache(col);
-        type: "set scatterplot x",
+    const { x, y, bins } = histogramCache;
-        data: field
+    this._histogram = { x, y, bins, svgRef };
-      });
-    };
-  }
-
-  handleSetGeneAsScatterplotY() {
-    return () => {
-      const { dispatch, field } = this.props;
-      dispatch({
-        type: "set scatterplot y",
-        data: field
-      });
-    };
   }
 
   renderAxesBrushBins(x, y, bins, svgRef, field) {
@@ -439,7 +435,7 @@ class HistogramBrush extends React.PureComponent {
               <ButtonGroup style={{ marginRight: 7 }}>
                 <Button
                   data-testid={`plot-x-${field}`}
-                  onClick={this.handleSetGeneAsScatterplotX(field).bind(this)}
+                  onClick={this.handleSetGeneAsScatterplotX}
                   active={scatterplotXXaccessor === field}
                   intent={scatterplotXXaccessor === field ? "primary" : "none"}
                 >
@@ -447,7 +443,7 @@ class HistogramBrush extends React.PureComponent {
                 </Button>
                 <Button
                   data-testid={`plot-y-${field}`}
-                  onClick={this.handleSetGeneAsScatterplotY(field).bind(this)}
+                  onClick={this.handleSetGeneAsScatterplotY}
                   active={scatterplotYYaccessor === field}
                   intent={scatterplotYYaccessor === field ? "primary" : "none"}
                 >
@@ -459,7 +455,7 @@ class HistogramBrush extends React.PureComponent {
           {isUserDefined ? (
             <Button
               minimal
-              onClick={this.removeHistogram.bind(this)}
+              onClick={this.removeHistogram}
               style={{
                 color: globals.blue,
                 cursor: "pointer",
@@ -475,7 +471,7 @@ class HistogramBrush extends React.PureComponent {
             hoverOpenDelay={globals.tooltipHoverOpenDelay}
           >
             <Button
-              onClick={this.handleColorAction.bind(this)}
+              onClick={this.handleColorAction}
               active={colorAccessor === field}
               intent={colorAccessor === field ? "primary" : "none"}
               data-testclass="colorby"

client/src/components/categorical/categorical.js

@@ -7,12 +7,14 @@ import {
   InputGroup,
   Dialog,
   Classes,
-  MenuItem
+  MenuItem,
+  Colors
 } from "@blueprintjs/core";
 import { Select } from "@blueprintjs/select";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
+import { AnnotationsHelpers } from "../../util/stateManager";
 
 @connect(state => ({
   categoricalSelection: state.categoricalSelection,
@@ -49,15 +51,66 @@ class Categories extends React.Component {
   };
 
   handleDisableAnnoMode = () => {
-    this.setState({ createAnnoModeActive: false });
+    this.setState({
+      createAnnoModeActive: false,
+      categoryToDuplicate: null,
+      newCategoryText: ""
+    });
   };
 
   handleModalDuplicateCategorySelection = d => {
     this.setState({ categoryToDuplicate: d });
   };
 
+  categoryNameError = name => {
+    /*
+    return false if this is a LEGAL/acceptable category name or NULL/empty string,
+    or return an error type.
+    */
+    if (!name) return false;
+
+    const { categoricalSelection } = this.props;
+    const allCategoryNames = Object.keys(categoricalSelection);
+
+    if (allCategoryNames.indexOf(name) !== -1) {
+      return "duplicate";
+    }
+
+    if (!AnnotationsHelpers.isLegalAnnotationName(name)) {
+      return "characters";
+    }
+
+    return false;
+  };
+
+  categoryNameErrorMessage = name => {
+    const err = this.categoryNameError(name);
+    if (err === false) return null;
+    if (err === "duplicate") {
+      return (
+        <span>
+          <span style={{ fontStyle: "italic" }}>{name}</span> already exists -
+          no duplicates allowed
+        </span>
+      );
+    }
+    if (err === "characters") {
+      return (
+        <span>
+          <span style={{ fontStyle: "italic" }}>{name}</span> contains illegal
+          characters. Hint: use alpha-numeric and underscore
+        </span>
+      );
+    }
+    return err;
+  };
+
   render() {
-    const { createAnnoModeActive, categoryToDuplicate } = this.state;
+    const {
+      createAnnoModeActive,
+      categoryToDuplicate,
+      newCategoryText
+    } = this.state;
     const {
       categoricalSelection,
       writableCategoriesEnabled,
@@ -108,7 +161,6 @@ class Categories extends React.Component {
               <form
                 onSubmit={e => {
                   e.preventDefault();
-                  this.handleCreateUserAnno();
                 }}
               >
                 <div className={Classes.DIALOG_BODY}>
@@ -116,12 +168,30 @@ class Categories extends React.Component {
                     <p>New, unique category name:</p>
                     <InputGroup
                       autoFocus
+                      value={newCategoryText}
+                      intent={
+                        this.categoryNameError(newCategoryText)
+                          ? "warning"
+                          : "none"
+                      }
                       onChange={e =>
                         this.setState({ newCategoryText: e.target.value })
                       }
                       leftIcon="tag"
                     />
+                    <p
+                      style={{
+                        marginTop: 7,
+                        visibility: this.categoryNameError(newCategoryText)
+                          ? "visible"
+                          : "hidden",
+                        color: Colors.ORANGE3
+                      }}
+                    >
+                      {this.categoryNameErrorMessage(newCategoryText)}
+                    </p>
                   </div>
+
                   <p>
                     Optionally duplicate all labels & cell assignments from
                     existing category into new category:
@@ -157,6 +227,10 @@ class Categories extends React.Component {
                     </Tooltip>
                     <Button
                       onClick={this.handleCreateUserAnno}
+                      disabled={
+                        !newCategoryText ||
+                        this.categoryNameError(newCategoryText)
+                      }
                       intent="primary"
                       type="submit"
                     >

client/src/components/categorical/category.js

@@ -2,7 +2,7 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { Flipper, Flipped, Spring } from "react-flip-toolkit";
+import { Flipper, Flipped } from "react-flip-toolkit";
 import {
   Button,
   Tooltip,
@@ -14,12 +14,14 @@ import {
   Classes,
   Icon,
   Position,
-  PopoverInteractionKind
+  PopoverInteractionKind,
+  Colors
 } from "@blueprintjs/core";
 
 import * as globals from "../../globals";
 import Value from "./value";
 import sortedCategoryValues from "./util";
+import { AnnotationsHelpers } from "../../util/stateManager";
 
 @connect(state => ({
   colorAccessor: state.colors.colorAccessor,
@@ -33,7 +35,7 @@ class Category extends React.Component {
     this.state = {
       isChecked: true,
       isExpanded: false,
-      newCategoryText: "",
+      newCategoryText: props.metadataField,
       newLabelText: ""
     };
   }
@@ -81,18 +83,14 @@ class Category extends React.Component {
     dispatch({
       type: "annotation: disable add new label mode"
     });
+    this.setState({
+      newLabelText: ""
+    });
   };
 
   handleAddNewLabelToCategory = () => {
     const { dispatch, metadataField } = this.props;
     const { newLabelText } = this.state;
-    /*
-    XXX TODO - temporary code generates random label string. Remove
-    when the label creation UI is implemented.
-
-    const { newLabelText } = this.state;
-    */
-    // const newLabelText = `label${Math.random()}`;
     dispatch({
       type: "annotation: add new label to category",
       metadataField,
@@ -118,9 +116,19 @@ class Category extends React.Component {
   };
 
   handleEditCategory = () => {
-    const { dispatch, metadataField } = this.props;
+    const { dispatch, metadataField, categoricalSelection } = this.props;
     const { newCategoryText } = this.state;
 
+    const allCategoryNames = _.keys(categoricalSelection);
+
+    if (
+      (allCategoryNames.indexOf(newCategoryText) > -1 &&
+        newCategoryText !== metadataField) ||
+      newCategoryText === ""
+    ) {
+      return;
+    }
+
     dispatch({
       type: "annotation: category edited",
       metadataField,
@@ -145,6 +153,126 @@ class Category extends React.Component {
     });
   };
 
+  labelNameError = name => {
+    /*
+    return false if this is a LEGAL/acceptable category name or NULL/empty string,
+    or return an error type.
+    */
+    let error = false;
+    if (name) {
+      const { metadataField, universe } = this.props;
+      const { obsByName } = universe.schema.annotations;
+
+      if (obsByName[metadataField].categories.indexOf(name) !== -1) {
+        error = "duplicate";
+      } else if (!AnnotationsHelpers.isLegalAnnotationName(name)) {
+        error = "characters";
+      }
+    }
+    return error;
+  };
+
+  labelNameErrorMessage = name => {
+    const { metadataField } = this.props;
+    const err = this.labelNameError(name);
+    if (err === false) return null;
+    if (err === "duplicate") {
+      return (
+        <span>
+          <span style={{ fontStyle: "italic" }}>{name}</span> already exists
+          already exists within{" "}
+          <span style={{ fontStyle: "italic" }}>{metadataField}</span>{" "}
+        </span>
+      );
+    }
+    if (err === "characters") {
+      return (
+        <span>
+          <span style={{ fontStyle: "italic" }}>{name}</span> contains illegal
+          characters. Hint: use alpha-numeric and underscore
+        </span>
+      );
+    }
+    return err;
+  };
+
+  categoryNameErrorMessage = () => {
+    const { newCategoryText } = this.state;
+    const err = this.editedCategoryNameError();
+    if (err === false) return null;
+
+    let markup = null;
+
+    if (err === "empty_string") {
+      markup = (
+        <span
+          style={{
+            display: "block",
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3
+          }}
+        >
+          {"Category name cannot be blank"}
+        </span>
+      );
+    } else if (err === "already_exists") {
+      markup = (
+        <span
+          style={{
+            display: "block",
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3
+          }}
+        >
+          {"Category name must be unique"}
+        </span>
+      );
+    } else if (err === "characters") {
+      markup = (
+        <span
+          style={{
+            display: "block",
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3
+          }}
+        >
+          {"Only alphanumeric and underscore allowed"}
+        </span>
+      );
+    }
+
+    return markup;
+  };
+
+  editedCategoryNameError = () => {
+    const { metadataField, categoricalSelection } = this.props;
+    const { newCategoryText } = this.state;
+    const allCategoryNames = _.keys(categoricalSelection);
+
+    const isEmptyString = newCategoryText === "";
+    const categoryNameAlreadyExists =
+      allCategoryNames.indexOf(newCategoryText) > -1;
+    const sameName = newCategoryText === metadataField;
+
+    let error = false;
+
+    if (isEmptyString) {
+      error = "empty_string";
+    } else if (categoryNameAlreadyExists && !sameName) {
+      error = "already_exists";
+    } else if (!AnnotationsHelpers.isLegalAnnotationName(newCategoryText)) {
+      error = "characters";
+    }
+
+    return error;
+  };
+
   toggleAll() {
     const { dispatch, metadataField } = this.props;
     dispatch({
@@ -202,8 +330,7 @@ class Category extends React.Component {
       colorAccessor,
       categoricalSelection,
       isUserAnno,
-      annotations,
+      annotations
-      universe
     } = this.props;
     const { isTruncated } = categoricalSelection[metadataField];
 
@@ -212,6 +339,7 @@ class Category extends React.Component {
       ...cat.categoryValueIndices
     ]);
     const optTuplesAsKey = _.map(optTuples, t => t[0]).join(""); // animation
+    const allCategoryNames = _.keys(categoricalSelection);
 
     return (
       <div
@@ -232,7 +360,7 @@ class Category extends React.Component {
             style={{
               display: "flex",
               justifyContent: "flex-start",
-              alignItems: "baseline"
+              alignItems: "flex-start"
             }}
           >
             <label className="bp3-control bp3-checkbox">
@@ -248,7 +376,6 @@ class Category extends React.Component {
                 type="checkbox"
               />
               <span className="bp3-control-indicator" />
-              {""}
             </label>
             <span
               data-testid={`category-expand-${metadataField}`}
@@ -294,7 +421,7 @@ class Category extends React.Component {
                     rightElement={
                       <Button
                         minimal
-                        disabled={newCategoryText.length === 0}
+                        disabled={this.editedCategoryNameError()}
                         style={{ position: "relative", top: -1 }}
                         type="button"
                         icon="small-tick"
@@ -304,6 +431,7 @@ class Category extends React.Component {
                       />
                     }
                   />
+                  {this.categoryNameErrorMessage()}
                 </form>
               ) : (
                 metadataField
@@ -345,11 +473,28 @@ class Category extends React.Component {
                         <p>New, unique label name:</p>
                         <InputGroup
                           autoFocus
+                          value={newLabelText}
+                          intent={
+                            this.labelNameError(newLabelText)
+                              ? "warning"
+                              : "none"
+                          }
                           onChange={e =>
                             this.setState({ newLabelText: e.target.value })
                           }
                           leftIcon="tag"
                         />
+                        <p
+                          style={{
+                            marginTop: 7,
+                            visibility: this.labelNameError(newLabelText)
+                              ? "visible"
+                              : "hidden",
+                            color: Colors.ORANGE3
+                          }}
+                        >
+                          {this.labelNameErrorMessage(newLabelText)}
+                        </p>
                       </div>
                     </div>
                     <div className={Classes.DIALOG_FOOTER}>
@@ -361,10 +506,7 @@ class Category extends React.Component {
                         </Tooltip>
                         <Button
                           disabled={
-                            newLabelText.length === 0 ||
+                            !newLabelText || this.labelNameError(newLabelText)
-                            universe.schema.annotations.obsByName[
-                              metadataField
-                            ].categories.indexOf(newLabelText) !== -1
                           }
                           onClick={this.handleAddNewLabelToCategory}
                           intent="primary"

client/src/components/categorical/occupancy.js

@@ -144,7 +144,9 @@ class Occupancy extends React.Component {
       categoryIndex
     } = this.props;
 
-    this.canvas?.getContext("2d").clearRect(0, 0, this._WIDTH, this._HEIGHT);
+    const { canvas } = this;
+    if (canvas)
+      canvas.getContext("2d").clearRect(0, 0, this._WIDTH, this._HEIGHT);
 
     const colorByIsCatagoricalData = !!categoricalSelection[colorAccessor];
 

client/src/components/categorical/value.js

@@ -9,13 +9,12 @@ import {
   MenuItem,
   Popover,
   Position,
-  Icon,
+  PopoverInteractionKind,
-  PopoverInteractionKind
+  Tooltip
 } from "@blueprintjs/core";
 import Occupancy from "./occupancy";
 import * as globals from "../../globals";
 import styles from "./categorical.css";
-import { Tooltip } from "@blueprintjs/core";
 
 import { AnnotationsHelpers } from "../../util/stateManager";
 
@@ -32,19 +31,26 @@ class CategoryValue extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      editedLabelText: ""
+      editedLabelText: String(
+        props.categoricalSelection[props.metadataField].categoryValues[
+          props.categoryIndex
+        ]
+      ).valueOf()
     };
   }
 
-  handleDeleteValue = () => {
+  getLabel = () => {
-    const {
+    const { metadataField, categoryIndex, categoricalSelection } = this.props;
-      dispatch,
-      metadataField,
-      categoryIndex,
-      categoricalSelection
-    } = this.props;
     const category = categoricalSelection[metadataField];
     const label = category.categoryValues[categoryIndex];
+
+    return label;
+  };
+
+  handleDeleteValue = () => {
+    const { dispatch, metadataField } = this.props;
+    const label = this.getLabel();
+
     dispatch({
       type: "annotation: delete label",
       metadataField,
@@ -53,14 +59,8 @@ class CategoryValue extends React.Component {
   };
 
   handleAddCurrentSelectionToThisLabel = () => {
-    const {
+    const { dispatch, metadataField, categoryIndex } = this.props;
-      dispatch,
+    const label = this.getLabel();
-      metadataField,
-      categoryIndex,
-      categoricalSelection
-    } = this.props;
-    const category = categoricalSelection[metadataField];
-    const label = category.categoryValues[categoryIndex];
     dispatch({
       type: "annotation: label current cell selection",
       metadataField,
@@ -70,15 +70,9 @@ class CategoryValue extends React.Component {
   };
 
   handleEditValue = () => {
-    const {
+    const { dispatch, metadataField, categoryIndex } = this.props;
-      dispatch,
-      metadataField,
-      categoryIndex,
-      categoricalSelection
-    } = this.props;
     const { editedLabelText } = this.state;
-    const category = categoricalSelection[metadataField];
+    const label = this.getLabel();
-    const label = category.categoryValues[categoryIndex];
     dispatch({
       type: "annotation: label edited",
       editedLabel: editedLabelText,
@@ -86,7 +80,80 @@ class CategoryValue extends React.Component {
       categoryIndex,
       label
     });
-    this.setState({ editedLabelText: "" });
+  };
+
+  valueNameErrorMessage = () => {
+    const { editedLabelText } = this.state;
+    const err = this.valueNameError();
+    if (!err) return null;
+
+    let markup = null;
+
+    if (err === "empty_string") {
+      markup = (
+        <span
+          style={{
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3
+          }}
+        >
+          {"Label cannot be blank"}
+        </span>
+      );
+    } else if (err === "duplicate") {
+      markup = (
+        <span
+          style={{
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3
+          }}
+        >
+          {"Label must be unique"}
+        </span>
+      );
+    } else if (err === "characters") {
+      markup = (
+        <span
+          style={{
+            fontStyle: "italic",
+            fontSize: 12,
+            marginTop: 5,
+            color: Colors.ORANGE3
+          }}
+        >
+          {"Only alphanumeric and underscore allowed"}
+        </span>
+      );
+    }
+    return markup;
+  };
+
+  valueNameError = () => {
+    const { editedLabelText } = this.state;
+    const { categoricalSelection, metadataField, categoryIndex } = this.props;
+
+    let err = null;
+
+    const category = categoricalSelection[metadataField];
+    const displayString = String(
+      category.categoryValues[categoryIndex]
+    ).valueOf();
+
+    if (editedLabelText === "") {
+      err = "empty_string";
+    } else if (
+      category.categoryValues.indexOf(editedLabelText) > -1 &&
+      editedLabelText !== displayString
+    ) {
+      err = "duplicate";
+    } else if (!AnnotationsHelpers.isLegalAnnotationName(editedLabelText)) {
+      err = "characters";
+    }
+    return err;
   };
 
   activateEditLabelMode = () => {
@@ -116,7 +183,7 @@ class CategoryValue extends React.Component {
     });
   };
 
-  shouldComponentUpdate = nextProps => {
+  shouldComponentUpdate = (nextProps, nextState) => {
     /*
     Checks to see if at least one of the following changed:
     * world state
@@ -126,7 +193,7 @@ class CategoryValue extends React.Component {
 
     If and only if true, update the component
     */
-    const { props } = this;
+    const { props, state } = this;
     const { metadataField, categoryIndex, categoricalSelection } = props;
     const { categoricalSelection: newCategoricalSelection } = nextProps;
 
@@ -142,13 +209,15 @@ class CategoryValue extends React.Component {
     const colorAccessorChange = props.colorAccessor !== nextProps.colorAccessor;
     const annotationsChange = props.annotations !== nextProps.annotations;
     const crossfilterChange = props.crossfilter !== nextProps.crossfilter;
+    const editingLabel = state.editedLabelText !== nextState.editedLabelText;
 
     return (
       valueSelectionChange ||
       worldChange ||
       colorAccessorChange ||
       annotationsChange ||
-      crossfilterChange
+      crossfilterChange ||
+      editingLabel
     );
   };
 
@@ -223,6 +292,8 @@ class CategoryValue extends React.Component {
       flippedProps
     } = this.props;
 
+    const { editedLabelText } = this.state;
+
     if (!categoricalSelection) return null;
 
     const category = categoricalSelection[metadataField];
@@ -348,6 +419,9 @@ class CategoryValue extends React.Component {
               <form
                 onSubmit={e => {
                   e.preventDefault();
+                  if (this.valueNameError()) {
+                    return;
+                  }
                   this.handleEditValue();
                 }}
               >
@@ -358,6 +432,7 @@ class CategoryValue extends React.Component {
                   }}
                   small
                   autoFocus
+                  intent={this.valueNameError() ? "warning" : "none"}
                   onChange={e => {
                     this.setState({ editedLabelText: e.target.value });
                   }}
@@ -366,6 +441,7 @@ class CategoryValue extends React.Component {
                     <Button
                       minimal
                       style={{ position: "relative", top: -1 }}
+                      disabled={this.valueNameError()}
                       type="button"
                       icon="small-tick"
                       data-testclass="submitEdit"
@@ -374,6 +450,7 @@ class CategoryValue extends React.Component {
                     />
                   }
                 />
+                {this.valueNameErrorMessage()}
               </form>
             ) : null}
             {/*
@@ -478,16 +555,19 @@ class CategoryValue extends React.Component {
                         data-testclass="handleDeleteValue"
                         data-testid={`handleDeleteValue-${metadataField}`}
                         onClick={this.handleDeleteValue}
-                        text={`Delete this label, and reassign all cells to type '${
+                        text={`Delete this label, and reassign all cells to type '${globals.unassignedCategoryLabel}'`}
-                          globals.unassignedCategoryLabel
-                        }'`}
                       />
                     ) : null}
                   </Menu>
                 }
               >
                 <Button
-                  style={{ marginLeft: 0, position: "relative", top: -1 }}
+                  style={{
+                    marginLeft: 0,
+                    position: "relative",
+                    top: -1,
+                    minHeight: 16
+                  }}
                   data-testclass="seeActions"
                   data-testid={`seeActions-${metadataField}`}
                   icon="more"

client/src/components/continuous/continuous.js

@@ -25,7 +25,7 @@ class Continuous extends React.Component {
 
   componentDidUpdate() {}
 
-  handleColorAction(key) {
+  handleColorAction = key => {
     return () => {
       const { dispatch, obsAnnotations } = this.props;
       const summary = obsAnnotations.col(key).summarize();
@@ -35,7 +35,7 @@ class Continuous extends React.Component {
         rangeForColorAccessor: summary
       });
     };
-  }
+  };
 
   render() {
     const { obsAnnotations, schema } = this.props;
@@ -54,10 +54,11 @@ class Continuous extends React.Component {
       <div>
         {this.hasContinuous ? (
           <p
-            style={Object.assign({}, globals.leftSidebarSectionHeading, {
+            style={{
+              ...globals.leftSidebarSectionHeading,
               marginTop: 40,
               paddingLeft: globals.leftSidebarSectionPadding
-            })}
+            }}
           >
             Continuous metadata
           </p>
@@ -84,7 +85,7 @@ class Continuous extends React.Component {
                     isObs
                     zebra={zebra % 2 === 0}
                     ranges={summary}
-                    handleColorAction={this.handleColorAction(key).bind(this)}
+                    handleColorAction={this.handleColorAction(key)}
                   />
                 );
               }

client/src/components/framework/custom-icons.js

@@ -1,16 +0,0 @@
-/* https://github.com/palantir/blueprint/issues/2348 */
-
-<defs>
-<clipPath id="clip0">
-<rect width="16" height="16" fill="white"/>
-</clipPath>
-</defs>
-<g clip-path="url(#clip0)">
-<rect width="16" height="16" fill="white"/>
-<path d="M1.33415 8.75877C0.939491 8.36411 0.727699 7.82249 0.749957 7.2648L0.926361 2.84501C0.967947 1.80308 1.80308 0.967947 2.84501 0.926361L7.2648 0.749958C7.82249 0.727699 8.36411 0.939492 8.75877 1.33415L14.3595 6.93485C15.1405 7.7159 15.1405 8.98223 14.3595 9.76328L9.76328 14.3595C8.98223 15.1405 7.7159 15.1405 6.93485 14.3595L1.33415 8.75877Z" fill="black"/>
-<circle cx="4.5" cy="4.5" r="1.5" fill="white"/>
-<circle cx="4.5" cy="11.5" r="3.75" stroke="white" stroke-width="0.5"/>
-<circle cx="4.5" cy="11.5" r="3.5" fill="black"/>
-<line x1="4.5" y1="10" x2="4.5" y2="13" stroke="white"/>
-<line x1="3" y1="11.5" x2="6" y2="11.5" stroke="white"/>
-</g>

client/src/components/geneExpression/index.js

@@ -24,7 +24,7 @@ import {
 
 import { memoize } from "../../util/dataframe/util";
 
-const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
+const renderGene = (fuzzySortResult, { handleClick, modifiers }) => {
   if (!modifiers.matchesPredicate) {
     return null;
   }
@@ -89,6 +89,59 @@ class GeneExpression extends React.Component {
   // eslint-disable-next-line react/sort-comp
   _memoGenesToUpper = memoize(this._genesToUpper, arr => arr);
 
+  handleBulkAddClick = () => {
+    const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
+    const { bulkAdd } = this.state;
+
+    /*
+      test:
+      Apod,,, Cd74,,    ,,,    Foo,    Bar-2,,
+    */
+    if (bulkAdd !== "") {
+      const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
+      if (genes.length === 0) {
+        return keepAroundErrorToast("Must enter a gene name.");
+      }
+      const worldGenes = world.varAnnotations.col(varIndexName).asArray();
+
+      // These gene lists are unique enough where memoization is useless
+      const upperGenes = this._genesToUpper(genes);
+      const upperUserDefinedGenes = this._genesToUpper(userDefinedGenes);
+
+      const upperWorldGenes = this._memoGenesToUpper(worldGenes);
+
+      dispatch({ type: "bulk user defined gene start" });
+
+      Promise.all(
+        [...upperGenes.keys()].map(upperGene => {
+          if (upperUserDefinedGenes.get(upperGene) !== undefined) {
+            return keepAroundErrorToast("That gene already exists");
+          }
+
+          const indexOfGene = upperWorldGenes.get(upperGene);
+
+          if (indexOfGene === undefined) {
+            return keepAroundErrorToast(
+              `${
+                genes[upperGenes.get(upperGene)]
+              } doesn't appear to be a valid gene name.`
+            );
+          }
+          return dispatch(
+            actions.requestUserDefinedGene(worldGenes[indexOfGene])
+          );
+        })
+      ).then(
+        () => dispatch({ type: "bulk user defined gene complete" }),
+        () => dispatch({ type: "bulk user defined gene error" })
+      );
+    }
+
+    this.setState({ bulkAdd: "" });
+    return undefined;
+  };
+
   placeholderGeneNames() {
     /*
     return a string containing gene name suggestions for use as a user hint.
@@ -145,58 +198,6 @@ class GeneExpression extends React.Component {
     }
   }
 
-  handleBulkAddClick() {
-    const { world, dispatch, userDefinedGenes } = this.props;
-    const varIndexName = world.schema.annotations.var.index;
-    const { bulkAdd } = this.state;
-
-    /*
-      test:
-      Apod,,, Cd74,,    ,,,    Foo,    Bar-2,,
-    */
-    if (bulkAdd !== "") {
-      const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
-      if (genes.length === 0) {
-        return keepAroundErrorToast("Must enter a gene name.");
-      }
-      const worldGenes = world.varAnnotations.col(varIndexName).asArray();
-
-      // These gene lists are unique enough where memoization is useless
-      const upperGenes = this._genesToUpper(genes);
-      const upperUserDefinedGenes = this._genesToUpper(userDefinedGenes);
-
-      const upperWorldGenes = this._memoGenesToUpper(worldGenes);
-
-      dispatch({ type: "bulk user defined gene start" });
-
-      Promise.all(
-        [...upperGenes.keys()].map(upperGene => {
-          if (upperUserDefinedGenes.get(upperGene) !== undefined) {
-            return keepAroundErrorToast("That gene already exists");
-          }
-
-          const indexOfGene = upperWorldGenes.get(upperGene);
-
-          if (indexOfGene === undefined) {
-            return keepAroundErrorToast(
-              `${
-                genes[upperGenes.get(upperGene)]
-              } doesn't appear to be a valid gene name.`
-            );
-          }
-          return dispatch(
-            actions.requestUserDefinedGene(worldGenes[indexOfGene])
-          );
-        })
-      ).then(
-        () => dispatch({ type: "bulk user defined gene complete" }),
-        () => dispatch({ type: "bulk user defined gene error" })
-      );
-    }
-
-    this.setState({ bulkAdd: "" });
-  }
-
   render() {
     const {
       world,
@@ -261,7 +262,7 @@ class GeneExpression extends React.Component {
                 }}
                 initialContent={<MenuItem disabled text="Enter a gene…" />}
                 inputProps={{ "data-testid": "gene-search" }}
-                inputValueRenderer={g => {
+                inputValueRenderer={() => {
                   return "";
                 }}
                 itemListPredicate={filterGenes}
@@ -276,7 +277,7 @@ class GeneExpression extends React.Component {
               />
               <Button
                 className="bp3-button bp3-intent-primary"
-                data-testid={"add-gene"}
+                data-testid="add-gene"
                 loading={userDefinedGenesLoading}
                 onClick={() => this.handleClick(activeItem)}
               >
@@ -308,7 +309,7 @@ class GeneExpression extends React.Component {
                     />
                     <Button
                       intent="primary"
-                      onClick={this.handleBulkAddClick.bind(this)}
+                      onClick={this.handleBulkAddClick}
                       loading={userDefinedGenesLoading}
                     >
                       Add genes

client/src/components/leftSidebar/index.js

@@ -2,8 +2,6 @@
 import React from "react";
 import { connect } from "react-redux";
 import Categorical from "../categorical/categorical";
-import Continuous from "../continuous/continuous";
-import GeneExpression from "../geneExpression";
 import * as globals from "../../globals";
 import DynamicScatterplot from "../scatterplot/scatterplot";
 import TopLeftLogoAndTitle from "./topLeftLogoAndTitle";

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -74,7 +74,7 @@ class LeftSideBar extends React.Component {
             display: "inline-block",
             width: "190px",
             marginLeft: "7px",
-            height: "1.1em",
+            height: "1.2em",
             overflow: "hidden",
             wordBreak: "break-all"
           }}

client/src/components/menubar/index.js

@@ -266,11 +266,10 @@ class MenuBar extends React.Component {
     const haveBothCellSets =
       !!differential.celllist1 && !!differential.celllist2;
 
-    const tipMessage =
+    const slowMsg = diffexpMayBeSlow
-      "See top 10 differentially expressed genes" +
+      ? " (CAUTION: large dataset - may take longer or fail)"
-      (diffexpMayBeSlow
+      : "";
-        ? " (CAUTION: large dataset - may take longer or fail)"
+    const tipMessage = `See top 10 differentially expressed genes${slowMsg}`;
-        : "");
 
     return (
       <div className="bp3-button-group" style={{ marginRight: 10 }}>
@@ -318,7 +317,6 @@ class MenuBar extends React.Component {
   render() {
     const {
       dispatch,
-      differential,
       crossfilter,
       resettingInterface,
       libraryVersions,

client/src/components/rightSidebar/index.js

@@ -12,17 +12,7 @@ import * as globals from "../../globals";
 }))
 class RightSidebar extends React.Component {
   render() {
-    const {
+    const { responsive } = this.props;
-      responsive,
-      scatterplotXXaccessor,
-      scatterplotYYaccessor
-    } = this.props;
-
-    /*
-    this magic number should be made less fragile,
-    if cellxgene logo or tabs change, this must as well
-    */
-    const logoRelatedPadding = 50;
 
     return (
       <div

client/src/components/scatterplot/scatterplot.js

@@ -381,7 +381,7 @@ class Scatterplot extends React.PureComponent {
       <div
         style={{
           position: "fixed",
-          bottom: minimized ? -height + -margin.top : 0,
+          bottom: minimized ? -height + -margin.top - 2 : 0,
           borderRadius: "3px 3px 0px 0px",
           left: globals.leftSidebarWidth + globals.scatterplotMarginLeft,
           padding: "0px 20px 20px 0px",

client/src/reducers/annotations.js

@@ -5,7 +5,7 @@ const Annotations = (
   state = {
     isEditingCategoryName: false,
     isEditingLabelName: false,
-    categoryBeingEdited: false,
+    categoryBeingEdited: null,
     categoryAddingNewLabel: null,
     labelEditable: { category: null, label: null }
   },
@@ -46,7 +46,7 @@ const Annotations = (
     case "annotation: category edited":
       return {
         ...state,
-        isEditingCategoryName: true,
+        isEditingCategoryName: false,
         categoryBeingEdited: null
       };
 

client/src/reducers/centroidLabel.js

@@ -1,5 +1,3 @@
-import calcCentroid from "../util/centroid";
-
 const initialState = {
   metadataField: "",
   categoryIndex: -1,
@@ -8,7 +6,7 @@ const initialState = {
 };
 
 const CentroidLabel = (state = initialState, action, sharedNextState) => {
-  const { categoricalSelection, world, layoutChoice } = sharedNextState;
+  const { categoricalSelection } = sharedNextState;
   const { metadataField, categoryIndex } = action;
   const categoryField =
     categoricalSelection?.[metadataField]?.categoryValues[categoryIndex];
@@ -19,12 +17,7 @@ const CentroidLabel = (state = initialState, action, sharedNextState) => {
         metadataField,
         categoryIndex,
         categoryField,
-        centroidXY: null /* calcCentroid(  This function call is computationally heavy and also leading to large GC. Before reimplementation, look into optimization and memoization
+        centroidXY: null
-          world,
-          metadataField,
-          categoryField,
-          layoutChoice.currentDimNames
-        ) */
       };
 
     case "category value mouse hover end":

client/src/reducers/colors.js

@@ -53,6 +53,18 @@ const ColorsReducer = (
       };
     }
 
+    case "annotation: delete category": {
+      const { colorAccessor } = state;
+      if (action.metadataField !== colorAccessor) {
+        return state;
+      }
+      /* else reset */
+      return {
+        ...state,
+        ...ColorHelpers.resetColors(prevSharedState.world)
+      };
+    }
+
     case "reset colorscale": {
       return {
         ...state,

client/src/util/centroid.js

@@ -3,6 +3,8 @@ import quantile from "./quantile";
 /*
   Centroid coordinate calculation
 */
+
+/* Unused - please cleanup
 const calcMeanCentroid = (world, annoName, annoValue, layoutDimNames) => {
   const centroid = { x: 0, y: 0, size: 0 };
   const annoArray = world.obsAnnotations.col(annoName).asArray();
@@ -24,6 +26,7 @@ const calcMeanCentroid = (world, annoName, annoValue, layoutDimNames) => {
 
   return [centroid.x, centroid.y];
 };
+*/
 
 const calcMedianCentroid = (world, annoName, annoValue, layoutDimNames) => {
   const centroidX = [];

client/src/util/stateManager/annotationsHelpers.js

@@ -138,7 +138,7 @@ export function allHaveLabelByMask(df, colName, label, mask) {
 	const col = df.col(colName);
 	if (!col) return false;
 	if (df.length !== mask.length)
-		throw new InternalError("mismatch on mask length");
+		throw new RangeError("mismatch on mask length");
 
 	for (let i = 0; i < df.length; i += 1) {
 		if (mask[i]) {
@@ -182,3 +182,8 @@ export function createWritableAnnotationDimensions(world, crossfilter) {
 	}, crossfilter);
 	return crossfilter;
 }
+
+const legalNames = /^\w+$/;
+export function isLegalAnnotationName(name) {
+	return legalNames.test(name);
+}

client/src/util/typedCrossfilter/crossfilter.js

@@ -62,7 +62,7 @@ export default class ImmutableTypedCrossfilter {
 
   setData(data) {
     const { selectionCache } = this;
-    this.selectionCache = null;
+    this.selectionCache = {};
     return new ImmutableTypedCrossfilter(data, this.dimensions, selectionCache);
   }
 
@@ -108,7 +108,7 @@ export default class ImmutableTypedCrossfilter {
     };
 
     return new ImmutableTypedCrossfilter(data, dimensions, {
-      bitArray: bitArray
+      bitArray
     });
   }
 
@@ -128,7 +128,7 @@ export default class ImmutableTypedCrossfilter {
     }
 
     return new ImmutableTypedCrossfilter(data, dimensions, {
-      bitArray: bitArray
+      bitArray
     });
   }
 
@@ -159,7 +159,7 @@ export default class ImmutableTypedCrossfilter {
       select("blort", {mode: "range", lo: 0, hi: 999.99});
     */
     const { data, selectionCache } = this;
-    this.selectionCache = null;
+    this.selectionCache = {};
     const dimensions = { ...this.dimensions };
     const { dim, id, selection: oldSelection } = dimensions[name];
     const newSelection = dim.select(spec);

server/app/driver/driver.py

@@ -11,18 +11,20 @@ Sort order for methods
 
 
 class CXGDriver(metaclass=ABCMeta):
-    def __init__(self, data=None, args={}):
+    def __init__(self, data_locator=None, args={}):
         self.config = self._get_default_config()
         self.config.update(args)
-        if data:
+        if data_locator:
-            self._load_data(data)
+            self._load_data(data_locator)
+            self.data_locator = data_locator
         else:
             self.data = None
 
-    def update(self, data=None, args={}):
+    def update(self, data_locator=None, args={}):
         self.config.update(args)
-        if data:
+        if data_locator:
-            self._load_data(data)
+            self._load_data(data_locator)
+            self.data_locator = data_locator
 
     @staticmethod
     def _get_default_config():

server/app/scanpy_engine/labels.py

@@ -8,15 +8,18 @@ import pandas as pd
 
 def read_labels(fname):
     if exists(fname) and getsize(fname) > 0:
-        return pd.read_csv(fname, dtype='category', index_col=0)
+        return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#')
     else:
         return pd.DataFrame()
 
 
-def write_labels(fname, df):
+def write_labels(fname, df, header=None):
     rotate_fname(fname)
     if not df.empty:
-        df.to_csv(fname)
+        f = open(fname, 'a', newline="")
+        if header is not None:
+            f.write(header)
+        df.to_csv(f)
     else:
         open(fname, 'a').close()
 

server/app/scanpy_engine/scanpy_engine.py

@@ -1,6 +1,7 @@
 import warnings
 import copy
 import threading
+from datetime import datetime
 
 import numpy as np
 import pandas
@@ -8,6 +9,7 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 import anndata
 from scipy import sparse
 
+from server import __version__ as cellxgene_version
 from server.app.driver.driver import CXGDriver
 from server.app.util.constants import Axis, DEFAULT_TOP_N, MAX_LAYOUTS
 from server.app.util.errors import (
@@ -32,15 +34,15 @@ def has_method(o, name):
 
 
 class ScanpyEngine(CXGDriver):
-    def __init__(self, data=None, args={}):
+    def __init__(self, data_locator=None, args={}):
-        super().__init__(data, args)
+        super().__init__(data_locator, args)
         # lock used to protect label file write ops
         self.label_lock = threading.Lock()
         if self.data:
             self._validate_and_initialize()
 
-    def update(self, data=None, args={}):
+    def update(self, data_locator=None, args={}):
-        super().__init__(data, args)
+        super().__init__(data_locator, args)
         if self.data:
             self._validate_and_initialize()
 
@@ -484,7 +486,13 @@ class ScanpyEngine(CXGDriver):
         # so treat this as a critical section.
         with self.label_lock:
             self.labels = new_label_df
-            write_labels(fname, self.labels)
+            lastmod = self.data_locator.lastmodtime()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \
+                     f"using cellxgene version {cellxgene_version}\n" \
+                     f"# Input data file was {self.data_locator.uri_or_path}, " \
+                     f"which was last modified on {lastmodstr}\n"
+            write_labels(fname, self.labels, header)
 
         return jsonify_scanpy({"status": "OK"})
 

server/app/util/data_locator.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 import fsspec
+from datetime import datetime
 
 
 class DataLocator():
@@ -48,6 +49,14 @@ class DataLocator():
     def size(self):
         return self.fs.size(self.cname)
 
+    def lastmodtime(self):
+        """ return datetime object representing last modification time, or None if unavailable """
+        info = self.fs.info(self.cname)
+        if self.islocal() and info is not None:
+            return datetime.fromtimestamp(info['mtime'])
+        else:
+            return getattr(info, 'LastModified', None)
+
     def isfile(self):
         return self.fs.isfile(self.cname)
 

server/app/util/fbs/matrix.py

@@ -95,7 +95,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
         if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr):
             arr = arr.toarray()
         elif isinstance(arr, pd.Series):
-            arr = arr.get_values()
+            arr = arr.to_numpy()
         if arr.dtype != as_type:
             arr = arr.astype(as_type)
 

server/cli/cli.py

@@ -4,8 +4,17 @@ from .launch import launch
 from .prepare import prepare
 
 
-@click.group(name="cellxgene", context_settings=dict(max_content_width=85))
+@click.group(name="cellxgene",
-@click.version_option(version="0.12.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+             subcommand_metavar="COMMAND <args>",
+             options_metavar="<options>",
+             context_settings=dict(max_content_width=85,
+                                   help_option_names=['-h', '--help']))
+@click.help_option("--help", "-h", help="Show this message and exit.")
+@click.version_option(
+    version="0.12.0",
+    prog_name="cellxgene",
+    message="[%(prog)s] Version %(version)s",
+    help="Show the software version and exit.")
 def cli():
     pass
 

server/cli/launch.py

@@ -13,7 +13,7 @@ import click
 from server.app.app import Server
 from server.app.util.errors import ScanpyFileError
 from server.app.util.utils import custom_format_warning
-from server.utils.utils import find_available_port, is_port_available
+from server.utils.utils import find_available_port, is_port_available, sort_options
 from server.app.util.data_locator import DataLocator
 
 # anything bigger than this will generate a special message
@@ -25,55 +25,70 @@ def common_args(func):
     Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args.
     """
 
-    @click.option("--title", "-t", help="Title to display (if omitted will use file name).")
+    @click.option(
-    @click.option("--about",
+        "--title",
-                  help="A URL to more information about the dataset."
+        "-t",
-                       "(This must be an absolute URL including HTTP(S) protocol)")
+        metavar="<text>",
+        help="Title to display. If omitted will use file name.")
+    @click.option(
+        "--about",
+        metavar="<URL>",
+        help="URL providing more information about the dataset "
+             "(hint: must be a fully specified absolute URL).")
     @click.option(
         "--embedding",
         "-e",
         default=[],
         multiple=True,
         show_default=False,
+        metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all."
     )
-    @click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
+    @click.option(
-    @click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
+        "--obs-names",
+        "-obs",
+        default=None,
+        metavar="<text>",
+        help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.")
+    @click.option(
+        "--var-names",
+        "-var",
+        default=None,
+        metavar="<text>",
+        help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.")
     @click.option(
         "--max-category-items",
         default=1000,
-        metavar="",
+        metavar="<integer>",
         show_default=True,
-        help="Categories with more distinct values than this will not be displayed.",
+        help="Will not display categories with more distinct values than specified.",)
-    )
     @click.option(
         "--diffexp-lfc-cutoff",
+        "-de",
         default=0.01,
         show_default=True,
-        help="Relative expression cutoff used when selecting top N differentially expressed genes",
+        metavar="<float>",
-    )
+        help="Minimum log fold change threshold for differential expression.",)
     @click.option(
         "--experimental-label-file",
         default=None,
         show_default=True,
         multiple=False,
-        metavar="<user labels CSV file>",
+        metavar="<path>",
-        help="CSV file containing user annotations; will be overwritten.  Created if does not exist.",
+        help="CSV file containing user annotations; will be overwritten.  Created if does not exist.",)
-    )
     @click.option(
         "--backed",
+        "-b",
         is_flag=True,
         default=False,
         show_default=False,
-        help="Load data in file-backed mode, which may save memory, but result in slower overall performance."
+        help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.")
-    )
     @click.option(
         "--disable-diffexp",
         is_flag=True,
         default=False,
         show_default=False,
-        help="Disable on-demand differential expression."
+        help="Disable on-demand differential expression.")
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -96,17 +111,26 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items,
     }
 
 
-@click.command()
+@sort_options
-@click.argument("data", nargs=1, metavar="<data file>", required=True)
+@click.command(short_help="Launch the cellxgene data viewer. "
+                          "Run `cellxgene launch --help` for more information.",
+               options_metavar="<options>",)
+@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
 @click.option(
     "--verbose",
     "-v",
     is_flag=True,
     default=False,
     show_default=True,
-    help="Provide verbose output, including warnings and all server requests.",
+    help="Provide verbose output, including warnings and all server requests.",)
-)
+@click.option(
-@click.option("--debug", is_flag=True, default=False, show_default=True, help="Run in debug mode.")
+    "--debug",
+    "-d",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Run in debug mode. This is helpful for cellxgene developers, "
+         "or when you want more information about an error condition.",)
 @click.option(
     "--open",
     "-o",
@@ -114,18 +138,29 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items,
     is_flag=True,
     default=False,
     show_default=True,
-    help="Open the web browser after launch.",
+    help="Open web browser after launch.",)
-)
+@click.option(
-@click.option("--port", "-p", help="Port to run server on, if not specified cellxgene will find an available port.",
+    "--port",
-              metavar="", show_default=True)
+    "-p",
-@click.option("--host", default="127.0.0.1", help="Host IP address")
+    metavar="<port>",
+    show_default=True,
+    help="Port to run server on. If not specified cellxgene will find an available port.",)
+@click.option(
+    "--host",
+    metavar="<IP address>",
+    default="127.0.0.1",
+    show_default=False,
+    help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).")
 @click.option(
     "--scripts",
+    "-s",
     default=[],
     multiple=True,
-    help="Additional script files to include in html page",
+    metavar="<text>",
-    show_default=True,
+    help="Additional script files to include in HTML page. If not specified, "
-)
+         "no additional script files will be included.",
+    show_default=False,)
+@click.help_option("--help", "-h", help="Show this message and exit.")
 @common_args
 def launch(
         data,
@@ -148,8 +183,9 @@ def launch(
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
-    Data must be in a format that cellxgene expects, read the
+    Data must be in a format that cellxgene expects.
-    "getting started" guide.
+    Read the "getting started" guide to learn more:
+    https://chanzuckerberg.github.io/cellxgene/getting-started.html
 
     Examples:
 

server/cli/prepare.py

@@ -4,16 +4,21 @@ import click
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
+from server.utils.utils import sort_options
 
-@click.command()
+
-@click.argument("data", nargs=1, metavar="<dataset: file or path to data>", required=True)
+@sort_options
+@click.command(short_help="Preprocess data for use with cellxgene. "
+                          "Run `cellxgene prepare --help` for more information.",
+               options_metavar="<options>",)
+@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
 @click.option(
     "--embedding",
     "-e",
     default=["umap", "tsne"],
     multiple=True,
     type=click.Choice(["umap", "tsne"]),
-    help="Embedding algorithm",
+    help="Embedding algorithm(s). Repeat option for multiple embeddings.",
     show_default=True,
 )
 @click.option(
@@ -25,21 +30,29 @@ from scipy.sparse.csc import csc_matrix
     show_default=True,
 )
 @click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
-@click.option("--plotting", "-p", default=False, is_flag=True, help="Whether to generate plots.", show_default=True)
+@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
-@click.option("--sparse", default=False, is_flag=True, help="Whether to force sparsity.", show_default=True)
+@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
 @click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
 @click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
 @click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
+@click.option("--skip-qc", default=False, is_flag=True,
+              help="Do not run quality control metrics. By default cellxgene runs them "
+                   "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).")
 @click.option(
-    "--run-qc/--skip-qc", default=True, is_flag=True,
+    "--make-obs-names-unique",
-    help="Whether to calculate QC metrics (saved to adata.obs and adata.var). \
+    default=True,
-    See scanpy.pp.calculate_qc_metrics for details.", show_default=True)
+    is_flag=True,
-@click.option(
+    help="Ensure obs index is unique.",
-    "--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True
+    show_default=True
 )
 @click.option(
-    "--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True
+    "--make-var-names-unique",
+    default=True,
+    is_flag=True,
+    help="Ensure var index is unique.",
+    show_default=True
 )
+@click.help_option("--help", "-h", help="Show this message and exit.")
 def prepare(
         data,
         embedding,
@@ -50,18 +63,18 @@ def prepare(
         overwrite,
         set_obs_names,
         set_var_names,
-        run_qc,
+        skip_qc,
         make_obs_names_unique,
         make_var_names_unique,
 ):
-    """Preprocesses data for use with cellxgene.
+    """
-
+    Preprocess data for use with cellxgene.
-    This tool runs a series of scanpy routines for preparing a dataset
+    This tool runs a series of scanpy routines for preparing a dataset for use
-    for use with cellxgene. It loads data from different formats
+    with cellxgene. It loads data from different formats
     (h5ad, loom, or a 10x directory), runs dimensionality reduction,
     computes nearest neighbors, computes an embedding, performs clustering,
-    and saves the results. Includes additional options for naming
+    and saves the results. Includes additional options for naming annotations,
-    annotations, ensuring sparsity, and plotting results."""
+    ensuring sparsity, and plotting results."""
 
     # collect slow imports here to make CLI startup more responsive
     click.echo("[cellxgene] Starting CLI...")
@@ -129,7 +142,7 @@ def prepare(
         return adata
 
     def calculate_qc_metrics(adata):
-        if run_qc:
+        if not skip_qc:
             sc.pp.calculate_qc_metrics(adata, inplace=True)
         return adata
 
@@ -179,7 +192,7 @@ def prepare(
                 sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
 
     def show_step(item):
-        if run_qc:
+        if not skip_qc:
             qc_name = "Calculating QC metrics"
         else:
             qc_name = "Skipping QC"

server/requirements.txt

@@ -8,7 +8,7 @@ Flask-RESTful>=0.3.6
 flatbuffers>=1.10.0
 fsspec>=0.4.4
 numpy>=1.15.2
-pandas>=0.23.1
+pandas>=0.24.2
 scipy>=1.3.0
 tables==3.5.1
 # TEMP workaround for https://github.com/theislab/scanpy/issues/832 aka h5py regression

server/test/test_scanpy_engine_data_load.py

@@ -37,7 +37,7 @@ class DataLoadEngineTest(unittest.TestCase):
             self.data._create_schema()
 
     def test_delayed_load_data(self):
-        self.data.update(data=self.data_file)
+        self.data.update(data_locator=self.data_file)
         self.data._create_schema()
         self.assertEqual(self.data.cell_count, 2638)
         self.assertEqual(self.data.gene_count, 1838)
@@ -45,7 +45,7 @@ class DataLoadEngineTest(unittest.TestCase):
         self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
 
     def test_diffexp_topN(self):
-        self.data.update(data=self.data_file)
+        self.data.update(data_locator=self.data_file)
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))

server/test/test_writable_annotation.py

@@ -60,7 +60,7 @@ class WritableAnnotationTest(unittest.TestCase):
         res = self.data.annotation_put_fbs("obs", fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
         self.assertTrue(path.exists(self.label_file))
-        df = pd.read_csv(self.label_file, index_col=0)
+        df = pd.read_csv(self.label_file, index_col=0, header=0, comment='#')
         self.assertEqual(df.shape, (n_rows, 2))
         self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
         self.assertTrue(self.data.original_obs_index.equals(df.index))
@@ -75,7 +75,7 @@ class WritableAnnotationTest(unittest.TestCase):
         res = self.data.annotation_put_fbs("obs", fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
         self.assertTrue(path.exists(self.label_file))
-        df = pd.read_csv(self.label_file, index_col=0)
+        df = pd.read_csv(self.label_file, index_col=0, header=0, comment='#')
         self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
         self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
         self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))

server/utils/utils.py

@@ -24,3 +24,12 @@ def is_port_available(host, port):
         except socket.error:
             pass
     return is_available
+
+
+def sort_options(command):
+    """
+    Helper for the click options - will sort options in a command, and can
+    be used as a decorator.
+    """
+    command.params.sort(key=lambda p: p.name)
+    return command