fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Experimental - manual annotations (#837)

Browse Source 
* icons, partway

* redux for values

* onChange

* cancel

* annotations lifecycle for category names

* copy categorical

* edit category

* add Dataframe.withColsFrom

* render user annotations; default add/delete annotation category

* add label name to actions

* category name edit

* error checking improvements

* change schema field isUserAnnotation to writable

* always have an unassigned label; implement delete label

* implement add new label and edit label name

* label current cell selection

* fix select exact bug in crossfilter

* clean up categorical reducer

* fix tests

* remove debugging printf

* implement subset/reset for user annotations

* undo redo support for user annotations

* remove duplicate button from categories

* add modal

* remove obsolete duplicate annotation reducers

* remove old debugging printf

* connect modal to annotation create and dup

* initial full-stack wiring

* finish up end-to-end wiring

* fix existing unit tests

* fix pytests to match new schema API

* remove debugging printfs

* add label file rotation

* remove obsolete comment

* add fbs encode/decode tests

* add tests for writable annotations

* simplify code

* fix hashing bug with FBS encoding

* lint

* fix smoke tests

* improve error checking in Dataframe.withColsFrom

* add unit test for Dataframe.withColsFrom

* add unit test for Dataframe.columns and Dataframe.renameCol

* fix bug in FBS encode, add better error checks, refactor

* add FBS encode/decode test

* add clarifying comment

* clean up action type names; fix state inconsistency in crossfilter update

* change autosave timer to 2.5sec

* sort categorical metadata render order so it remains consistent

* add temporary autogenerated label for add-new-label operation

* fix hover-over label menu interference with cell highlighting

* remove debugging code

* add missing reducer cases & fix typo

* make dataframe memoize more general purpose

* add dev mode for annos

* fix error on select duplicate

* handle zero occupancy categories

* correctly maintain unclipped AND clipped world

* correctly handle zero length FBS matrix and label files

* ensure all writable categorical schema contains an unassigned category

* handle case where building occupancy stack for category with no members

* dialog for creating label, disable button if duplicate or empty

* visually separate writeable

* edit category

* fix edit category name

* remove debugging code

* fix edit annotation label

* visually define unassigned, change options

* Pull in requirements.txt from `master`

* label currently selected cells

* duplicate label

* lint

* fix pytest merge issues

* rename --label-file to --experimental-label-file

* remove debugging console log

* spelling error fix; fix bug found in PR review.

* lint
This commit is contained in:
Bruce Martin
2019-09-18 04:33:41 -07:00
committed by Colin Megill
parent ab2c423006
commit 3660a6cc27
51 changed files with 2823 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
client/__tests__/e2e/data.js
View File

@@ -87,7 +87,13 @@ export const datasets = {
categorical: {
louvain: {
"B cells": "342",
Megakaryocytes: "15"
"CD14+ Monocytes": "0",
"CD4 T cells": "0",
"CD8 T cells": "0",
"Dendritic cells": "0",
"FCGR3A+ Monocytes": "0",
Megakaryocytes: "15",
"NK cells": "0"
}
},
lasso: {

79
client/__tests__/util/dataframe/dataframe.test.js
View File

@@ -452,6 +452,59 @@ describe("dataframe factories", () => {
});
});
describe("withColsFrom", () => {
test("error conditions", () => {
/*
make sure we catch common errors:
- duplicate column names
- dimensionality difference
*/
const dfA = new Dataframe.Dataframe(
[2, 3],
[["red", "blue"], [true, false], [1, 0]],
null,
new Dataframe.KeyIndex(["colors", "bools", "numbers"])
);
/* different dimensionality should throw error */
const dfB = new Dataframe.Dataframe(
[3, 1],
[["red", "blue", "green"]],
null,
new Dataframe.KeyIndex(["colorsA"])
);
expect(() => dfA.withColsFrom(dfB)).toThrow(RangeError);
/* duplicate labels should throw an error */
expect(() => dfA.withColsFrom(dfA)).toThrow(Error);
});
test("simple", () => {
/* simple test that it works as expected in common case */
const dfA = new Dataframe.Dataframe(
[2, 1],
[["red", "blue"]],
null,
new Dataframe.KeyIndex(["colors"])
);
const dfB = new Dataframe.Dataframe(
[2, 1],
[[true, false]],
null,
new Dataframe.KeyIndex(["bools"])
);
const dfC = dfA.withColsFrom(dfB);
expect(dfC).toBeDefined();
expect(dfC.dims).toEqual([2, 2]);
expect(dfC.colIndex.keys()).toEqual(["colors", "bools"]);
expect(dfC.rowIndex).toEqual(dfA.rowIndex);
expect(dfC.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
expect(dfC.col("colors").asArray()).toEqual(["red", "blue"]);
expect(dfC.col("bools").asArray()).toEqual([true, false]);
});
});
describe("dropCol", () => {
test("KeyIndex", () => {
const df = new Dataframe.Dataframe(
@@ -567,6 +620,32 @@ describe("dataframe factories", () => {
expect(dfB.iat(0, 1)).toEqual(1);
expect(dfB.iat(0, 2)).toEqual(1);
});
test("columns", () => {
const df = Dataframe.Dataframe.create(
[3, 3],
[new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
);
expect(df).toBeDefined();
expect(df.columns()).toHaveLength(3);
expect(df.columns()[0]).toEqual(df.icol(0));
expect(df.columns()[2]).toEqual(df.icol(2));
});
test("renameCol", () => {
const dfA = new Dataframe.Dataframe(
[2, 2],
[[true, false], [1, 0]],
null,
new Dataframe.KeyIndex(["A", "B"])
);
const dfB = dfA.renameCol("B", "C");
expect(dfA.colIndex.keys()).toEqual(["A", "B"]);
expect(dfB.colIndex.keys()).toEqual(["A", "C"]);
expect(dfA.dims).toMatchObject(dfB.dims);
expect(dfA.columns()).toMatchObject(dfB.columns());
});
});
});

34
client/__tests__/util/stateManager/fbs.test.js Normal file
View File

@@ -0,0 +1,34 @@
/*
test FBS encode/decode API
*/
import { Dataframe, KeyIndex } from "../../../src/util/dataframe";
import {
decodeMatrixFBS,
encodeMatrixFBS
} from "../../../src/util/stateManager/matrix";
describe("encode/decode", () => {
test("round trip", () => {
const columns = [
["red", "green", "blue"],
new Int32Array(3).fill(0),
new Uint32Array(3).fill(1),
new Float32Array(3).fill(2)
];
const dfNoColIdx = new Dataframe([3, 4], columns);
const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
expect(dfA.colIdx).toBeNull();
expect(dfA.rowIdx).toBeNull();
expect(dfA.columns).toEqual(columns);
const colIndex = new KeyIndex(["a", "b", "c", "d"]);
const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
expect(dfB.colIdx).toEqual(colIndex.keys());
expect(dfB.rowIdx).toBeNull();
expect(dfB.columns).toEqual(columns);
});
});

8
client/__tests__/util/stateManager/sampleResponses.js
View File

@@ -110,6 +110,14 @@ function encodeTypedArray(builder, uType, uData) {
}
function encodeMatrix(columns, colIndex = undefined) {
/*
IMPORTANT: this is not a general purpose encoder. in particular,
it doesn't correctly handle all column index types, nor does it
handle all column typedarray types.
encodeMatrixFBS in matrix.py is more general. This is used only
as a testing santity check (alt implementation).
*/
const utf8Encoder = new TextEncoder("utf-8");
const builder = new flatbuffers.Builder(1024);
const cols = _.map(columns, carr => {

20
client/__tests__/util/stateManager/world.test.js
View File

@@ -18,11 +18,11 @@ const defaultBigBang = () => {
/* create unverse, world, crossfilter and dimensionMap */
/* create universe */
const universe = Universe.createUniverseFromResponse(
REST.config,
REST.schema,
REST.annotationsObs,
REST.annotationsVar,
REST.layoutObs
_.cloneDeep(REST.config),
_.cloneDeep(REST.schema),
_.cloneDeep(REST.annotationsObs),
_.cloneDeep(REST.annotationsVar),
_.cloneDeep(REST.layoutObs)
);
/* create world */
const world = World.createWorldFromEntireUniverse(universe);
@@ -43,11 +43,11 @@ const defaultBigBang = () => {
describe("createWorldFromEntireUniverse", () => {
test("create from REST sample", () => {
const universe = Universe.createUniverseFromResponse(
REST.config,
REST.schema,
REST.annotationsObs,
REST.annotationsVar,
REST.layoutObs
_.cloneDeep(REST.config),
_.cloneDeep(REST.schema),
_.cloneDeep(REST.annotationsObs),
_.cloneDeep(REST.annotationsVar),
_.cloneDeep(REST.layoutObs)
);
expect(universe).toBeDefined();

2
client/__tests__/util/typedCrossfilter/sort.test.js
View File

@@ -180,6 +180,7 @@ describe("lowerBound", () => {
expect(lowerBound([0, 1, 2, 3], 1, 0, 4)).toEqual(1);
expect(lowerBound([0, 1, 2, 3], 3, 0, 4)).toEqual(3);
expect(lowerBound([0, 1, 2, 3], 4, 0, 4)).toEqual(4);
expect(lowerBound([0, 1, 2, 3], 4, 0, 3)).toEqual(3);
expect(lowerBound([0, 1, 2, 3, 4], -1, 0, 5)).toEqual(0);
expect(lowerBound([0, 1, 2, 3, 4], 0, 0, 5)).toEqual(0);
@@ -204,6 +205,7 @@ describe("lowerBound", () => {
expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
expect(lowerBound(new Float32Array([0, 1, 2, 3]), 3, 0, 4)).toEqual(3);
expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 4)).toEqual(4);
expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 3)).toEqual(3);
expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), -1, 0, 5)).toEqual(0);
expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 0, 0, 5)).toEqual(0);

1
client/package.json
View File

@@ -6,6 +6,7 @@
"repository": "https://github.com/chanzuckerberg/cellxgene",
"scripts": {
"backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
"backend-dev-anno": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-label-file labels.csv ",
"build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
"clean": "rimraf build",
"dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",

50
client/src/actions/index.js
View File

@@ -1,7 +1,7 @@
// jshint esversion: 6
import _ from "lodash";
import * as globals from "../globals";
import { Universe } from "../util/stateManager";
import { Universe, MatrixFBS } from "../util/stateManager";
import {
catchErrorsWrap,
doJsonRequest,
@@ -340,11 +340,57 @@ const resetInterface = () => (dispatch, getState) => {
});
};
const saveObsAnnotations = () => async (dispatch, getState) => {
const { universe } = getState();
const { obsAnnotations, schema } = universe;
dispatch({
type: "writable obs annotations - save started"
});
const writableAnnotations = schema.annotations.obs.columns
.filter(s => s.writable)
.map(s => s.name);
const df = obsAnnotations.subset(null, writableAnnotations);
const matrix = MatrixFBS.encodeMatrixFBS(df);
try {
const res = await fetch(
`${globals.API.prefix}${globals.API.version}annotations/obs`,
{
method: "PUT",
body: matrix,
headers: new Headers({
"Content-Type": "application/octet-stream"
})
}
);
if (res.ok) {
dispatch({
type: "writable obs annotations - save complete",
obsAnnotations
});
} else {
dispatch({
type: "writable obs annotations - save error",
message: `HTTP error ${res.status} - ${res.statusText}`,
res
});
}
} catch (error) {
dispatch({
type: "writable obs annotations - save error",
message: error.toString(),
error
});
}
};
export default {
regraph,
resetInterface,
requestSingleGeneExpressionCountsForColoringPOST,
requestDifferentialExpression,
requestUserDefinedGene,
doInitialDataLoad
doInitialDataLoad,
saveObsAnnotations
};

2
client/src/components/app.js
View File

@@ -8,6 +8,7 @@ import LeftSideBar from "./leftSidebar";
import Legend from "./continuousLegend";
import Graph from "./graph/graph";
import MenuBar from "./menubar";
import Autosave from "./autosave";
import actions from "../actions";
@@ -89,6 +90,7 @@ class App extends React.Component {
{loading ? null : <LeftSideBar />}
{loading ? null : <MenuBar />}
{loading ? null : <Graph key={graphRenderCounter} />}
{loading ? null : <Autosave />}
<Legend />
</div>
</Container>

81
client/src/components/autosave/index.js Normal file
View File

@@ -0,0 +1,81 @@
import React from "react";
import { connect } from "react-redux";
import actions from "../../actions";
@connect(state => ({
universe: state.universe,
obsAnnotations: state.universe.obsAnnotations,
saveInProgress: state.autosave?.saveInProgress ?? false,
lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
error: state.autosave?.error,
writableCategoriesEnabled: state.config?.parameters?.["label_file"] ?? false
}))
class Autosave extends React.Component {
constructor(props) {
super(props);
this.state = {
timer: null
};
}
componentDidMount() {
const { writableCategoriesEnabled } = this.props;
let { timer } = this.state;
if (timer) clearInterval(timer);
if (writableCategoriesEnabled) {
timer = setInterval(this.tick, 2500);
} else {
timer = null;
}
this.setState({ timer });
}
componentWillUnmount() {
const { timer } = this.state;
if (timer) this.clearInterval(timer);
}
tick = () => {
const { dispatch, saveInProgress } = this.props;
if (this.needToSave() && !saveInProgress) {
dispatch(actions.saveObsAnnotations());
}
};
needToSave = () => {
/* return true if we need to save, false if we don't */
const { obsAnnotations, lastSavedObsAnnotations } = this.props;
return (
lastSavedObsAnnotations && obsAnnotations !== lastSavedObsAnnotations
);
};
statusMessage() {
const { error } = this.props;
if (error) {
return `Autosave error: ${error}`;
}
return this.needToSave() ? "Unsaved" : "All saved";
}
render() {
const { writableCategoriesEnabled } = this.props;
return writableCategoriesEnabled ? (
<div
id="autosave"
style={{
position: "fixed",
display: "inherit",
right: 0,
bottom: 0
}}
>
{this.statusMessage()}
</div>
) : null;
}
}
export default Autosave;

144
client/src/components/categorical/categorical.js
View File

@@ -1,27 +1,161 @@
// jshint esversion: 6
import React from "react";
import _ from "lodash";
import {
Button,
Tooltip,
InputGroup,
Dialog,
Classes,
MenuItem
} from "@blueprintjs/core";
import { Select } from "@blueprintjs/select";
import { connect } from "react-redux";
import * as globals from "../../globals";
import Category from "./category";
@connect(state => ({
categoricalSelection: state.categoricalSelection
categoricalSelection: state.categoricalSelection,
writableCategoriesEnabled: state.config?.parameters?.["label_file"] ?? false,
schema: state.world?.schema
}))
class Categories extends React.Component {
constructor(props) {
super(props);
this.state = {
createAnnoModeActive: false,
newCategoryText: "",
categoryToDuplicate: null
};
}
handleCreateUserAnno = () => {
const { dispatch } = this.props;
const { newCategoryText, categoryToDuplicate } = this.state;
dispatch({
type: "annotation: create category",
data: newCategoryText,
categoryToDuplicate
});
this.setState({
createAnnoModeActive: false,
categoryToDuplicate: null,
newCategoryText: ""
});
};
handleEnableAnnoMode = () => {
this.setState({ createAnnoModeActive: true });
};
handleDisableAnnoMode = () => {
this.setState({ createAnnoModeActive: false });
};
handleModalDuplicateCategorySelection = d => {
this.setState({ categoryToDuplicate: d });
};
render() {
const { categoricalSelection } = this.props;
const { createAnnoModeActive, categoryToDuplicate } = this.state;
const {
categoricalSelection,
writableCategoriesEnabled,
schema
} = this.props;
if (!categoricalSelection) return null;
/* all names, sorted in display order. Will be rendered in this order */
const allCategoryNames = Object.keys(categoricalSelection).sort();
return (
<div
style={{
padding: globals.leftSidebarSectionPadding
}}
>
{_.map(categoricalSelection, (catState, catName) => (
<Category key={catName} metadataField={catName} />
))}
{/* READ ONLY CATEGORICAL FIELDS */}
{/* this is duplicative but flat, could be abstracted */}
{_.map(allCategoryNames, catName =>
!schema.annotations.obsByName[catName].writable ? (
<Category
key={catName}
metadataField={catName}
createAnnoModeActive={createAnnoModeActive}
isUserAnno={false}
/>
) : null
)}
{/* WRITEABLE FIELDS */}
{_.map(allCategoryNames, catName =>
schema.annotations.obsByName[catName].writable ? (
<Category
key={catName}
metadataField={catName}
createAnnoModeActive={createAnnoModeActive}
isUserAnno
/>
) : null
)}
{writableCategoriesEnabled ? (
<div>
<Dialog
icon="tag"
title="Create new category"
isOpen={createAnnoModeActive}
onClose={this.handleDisableAnnoMode}
>
<div className={Classes.DIALOG_BODY}>
<div style={{ marginBottom: 20 }}>
<p>New, unique category name:</p>
<InputGroup
autoFocus
onChange={e =>
this.setState({ newCategoryText: e.target.value })
}
leftIcon="tag"
/>
</div>
<p>
Optionally duplicate all labels & cell assignments from
existing category into new category:
</p>
<Select
items={allCategoryNames}
filterable={false}
itemRenderer={(d, { handleClick }) => {
return <MenuItem onClick={handleClick} key={d} text={d} />;
}}
noResults={<MenuItem disabled text="No results." />}
onItemSelect={d => {
this.handleModalDuplicateCategorySelection(d);
}}
>
{/* children become the popover target; render value here */}
<Button
text={
categoryToDuplicate || "None (all cells 'unassigned')"
}
rightIcon="double-caret-vertical"
/>
</Select>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<Tooltip content="Close this dialog without creating a category.">
<Button onClick={this.handleDisableAnnoMode}>Cancel</Button>
</Tooltip>
<Button onClick={this.handleCreateUserAnno} intent="primary">
Create new category
</Button>
</div>
</div>
</Dialog>
<Button onClick={this.handleEnableAnnoMode} intent="primary">
Create new category
</Button>
</div>
) : null}
</div>
);
}

261
client/src/components/categorical/category.js
View File

@@ -2,7 +2,19 @@ import React from "react";
import _ from "lodash";
import { connect } from "react-redux";
import { FaChevronRight, FaChevronDown } from "react-icons/fa";
import { Button, Tooltip } from "@blueprintjs/core";
import {
Button,
Tooltip,
InputGroup,
Menu,
Dialog,
MenuItem,
Popover,
Classes,
Icon,
Position,
PopoverInteractionKind
} from "@blueprintjs/core";
import * as globals from "../../globals";
import Value from "./value";
@@ -10,14 +22,18 @@ import sortedCategoryValues from "./util";
@connect(state => ({
colorAccessor: state.colors.colorAccessor,
categoricalSelection: state.categoricalSelection
categoricalSelection: state.categoricalSelection,
annotations: state.annotations,
universe: state.universe
}))
class Category extends React.Component {
constructor(props) {
super(props);
this.state = {
isChecked: true,
isExpanded: false
isExpanded: false,
newCategoryText: "",
newLabelText: ""
};
}
@@ -51,6 +67,74 @@ class Category extends React.Component {
}
}
activateAddNewLabelMode = () => {
const { dispatch, metadataField } = this.props;
dispatch({
type: "annotation: activate add new label mode",
data: metadataField
});
};
disableAddNewLabelMode = () => {
const { dispatch } = this.props;
dispatch({
type: "annotation: disable add new label mode"
});
};
handleAddNewLabelToCategory = () => {
const { dispatch, metadataField } = this.props;
const { newLabelText } = this.state;
/*
XXX TODO - temporary code generates random label string. Remove
when the label creation UI is implemented.
const { newLabelText } = this.state;
*/
// const newLabelText = `label${Math.random()}`;
dispatch({
type: "annotation: add new label to category",
metadataField,
newLabelText
});
this.setState({ newLabelText: "" });
};
activateEditCategoryMode = () => {
const { dispatch, metadataField } = this.props;
dispatch({
type: "annotation: activate category edit mode",
data: metadataField
});
};
disableEditCategoryMode = () => {
const { dispatch } = this.props;
dispatch({
type: "annotation: disable category edit mode"
});
};
handleEditCategory = () => {
const { dispatch, metadataField } = this.props;
const { newCategoryText } = this.state;
dispatch({
type: "annotation: category edited",
metadataField,
newCategoryText,
data: newCategoryText
});
};
handleDeleteCategory = () => {
const { dispatch, metadataField } = this.props;
dispatch({
type: "annotation: delete category",
metadataField
});
};
handleColorChange = () => {
const { dispatch, metadataField } = this.props;
dispatch({
@@ -88,12 +172,13 @@ class Category extends React.Component {
}
renderCategoryItems() {
const { categoricalSelection, metadataField } = this.props;
const { categoricalSelection, metadataField, isUserAnno } = this.props;
const cat = categoricalSelection[metadataField];
const optTuples = sortedCategoryValues([...cat.categoryValueIndices]);
return _.map(optTuples, (tuple, i) => (
<Value
isUserAnno={isUserAnno}
optTuples={optTuples}
key={tuple[1]}
metadataField={metadataField}
@@ -104,9 +189,17 @@ class Category extends React.Component {
}
render() {
const { isExpanded, isChecked } = this.state;
const { metadataField, colorAccessor, categoricalSelection } = this.props;
const { isExpanded, isChecked, newLabelText, newCategoryText } = this.state;
const {
metadataField,
colorAccessor,
categoricalSelection,
isUserAnno,
annotations,
universe
} = this.props;
const { isTruncated } = categoricalSelection[metadataField];
return (
<div
style={{
@@ -144,7 +237,45 @@ class Category extends React.Component {
<span className="bp3-control-indicator" />
{""}
</label>
{/* Dialog uses portal, can be anywhere/factored out */}
<Dialog
icon="tag"
title="Edit category name"
isOpen={
annotations.isEditingCategoryName &&
annotations.categoryEditable === metadataField
}
onClose={this.disableEditCategoryMode}
>
<div className={Classes.DIALOG_BODY}>
<div style={{ marginBottom: 20 }}>
<p>New, unique category name:</p>
<InputGroup
autoFocus
onChange={e =>
this.setState({ newCategoryText: e.target.value })
}
leftIcon="tag"
/>
</div>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<Tooltip content="Close this dialog without editing the category.">
<Button onClick={this.disableEditCategoryMode}>
Cancel
</Button>
</Tooltip>
<Button
disabled={newCategoryText.length === 0}
onClick={this.handleEditCategory}
intent="primary"
>
Edit category name
</Button>
</div>
</div>
</Dialog>
<span
data-testid={`category-expand-${metadataField}`}
style={{
@@ -155,6 +286,9 @@ class Category extends React.Component {
this.setState({ isExpanded: !isExpanded });
}}
>
{isUserAnno ? (
<Icon style={{ marginRight: 5 }} icon="tag" iconSize={16} />
) : null}
{metadataField}
{isExpanded ? (
<FaChevronDown
@@ -169,20 +303,105 @@ class Category extends React.Component {
)}
</span>
</div>
<Tooltip
content="Use as color scale"
position="bottom"
hoverOpenDelay={globals.tooltipHoverOpenDelay}
>
<Button
data-testclass="colorby"
data-testid={`colorby-${metadataField}`}
onClick={this.handleColorChange}
active={colorAccessor === metadataField}
intent={colorAccessor === metadataField ? "primary" : "none"}
icon="tint"
/>
</Tooltip>
<div>
{isUserAnno ? (
<>
<Dialog
icon="tag"
title="Add new label"
isOpen={annotations.isAddingNewLabel}
onClose={this.disableAddNewLabelMode}
>
<div className={Classes.DIALOG_BODY}>
<div style={{ marginBottom: 20 }}>
<p>New, unique label name:</p>
<InputGroup
autoFocus
onChange={e =>
this.setState({ newLabelText: e.target.value })
}
leftIcon="tag"
/>
</div>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<Tooltip content="Close this dialog without adding a label.">
<Button onClick={this.disableAddNewLabelMode}>
Cancel
</Button>
</Tooltip>
<Button
disabled={
newLabelText.length === 0 ||
universe.schema.annotations.obsByName[
metadataField
].categories.indexOf(newLabelText) !== -1
}
onClick={this.handleAddNewLabelToCategory}
intent="primary"
>
Add new label to category
</Button>
</div>
</div>
</Dialog>
<Popover
interactionKind={PopoverInteractionKind.HOVER}
boundary="window"
position={Position.RIGHT}
content={
<Menu>
<MenuItem
icon="tag"
data-testclass="handleAddNewLabelToCategory"
data-testid={`handleAddNewLabelToCategory-${metadataField}`}
onClick={this.activateAddNewLabelMode}
text="Add a new label to this category"
/>
<MenuItem
icon="edit"
data-testclass="activateEditCategoryMode"
data-testid={`activateEditCategoryMode-${metadataField}`}
onClick={this.activateEditCategoryMode}
text="Edit this category's name"
/>
<MenuItem
icon="delete"
intent="danger"
data-testclass="handleDeleteCategory"
data-testid={`handleDeleteCategory-${metadataField}`}
onClick={this.handleDeleteCategory}
text="Delete this category, all associated labels, and remove all cell assignments"
/>
</Menu>
}
>
<Button
style={{ marginLeft: 0 }}
data-testclass="seeActions"
data-testid={`seeActions-${metadataField}`}
icon="more"
minimal
/>
</Popover>
</>
) : null}
<Tooltip
content="Use as color scale"
position="bottom"
hoverOpenDelay={globals.tooltipHoverOpenDelay}
>
<Button
data-testclass="colorby"
data-testid={`colorby-${metadataField}`}
onClick={this.handleColorChange}
active={colorAccessor === metadataField}
intent={colorAccessor === metadataField ? "primary" : "none"}
icon="tint"
/>
</Tooltip>
</div>
</div>
<div style={{ marginLeft: 26 }}>
{isExpanded ? this.renderCategoryItems() : null}

51
client/src/components/categorical/occupancy.js
View File

@@ -45,7 +45,10 @@ class Occupancy extends React.Component {
groupBy
); /* Because the signature changes we really need different names for histogram to differentiate signatures */
const bins = histogramMap.get(category.categoryValues[categoryIndex]);
const categoryValue = category.categoryValues[categoryIndex];
const bins = histogramMap.has(categoryValue)
? histogramMap.get(categoryValue)
: new Array(50).fill(0);
const xScale = d3
.scaleLinear()
@@ -102,30 +105,34 @@ class Occupancy extends React.Component {
const occupancy = occupancyMap.get(category.categoryValues[categoryIndex]);
const x = d3
.scaleLinear()
/* get all the keys d[1] as an array, then find the sum */
.domain([0, d3.sum(Array.from(occupancy.values()))])
.range([0, this._WIDTH]);
const categories = schema.annotations.obsByName[colorAccessor]?.categories;
if (occupancy && occupancy.size > 0) {
// not all categories have occupancy, so occupancy may be undefined.
const x = d3
.scaleLinear()
/* get all the keys d[1] as an array, then find the sum */
.domain([0, d3.sum(Array.from(occupancy.values()))])
.range([0, this._WIDTH]);
const categories =
schema.annotations.obsByName[colorAccessor]?.categories;
let currentOffset = 0;
const dfColumn = world.obsAnnotations.col(colorAccessor);
const categoryValues = dfColumn.summarize().categories;
let currentOffset = 0;
const dfColumn = world.obsAnnotations.col(colorAccessor);
const categoryValues = dfColumn.summarize().categories;
let o;
let scaledValue;
let value;
let o;
let scaledValue;
let value;
for (let i = 0, { length } = categoryValues; i < length; i += 1) {
value = categoryValues[i];
o = occupancy.get(value);
scaledValue = x(o);
ctx.fillStyle = o
? colorScale(categories.indexOf(value))
: "rgb(255,255,255)";
ctx.fillRect(currentOffset, 0, o ? scaledValue : 0, this._HEIGHT);
currentOffset += o ? scaledValue : 0;
for (let i = 0, { length } = categoryValues; i < length; i += 1) {
value = categoryValues[i];
o = occupancy.get(value);
scaledValue = x(o);
ctx.fillStyle = o
? colorScale(categories.indexOf(value))
: "rgb(255,255,255)";
ctx.fillRect(currentOffset, 0, o ? scaledValue : 0, this._HEIGHT);
currentOffset += o ? scaledValue : 0;
}
}
};

295
client/src/components/categorical/value.js
View File

@@ -1,18 +1,108 @@
// jshint esversion: 6
import { connect } from "react-redux";
import React from "react";
import {
Button,
InputGroup,
Menu,
MenuItem,
Popover,
Position,
Icon,
PopoverInteractionKind
} from "@blueprintjs/core";
import Occupancy from "./occupancy";
import * as globals from "../../globals";
import styles from "./categorical.css";
@connect(state => ({
categoricalSelection: state.categoricalSelection,
annotations: state.annotations,
colorScale: state.colors.scale,
colorAccessor: state.colors.colorAccessor,
schema: state.world?.schema,
world: state.world
}))
class CategoryValue extends React.Component {
constructor(props) {
super(props);
this.state = {
editedLabelText: ""
};
}
handleDeleteValue = () => {
const {
dispatch,
metadataField,
categoryIndex,
categoricalSelection
} = this.props;
const category = categoricalSelection[metadataField];
const label = category.categoryValues[categoryIndex];
dispatch({
type: "annotation: delete label",
metadataField,
label
});
};
handleAddCurrentSelectionToThisLabel = () => {
const {
dispatch,
metadataField,
categoryIndex,
categoricalSelection
} = this.props;
const category = categoricalSelection[metadataField];
const label = category.categoryValues[categoryIndex];
dispatch({
type: "annotation: label current cell selection",
metadataField,
categoryIndex,
label
});
};
handleEditValue = () => {
const {
dispatch,
metadataField,
categoryIndex,
categoricalSelection
} = this.props;
const { editedLabelText } = this.state;
const category = categoricalSelection[metadataField];
const label = category.categoryValues[categoryIndex];
dispatch({
type: "annotation: label edited",
editedLabel: editedLabelText,
metadataField,
categoryIndex,
label
});
this.setState({ editedLabelText: "" });
};
activateEditLabelMode = () => {
const { dispatch, metadataField, categoryIndex } = this.props;
dispatch({
type: "annotation: activate edit label mode",
metadataField,
categoryIndex
});
};
cancelEdit = () => {
const { dispatch, metadataField, categoryIndex } = this.props;
dispatch({
type: "annotation: cancel edit label mode",
metadataField,
categoryIndex
});
};
toggleOff = () => {
const { dispatch, metadataField, categoryIndex } = this.props;
dispatch({
@@ -23,12 +113,12 @@ class CategoryValue extends React.Component {
};
shouldComponentUpdate = nextProps => {
/*
/*
Checks to see if at least one of the following changed:
* world state
* the color accessor (what is currently being colored by)
* if this catagorical value's selection status has changed
If and only if true, update the component
*/
const { props } = this;
@@ -45,8 +135,14 @@ class CategoryValue extends React.Component {
const worldChange = props.world !== nextProps.world;
const colorAccessorChange = props.colorAccessor !== nextProps.colorAccessor;
const annotationsChange = props.annotations !== nextProps.annotations;
return valueSelectionChange || worldChange || colorAccessorChange;
return (
valueSelectionChange ||
worldChange ||
colorAccessorChange ||
annotationsChange
);
};
toggleOn = () => {
@@ -84,7 +180,9 @@ class CategoryValue extends React.Component {
colorAccessor,
colorScale,
i,
schema
schema,
isUserAnno,
annotations
} = this.props;
if (!categoricalSelection) return null;
@@ -121,41 +219,103 @@ class CategoryValue extends React.Component {
onMouseEnter={this.handleMouseEnter}
onMouseLeave={this.handleMouseExit}
>
<div
style={{
margin: 0,
padding: 0,
userSelect: "none",
width: globals.leftSidebarWidth - 130,
display: "flex",
justifyContent: "space-between"
}}
>
<div style={{ display: "flex" }}>
<label className="bp3-control bp3-checkbox" style={{ margin: 0 }}>
<input
onChange={selected ? this.toggleOff : this.toggleOn}
data-testclass="categorical-value-select"
data-testid={`categorical-value-select-${metadataField}-${displayString}`}
checked={selected}
type="checkbox"
/>
<div style={{ display: "flex", justifyContent: "space-between" }}>
<div
style={{
margin: 0,
padding: 0,
userSelect: "none",
width: globals.leftSidebarWidth - 240,
display: "flex",
justifyContent: "flex-start"
}}
>
<div style={{ display: "flex" }}>
<label className="bp3-control bp3-checkbox" style={{ margin: 0 }}>
<input
onChange={selected ? this.toggleOff : this.toggleOn}
data-testclass="categorical-value-select"
data-testid={`categorical-value-select-${metadataField}-${displayString}`}
checked={selected}
type="checkbox"
/>
<span
className="bp3-control-indicator"
onMouseEnter={this.handleMouseExit}
onMouseLeave={this.handleMouseEnter}
/>
</label>
<span
className="bp3-control-indicator"
onMouseEnter={this.handleMouseExit}
onMouseLeave={this.handleMouseEnter}
/>
</label>
<span
data-testid={`categorical-value-${metadataField}-${displayString}`}
data-testclass="categorical-value"
style={{ wordBreak: "break-all" }}
>
{displayString}
</span>
data-testid={`categorical-value-${metadataField}-${displayString}`}
data-testclass="categorical-value"
style={{
wordBreak: "break-all",
color:
displayString === globals.unassignedCategoryLabel
? "#ababab"
: "black",
fontStyle:
displayString === globals.unassignedCategoryLabel
? "italic"
: "auto"
}}
>
{annotations.isEditingLabelName &&
annotations.labelEditable.category === metadataField &&
annotations.labelEditable.label === categoryIndex
? null
: displayString}
</span>
</div>
{isUserAnno &&
annotations.isEditingLabelName &&
annotations.labelEditable.category === metadataField &&
annotations.labelEditable.label === categoryIndex ? (
<form
onSubmit={e => {
e.preventDefault();
this.handleEditValue();
}}
>
<InputGroup
style={{ position: "relative", top: -1 }}
ref={input => {
this.editableInput = input;
}}
small
onChange={e => {
this.setState({ editedLabelText: e.target.value });
}}
defaultValue={displayString}
rightElement={
<Button
minimal
style={{ position: "relative", top: -1 }}
type="button"
icon="small-tick"
data-testclass="submitEdit"
data-testid="submitEdit"
onClick={this.handleEditValue}
/>
}
/>
</form>
) : null}
{/*
CANCEL IT, WITH BUTTON, ESCAPE KEY, CLICK OUT, UNDO?
<Button
minimal
style={{ position: "relative", top: -1 }}
type="button"
icon="cross"
data-testclass="submitEdit"
data-testid="submitEdit"
onClick={this.cancelEdit}
/> */}
</div>
<span style={{ flexShrink: 0 }}>
{colorAccessor && !isColorBy ? (
{colorAccessor && !isColorBy && !annotations.isEditingLabelName ? (
<Occupancy category={category} {...this.props} />
) : null}
</span>
@@ -164,10 +324,22 @@ class CategoryValue extends React.Component {
<span
data-testclass="categorical-value-count"
data-testid={`categorical-value-count-${metadataField}-${displayString}`}
style={{
color:
displayString === globals.unassignedCategoryLabel
? "#ababab"
: "black",
fontStyle:
displayString === globals.unassignedCategoryLabel
? "italic"
: "auto"
}}
>
{count}
</span>
<svg
display={isColorBy && categories ? "auto" : "none"}
style={{
marginLeft: 5,
width: 11,
@@ -178,6 +350,57 @@ class CategoryValue extends React.Component {
: "inherit"
}}
/>
{isUserAnno ? (
<span
onMouseEnter={this.handleMouseExit}
onMouseLeave={this.handleMouseEnter}
>
<Popover
interactionKind={PopoverInteractionKind.HOVER}
boundary="window"
position={Position.RIGHT_TOP}
content={
<Menu>
<MenuItem
icon="plus"
data-testclass="handleAddCurrentSelectionToThisLabel"
data-testid={`handleAddCurrentSelectionToThisLabel-${metadataField}`}
onClick={this.handleAddCurrentSelectionToThisLabel}
text={`Label currently selected cells as ${displayString}`}
/>
{displayString !== globals.unassignedCategoryLabel ? (
<MenuItem
icon="edit"
text="Edit this label's name"
data-testclass="handleEditValue"
data-testid={`handleEditValue-${metadataField}`}
onClick={this.activateEditLabelMode}
/>
) : null}
{displayString !== globals.unassignedCategoryLabel ? (
<MenuItem
icon="delete"
intent="danger"
data-testclass="handleDeleteValue"
data-testid={`handleDeleteValue-${metadataField}`}
onClick={this.handleDeleteValue}
text="Delete this value, and reassign all cells to type 'unknown'"
/>
) : null}
</Menu>
}
>
<Button
style={{ marginLeft: 0, position: "relative", top: -1 }}
data-testclass="seeActions"
data-testid={`seeActions-${metadataField}`}
icon="more"
small
minimal
/>
</Popover>
</span>
) : null}
</span>
</div>
);

16
client/src/components/framework/custom-icons.js Normal file
View File

@@ -0,0 +1,16 @@
/* https://github.com/palantir/blueprint/issues/2348 */
<defs>
<clipPath id="clip0">
<rect width="16" height="16" fill="white"/>
</clipPath>
</defs>
<g clip-path="url(#clip0)">
<rect width="16" height="16" fill="white"/>
<path d="M1.33415 8.75877C0.939491 8.36411 0.727699 7.82249 0.749957 7.2648L0.926361 2.84501C0.967947 1.80308 1.80308 0.967947 2.84501 0.926361L7.2648 0.749958C7.82249 0.727699 8.36411 0.939492 8.75877 1.33415L14.3595 6.93485C15.1405 7.7159 15.1405 8.98223 14.3595 9.76328L9.76328 14.3595C8.98223 15.1405 7.7159 15.1405 6.93485 14.3595L1.33415 8.75877Z" fill="black"/>
<circle cx="4.5" cy="4.5" r="1.5" fill="white"/>
<circle cx="4.5" cy="11.5" r="3.75" stroke="white" stroke-width="0.5"/>
<circle cx="4.5" cy="11.5" r="3.5" fill="black"/>
<line x1="4.5" y1="10" x2="4.5" y2="13" stroke="white"/>
<line x1="3" y1="11.5" x2="6" y2="11.5" stroke="white"/>
</g>

3
client/src/globals.js
View File

@@ -3,6 +3,9 @@ import { Colors } from "@blueprintjs/core";
/* if a categorical metadata field has more options than this, truncate */
export const maxCategoricalOptionsToDisplay = 100;
/* default "unassigned" value for user-created categorical metadata */
export const unassignedCategoryLabel = "unassigned";
/*
these are default values for configuration the CLI may supply.
See the REST API and CLI specs for more info.

88
client/src/reducers/annotations.js Normal file
View File

@@ -0,0 +1,88 @@
/*
Reducers for annotation UI-state.
*/
const Annotations = (
state = {
isEditingCategoryName: false,
isEditingLabelName: false,
categoryEditable: false,
labelEditable: { category: null, label: null }
},
action
) => {
switch (action.type) {
/* CATEGORY */
case "annotation: activate add new label mode":
console.log(action.type, action);
return {
...state,
isAddingNewLabel: true,
categoryAddingNewLabel: action.data
};
case "annotation: disable add new label mode":
return {
...state,
isAddingNewLabel: false,
categoryAddingNewLabel: null
};
case "annotation: add new label to category":
console.log(action.type, action);
return {
...state,
isAddingNewLabel: false,
categoryAddingNewLabel: null
};
case "annotation: activate category edit mode":
console.log(action.type, action);
return {
...state,
isEditingCategoryName: true,
categoryEditable: action.data
};
case "annotation: disable category edit mode":
console.log(action.type, action);
return {
...state,
isEditingCategoryName: false,
categoryEditable: null
};
case "annotation: category edited":
console.log(action.type, action);
return {
...state,
isEditingCategoryName: true,
categoryEditable: null
};
/* LABEL */
case "annotation: activate edit label mode":
console.log(action.type, action);
return {
...state,
isEditingLabelName: true,
labelEditable: {
category: action.metadataField,
label: action.categoryIndex
}
};
case "annotation: cancel edit label mode":
console.log(action.type, action);
return {
...state,
isEditingLabelName: false,
labelEditable: { category: null, label: null }
};
case "annotation: label edited":
console.log(action.type, action);
return {
...state,
isEditingLabelName: false,
labelEditable: { category: null, label: null }
/* Bruce to persist new label name */
};
default:
return state;
}
};
export default Annotations;

53
client/src/reducers/autosave.js Normal file
View File

@@ -0,0 +1,53 @@
const Autosave = (
state = {
saveInProgress: false,
error: false,
lastSavedObsAnnotations: null
},
action,
nextSharedState
) => {
switch (action.type) {
case "initial data load complete (universe exists)": {
/* don't save on init */
const { universe } = nextSharedState;
return {
...state,
error: false,
saveInProgress: false,
lastSavedObsAnnotations: universe.obsAnnotations
};
}
case "writable obs annotations - save started": {
return {
...state,
saveInProgress: true
};
}
case "writable obs annotations - save error": {
const { message } = action;
return {
...state,
error: message,
saveInProgress: false
};
}
case "writable obs annotations - save complete": {
const lastSavedObsAnnotations = action.obsAnnotations;
return {
...state,
saveInProgress: false,
error: false,
lastSavedObsAnnotations
};
}
default:
return { ...state };
}
};
export default Autosave;

46
client/src/reducers/categoricalSelection.js
View File

@@ -1,4 +1,4 @@
import { ControlsHelpers } from "../util/stateManager";
import { ControlsHelpers as CH } from "../util/stateManager";
import * as globals from "../globals";
function maxCategoryItems(state) {
@@ -20,10 +20,11 @@ const CategoricalSelection = (
case "reset World to eq Universe":
case "set clip quantiles": {
const { world } = nextSharedState;
return ControlsHelpers.createCategoricalSelection(
maxCategoryItems(prevSharedState),
world
const newState = CH.createCategoricalSelection(
world,
CH.selectableCategoryNames(world, maxCategoryItems(prevSharedState))
);
return newState;
}
case "categorical metadata filter select": {
@@ -96,6 +97,43 @@ const CategoricalSelection = (
return newCategoricalSelection;
}
case "annotation: create category": {
const { world } = nextSharedState;
const name = action.data;
return {
...state,
...CH.createCategoricalSelection(world, [name])
};
}
case "annotation: category edited": {
const name = action.metadataField;
const newName = action.newCategoryText;
const { [name]: catSeln, ...newState } = state;
newState[newName] = catSeln;
return newState;
}
case "annotation: delete category": {
const name = action.metadataField;
const { [name]: _, ...newState } = state;
return newState;
}
case "annotation: label current cell selection":
case "annotation: add new label to category":
case "annotation: label edited":
case "annotation: delete label": {
/* need to rebuild the state for this annotation */
const { world } = nextSharedState;
const name = action.metadataField;
const { [name]: _, ...partialState } = state;
return {
...partialState,
...CH.createCategoricalSelection(world, [name])
};
}
default: {
return state;
}

21
client/src/reducers/colors.js
View File

@@ -108,6 +108,27 @@ const ColorsReducer = (
};
}
case "annotation: add new label to category":
case "annotation: label current cell selection":
case "annotation: delete label": {
const { world } = nextSharedState;
const { colorMode, colorAccessor } = state;
const { metadataField } = action;
if (
colorMode !== "color by categorical metadata" ||
colorAccessor !== metadataField
)
return state;
/* else, we need to rebuild colors as labels have changed! */
const { rgb, scale } = ColorHelpers.createColors(
world,
colorMode,
colorAccessor
);
return { ...state, rgb, scale };
}
case "clear differential expression": {
const { world: prevWorld, controls: prevControls } = prevSharedState;
const resetColorState = ColorHelpers.checkIfColorByDiffexpAndResetColors(

2
client/src/reducers/controls.js vendored
View File

@@ -7,7 +7,7 @@ import { WorldUtil } from "../util/stateManager";
const Controls = (
state = {
// data loading flag
loading: false,
loading: true,
error: null,
// all of the data + selection state

78
client/src/reducers/crossfilter.js
View File

@@ -1,7 +1,11 @@
import _ from "lodash";
import Crossfilter from "../util/typedCrossfilter";
import { World, ControlsHelpers } from "../util/stateManager";
import {
World,
ControlsHelpers as CH,
AnnotationsHelpers as AH
} from "../util/stateManager";
import {
layoutDimensionName,
obsAnnoDimensionName,
@@ -12,7 +16,7 @@ import {
const XYDimName = layoutDimensionName("XY");
const CrossfilterReducer = (
const CrossfilterReducerBase = (
state = null,
action,
nextSharedState,
@@ -32,12 +36,14 @@ const CrossfilterReducer = (
case "reset World to eq Universe": {
const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
const { world } = nextSharedState;
const crossfilter = ControlsHelpers.createGeneDimensions(
let { crossfilter } = prevSharedState.resetCache;
crossfilter = CH.createGeneDimensions(
userDefinedGenes,
diffexpGenes,
world,
prevSharedState.resetCache.crossfilter
crossfilter
);
crossfilter = AH.createWritableAnnotationDimensions(world, crossfilter);
return crossfilter;
}
@@ -51,7 +57,7 @@ const CrossfilterReducer = (
world,
layoutChoice.currentDimNames
);
crossfilter = ControlsHelpers.createGeneDimensions(
crossfilter = CH.createGeneDimensions(
userDefinedGenes,
diffexpGenes,
world,
@@ -136,6 +142,37 @@ const CrossfilterReducer = (
return crossfilter;
}
case "annotation: create category": {
const name = action.data;
const { world } = nextSharedState;
const colData = world.obsAnnotations.col(name).asArray();
return state.addDimension(obsAnnoDimensionName(name), "enum", colData);
}
case "annotation: category edited": {
const name = action.metadataField;
const newName = action.newCategoryText;
return state.renameDimension(
obsAnnoDimensionName(name),
obsAnnoDimensionName(newName)
);
}
case "annotation: delete category": {
return state.delDimension(obsAnnoDimensionName(action.metadataField));
}
case "annotation: label current cell selection":
case "annotation: label edited":
case "annotation: delete label": {
/* we need to reindex the dimension. For now, just drop it and add another */
const name = action.metadataField;
const dimName = obsAnnoDimensionName(name);
const { world } = nextSharedState;
const colData = world.obsAnnotations.col(name).asArray();
return state.delDimension(dimName).addDimension(dimName, "enum", colData);
}
case "graph brush end":
case "graph brush change": {
const [minX, maxY] = action.brushCoords.northwest;
@@ -185,12 +222,12 @@ const CrossfilterReducer = (
case "categorical metadata filter select":
case "categorical metadata filter deselect": {
const { categoricalSelection } = nextSharedState;
const { world } = prevSharedState;
const cat = categoricalSelection[action.metadataField];
const col = world.obsAnnotations.col(action.metadataField);
const { categoryValues, categoryValueSelected } = cat;
const values = categoryValues.filter((v, i) => categoryValueSelected[i]);
return state.select(obsAnnoDimensionName(action.metadataField), {
mode: "exact",
values: ControlsHelpers.selectedValuesForCategory(cat, col)
values
});
}
@@ -212,4 +249,29 @@ const CrossfilterReducer = (
}
};
/*
IMPORTANT: the system assumes that crossfilter.data() will point at the
same value as world.obsAnnotations. For actions handled in this reducer,
make sure that this remains true.
This wrapper performs only this function.
*/
const CrossfilterReducer = (
state,
action,
nextSharedState,
prevSharedState
) => {
const nextState = CrossfilterReducerBase(
state,
action,
nextSharedState,
prevSharedState
);
if (!nextState || nextState.all() === nextSharedState.world.obsAnnotations) {
return nextState;
}
return nextState.setData(nextSharedState.world.obsAnnotations);
};
export default CrossfilterReducer;

62
client/src/reducers/index.js
View File

@@ -17,38 +17,44 @@ import responsive from "./responsive";
import controls from "./controls";
import resetCache from "./resetCache";
import centroidLabel from "./centroidLabel";
import annotations from "./annotations";
import autosave from "./autosave";
import undoableConfig from "./undoableConfig";
const Reducer = undoable(
cascadeReducers([
["config", config],
["universe", universe],
["world", world],
["layoutChoice", layoutChoice],
["categoricalSelection", categoricalSelection],
["continuousSelection", continuousSelection],
["graphSelection", graphSelection],
["crossfilter", crossfilter],
["colors", colors],
["controls", controls],
["differential", differential],
["responsive", responsive],
["centroidLabel", centroidLabel],
["resetCache", resetCache]
]),
[
"world",
"categoricalSelection",
"continuousSelection",
"graphSelection",
"crossfilter",
"colors",
"controls",
"differential",
"layoutChoice"
],
undoableConfig
cascadeReducers([
["config", config],
["universe", universe],
["world", world],
["annotations", annotations],
["layoutChoice", layoutChoice],
["categoricalSelection", categoricalSelection],
["continuousSelection", continuousSelection],
["graphSelection", graphSelection],
["crossfilter", crossfilter],
["colors", colors],
["controls", controls],
["differential", differential],
["responsive", responsive],
["centroidLabel", centroidLabel],
["autosave", autosave],
["resetCache", resetCache]
]),
[
"universe",
"world",
"categoricalSelection",
"continuousSelection",
"graphSelection",
"crossfilter",
"colors",
"controls",
"differential",
"layoutChoice",
"annotations"
],
undoableConfig
);
const store = createStore(Reducer, applyMiddleware(thunk));

11
client/src/reducers/resetCache.js
View File

@@ -1,13 +1,13 @@
/*
Reducer which caches derived state to be used in a reset or other
recomputation.
recomputation. Add stuff here you want stashed at init time (or whenever),
for later use.
Currently this only caches the baseline (full universe) world & crossfilter,
for use in a Reset.
Currently this only caches the baseline (full universe) crossfilter,
which improves Reset UI performance.
*/
const ResetCacheReducer = (
state = {
world: null,
crossfilter: null
},
action,
@@ -15,10 +15,9 @@ const ResetCacheReducer = (
) => {
switch (action.type) {
case "initial data load complete (universe exists)": {
const { world, crossfilter } = nextSharedState;
const { crossfilter } = nextSharedState;
return {
...state,
world,
crossfilter
};
}

10
client/src/reducers/undoableConfig.js
View File

@@ -78,7 +78,15 @@ const saveOnActions = new Set([
"set clip quantiles",
"set layout choice",
"change graph interaction mode"
"change graph interaction mode",
// user editable annotations
"annotation: create category",
"annotation: add new label to category",
"annotation: delete category",
"annotation: label edited",
"annotation: label current cell selection",
"annotation: delete label"
]);
/**

189
client/src/reducers/universe.js
View File

@@ -1,4 +1,9 @@
import { ControlsHelpers } from "../util/stateManager";
import { unassignedCategoryLabel } from "../globals";
import {
World,
ControlsHelpers as CH,
AnnotationsHelpers as AH
} from "../util/stateManager";
const Universe = (state = null, action, nextSharedState, prevSharedState) => {
switch (action.type) {
@@ -30,12 +35,186 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
Object.keys(action.expressionData)
)
];
varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);
varData = CH.pruneVarDataCache(varData, allTheGenesWeNeed);
return { ...state, varData };
}
return {
...state,
varData
case "annotation: create category": {
/* create a new annotation category, with all values set to 'unassigned' */
const name = action.data;
const { categoryToDuplicate } = action;
/* name must be a string, non-zero length */
if (typeof name !== "string" || name.length === 0)
throw new Error("user annotations require string name");
/* ensure the name isn't already in use! */
if (state.obsAnnotations.hasCol(name))
throw new Error("name collision on annotation category create");
/* ensure the duplicate col exists */
if (
categoryToDuplicate &&
!state.obsAnnotations.hasCol(categoryToDuplicate)
)
throw new Error("categoryToDuplicate does not exist");
let schema;
let data;
if (categoryToDuplicate) {
/* duplicate the named annotation */
schema = AH.dupObsAnnoSchema(state.schema, categoryToDuplicate, name, {
writable: true
});
/* if we are duplicating a non-writable annotation, it may not have an unassigned category */
const s = schema.annotations.obsByName[categoryToDuplicate];
if (s.categories.indexOf(unassignedCategoryLabel) === -1) {
s.categories = s.categories.concat(unassignedCategoryLabel);
}
data = state.obsAnnotations.col(categoryToDuplicate).asArray();
} else {
/* else, all are unassined */
const categories = [unassignedCategoryLabel];
schema = AH.addObsAnnoSchema(state.schema, name, {
name,
categories,
type: "categorical",
writable: true
});
data = new Array(state.nObs).fill(unassignedCategoryLabel);
}
const obsAnnotations = state.obsAnnotations.withCol(name, data);
return { ...state, obsAnnotations, schema };
}
case "annotation: category edited": {
/* change the name of an obs annotation category */
const name = action.metadataField;
const newName = action.newCategoryText;
if (!AH.isUserAnnotation(state, name))
throw new Error("unable to edit read-only annotation");
if (typeof newName !== "string" || newName.length === 0)
throw new Error("user annotations require string name");
const colSchema = {
...state.schema.annotations.obsByName[name],
name: newName
};
const schema = AH.addObsAnnoSchema(
AH.removeObsAnnoSchema(state.schema, name),
newName,
colSchema
);
const obsAnnotations = state.obsAnnotations.renameCol(name, newName);
return { ...state, schema, obsAnnotations };
}
case "annotation: delete category": {
/* delete annotation category from schema and obsAnnotations */
const name = action.metadataField;
if (!AH.isUserAnnotation(state, name))
throw new Error("unable to delete read-only annotation");
const schema = AH.removeObsAnnoSchema(state.schema, name);
const obsAnnotations = state.obsAnnotations.dropCol(name);
return { ...state, schema, obsAnnotations };
}
case "annotation: add new label to category": {
const annotationName = action.metadataField;
const newLabelName = action.newLabelText;
if (!AH.isUserAnnotation(state, annotationName))
throw new Error("unable to modify read-only annotation");
if (typeof newLabelName !== "string" || newLabelName.length === 0)
throw new Error(
"user annotations require a non-zero length string name"
);
/* add the new label to the annotation */
const schema = AH.addObsAnnoCategory(
state.schema,
annotationName,
newLabelName
);
return { ...state, schema };
}
case "annotation: label edited": {
const annotationName = action.metadataField;
const oldLabelName = action.label;
const newLabelName = action.editedLabel;
if (!AH.isUserAnnotation(state, annotationName))
throw new Error("unable to modify read-only annotation");
if (typeof newLabelName !== "string" || newLabelName.length === 0)
throw new Error(
"user annotations require a non-zero length string name"
);
/* remove old label, add new label */
const schema = AH.addObsAnnoCategory(
AH.removeObsAnnoCategory(state.schema, annotationName, oldLabelName),
annotationName,
newLabelName
);
/* change all values in obsAnnotation */
const obsAnnotations = AH.setLabelByValue(
state.obsAnnotations,
annotationName,
oldLabelName,
newLabelName
);
return { ...state, schema, obsAnnotations };
}
case "annotation: delete label": {
/* delete the label from the annotation, and set all cells with this value to unassigned */
const annotationName = action.metadataField;
const labelName = action.label;
if (!AH.isUserAnnotation(state, annotationName))
throw new Error("unable to modify read-only annotation");
if (labelName === unassignedCategoryLabel)
throw new Error("may not remove the unassigned label");
/* remove the category from the schema */
const schema = AH.removeObsAnnoCategory(
state.schema,
annotationName,
labelName
);
/* set all values to unassigned in obsAnnotations */
const obsAnnotations = AH.setLabelByValue(
state.obsAnnotations,
annotationName,
labelName,
unassignedCategoryLabel
);
return { ...state, schema, obsAnnotations };
}
case "annotation: label current cell selection": {
const { metadataField, label } = action;
const { world, crossfilter } = prevSharedState;
/*
selection state is relative to world. We need to convert it
to a mask for Universe before applying it.
*/
const worldMask = crossfilter.allSelectedMask();
const mask = World.worldEqUniverse(world, state)
? worldMask
: AH.worldToUniverseMask(worldMask, world.obsAnnotations, state.nObs);
const obsAnnotations = AH.setLabelByMask(
state.obsAnnotations,
metadataField,
mask,
label
);
return { ...state, obsAnnotations };
}
default: {

155
client/src/reducers/world.js
View File

@@ -1,7 +1,22 @@
import { World, ControlsHelpers } from "../util/stateManager";
import { unassignedCategoryLabel } from "../globals";
import {
World,
ControlsHelpers as CH,
AnnotationsHelpers as AH
} from "../util/stateManager";
import clip from "../util/clip";
import quantile from "../util/quantile";
/*
important note: much of this code assumes that wriable (user) annotations
will NOT contain scalar data (ie, will only contain categorical labelled
data), and therefore will never need to be clipped. Put another way, it
assumes that for these annotations, the clipped & unclipped data is equal.
If we ever start allowing user editable scalar data, this assumption will
need to be revisited.
*/
const WorldReducer = (
state = null,
action,
@@ -9,16 +24,13 @@ const WorldReducer = (
prevSharedState
) => {
switch (action.type) {
case "initial data load complete (universe exists)": {
case "initial data load complete (universe exists)":
case "reset World to eq Universe": {
const { universe } = nextSharedState;
const world = World.createWorldFromEntireUniverse(universe);
return world;
}
case "reset World to eq Universe": {
return prevSharedState.resetCache.world;
}
case "set World to current selection": {
/* Set viewable world to be the currently selected data */
const world = World.createWorldBySelection(
@@ -80,7 +92,7 @@ const WorldReducer = (
Object.keys(action.expressionData)
)
];
unclippedVarData = ControlsHelpers.pruneVarDataCache(
unclippedVarData = CH.pruneVarDataCache(
unclippedVarData,
allTheGenesWeNeed
);
@@ -122,6 +134,135 @@ const WorldReducer = (
};
}
case "annotation: create category": {
const name = action.data;
const { universe } = nextSharedState;
const { schema } = universe;
/*
if world !== universe, we have to subset the newly created annotation,
else, just use it as is.
*/
let newAnnotation = null;
if (!World.worldEqUniverse(state, universe)) {
newAnnotation = universe.obsAnnotations
.subset(state.obsAnnotations.rowIndex.keys(), [name], null)
.icol(0)
.asArray();
} else {
newAnnotation = universe.obsAnnotations.col(name).asArray();
}
const obsAnnotations = state.obsAnnotations.withCol(
name,
newAnnotation,
state.obsAnnotations.rowIndex
);
const unclipped = {
...state.unclipped,
obsAnnotations: state.unclipped.obsAnnotations.withCol(
name,
newAnnotation,
state.unclipped.obsAnnotations.rowIndex
)
};
return { ...state, schema, obsAnnotations, unclipped };
}
case "annotation: category edited": {
/* change the name of an obs annotation */
const name = action.metadataField;
const newName = action.newCategoryText;
const { schema } = nextSharedState.universe;
const obsAnnotations = state.obsAnnotations.renameCol(name, newName);
const unclipped = {
...state.unclipped,
obsAnnotations: state.unclipped.obsAnnotations.renameCol(name, newName)
};
return { ...state, schema, obsAnnotations, unclipped };
}
case "annotation: delete category": {
/* remove a category from obs annotation */
const { schema } = nextSharedState.universe;
const name = action.metadataField;
const obsAnnotations = state.obsAnnotations.dropCol(name);
const unclipped = {
...state.unclipped,
obsAnnotations: state.unclipped.obsAnnotations.dropCol(name)
};
return { ...state, schema, obsAnnotations, unclipped };
}
case "annotation: add new label to category": {
/* add a new label to the schema - schema updated by universe reducer, we just need to note it */
const { schema } = nextSharedState.universe;
return { ...state, schema };
}
case "annotation: label edited": {
const { schema } = nextSharedState.universe;
const { metadataField } = action;
const oldLabelName = action.label;
const newLabelName = action.editedLabel;
/* set all values to to new label */
const unclipped = {
...state.unclipped,
obsAnnotations: AH.setLabelByValue(
state.unclipped.obsAnnotations,
metadataField,
oldLabelName,
newLabelName
)
};
const obsAnnotations = state.obsAnnotations.replaceColData(
metadataField,
unclipped.obsAnnotations.col(metadataField).asArray()
);
return { ...state, schema, obsAnnotations, unclipped };
}
case "annotation: delete label": {
const { schema } = nextSharedState.universe;
const { label, metadataField } = action;
/* set all values to unassigned in obsAnnotations */
const unclipped = {
...state.unclipped,
obsAnnotations: AH.setLabelByValue(
state.unclipped.obsAnnotations,
metadataField,
label,
unassignedCategoryLabel
)
};
const obsAnnotations = state.obsAnnotations.replaceColData(
metadataField,
unclipped.obsAnnotations.col(metadataField).asArray()
);
return { ...state, schema, obsAnnotations, unclipped };
}
case "annotation: label current cell selection": {
const { metadataField, label } = action;
const { crossfilter } = prevSharedState;
const mask = crossfilter.allSelectedMask();
const unclipped = {
...state.unclipped,
obsAnnotations: AH.setLabelByMask(
state.unclipped.obsAnnotations,
metadataField,
mask,
label
)
};
const obsAnnotations = state.obsAnnotations.replaceColData(
metadataField,
unclipped.obsAnnotations.col(metadataField).asArray()
);
return { ...state, obsAnnotations, unclipped };
}
default: {
return state;
}

89
client/src/util/dataframe/dataframe.js
View File

@@ -345,12 +345,50 @@ class Dataframe {
);
}
withColsFrom(dataframe) {
/*
return a new dataframe containing all columns from both `this` and the
provided dataframe.
The row index from `this` will be used. Both dataframes must have identical
dimensionality, and no overlapping columns labels.
*/
const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
const { rowIndex } = this;
const columns = [...this.__columns, ...dataframe.__columns];
const colIndex = this.colIndex.withLabels(dataframe.colIndex.keys());
const columnsAccessor = [
...this.__columnsAccessor,
...dataframe.__columnsAccessor
];
return new this.constructor(
dims,
columns,
rowIndex,
colIndex,
columnsAccessor
);
}
dropCol(label) {
/*
Create a new dataframe, omitting one columns.
const newDf = df.dropCol("colors");
Corner case to manage: if dropping the last column, return an empty dataframe.
*/
if (!this.hasCol(label)) {
throw new RangeError(`unknown label: ${label}`);
}
/*
Corner case to manage: if dropping the last column, return an empty dataframe.
*/
if (this.dims[1] === 1) {
return Dataframe.empty();
}
const dims = [this.dims[0], this.dims[1] - 1];
const coffset = this.colIndex.getOffset(label);
const columns = [...this.__columns];
@@ -367,6 +405,50 @@ class Dataframe {
);
}
renameCol(oldLabel, newLabel) {
/*
Accelerator for dropping a column and then adding it again with a new label
*/
const coffset = this.colIndex.getOffset(oldLabel);
const colIndex = this.colIndex.dropLabel(oldLabel).withLabel(newLabel);
const columns = [...this.__columns];
columns.push(columns[coffset]);
columns.splice(coffset, 1);
const columnsAccessor = [...this.__columnsAccessor];
columnsAccessor.push(columnsAccessor[coffset]);
columnsAccessor.splice(coffset, 1);
return new this.constructor(
this.dims,
columns,
this.rowIndex,
colIndex,
columnsAccessor
);
}
replaceColData(label, newColData) {
/*
Accelerator for dropping a column then adding it again with same
label and different values.
*/
const coffset = this.colIndex.getOffset(label);
const columns = [...this.__columns];
columns[coffset] = newColData;
const columnsAccessor = [...this.__columnsAccessor];
columnsAccessor[coffset] = null;
return new this.constructor(
this.dims,
columns,
this.rowIndex,
this.colIndex,
columnsAccessor
);
}
static empty(rowIndex = null, colIndex = null) {
return new Dataframe([0, 0], [], rowIndex, colIndex);
}
@@ -443,6 +525,8 @@ class Dataframe {
return newCol;
});
}
if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
return new Dataframe(dims, columns, rowIndex, colIndex);
}
@@ -526,6 +610,11 @@ class Dataframe {
Data access with row/col.
**/
columns() {
/* return all column accessors as an array, in offset order */
return [...this.__columnsAccessor];
}
col(columnLabel) {
/*
Return accessor bound to a column. Allows random row access

17
client/src/util/dataframe/labelIndex.js
View File

@@ -80,6 +80,10 @@ class IdentityInt32Index {
return this.__promote([...this.keys(), label]);
}
withLabels(labels) {
return this.__promote([...this.keys(), ...labels]);
}
dropLabel(label) {
if (label === this.maxOffset - 1) {
return new IdentityInt32Index(label);
@@ -163,6 +167,10 @@ class DenseInt32Index {
return this.__promote([...this.keys(), label]);
}
withLabels(labels) {
return this.__promote([...this.keys(), ...labels]);
}
dropLabel(label) {
const labelArray = [...this.keys()];
labelArray.splice(labelArray.indexOf(label), 1);
@@ -187,6 +195,11 @@ class KeyIndex {
index.set(v, i);
});
if (index.size !== rindex.length) {
/* if true, there was a duplicate in the keys */
throw new Error("duplicate label provided to KeyIndex");
}
this.index = index;
this.rindex = rindex;
this.__compile();
@@ -218,6 +231,10 @@ class KeyIndex {
return new KeyIndex([...this.rindex, label]);
}
withLabels(labels) {
return new KeyIndex([...this.rindex, ...labels]);
}
dropLabel(label) {
const idx = this.rindex.indexOf(label);
const labelArray = [...this.rindex];

14
client/src/util/dataframe/util.js
View File

@@ -21,7 +21,7 @@ export function callOnceLazy(f) {
return result;
}
export function memoize(fn, hashFn) {
export function memoize(fn, hashFn, maxResultsCached = -1) {
/*
function memoization, with user-provided hash. hashFn must return a
key which will be unique as a Map key (ie, obeys "sameValueZero" algorithm
@@ -36,7 +36,19 @@ export function memoize(fn, hashFn) {
}
const result = fn(...args);
cache.set(key, result);
if (maxResultsCached > -1 && cache.size > maxResultsCached) {
/* Least recent insertion deletion */
cache.delete(cache.keys().next().value);
}
return result;
};
wrap.clear = function clear() {
/* clear memoization cache */
cache.clear();
};
return wrap;
}

168
client/src/util/stateManager/annotationsHelpers.js Normal file
View File

@@ -0,0 +1,168 @@
/*
Helper functions for user-editable nnotations state management.
See also reducers/annotations.js
*/
import { unassignedCategoryLabel } from "../../globals";
import * as SchemaHelpers from "./schemaHelpers";
import { obsAnnoDimensionName } from "../nameCreators";
/*
There are a number of state constraints assumed throughout the
application:
- all obs annotations are in {world|universe}.obsAnnotations,
regardless of whether or not they are user editable.
- the {world|universe}.schema is always up to date and matches
the data
- the schema flag `writable` correctly indicates whether
the annotation is editable/mutable.
In addition, the current state management only allows for
categorical annotations to be writable.
*/
export function isCategoricalAnnotation(schema, name) {
/* we treat any string, categorical or boolean as a categorical */
const { type } = schema.annotations.obsByName[name];
return type === "string" || type === "boolean" || type === "categorical";
}
export function isContinuousAnnotation(schema, name) {
return !isCategoricalAnnotation(schema, name);
}
function _isUserAnnotation(schema, name) {
return schema.annotations.obsByName[name]?.writable;
}
export function isUserAnnotation(worldOrUniverse, name) {
return _isUserAnnotation(worldOrUniverse.schema, name);
}
export function removeObsAnnoSchema(schema, name) {
/*
remove named annotation from obs annotation schema
*/
/* only remove if it exists and is a user annotation */
if (!_isUserAnnotation(schema, name))
throw new Error("removing non-user-defined schema");
return SchemaHelpers.removeObsAnnoColumn(schema, name);
}
export function addObsAnnoSchema(schema, name, colSchema) {
/*
add a categorical type to the obs annotation schema
*/
/* collision detection */
if (schema.annotations.obs.columns.some(v => v.name === name))
throw Error("annotations may not contain duplicate category names");
if (name !== colSchema.name) throw Error("column schema does not match");
return SchemaHelpers.addObsAnnoColumn(schema, name, colSchema);
}
export function dupObsAnnoSchema(schema, sourceName, dupName, defaultSchema) {
/*
duplicate the obs annotation `sourceName` schema, but with the name `dupName`
*/
const colSchema = {
...schema.annotations.obsByName[sourceName],
...defaultSchema,
name: dupName
};
/* existance check */
if (!colSchema) throw Error("source annotation does not exist");
/* collision detection */
if (schema.annotations.obs.columns.some(v => v.name === dupName))
throw Error("annotations may not contain duplicate category names");
return SchemaHelpers.addObsAnnoColumn(schema, dupName, colSchema);
}
export function removeObsAnnoCategory(schema, name, category) {
/* don't allow deletion of unassigned category on writable annotations */
if (!_isUserAnnotation(schema, name))
throw new Error("unable to modify read-only schema");
if (category === unassignedCategoryLabel)
throw new Error("may not remove unassigned category label");
return SchemaHelpers.removeObsAnnoCategory(schema, name, category);
}
export function addObsAnnoCategory(schema, name, category) {
if (!_isUserAnnotation(schema, name))
throw new Error("unable to modify read-only schema");
return SchemaHelpers.addObsAnnoCategory(schema, name, category);
}
export function setLabelByValue(df, colName, fromLabel, toLabel) {
/*
in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
*/
const keys = df.colIndex.keys();
const ndf = df.mapColumns((col, colIdx) => {
if (colName !== keys[colIdx]) return col;
/* clone data and return it. */
const newCol = col.slice();
for (let i = 0, l = newCol.length; i < l; i += 1) {
if (newCol[i] === fromLabel) newCol[i] = toLabel;
}
return newCol;
});
return ndf;
}
export function setLabelByMask(df, colName, mask, label) {
/*
in the dataframe column `colName`, set the masked rows to 'label'
*/
const keys = df.colIndex.keys();
const ndf = df.mapColumns((col, colIdx) => {
if (colName !== keys[colIdx]) return col;
/* clone data and return it. */
const newCol = col.slice();
for (let i = 0, l = newCol.length; i < l; i += 1) {
if (mask[i]) newCol[i] = label;
}
return newCol;
});
return ndf;
}
export function worldToUniverseMask(worldMask, worldObsAnnotations, nObs) {
/*
given world seleciton mask, return a selection mask for entire universe
that has same selection state.
*/
const mask = new Uint8Array(nObs);
const { rowIndex } = worldObsAnnotations;
for (let i = 0, l = worldMask.length; i < l; i += 1) {
if (worldMask[i]) {
const label = rowIndex.getLabel(i);
mask[label] = 1;
}
}
return mask;
}
export function createWritableAnnotationDimensions(world, crossfilter) {
const { obsAnnotations, schema } = world;
const writableAnnotations = schema.annotations.obs.columns
.filter(s => s.writable)
.map(s => s.name);
crossfilter = writableAnnotations.reduce((xflt, anno) => {
const dimName = obsAnnoDimensionName(anno);
if (xflt.hasDimension(dimName)) xflt = xflt.delDimension(dimName);
return xflt.addDimension(
dimName,
"enum",
obsAnnotations.col(anno).asArray()
);
}, crossfilter);
return crossfilter;
}

5
client/src/util/stateManager/colorHelpers.js
View File

@@ -50,8 +50,9 @@ function createColorsByCategoricalMetadata(world, accessor) {
}, {});
const rgb = new Array(world.nObs);
const data = world.obsAnnotations.col(accessor).asArray();
for (let i = 0, len = world.obsAnnotations.length; i < len; i += 1) {
const df = world.obsAnnotations;
const data = df.col(accessor).asArray();
for (let i = 0, len = df.length; i < len; i += 1) {
const cat = data[i];
rgb[i] = colors[cat];
}

116
client/src/util/stateManager/controlsHelpers.js
View File

@@ -37,16 +37,14 @@ Remember that option values can be ANY js type, except undefined/null.
}
}
*/
function topNCategories(summary) {
const counts = _.map(summary.categories, cat =>
summary.categoryCounts.get(cat)
);
const sortIndex = fillRange(new Array(summary.numCategories)).sort(
function topNCategories(colSchema, summary, N) {
const { categories } = colSchema;
const counts = _.map(categories, cat => summary.categoryCounts.get(cat) ?? 0);
const sortIndex = fillRange(new Array(categories.length)).sort(
(a, b) => counts[b] - counts[a]
);
const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
const sortedCategories = _.map(sortIndex, i => categories[i]);
const sortedCounts = _.map(sortIndex, i => counts[i]);
const N = globals.maxCategoricalOptionsToDisplay;
if (sortedCategories.length < N) {
return [sortedCategories, sortedCounts];
@@ -54,64 +52,56 @@ function topNCategories(summary) {
return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
}
export function createCategoricalSelection(maxCategoryItems, world) {
const res = {};
const obsIndexName = world.schema.annotations.obs.index;
_.forEach(world.obsAnnotations.colIndex.keys(), key => {
const summary = world.obsAnnotations.col(key).summarize();
if (summary.categories) {
const isColorField = key.includes("color") || key.includes("Color");
const isSelectableCategory =
!isColorField &&
key !== obsIndexName &&
summary.categories.length < maxCategoryItems;
if (isSelectableCategory) {
const [categoryValues, categoryValueCounts] = topNCategories(summary);
const categoryValueIndices = new Map(
categoryValues.map((v, i) => [v, i])
);
const numCategoryValues = categoryValueIndices.size;
const categoryValueSelected = new Array(numCategoryValues).fill(true);
const isTruncated = categoryValues.length < summary.numCategories;
res[key] = {
categoryValues, // array: of natively typed category values
categoryValueIndices, // map: category value (native type) -> category index
categoryValueSelected, // array: t/f selection state
numCategoryValues, // number: of values in the category
isTruncated, // bool: true if list was truncated
categoryValueCounts, // array: cardinality of each category,
categorySelected: true // bool - default state for entire category
};
}
}
});
return res;
export function selectableCategoryNames(world, maxCategoryItems) {
const { schema } = world;
const { index, columns } = schema.annotations.obs;
return columns
.filter(colSchema => {
const { name, categories } = colSchema;
return (
categories && categories.length < maxCategoryItems && name !== index
);
})
.map(v => v.name);
}
/*
given a categoricalSelection, return the list of all category values
where selection state is true (ie, they are selected).
*/
export function selectedValuesForCategory(categorySelectionState, dfColumn) {
const {
categorySelected,
categoryValueSelected,
categoryValueIndices
} = categorySelectionState;
let selectedValues;
if (categorySelected) {
selectedValues = new Set(dfColumn.summarize().categories);
} else {
selectedValues = new Set();
}
categoryValueIndices.forEach((catIndex, catValue) => {
if (!categoryValueSelected[catIndex]) {
selectedValues.delete(catValue);
} else {
selectedValues.add(catValue);
}
});
return [...selectedValues.values()];
export function createCategoricalSelection(world, names) {
const N = globals.maxCategoricalOptionsToDisplay;
const { obsAnnotations, schema } = world;
const res = names.reduce((acc, name) => {
const colSchema = schema.annotations.obsByName[name];
const { writable: isUserAnno } = colSchema;
/*
Summarize the annotation data currently in world. Must return categoryValues
in sorted order, and must include all category values even if they are not
actively used in the current world.
*/
const summary = obsAnnotations.col(name).summarize();
const [categoryValues, categoryValueCounts] = topNCategories(
colSchema,
summary,
N
);
const categoryValueIndices = new Map(categoryValues.map((v, i) => [v, i]));
const numCategoryValues = categoryValueIndices.size;
const categoryValueSelected = new Array(numCategoryValues).fill(true);
const isTruncated = categoryValues.length < summary.numCategories;
acc[name] = {
categoryValues, // array: of natively typed category values
categoryValueIndices, // map: category value (native type) -> category index
categoryValueSelected, // array: t/f selection state
numCategoryValues, // number: of values in the category
isTruncated, // bool: true if list was truncated
categoryValueCounts, // array: cardinality of each category,
categorySelected: true, // bool - default state for entire category
isUserAnno // bool
};
return acc;
}, {});
return res;
}
/*

3
client/src/util/stateManager/index.js
View File

@@ -19,3 +19,6 @@ export * as Universe from "./universe";
export * as World from "./world";
export * as WorldUtil from "./worldUtil";
export * as ControlsHelpers from "./controlsHelpers";
export * as AnnotationsHelpers from "./annotationsHelpers";
export * as SchemaHelpers from "./schemaHelpers";
export * as MatrixFBS from "./matrix";

107
client/src/util/stateManager/matrix.js
View File

@@ -1,5 +1,7 @@
import { flatbuffers } from "flatbuffers";
import { NetEncoding } from "./matrix_generated";
import { isTypedArray } from "../typeHelpers";
import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe";
const utf8Decoder = new TextDecoder("utf-8");
@@ -41,25 +43,25 @@ Returns: object containing decoded Matrix:
colIdx: []|null
}
*/
function decodeMatrixFBS(arrayBuffer, inplace = false) {
export function decodeMatrixFBS(arrayBuffer, inplace = false) {
const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
const df = NetEncoding.Matrix.getRootAsMatrix(bb);
const matrix = NetEncoding.Matrix.getRootAsMatrix(bb);
const nRows = df.nRows();
const nCols = df.nCols();
const nRows = matrix.nRows();
const nCols = matrix.nCols();
/* decode columns */
const columnsLength = df.columnsLength();
const columnsLength = matrix.columnsLength();
const columns = Array(columnsLength).fill(null);
for (let c = 0; c < columnsLength; c += 1) {
const col = df.columns(c);
const col = matrix.columns(c);
columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
}
/* decode col_idx */
const colIdx = decodeTypedArray(
df.colIndexType(),
df.colIndex.bind(df),
matrix.colIndexType(),
matrix.colIndex.bind(matrix),
inplace
);
@@ -72,4 +74,91 @@ function decodeMatrixFBS(arrayBuffer, inplace = false) {
};
}
export default decodeMatrixFBS;
function encodeTypedArray(builder, uType, uData) {
const uTypeName = NetEncoding.TypedArray[uType];
const ArrayType = NetEncoding[uTypeName];
const dv = ArrayType.createDataVector(builder, uData);
builder.startObject(1);
builder.addFieldOffset(0, dv, 0);
return builder.endObject();
}
export function encodeMatrixFBS(df) {
/*
encode the dataframe as an FBS Matrix
*/
/* row indexing not supported currently */
if (df.rowIndex.constructor !== IdentityInt32Index) {
throw new Error("FBS does not support row index encoding at this time");
}
const shape = df.dims;
const utf8Encoder = new TextEncoder("utf-8");
const builder = new flatbuffers.Builder(1024);
let encColIndex;
let encColIndexUType;
let encColumns;
if (shape[0] > 0 && shape[1] > 0) {
const columns = df.columns().map(col => col.asArray());
const cols = columns.map(carr => {
let uType;
let tarr;
if (isTypedArray(carr)) {
uType = NetEncoding.TypedArray[carr.constructor.name];
tarr = encodeTypedArray(builder, uType, carr);
} else {
uType = NetEncoding.TypedArray.JSONEncodedArray;
const json = JSON.stringify(carr);
const jsonUTF8 = utf8Encoder.encode(json);
tarr = encodeTypedArray(builder, uType, jsonUTF8);
}
NetEncoding.Column.startColumn(builder);
NetEncoding.Column.addUType(builder, uType);
NetEncoding.Column.addU(builder, tarr);
return NetEncoding.Column.endColumn(builder);
});
encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
if (df.colIndex && shape[1] > 0) {
const colIndexType = df.colIndex.constructor;
if (colIndexType === IdentityInt32Index) {
encColIndex = undefined;
} else if (colIndexType === DenseInt32Index) {
encColIndexUType = NetEncoding.TypedArray.Int32Array;
encColIndex = encodeTypedArray(
builder,
encColIndexUType,
df.colIndex.keys()
);
} else if (colIndexType === KeyIndex) {
encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray;
encColIndex = encodeTypedArray(
builder,
encColIndexUType,
utf8Encoder.encode(JSON.stringify(df.colIndex.keys()))
);
} else {
throw new Error("Index type FBS encoding unsupported");
}
}
}
NetEncoding.Matrix.startMatrix(builder);
NetEncoding.Matrix.addNRows(builder, shape[0]);
NetEncoding.Matrix.addNCols(builder, shape[1]);
if (encColumns) {
NetEncoding.Matrix.addColumns(builder, encColumns);
}
if (encColIndexUType) {
NetEncoding.Matrix.addColIndexType(builder, encColIndexUType);
NetEncoding.Matrix.addColIndex(builder, encColIndex);
}
const root = NetEncoding.Matrix.endMatrix(builder);
builder.finish(root);
return builder.asUint8Array();
}

96
client/src/util/stateManager/schemaHelpers.js Normal file
View File

@@ -0,0 +1,96 @@
/*
Helpers for schema management
*/
import _ from "lodash";
import fromEntries from "../fromEntries";
/*
System wide schema assumptions:
- schema and data wil be consistent (eg, for user-created annotations)
- schema will be internally self-consistent (eg, index matches columns)
- world & universe schema are same - only data is subset
*/
export function indexEntireSchema(schema) {
/* Index schema for ease of use */
schema.annotations.obsByName = fromEntries(
schema.annotations.obs.columns.map(v => [v.name, v])
);
schema.annotations.varByName = fromEntries(
schema.annotations.var.columns.map(v => [v.name, v])
);
schema.layout.obsByName = fromEntries(
schema.layout.obs.map(v => [v.name, v])
);
schema.layout.varByName = fromEntries(
schema.layout.var.map(v => [v.name, v])
);
return schema;
}
function _copy(schema) {
/* redux copy conventions - WARNING, only for modifyign obs annotations */
return {
...schema,
annotations: {
...schema.annotations,
obs: _.cloneDeep(schema.annotations.obs)
}
};
}
function _reindex(schema) {
/* reindex obs annotations ONLY */
schema.annotations.obsByName = fromEntries(
schema.annotations.obs.columns.map(v => [v.name, v])
);
return schema;
}
export function removeObsAnnoColumn(schema, name) {
const newSchema = _copy(schema);
newSchema.annotations.obs.columns = schema.annotations.obs.columns.filter(
v => v.name !== name
);
return _reindex(newSchema);
}
export function addObsAnnoColumn(schema, name, defn) {
const newSchema = _copy(schema);
newSchema.annotations.obs.columns.push(defn);
return _reindex(newSchema);
}
export function removeObsAnnoCategory(schema, name, category) {
/* remove a category from a categorical annotation */
const categories = schema.annotations.obsByName[name]?.categories;
if (!categories)
throw new Error("column does not exist or is not categorical");
const idx = categories.indexOf(category);
if (idx === -1) throw new Error("category does not exist");
const newSchema = _reindex(_copy(schema));
/* remove category */
newSchema.annotations.obsByName[name].categories.splice(idx, 1);
return newSchema;
}
export function addObsAnnoCategory(schema, name, category) {
/* add a category to a categorical annotation */
const categories = schema.annotations.obsByName[name]?.categories;
if (!categories)
throw new Error("column does not exist or is not categorical");
const idx = categories.indexOf(category);
if (idx !== -1) throw new Error("category already exists");
const newSchema = _reindex(_copy(schema));
/* remove category */
newSchema.annotations.obsByName[name].categories.push(category);
return newSchema;
}

48
client/src/util/stateManager/universe.js
View File

@@ -1,11 +1,11 @@
// jshint esversion: 6
import _ from "lodash";
import decodeMatrixFBS from "./matrix";
import { unassignedCategoryLabel } from "../../globals";
import { decodeMatrixFBS } from "./matrix";
import * as Dataframe from "../dataframe";
import fromEntries from "../fromEntries";
import { isFpTypedArray } from "../typeHelpers";
import { indexEntireSchema } from "./schemaHelpers";
import { isCategoricalAnnotation } from "./annotationsHelpers";
/*
Private helper function - create and return a template Universe
@@ -18,10 +18,13 @@ function templateUniverse() {
schema: {},
/*
Annotations
annotations
*/
obsAnnotations: Dataframe.Dataframe.empty(),
varAnnotations: Dataframe.Dataframe.empty(),
/*
layout
*/
obsLayout: Dataframe.Dataframe.empty(),
/*
@@ -122,6 +125,10 @@ function reconcileSchemaCategoriesWithSummary(universe) {
For example, boolean defined fields in the schema do not contain
explicit declaration of categories (nor do string fields). In these
cases, add a 'categories' field to the schema so it is accessible.
In addition, we have a client-side convention (UI) that all writable
annotations must have an 'unassigned' category, even if it is not currently
in use.
*/
universe.schema.annotations.obs.columns.forEach(s => {
@@ -136,6 +143,10 @@ function reconcileSchemaCategoriesWithSummary(universe) {
);
s.categories = categories;
}
if (s.writable && s.categories.indexOf(unassignedCategoryLabel) === -1) {
s.categories = s.categories.concat(unassignedCategoryLabel);
}
});
}
@@ -166,7 +177,7 @@ export function createUniverseFromResponse(
/* layout */
universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
/* sanity check */
/* sanity checks */
if (
universe.nObs !== universe.obsLayout.length ||
universe.nObs !== universe.obsAnnotations.length ||
@@ -176,20 +187,19 @@ export function createUniverseFromResponse(
}
reconcileSchemaCategoriesWithSummary(universe);
indexEntireSchema(universe.schema);
/* sanity checks */
if (
schema.annotations.obs.columns.some(
s => s.writable && !isCategoricalAnnotation(schema, s.name)
)
) {
throw new Error(
"Writable continuous obs annotations are not supproted - failed to laod"
);
}
/* Index schema for ease of use */
universe.schema.annotations.obsByName = fromEntries(
universe.schema.annotations.obs.columns.map(v => [v.name, v])
);
universe.schema.annotations.varByName = fromEntries(
universe.schema.annotations.var.columns.map(v => [v.name, v])
);
universe.schema.layout.obsByName = fromEntries(
universe.schema.layout.obs.map(v => [v.name, v])
);
universe.schema.layout.varByName = fromEntries(
universe.schema.layout.var.map(v => [v.name, v])
);
return universe;
}

15
client/src/util/stateManager/world.js
View File

@@ -1,14 +1,7 @@
// jshint esversion: 6
import clip from "../clip";
import {
layoutDimensionName,
obsAnnoDimensionName,
diffexpDimensionName,
userDefinedDimensionName
} from "../nameCreators";
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
import * as Dataframe from "../dataframe";
import ImmutableTypedCrossfilter from "../typedCrossfilter/crossfilter";
import { isContinuousAnnotation } from "./annotationsHelpers";
/*
@@ -158,7 +151,7 @@ and world.varData.
function setClippedDataframes(world) {
const { schema } = world;
const isContinuousObsAnnotation = (df, idx, label) =>
deduceDimensionType(schema.annotations.obsByName[label], label) !== "enum";
isContinuousAnnotation(schema, label);
const obsQuantile = (label, q) =>
world.unclipped.obsAnnotations.col(label).summarize().percentiles[100 * q];
world.obsAnnotations = clipDataframe(
@@ -183,7 +176,7 @@ function setClippedDataframes(world) {
/*
Subset the current world based upon the current selection, maintaining any existing
clip. Returns new world. Parameters:
* unvierse
* universe
* world - the current world
* crossfilter - the selection state
*/

32
client/src/util/typedCrossfilter/crossfilter.js
View File

@@ -5,6 +5,7 @@ import BitArray from "./bitArray";
import {
sortArray,
lowerBound,
binarySearch,
lowerBoundIndirect,
upperBoundIndirect
} from "./sort";
@@ -56,6 +57,14 @@ export default class ImmutableTypedCrossfilter {
return this.data;
}
setData(data) {
return new ImmutableTypedCrossfilter(
data,
this.dimensions,
this.selectionCache
);
}
dimensionNames() {
/* return array of all dimensions (by name) */
return Object.keys(this.dimensions);
@@ -113,6 +122,23 @@ export default class ImmutableTypedCrossfilter {
return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
}
renameDimension(oldName, newName) {
/*
rename a dimension
*/
const { [oldName]: dim, ...dimensions } = this.dimensions;
const { data, selectionCache } = this;
dim.dim.rename(newName);
return new ImmutableTypedCrossfilter(
data,
{
...dimensions,
[newName]: dim
},
selectionCache
);
}
select(name, spec) {
/*
select on named dimension, as indicated by `spec`. Spec is an object
@@ -288,6 +314,10 @@ class _ImmutableBaseDimension {
this.name = name;
}
rename(name) {
this.name = name;
}
select(spec) {
const { mode } = spec;
if (mode === undefined) {
@@ -436,7 +466,7 @@ class ImmutableEnumDimension extends ImmutableScalarDimension {
const { values } = spec;
return super.selectExact({
mode: spec.mode,
values: values.map(v => lowerBound(enumIndex, v, 0, enumIndex.length))
values: values.map(v => binarySearch(enumIndex, v, 0, enumIndex.length))
});
}

13
client/src/util/typedCrossfilter/sort.js
View File

@@ -413,3 +413,16 @@ export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
}
return upperBoundNonFloatIndirect(valueArray, indexArray, value, first, last);
}
// Search for `value` in the sorted array `arr`, in the range [first, last).
// Return the first index where arr[index] == value, OR if value not present,
// return `last`
//
// The same semantics/behavior as:
// C++: binary_search()
//
export function binarySearch(valueArray, value, first, last) {
const index = lowerBound(valueArray, value, first, last);
if (index !== last && value === valueArray[index]) return index;
return last;
}

14
server/app/driver/driver.py
View File

@@ -45,6 +45,13 @@ class CXGDriver(metaclass=ABCMeta):
features["layout"]["obs"] = {"available": True, "interactiveLimit": 50000}
return features
@abstractmethod
def get_schema(self):
"""
Return current schema
"""
pass
@abstractmethod
def _load_data(self, data_locator):
pass
@@ -59,6 +66,13 @@ class CXGDriver(metaclass=ABCMeta):
"""
pass
@abstractmethod
def annotation_put_fbs(self, axis, fbs):
"""
Put/save FBS as user-defined labels
"""
pass
@abstractmethod
def data_frame_to_fbs_matrix(self, filter, axis):
pass

21
server/app/rest_api/rest.py
View File

@@ -1,5 +1,6 @@
from http import HTTPStatus
import warnings
from os.path import basename
from flask import Blueprint, current_app, jsonify, make_response, request
from flask_restful import Api, Resource
@@ -16,6 +17,7 @@ from server.app.util.errors import (
InteractiveError,
JSONEncodingValueError,
PrepareError,
DisabledFeatureError,
)
"""
@@ -29,7 +31,7 @@ Sort order for routes
class SchemaAPI(Resource):
def get(self):
return make_response(
jsonify({"schema": current_app.data.schema}), HTTPStatus.OK
jsonify({"schema": current_app.data.get_schema()}), HTTPStatus.OK
)
@@ -72,6 +74,11 @@ class ConfigAPI(Resource):
}
}
}
label_file = current_app.data.config["label_file"]
if label_file:
config["config"]["parameters"]["label_file"] = basename(label_file)
return make_response(jsonify(config), HTTPStatus.OK)
@@ -93,6 +100,18 @@ class AnnotationsObsAPI(Resource):
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
def put(self):
try:
fbs = request.get_data()
res = current_app.data.annotation_put_fbs("obs", fbs)
return make_response(
res, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except (ValueError, DisabledFeatureError, KeyError) as e:
return make_response(str(e), HTTPStatus.BAD_REQUEST)
except Exception as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class AnnotationsVarAPI(Resource):
def get(self):

49
server/app/scanpy_engine/labels.py Normal file
View File

@@ -0,0 +1,49 @@
"""
Helpers for user annotations / label_file parameter
"""
from os.path import exists, splitext, getsize
from os import remove, rename
import pandas as pd
def read_labels(fname):
if exists(fname) and getsize(fname) > 0:
return pd.read_csv(fname, dtype='category')
else:
return pd.DataFrame()
def write_labels(fname, df):
rotate_fname(fname)
if not df.empty:
df.to_csv(fname, index=False)
else:
open(fname, 'a').close()
def rotate_fname(fname):
"""
save N backups of file.
fname -> fname-0
fname-0 -> fname->1
...
fname-(N-1) -> fname-N
"""
def rotate(src, dst):
if exists(src):
if exists(dst):
remove(dst)
rename(src, dst)
rotation_size = 9 # rotation size
name, ext = splitext(fname)
# rotate existing files
for i in range(rotation_size - 1, 0, -1):
src = f"{name}-{i}{ext}"
tgt = f"{name}-{i+1}{ext}"
rotate(src, tgt)
tgt = f"{name}-1{ext}"
rotate(fname, tgt)

132
server/app/scanpy_engine/scanpy_engine.py
View File

@@ -1,4 +1,6 @@
import warnings
import copy
import threading
import numpy as np
import pandas
@@ -13,10 +15,12 @@ from server.app.util.errors import (
JSONEncodingValueError,
PrepareError,
ScanpyFileError,
DisabledFeatureError,
)
from server.app.util.utils import jsonify_scanpy, requires_data
from server.app.scanpy_engine.diffexp import diffexp_ttest
from server.app.util.fbs.matrix import encode_matrix_fbs
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
from server.app.scanpy_engine.labels import read_labels, write_labels
"""
Sort order for methods
@@ -31,6 +35,8 @@ Sort order for methods
class ScanpyEngine(CXGDriver):
def __init__(self, data=None, args={}):
super().__init__(data, args)
# lock used to protect label file write ops
self.label_lock = threading.Lock()
if self.data:
self._validate_and_initialize()
@@ -47,6 +53,7 @@ class ScanpyEngine(CXGDriver):
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"label_file": None,
}
@staticmethod
@@ -125,6 +132,29 @@ class ScanpyEngine(CXGDriver):
return True
return False
@staticmethod
def _get_col_type(col):
dtype = col.dtype
data_kind = dtype.kind
schema = {}
if ScanpyEngine._can_cast_to_float32(col):
schema["type"] = "float32"
elif ScanpyEngine._can_cast_to_int32(col):
schema["type"] = "int32"
elif dtype == np.bool_:
schema["type"] = "boolean"
elif data_kind == "O" and dtype == "object":
schema["type"] = "string"
elif data_kind == "O" and dtype == "category":
schema["type"] = "categorical"
schema["categories"] = dtype.categories.tolist()
else:
raise TypeError(
f"Annotations of type {dtype} are unsupported by cellxgene."
)
return schema
@requires_data
def _create_schema(self):
self.schema = {
@@ -148,25 +178,8 @@ class ScanpyEngine(CXGDriver):
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
ann_schema = {"name": ann}
dtype = curr_axis[ann].dtype
data_kind = dtype.kind
if self._can_cast_to_float32(curr_axis[ann]):
ann_schema["type"] = "float32"
elif self._can_cast_to_int32(curr_axis[ann]):
ann_schema["type"] = "int32"
elif dtype == np.bool_:
ann_schema["type"] = "boolean"
elif data_kind == "O" and dtype == "object":
ann_schema["type"] = "string"
elif data_kind == "O" and dtype == "category":
ann_schema["type"] = "categorical"
ann_schema["categories"] = curr_axis[ann].dtype.categories.tolist()
else:
raise TypeError(
f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
)
ann_schema = {"name": ann, "writable": False}
ann_schema.update(self._get_col_type(curr_axis[ann]))
self.schema["annotations"][ax]["columns"].append(ann_schema)
for layout in self.config['layout']:
@@ -177,7 +190,24 @@ class ScanpyEngine(CXGDriver):
}
self.schema["layout"]["obs"].append(layout_schema)
@requires_data
def get_schema(self):
schema = self.schema # base schema
# add label obs annotations as needed
if self.labels is not None:
schema = copy.deepcopy(schema)
for col in self.labels.columns:
col_schema = {
"name": col,
"writable": True,
}
col_schema.update(self._get_col_type(self.labels[col]))
schema["annotations"]["obs"]["columns"].append(col_schema)
return schema
def _load_data(self, data_locator):
# as of AnnData 0.6.19, backed mode performs initial load fast, but at the
# cost of significantly slower access to X data.
try:
# there is no guarantee data_locator indicates a local file. The AnnData
# API will only consume local file objects. If we get a non-local object,
@@ -203,6 +233,17 @@ class ScanpyEngine(CXGDriver):
f"Please check your input and try again."
)
if self.config["label_file"]:
try:
self.labels = read_labels(self.config["label_file"])
except Exception as e:
raise ScanpyFileError(
f"Error while loading label file: {e}, File must be in the .csv format, please check "
f"your input and try again."
)
else:
self.labels = None
@requires_data
def _validate_and_initialize(self):
# var and obs column names must be unique
@@ -214,6 +255,7 @@ class ScanpyEngine(CXGDriver):
self.cell_count = self.data.shape[0]
self.gene_count = self.data.shape[1]
self._default_and_validate_layouts()
self._validate_label_file()
self._create_schema()
@requires_data
@@ -297,6 +339,26 @@ class ScanpyEngine(CXGDriver):
f"annotations with more than 500 categories in the UI"
)
@requires_data
def _validate_label_file(self):
"""
labels is None if disabled, empty if enabled by no data
"""
if self.labels is None or self.labels.empty:
return
# all lables must have a name, which must be unique and not used in obs column names
if not self.labels.columns.is_unique:
raise KeyError(f"All column names specified in {self.config['label_file']} must be unique.")
duplicate_columns = list(set(self.labels.columns) & set(self.data.obs.columns))
if len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# labels must have same count as obs annotations
if self.labels.shape[0] != self.data.obs.shape[0]:
raise ValueError("Labels file must have same number of rows as h5ad file.")
@staticmethod
def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool)
@@ -364,13 +426,41 @@ class ScanpyEngine(CXGDriver):
@requires_data
def annotation_to_fbs_matrix(self, axis, fields=None):
if axis == Axis.OBS:
df = self.data.obs
if self.labels is not None and not self.labels.empty:
df = pandas.concat([self.data.obs, self.labels], axis=1, join_axes=[self.data.obs.index], copy=False)
else:
df = self.data.obs
else:
df = self.data.var
if fields is not None and len(fields) > 0:
df = df[fields]
return encode_matrix_fbs(df, col_idx=df.columns)
@requires_data
def annotation_put_fbs(self, axis, fbs):
fname = self.config["label_file"]
if not fname or self.labels is None:
raise DisabledFeatureError("Writable annotations are not enabled")
if axis != Axis.OBS:
raise ValueError("Only OBS dimension access is supported")
new_label_df = decode_matrix_fbs(fbs)
# if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))
if not new_label_df.columns.is_unique or len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section critical section.
with self.label_lock:
self.labels = new_label_df
write_labels(fname, self.labels)
return jsonify_scanpy({"status": "OK"})
@staticmethod
def slice_columns(X, var_mask):
"""

9
server/app/util/errors.py
View File

@@ -59,3 +59,12 @@ class DriverError(Exception):
def __init__(self, message):
self.message = message
class DisabledFeatureError(Exception):
"""
Raised when an attempt to use a disabled feature occurs
"""
def __init__(self, message):
self.message = message

132
server/app/util/fbs/matrix.py
View File

@@ -2,10 +2,16 @@ import flatbuffers
import numpy as np
from scipy import sparse
import pandas as pd
import json
import server.app.util.fbs.NetEncoding.Column as Column
import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
import server.app.util.fbs.NetEncoding.Matrix as Matrix
import server.app.util.fbs.NetEncoding.Int32Array as Int32Array
import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
# Placeholder until recent enhancements to flatbuffers Python
@@ -104,38 +110,42 @@ def serialize_typed_array(builder, source_array, encoding_info):
return (array_type, array_value)
column_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
def column_encoding(arr):
type_map = {
# dtype: ( array_type, as_type )
np.float64: (TypedArray.TypedArray.Float32Array, np.float32),
np.float32: (TypedArray.TypedArray.Float32Array, np.float32),
np.float16: (TypedArray.TypedArray.Float32Array, np.float32),
return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
np.int8: (TypedArray.TypedArray.Int32Array, np.int32),
np.int16: (TypedArray.TypedArray.Int32Array, np.int32),
np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
np.uint8: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint16: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
return type_map.get(arr.dtype.type, type_map_default)
index_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
def index_encoding(arr):
type_map = {
# dtype: ( array_type, as_type )
np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
return type_map.get(arr.dtype.type, type_map_default)
return index_encoding_type_map.get(arr.dtype.str, index_encoding_default)
def guess_at_mem_needed(matrix):
@@ -205,3 +215,73 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
builder.Finish(matrix)
return builder.Output()
def deserialize_typed_array(tarr):
type_map = {
TypedArray.TypedArray.NONE: None,
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
}
(u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE:
return None
TarType = type_map.get(u_type, None)
if TarType is None:
raise TypeError(f"FBS contains unknown data type: {u_type}")
arr = TarType()
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8'))
return narr
def decode_matrix_fbs(fbs):
"""
Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data
and indices.
"""
matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0)
n_rows = matrix.NRows()
n_cols = matrix.NCols()
if n_rows == 0 or n_cols == 0:
return pd.DataFrame()
if matrix.RowIndexType() is not TypedArray.TypedArray.NONE:
raise ValueError("row indexing not supported for FBS Matrix")
columns_length = matrix.ColumnsLength()
columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
if columns_index is None:
columns_index = range(0, n_cols)
# sanity checks
if len(columns_index) != n_cols or columns_length != n_cols:
raise ValueError("FBS column count does not match number of columns in underlying matrix")
columns_data = {}
columns_type = {}
for col_idx in range(0, columns_length):
col = matrix.Columns(col_idx)
tarr = (col.UType(), col.U())
data = deserialize_typed_array(tarr)
columns_data[columns_index[col_idx]] = data
if len(data) != n_rows:
raise ValueError("FBS column length does not match number of rows")
if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
columns_type[columns_index[col_idx]] = "category"
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
# more sanity checks
if not df.columns.is_unique or len(df.columns) != n_cols:
raise KeyError("FBS column indices are not unique")
return df

22
server/cli/launch.py
View File

@@ -47,19 +47,28 @@ def common_args(func):
show_default=True,
help="Relative expression cutoff used when selecting top N differentially expressed genes",
)
@click.option(
"--experimental-label-file",
default=None,
show_default=True,
multiple=False,
metavar="<user labels CSV file>",
help="CSV file containing user annotations; will be overwritten. Created if does not exist.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff):
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, experimental_label_file):
return {
"layout": embedding,
"max_category_items": max_category_items,
"diffexp_lfc_cutoff": diffexp_lfc_cutoff,
"obs_names": obs_names,
"var_names": var_names,
"label_file": experimental_label_file,
}
@@ -107,7 +116,8 @@ def launch(
max_category_items,
diffexp_lfc_cutoff,
title,
scripts
scripts,
experimental_label_file
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -122,7 +132,8 @@ def launch(
> cellxgene launch <url>"""
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff)
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items,
diffexp_lfc_cutoff, experimental_label_file)
try:
data_locator = DataLocator(data)
except RuntimeError as re:
@@ -181,6 +192,11 @@ def launch(
else:
port = find_available_port(host)
if experimental_label_file:
lf_name, lf_ext = splitext(experimental_label_file)
if lf_ext and lf_ext != ".csv":
raise click.FileError(basename(experimental_label_file), hint="label file type must be .csv")
# Setup app
cellxgene_url = f"http://{host}:{port}"

21
server/test/schema.json
View File

@@ -10,19 +10,23 @@
"columns": [
{
"name": "name_0",
"type": "string"
"type": "string",
"writable": false
},
{
"name": "n_genes",
"type": "int32"
"type": "int32",
"writable": false
},
{
"name": "percent_mito",
"type": "float32"
"type": "float32",
"writable": false
},
{
"name": "n_counts",
"type": "float32"
"type": "float32",
"writable": false
},
{
"name": "louvain",
@@ -36,7 +40,8 @@
"FCGR3A+ Monocytes",
"Dendritic cells",
"Megakaryocytes"
]
],
"writable": false
}
]
},
@@ -45,11 +50,13 @@
"columns": [
{
"name": "name_0",
"type": "string"
"type": "string",
"writable": false
},
{
"name": "n_cells",
"type": "int32"
"type": "int32",
"writable": false
}
]
}

94
server/test/test_fbs.py Normal file
View File

@@ -0,0 +1,94 @@
import unittest
import pandas as pd
import numpy as np
from scipy import sparse
import decode_fbs
from server.app.util.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
class FbsTests(unittest.TestCase):
"""Test Case for Matrix FBS data encode/decode """
def test_encode_boundary(self):
""" test various boundary checks """
# row indexing is unsupported
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
# matrix must be 2D
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.ones((10,)))
def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
d = decode_fbs.decode_matrix_FBS(fbs)
print(d)
self.assertEqual(d["n_rows"], dims[0])
self.assertEqual(d["n_cols"], dims[1])
self.assertIsNone(d["row_idx"])
self.assertEqual(len(d["columns"]), dims[1])
for i in range(0, len(d["columns"])):
self.assertEqual(len(d["columns"][i]), dims[0])
self.assertIsInstance(d["columns"][i], expected_types[i][0])
if (expected_types[i][1] is not None):
self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
if expected_column_idx is not None:
self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
def test_encode_DataFrame(self):
df = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.int32),
(np.ndarray, np.uint32),
(list, None)
)
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd'])
def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32)
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.float32),
(np.ndarray, np.float32)
)
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = (
(np.ndarray, np.int32),
(np.ndarray, np.int32),
(np.ndarray, np.int32)
)
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self):
dfSrc = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
for c in dfSrc.columns:
self.assertTrue(c in dfDst.columns)
if isinstance(dfSrc[c], pd.Series):
self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
else:
self.assertEqual(dfSrc[c], dfDst[c])

172
server/test/test_scanpy_engine.py
View File

@@ -1,15 +1,18 @@
import json
from os import path
from os import path, listdir
import pytest
import time
import unittest
import decode_fbs
import tempfile
import shutil
import numpy as np
from pandas import Series
import pandas as pd
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import FilterError
from server.app.util.errors import FilterError, DisabledFeatureError
from server.app.util.fbs.matrix import encode_matrix_fbs
from server.app.util.data_locator import DataLocator
@@ -22,6 +25,7 @@ class EngineTest(unittest.TestCase):
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"layout_file": None,
}
self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
@@ -32,10 +36,10 @@ class EngineTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self):
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertIn(obs_index_col_name, self.data.data.obs)
self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertIn(var_index_col_name, self.data.data.var)
self.assertEqual(list(self.data.data.var.index), list(range(1838)))
@@ -73,16 +77,16 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 91)
def test_obs_and_var_names(self):
self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
def test_schema(self):
def test_get_schema(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh:
schema = json.load(fh)
self.assertEqual(self.data.schema, schema)
self.assertEqual(self.data.get_schema(), schema)
def test_schema_produces_error(self):
self.data.data.obs["time"] = Series(
self.data.data.obs["time"] = pd.Series(
list([time.time() for i in range(self.data.cell_count)]),
dtype="datetime64[ns]",
)
@@ -111,7 +115,7 @@ class EngineTest(unittest.TestCase):
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
@@ -121,7 +125,7 @@ class EngineTest(unittest.TestCase):
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
def test_annotation_fields(self):
@@ -130,12 +134,16 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 1)
def test_annotation_put(self):
with self.assertRaises(DisabledFeatureError):
self.data.annotation_put_fbs(None, "obs")
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
@@ -169,7 +177,7 @@ class EngineTest(unittest.TestCase):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self):
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
@@ -192,5 +200,137 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 3)
self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
if __name__ == "__main__":
unittest.main()
class WritableAnnotationTest(unittest.TestCase):
def setUp(self):
self.tmpDir = tempfile.mkdtemp()
self.label_file = path.join(self.tmpDir, "labels.csv")
args = {
"layout": ["umap"],
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"label_file": self.label_file
}
self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
def tearDown(self):
shutil.rmtree(self.tmpDir)
def make_fbs(self, data):
df = pd.DataFrame(data)
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
def test_error_checks(self):
# verify that the expected errors are generated
n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({
'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category')
})
# ensure attempt to change VAR annotation
with self.assertRaises(ValueError):
self.data.annotation_put_fbs("var", fbs_bad)
# ensure we catch attempt to overwrite non-writable data
with self.assertRaises(KeyError):
self.data.annotation_put_fbs("obs", fbs_bad)
def test_write_to_file(self):
# verify the file is written as expected
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.label_file))
df = pd.read_csv(self.label_file)
self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
# verify complete overwrite on second attempt, AND rotation occurs
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'),
'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category')
})
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.label_file))
df = pd.read_csv(self.label_file)
self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))
# rotation
name, ext = path.splitext(self.label_file)
self.assertTrue(path.exists(f"{name}-1{ext}"))
def test_file_rotation_to_max_9(self):
# verify we stop rotation at 9
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
name, ext = path.splitext(self.label_file)
expected_files = [self.label_file] + [f"{name}-{i}{ext}" for i in range(1, 10)]
found_files = [path.join(self.tmpDir, p) for p in listdir(self.tmpDir)]
self.assertEqual(set(expected_files), set(found_files))
def test_put_get_roundtrip(self):
# verify that OBS PUTs (annotation_put_fbs) are accessible via
# GET (annotation_to_fbs_matrix)
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
# put
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# get
fbsAll = self.data.annotation_to_fbs_matrix("obs")
schema = self.data.get_schema()
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
obs_index_col_name = schema["annotations"]["obs"]["index"]
self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"
])
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [
'label_A' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [
'label_B' for l in range(0, n_rows)
])
# verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(all_col_schema["cat_A"], {
"name": "cat_A",
"type": "categorical",
"categories": ["label_A"],
"writable": True
})
self.assertEqual(all_col_schema["cat_B"], {
"name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})

1
server/test/test_scanpy_engine_data_load.py
View File

@@ -24,6 +24,7 @@ class DataLoadEngineTest(unittest.TestCase):
"obs_names": "foo",
"var_names": "bar",
"diffexp_lfc_cutoff": 0.1,
"label_file": None,
}
self.data.update(args=args)
self.assertEqual(args, self.data.config)