diff --git a/.travis.yml b/.travis.yml index ccafd885..00cbed07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: python dist: xenial sudo: required -node_js: - - 10 +before_install: + - nvm install node cache: - pip - npm diff --git a/client/__tests__/util/dataframe/dataframe.test.js b/client/__tests__/util/dataframe/dataframe.test.js index 7bf54114..149c663d 100644 --- a/client/__tests__/util/dataframe/dataframe.test.js +++ b/client/__tests__/util/dataframe/dataframe.test.js @@ -481,6 +481,7 @@ describe("dataframe factories", () => { test("simple", () => { /* simple test that it works as expected in common case */ + const dfEmpty = Dataframe.Dataframe.empty(); const dfA = new Dataframe.Dataframe( [2, 1], [["red", "blue"]], @@ -494,6 +495,22 @@ describe("dataframe factories", () => { new Dataframe.KeyIndex(["bools"]) ); + const dfLikeA = dfEmpty.withColsFrom(dfA); + expect(dfLikeA).toBeDefined(); + expect(dfLikeA.dims).toEqual(dfA.dims); + expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys()); + expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex); + expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray()); + + const dfAlsoLikeA = dfA.withColsFrom(dfEmpty); + expect(dfAlsoLikeA).toBeDefined(); + expect(dfAlsoLikeA.dims).toEqual(dfA.dims); + expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys()); + expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex); + expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray()); + const dfC = dfA.withColsFrom(dfB); expect(dfC).toBeDefined(); expect(dfC.dims).toEqual([2, 2]); diff --git a/client/__tests__/util/stateManager/universe.test.js b/client/__tests__/util/stateManager/universe.test.js index 3bac0b69..f647d7e7 100644 --- a/client/__tests__/util/stateManager/universe.test.js +++ b/client/__tests__/util/stateManager/universe.test.js @@ -30,15 +30,39 @@ describe("createUniverseFromResponse", () => { create a universe from sample data nad validate its shape & contents */ const { nObs, nVar } = REST.schema.schema.dataframe; - const universe = Universe.createUniverseFromResponse( + let universe = Universe.createUniverseFromResponse( REST.config, - REST.schema, - REST.annotationsObs, - REST.annotationsVar, - REST.layoutObs + REST.schema + ); + expect(universe).toBeDefined(); + expect(universe).toMatchObject( + expect.objectContaining({ + nObs, + nVar, + schema: REST.schema.schema, + obsAnnotations: expect.any(Dataframe.Dataframe), + varAnnotations: expect.any(Dataframe.Dataframe), + obsLayout: expect.any(Dataframe.Dataframe), + varData: expect.any(Dataframe.Dataframe) + }) ); - expect(universe).toBeDefined(); + universe = { + ...universe, + ...Universe.addObsAnnotations( + universe, + Universe.matrixFBSToDataframe(REST.annotationsObs) + ), + ...Universe.addVarAnnotations( + universe, + Universe.matrixFBSToDataframe(REST.annotationsVar) + ), + ...Universe.addObsLayout( + universe, + Universe.matrixFBSToDataframe(REST.layoutObs) + ) + }; + expect(universe).toMatchObject( expect.objectContaining({ nObs, diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index 528fc9a9..62ea7626 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -17,13 +17,27 @@ the default REST test response. const defaultBigBang = () => { /* create unverse, world, crossfilter and dimensionMap */ /* create universe */ - const universe = Universe.createUniverseFromResponse( + let universe = Universe.createUniverseFromResponse( _.cloneDeep(REST.config), - _.cloneDeep(REST.schema), - _.cloneDeep(REST.annotationsObs), - _.cloneDeep(REST.annotationsVar), - _.cloneDeep(REST.layoutObs) + _.cloneDeep(REST.schema) ); + + universe = { + ...universe, + ...Universe.addObsAnnotations( + universe, + Universe.matrixFBSToDataframe(REST.annotationsObs) + ), + ...Universe.addVarAnnotations( + universe, + Universe.matrixFBSToDataframe(REST.annotationsVar) + ), + ...Universe.addObsLayout( + universe, + Universe.matrixFBSToDataframe(REST.layoutObs) + ) + }; + /* create world */ const world = World.createWorldFromEntireUniverse(universe); /* create crossfilter */ @@ -45,9 +59,9 @@ describe("createWorldFromEntireUniverse", () => { const universe = Universe.createUniverseFromResponse( _.cloneDeep(REST.config), _.cloneDeep(REST.schema), - _.cloneDeep(REST.annotationsObs), - _.cloneDeep(REST.annotationsVar), - _.cloneDeep(REST.layoutObs) + Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)), + Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)), + Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs)) ); expect(universe).toBeDefined(); diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 0bea93f4..2c2b46be 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -1,4 +1,3 @@ -// jshint esversion: 6 import _ from "lodash"; import * as globals from "../globals"; import { Universe, MatrixFBS } from "../util/stateManager"; @@ -9,6 +8,89 @@ import { dispatchNetworkErrorMessageToUser } from "../util/actionHelpers"; +/* +return promise to fetch the OBS annotations we need to load. Omit anything +we don't need. +*/ +function obsAnnotationFetchAndLoad(dispatch, schema, universe) { + const obsAnnotations = schema?.schema?.annotations?.obs ?? {}; + const columns = obsAnnotations.columns ?? []; + const index = obsAnnotations.index ?? false; + return Promise.all( + columns + .filter(col => col.name !== index) + .map(col => { + const path = `annotations/obs?annotation-name=${encodeURIComponent( + col.name + )}`; + const url = `${globals.API.prefix}${globals.API.version}${path}`; + return doBinaryRequest(url); + }) + .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer))) + .map(resp => + resp.then(df => + dispatch({ + type: "universe: column load success", + dim: "obsAnnotations", + dataframe: df + }) + ) + ) + ); +} + +/* +return promise fetching VAR annotations we need to load. Only index is currently used. +*/ +function varAnnotationFetchAndLoad(dispatch, schema, universe) { + const varAnnotations = schema?.schema?.annotations?.var ?? {}; + const index = varAnnotations.index ?? false; + const names = index ? [index] : []; + return Promise.all( + names + .map(name => { + const path = `annotations/var?annotation-name=${encodeURIComponent( + name + )}`; + const url = `${globals.API.prefix}${globals.API.version}${path}`; + return doBinaryRequest(url); + }) + .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer))) + .map(resp => + resp.then(df => + dispatch({ + type: "universe: column load success", + dim: "varAnnotations", + dataframe: df + }) + ) + ) + ); +} + +/* +return promise fetching layout we need +*/ +function layoutFetchAndLoad(dispatch, schema, universe) { + return Promise.all( + ["layout/obs"] + .map(path => { + const url = `${globals.API.prefix}${globals.API.version}${path}`; + return doBinaryRequest(url); + }) + .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer))) + .map(resp => + resp.then(df => + dispatch({ + type: "universe: column load success", + dim: "obsLayout", + dataframe: df + }) + ) + ) + ); +} + /* Bootstrap application with the initial data loading. * /config - application configuration @@ -31,34 +113,29 @@ const doInitialDataLoad = () => /* set config defaults */ const config = { ...globals.configDefaults, ...stepOneResults[0].config }; const schema = stepOneResults[1]; - - /* - Step 2 - dataframes, all binary. NOTE: uses results of step 1. - */ - /* only load names for var annotations, if possible*/ - const varIndexName = schema?.schema?.annotations?.var?.index; - const varAnnotationsQuery = varIndexName - ? `?annotation-name=${encodeURIComponent(varIndexName)}` - : ""; - const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`; - const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"] - .map(r => `${globals.API.prefix}${globals.API.version}${r}`) - .map(url => doBinaryRequest(url)); - const stepTwoResults = await Promise.all(requestBinary); - const [obsAnno, varAnno, obsLayout] = [...stepTwoResults]; - - const universe = Universe.createUniverseFromResponse( - config, - schema, - obsAnno, - varAnno, - obsLayout - ); - + const universe = Universe.createUniverseFromResponse(config, schema); + dispatch({ + type: "universe exists, but loading is still in progress", + universe + }); dispatch({ type: "configuration load complete", config }); + + /* + Step 2 - load the minimum stuff required to display. + */ + await Promise.all([ + layoutFetchAndLoad(dispatch, schema, universe), + varAnnotationFetchAndLoad(dispatch, schema, universe) + ]); + + /* + Step 3 - load everything else + */ + await obsAnnotationFetchAndLoad(dispatch, schema, universe); + dispatch({ type: "initial data load complete (universe exists)", universe diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js index 1b5b1a29..c26aba5a 100644 --- a/client/src/components/brushableHistogram/index.js +++ b/client/src/components/brushableHistogram/index.js @@ -351,8 +351,8 @@ class HistogramBrush extends React.PureComponent { const brushX = d3 .brushX() .extent([ - [x.range()[0], y.range()[1]], - [x.range()[1], this.marginTop + this.height + this.marginBottom] + [x.range()[0], y.range()[1]], + [x.range()[1], this.marginTop + this.height + this.marginBottom] ]) /* emit start so that the Undoable history can save an undo point diff --git a/client/src/components/categorical/categorical.js b/client/src/components/categorical/categorical.js index 5e80057a..ab54d53f 100644 --- a/client/src/components/categorical/categorical.js +++ b/client/src/components/categorical/categorical.js @@ -4,15 +4,15 @@ import { Button } from "@blueprintjs/core"; import { connect } from "react-redux"; import * as globals from "../../globals"; import Category from "./category"; -import { AnnotationsHelpers } from "../../util/stateManager"; +import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager"; import AnnoDialog from "./annoDialog"; import AnnoInputs from "./annoInputs"; import AnnoSelect from "./annoSelect"; @connect(state => ({ - categoricalSelection: state.categoricalSelection, writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false, - schema: state.world?.schema + schema: state.world?.schema, + config: state.config })) class Categories extends React.Component { constructor(props) { @@ -123,12 +123,15 @@ class Categories extends React.Component { const { categoricalSelection, writableCategoriesEnabled, - schema + schema, + config } = this.props; - if (!categoricalSelection) return null; /* all names, sorted in display order. Will be rendered in this order */ - const allCategoryNames = Object.keys(categoricalSelection).sort(); + const allCategoryNames = ControlsHelpers.selectableCategoryNames( + schema, + ControlsHelpers.maxCategoryItems(config) + ).sort(); return (
+
+
+ + + {metadataField} + +
+
+
+
+
+ ); + } + render() { const { isExpanded, isChecked } = this.state; const { metadataField, + categoricalSelection, colorAccessor, isUserAnno, annotations } = this.props; + const isStillLoading = !(categoricalSelection?.[metadataField] ?? false); + if (isStillLoading) { + return this.renderIsStillLoading(metadataField); + } + return ( { return () => { const { dispatch, obsAnnotations } = this.props; @@ -37,61 +28,82 @@ class Continuous extends React.Component { }; }; + renderIsStillLoading(zebra, key) { + return ( +
+
+
+
+ {key} +
+
+
+
+
+ ); + } + render() { const { obsAnnotations, schema } = this.props; - if (schema && !this.continuousChecked) { - this.hasContinuous = _.some( - schema.annotations.obs, - d => d.type === "int32" || d.type === "float32" - ); - this.continuousChecked = true; /* only do this once */ - } + + const obsIndex = schema.annotations.obs.index; + const allContinuousNames = schema.annotations.obs.columns + .filter(col => col.type === "int32" || col.type === "float32") + .filter(col => col.name != obsIndex) + .map(col => col.name); /* initial value for iterator to simulate index, ranges is an object */ let zebra = 0; return (
- {this.hasContinuous ? ( -

- Continuous metadata -

- ) : null} - {obsAnnotations - ? _.map(obsAnnotations.colIndex.keys(), key => { - const isColorField = - key.includes("color") || key.includes("Color"); - if (key === schema.annotations.obs.index || isColorField) - return null; - - const summary = obsAnnotations.col(key).summarize(); - const nonFiniteExtent = - summary.min === undefined || - summary.max === undefined || - Number.isNaN(summary.min) || - Number.isNaN(summary.max); - if (!summary.categorical && !nonFiniteExtent) { - zebra += 1; - return ( - - ); - } - return null; - }) - : null} + {allContinuousNames.map(key => { + if (!obsAnnotations.hasCol(key)) { + // still loading! + zebra += 1; + return this.renderIsStillLoading(zebra, key); + } else { + // data loaded and available + const summary = obsAnnotations.col(key).summarize(); + const nonFiniteExtent = + summary.min === undefined || + summary.max === undefined || + Number.isNaN(summary.min) || + Number.isNaN(summary.max); + if (!summary.categorical && !nonFiniteExtent) { + zebra += 1; + return ( + + ); + } + } + })}
); } diff --git a/client/src/components/geneExpression/index.js b/client/src/components/geneExpression/index.js index 2af9455a..dcfcb46b 100644 --- a/client/src/components/geneExpression/index.js +++ b/client/src/components/geneExpression/index.js @@ -103,7 +103,8 @@ class GeneExpression extends React.Component { if (genes.length === 0) { return keepAroundErrorToast("Must enter a gene name."); } - const worldGenes = world.varAnnotations.col(varIndexName).asArray(); + const worldGenes = + world.varAnnotations?.col(varIndexName)?.asArray() || []; // These gene lists are unique enough where memoization is useless const upperGenes = this._genesToUpper(genes); @@ -206,8 +207,12 @@ class GeneExpression extends React.Component { differential } = this.props; const varIndexName = world?.schema?.annotations?.var?.index; + const varIndex = world?.varAnnotations?.col(varIndexName)?.asArray(); const { tab, bulkAdd, activeItem } = this.state; + // may still be loading! + if (!varIndex) return null; + return (
@@ -268,11 +273,7 @@ class GeneExpression extends React.Component { itemListPredicate={filterGenes} onActiveItemChange={item => this.setState({ activeItem: item })} itemRenderer={renderGene.bind(this)} - items={ - world && world.varAnnotations - ? world.varAnnotations.col(varIndexName).asArray() - : ["No genes"] - } + items={varIndex || ["No genes"]} popoverProps={{ minimal: true }} />