diff --git a/client/src/components/util/truncate.js b/client/src/components/util/truncate.js
new file mode 100644
index 00000000..618c7831
--- /dev/null
+++ b/client/src/components/util/truncate.js
@@ -0,0 +1,85 @@
+import React, { cloneElement } from "react";
+import { Tooltip, Position } from "@blueprintjs/core";
+
+import { tooltipHoverOpenDelayQuick } from "../../globals";
+
+const SPLIT_STYLE = {
+ display: "flex",
+ overflow: "hidden",
+ justifyContent: "flex-start",
+};
+
+const FIRST_HALF_STYLE = {
+ overflow: "hidden",
+ textOverflow: "ellipsis",
+ whiteSpace: "nowrap",
+ flexShrink: 1,
+ minWidth: "5px",
+};
+
+const SECOND_HALF_INNER_STYLE = {
+ position: "absolute",
+ right: 0,
+};
+const SECOND_HALF_STYLE = {
+ color: "transparent",
+ position: "relative",
+ overflow: "hidden",
+ whiteSpace: "nowrap",
+};
+
+export default (props) => {
+ const { children } = props;
+ // Truncate only support a single child with a text child
+
+ if (
+ React.Children.count(children) !== 1 ||
+ React.Children.count(children.props?.children) !== 1
+ ) {
+ throw Error("Only pass a single child with text to Truncate");
+ }
+ const originalString = children.props.children;
+
+ const firstString = originalString.substr(0, originalString.length / 2);
+ const secondString = originalString.substr(originalString.length / 2);
+
+ const inheritedColor = children.props.style.color;
+
+ const splitStyle = { ...children.props.style, ...SPLIT_STYLE };
+ const secondHalfInnerStyle = {
+ ...SECOND_HALF_INNER_STYLE,
+ color: inheritedColor || "initial",
+ };
+
+ const truncatedJSX = (
+
+ {firstString}
+
+ {secondString}
+ {secondString}
+
+
+ );
+
+ // clone children, changing the children(text) to the truncated string
+ const newChildren = React.Children.map(children, (child) =>
+ cloneElement(child, {
+ children: truncatedJSX,
+ "aria-label": originalString,
+ })
+ );
+ return (
+
+ {newChildren}
+
+ );
+};
diff --git a/client/src/reducers/categoricalSelection.js b/client/src/reducers/categoricalSelection.js
index 39008666..3f3589f2 100644
--- a/client/src/reducers/categoricalSelection.js
+++ b/client/src/reducers/categoricalSelection.js
@@ -31,7 +31,7 @@ const CategoricalSelection = (
const names = CH.selectableCategoryNames(
world.schema,
CH.maxCategoryItems(prevSharedState.config),
- dataframe.colIndex.keys()
+ dataframe.colIndex.labels()
);
if (names.length === 0) return state;
return {
diff --git a/client/src/reducers/centroidLabels.js b/client/src/reducers/centroidLabels.js
index bfc35b4f..25d1d2d1 100644
--- a/client/src/reducers/centroidLabels.js
+++ b/client/src/reducers/centroidLabels.js
@@ -63,6 +63,7 @@ const centroidLabels = (state = initialState, action, sharedNextState) => {
};
case "color by continuous metadata":
+ case "color by expression":
return { ...state, labels: [] };
case "reset centroid labels":
diff --git a/client/src/reducers/world.js b/client/src/reducers/world.js
index 6a62df26..b1f2e410 100644
--- a/client/src/reducers/world.js
+++ b/client/src/reducers/world.js
@@ -93,7 +93,7 @@ const WorldReducer = (
let worldValSlice = val;
if (!World.worldEqUniverse(state, universe)) {
worldValSlice = universeVarData
- .subset(state.obsAnnotations.rowIndex.keys(), [key], null)
+ .subset(state.obsAnnotations.rowIndex.labels(), [key], null)
.icol(0)
.asArray();
}
@@ -129,10 +129,10 @@ const WorldReducer = (
//
let clippedVarData = state.varData;
const keysToDrop = clippedVarData.colIndex
- .keys()
+ .labels()
.filter((k) => !unclippedVarData.hasCol(k));
const keysToAdd = unclippedVarData.colIndex
- .keys()
+ .labels()
.filter((k) => !clippedVarData.hasCol(k));
keysToDrop.forEach((k) => {
clippedVarData = clippedVarData.dropCol(k);
@@ -171,7 +171,7 @@ const WorldReducer = (
let newAnnotation = null;
if (!World.worldEqUniverse(state, universe)) {
newAnnotation = universe.obsAnnotations
- .subset(state.obsAnnotations.rowIndex.keys(), [name], null)
+ .subset(state.obsAnnotations.rowIndex.labels(), [name], null)
.icol(0)
.asArray();
} else {
@@ -303,7 +303,7 @@ const WorldReducer = (
let schema = origSchema;
// alias the names the server sent us, in case they were not the same as the schema
- const embedingLabels = embedding.colIndex.keys();
+ const embedingLabels = embedding.colIndex.labels();
const labels = {
[embedingLabels[0]]: dims[0],
[embedingLabels[1]]: dims[1],
diff --git a/client/src/util/dataframe/dataframe.js b/client/src/util/dataframe/dataframe.js
index 162cfa6e..85a01e6d 100644
--- a/client/src/util/dataframe/dataframe.js
+++ b/client/src/util/dataframe/dataframe.js
@@ -1,6 +1,5 @@
import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
// weird cross-dependency that we should clean up someday...
-import { sortArray } from "../typedCrossfilter/sort";
import {
isTypedArray,
isArrayOrTypedArray,
@@ -389,7 +388,7 @@ class Dataframe {
let dstLabels;
if (!labels) {
// combine all columns
- dstLabels = dataframe.colIndex.keys();
+ dstLabels = dataframe.colIndex.labels();
srcLabels = dstLabels;
} else if (Array.isArray(labels)) {
// combine subset of keys with no aliasing
@@ -537,7 +536,12 @@ class Dataframe {
}
static empty(rowIndex = null, colIndex = null) {
- return new Dataframe([0, 0], [], rowIndex, colIndex);
+ const dims = [
+ rowIndex ? rowIndex.size() : 0,
+ colIndex ? colIndex.size() : 0,
+ ];
+ if (dims[0] && dims[1]) throw new Error("not an empty dataframe");
+ return new Dataframe(dims, new Array(dims[1]), rowIndex, colIndex);
}
static create(dims, columnarData) {
@@ -551,97 +555,59 @@ class Dataframe {
return new Dataframe(dims, columnarData, null, null);
}
- __subset(rowOffsets, colOffsets, withRowIndex) {
+ __subset(newRowIndex, newColIndex) {
const dims = [...this.dims];
- const getSortedLabelAndOffsets = (offsets, index) => {
- /*
- Given offsets, return both offsets and associated lables,
- sorted by offset.
- */
- if (!offsets) {
- return [null, null];
+ /* subset columns */
+ let { __columns, colIndex } = this;
+ if (newColIndex) {
+ const colOffsets = this.colIndex.getOffsets(newColIndex.labels());
+ __columns = new Array(colOffsets.length);
+ for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+ __columns[i] = this.__columns[colOffsets[i]];
}
- const sortedOffsets = sortArray(offsets);
- const sortedLabels = new Array(sortedOffsets.length);
- for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
- sortedLabels[i] = index.getLabel(sortedOffsets[i]);
- }
- return [sortedLabels, sortedOffsets];
- };
-
- let { colIndex } = this;
- if (colOffsets) {
- let colLabels;
- [colLabels, colOffsets] = getSortedLabelAndOffsets(
- colOffsets,
- this.colIndex
- );
+ colIndex = newColIndex;
dims[1] = colOffsets.length;
- colIndex = this.colIndex.subsetLabels(colLabels);
}
let { rowIndex } = this;
- if (withRowIndex) rowIndex = withRowIndex;
- if (rowOffsets) {
- let rowLabels;
- [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
- rowOffsets,
- this.rowIndex
- );
- dims[0] = rowLabels.length;
- if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
- }
-
- /* subset columns */
- let columns = this.__columns;
- if (colOffsets) {
- columns = new Array(colOffsets.length);
- for (let i = 0, l = colOffsets.length; i < l; i += 1) {
- columns[i] = this.__columns[colOffsets[i]];
- }
- }
-
- /* subset rows */
- if (rowOffsets) {
- columns = columns.map((col) => {
+ if (newRowIndex) {
+ const rowOffsets = this.rowIndex.getOffsets(newRowIndex.labels());
+ __columns = __columns.map((col) => {
const newCol = new col.constructor(rowOffsets.length);
for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
newCol[i] = col[rowOffsets[i]];
}
return newCol;
});
+ rowIndex = newRowIndex;
+ dims[0] = rowOffsets.length;
}
if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
- return new Dataframe(dims, columns, rowIndex, colIndex);
+ return new Dataframe(dims, __columns, rowIndex, colIndex);
}
subset(rowLabels, colLabels = null, withRowIndex = null) {
/*
Subset by row/col labels.
- withRowIndex allows assignment of new row index during subset operation.
- If withRowIndex === null, it will reset the index to identity (offset)
- indexing. if withRowIndex is a label index object, it will be used
- for the new dataframe.
+ withRowIndex allows subset with an index, rather than rowLabels.
+ If withRowIndex is specified, rowLabels is ignored.
*/
- const toOffsets = (labels, index) => {
- if (!labels) {
- return null;
- }
- return labels.map((label) => {
- const off = index.getOffset(label);
- if (off === undefined) {
- throw new RangeError(`unknown label: ${label}`);
- }
- return off;
- });
- };
+ let rowIndex = null;
+ if (withRowIndex) {
+ rowIndex = withRowIndex;
+ } else if (rowLabels) {
+ rowIndex = this.rowIndex.subset(rowLabels);
+ }
- const rowOffsets = toOffsets(rowLabels, this.rowIndex);
- const colOffsets = toOffsets(colLabels, this.colIndex);
- return this.__subset(rowOffsets, colOffsets, withRowIndex);
+ let colIndex = null;
+ if (colLabels) {
+ colIndex = this.colIndex.subset(colLabels);
+ }
+
+ return this.__subset(rowIndex, colIndex);
}
isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
@@ -653,7 +619,19 @@ class Dataframe {
indexing. If withRowIndex is a label index object, it will be used
for the new dataframe.
*/
- return this.__subset(rowOffsets, colOffsets, withRowIndex);
+ let rowIndex = null;
+ if (withRowIndex) {
+ rowIndex = withRowIndex;
+ } else if (rowOffsets) {
+ rowIndex = this.rowIndex.isubset(rowOffsets);
+ }
+
+ let colIndex = null;
+ if (colOffsets) {
+ colIndex = this.colIndex.isubset(colOffsets);
+ }
+
+ return this.__subset(rowIndex, colIndex);
}
isubsetMask(rowMask, colMask = null, withRowIndex = null) {
@@ -690,7 +668,7 @@ class Dataframe {
};
const rowOffsets = toList(rowMask, nRows);
const colOffsets = toList(colMask, nCols);
- return this.__subset(rowOffsets, colOffsets, withRowIndex);
+ return this.isubset(rowOffsets, colOffsets, withRowIndex);
}
/**
@@ -790,7 +768,7 @@ class Dataframe {
Return true if this is an empty dataframe, ie, has dimensions [0,0]
*/
const [rows, cols] = this.dims;
- return rows === 0 && cols === 0;
+ return rows === 0 || cols === 0;
}
/****
diff --git a/client/src/util/dataframe/index.js b/client/src/util/dataframe/index.js
index 68ccba29..15d95c1b 100644
--- a/client/src/util/dataframe/index.js
+++ b/client/src/util/dataframe/index.js
@@ -1,2 +1,7 @@
export { default as Dataframe } from "./dataframe";
-export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";
+export {
+ DenseInt32Index,
+ IdentityInt32Index,
+ KeyIndex,
+ isLabelIndex,
+} from "./labelIndex";
diff --git a/client/src/util/dataframe/labelIndex.js b/client/src/util/dataframe/labelIndex.js
index 79f69802..c3eae9ed 100644
--- a/client/src/util/dataframe/labelIndex.js
+++ b/client/src/util/dataframe/labelIndex.js
@@ -32,10 +32,10 @@ class IdentityInt32Index {
this.maxOffset = maxOffset;
}
- keys() {
+ labels() {
// memoize
const k = fillRange(new Int32Array(this.maxOffset));
- this.keys = function keys() {
+ this.labels = function labels() {
return k;
};
return k;
@@ -47,12 +47,24 @@ class IdentityInt32Index {
return i;
}
+ // eslint-disable-next-line class-methods-use-this
+ getOffsets(arr) {
+ // labels to offsets
+ return arr;
+ }
+
// eslint-disable-next-line class-methods-use-this
getLabel(i) {
// offset to label
return i;
}
+ // eslint-disable-next-line class-methods-use-this
+ getLabels(arr) {
+ // offsets to labels
+ return arr;
+ }
+
size() {
return this.maxOffset;
}
@@ -62,6 +74,9 @@ class IdentityInt32Index {
time/space decision - based on the resulting density
*/
const [minLabel, maxLabel] = extent(labelArray);
+ if (minLabel === 0 && maxLabel === labelArray.length - 1)
+ return new IdentityInt32Index(labelArray.length);
+
const labelSpaceSize = maxLabel - minLabel + 1;
const density = labelSpaceSize / this.maxOffset;
/* 0.1 is a magic number, that needs testing to optimize */
@@ -71,30 +86,43 @@ class IdentityInt32Index {
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
}
- subsetLabels(labelArray) {
- return this.__promote(labelArray);
+ subset(labels) {
+ /* validate subset */
+ const { maxOffset } = this;
+ for (let i = 0, l = labels.length; i < l; i += 1) {
+ const label = labels[i];
+ if (label < 0 || label >= maxOffset)
+ throw new RangeError(`offset or label: ${label}`);
+ }
+ return this.__promote(labels);
+ }
+
+ /* identity index - labels are offsets */
+ isubset(offsets) {
+ return this.subset(offsets);
}
withLabel(label) {
if (label === this.maxOffset) {
return new IdentityInt32Index(label + 1);
}
- return this.__promote([...this.keys(), label]);
+ return this.__promote([...this.labels(), label]);
}
withLabels(labels) {
- return this.__promote([...this.keys(), ...labels]);
+ return this.__promote([...this.labels(), ...labels]);
}
dropLabel(label) {
if (label === this.maxOffset - 1) {
return new IdentityInt32Index(label);
}
- const labelArray = [...this.keys()];
+ const labelArray = [...this.labels()];
labelArray.splice(labelArray.indexOf(label), 1);
return this.__promote(labelArray);
}
}
+
class DenseInt32Index {
/*
DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
@@ -129,12 +157,29 @@ class DenseInt32Index {
this.getOffset = function getOffset(l) {
return index[l - minLabel];
};
+
+ this.getOffsets = function getOffsets(arr) {
+ const res = new arr.constructor(arr.length);
+ for (let i = 0, len = arr.length; i < len; i += 1) {
+ res[i] = index[arr[i] - minLabel];
+ }
+ return res;
+ };
+
this.getLabel = function getLabel(i) {
return rindex[i];
};
+
+ this.getLabels = function getLabels(arr) {
+ const res = new arr.constructor(arr.length);
+ for (let i = 0, len = arr.length; i < len; i += 1) {
+ res[i] = rindex[arr[i]];
+ }
+ return res;
+ };
}
- keys() {
+ labels() {
return this.rindex;
}
@@ -158,20 +203,44 @@ class DenseInt32Index {
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
}
- subsetLabels(labelArray) {
- return this.__promote(labelArray);
+ subset(labels) {
+ /* validate subset */
+ for (let i = 0, l = labels.length; i < l; i += 1) {
+ const label = labels[i];
+ const offset = this.getOffset(label);
+ if (offset === undefined || offset === -1)
+ throw new RangeError(`unknown label: ${label}`);
+ }
+
+ return this.__promote(labels);
+ }
+
+ // eslint-disable-next-line class-methods-use-this
+ isubset(offsets) {
+ /* validate subset */
+ const { rindex } = this;
+ const maxOffset = rindex.length;
+ const labels = new Int32Array(offsets.length);
+ for (let i = 0, l = offsets.length; i < l; i += 1) {
+ const offset = offsets[i];
+ if (offset < 0 || offset >= maxOffset)
+ throw new RangeError(`out of bounds offset: ${offset}`);
+ labels[i] = rindex[offset];
+ }
+
+ return this.__promote(labels);
}
withLabel(label) {
- return this.__promote([...this.keys(), label]);
+ return this.__promote([...this.labels(), label]);
}
withLabels(labels) {
- return this.__promote([...this.keys(), ...labels]);
+ return this.__promote([...this.labels(), ...labels]);
}
dropLabel(label) {
- const labelArray = [...this.keys()];
+ const labelArray = [...this.labels()];
labelArray.splice(labelArray.indexOf(label), 1);
return this.__promote(labelArray);
}
@@ -207,12 +276,29 @@ class KeyIndex {
this.getOffset = function getOffset(k) {
return index.get(k);
};
+
+ this.getOffsets = function getOffsets(arr) {
+ const res = new arr.constructor(arr.length);
+ for (let i = 0, len = arr.length; i < len; i += 1) {
+ res[i] = index.get(arr[i]);
+ }
+ return res;
+ };
+
this.getLabel = function getLabel(i) {
return rindex[i];
};
+
+ this.getLabels = function getLabels(arr) {
+ const res = new arr.constructor(arr.length);
+ for (let i = 0, len = arr.length; i < len; i += 1) {
+ res[i] = rindex[arr[i]];
+ }
+ return res;
+ };
}
- keys() {
+ labels() {
return this.rindex;
}
@@ -220,9 +306,30 @@ class KeyIndex {
return this.rindex.length;
}
+ subset(labels) {
+ /* validate subset */
+ for (let i = 0, l = labels.length; i < l; i += 1) {
+ const label = labels[i];
+ const offset = this.getOffset(label);
+ if (offset === undefined) throw new RangeError(`unknown label: ${label}`);
+ }
+
+ return new KeyIndex(labels);
+ }
+
// eslint-disable-next-line class-methods-use-this
- subsetLabels(labelArray) {
- return new KeyIndex(labelArray);
+ isubset(offsets) {
+ const { rindex } = this;
+ const maxOffset = rindex.length;
+ const labels = new Array(offsets.length);
+ for (let i = 0, l = offsets.length; i < l; i += 1) {
+ const offset = offsets[i];
+ if (offset < 0 || offset >= maxOffset)
+ throw new RangeError(`out of bounds offset: ${offset}`);
+ labels[i] = rindex[offset];
+ }
+
+ return new KeyIndex(labels);
}
withLabel(label) {
diff --git a/client/src/util/maybeTruncateString.js b/client/src/util/maybeTruncateString.js
deleted file mode 100644
index 9165f4f7..00000000
--- a/client/src/util/maybeTruncateString.js
+++ /dev/null
@@ -1,12 +0,0 @@
-const maybeTruncateString = (str, maxLength) => {
- let truncatedString = null;
- if (str.length > maxLength) {
- truncatedString = `${str.slice(0, maxLength / 2)}…${str.slice(
- -maxLength / 2
- )}`;
- }
-
- return truncatedString;
-};
-
-export default maybeTruncateString;
diff --git a/client/src/util/stateManager/annotationsHelpers.js b/client/src/util/stateManager/annotationsHelpers.js
index 08249449..525e5b3b 100644
--- a/client/src/util/stateManager/annotationsHelpers.js
+++ b/client/src/util/stateManager/annotationsHelpers.js
@@ -100,7 +100,7 @@ export function setLabelByValue(df, colName, fromLabel, toLabel) {
/*
in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
*/
- const keys = df.colIndex.keys();
+ const keys = df.colIndex.labels();
const ndf = df.mapColumns((col, colIdx) => {
if (colName !== keys[colIdx]) return col;
@@ -118,7 +118,7 @@ export function setLabelByMask(df, colName, mask, label) {
/*
in the dataframe column `colName`, set the masked rows to 'label'
*/
- const keys = df.colIndex.keys();
+ const keys = df.colIndex.labels();
const ndf = df.mapColumns((col, colIdx) => {
if (colName !== keys[colIdx]) return col;
diff --git a/client/src/util/stateManager/controlsHelpers.js b/client/src/util/stateManager/controlsHelpers.js
index b88ab269..3206b63f 100644
--- a/client/src/util/stateManager/controlsHelpers.js
+++ b/client/src/util/stateManager/controlsHelpers.js
@@ -187,7 +187,7 @@ export function pruneVarDataCache(varData, needed) {
if (numOverWatermark <= 0) return varData;
const { colIndex } = varData;
- const all = colIndex.keys();
+ const all = colIndex.labels();
const unused = _.difference(all, needed);
if (unused.length > 0) {
// sort by offset in the dataframe - ie, psuedo-LRU
@@ -203,9 +203,9 @@ export function pruneVarDataCache(varData, needed) {
export function subsetAndResetGeneLists(state) {
const { userDefinedGenes, diffexpGenes } = state;
- const newUserDefinedGenes = []
- .concat(userDefinedGenes, diffexpGenes)
- .slice(0, globals.maxGenes);
+ const newUserDefinedGenes = _.uniq(
+ [].concat(userDefinedGenes, diffexpGenes)
+ ).slice(0, globals.maxGenes);
const newDiffExpGenes = [];
return [newUserDefinedGenes, newDiffExpGenes];
}
diff --git a/client/src/util/stateManager/matrix.js b/client/src/util/stateManager/matrix.js
index 9f467680..05d4e61d 100644
--- a/client/src/util/stateManager/matrix.js
+++ b/client/src/util/stateManager/matrix.js
@@ -1,7 +1,12 @@
import { flatbuffers } from "flatbuffers";
import { NetEncoding } from "./matrix_generated";
-import { isTypedArray } from "../typeHelpers";
-import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe";
+import { isTypedArray, isFpTypedArray } from "../typeHelpers";
+import {
+ Dataframe,
+ IdentityInt32Index,
+ DenseInt32Index,
+ KeyIndex,
+} from "../dataframe";
const utf8Decoder = new TextDecoder("utf-8");
@@ -133,14 +138,14 @@ export function encodeMatrixFBS(df) {
encColIndex = encodeTypedArray(
builder,
encColIndexUType,
- df.colIndex.keys()
+ df.colIndex.labels()
);
} else if (colIndexType === KeyIndex) {
encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray;
encColIndex = encodeTypedArray(
builder,
encColIndexUType,
- utf8Encoder.encode(JSON.stringify(df.colIndex.keys()))
+ utf8Encoder.encode(JSON.stringify(df.colIndex.labels()))
);
} else {
throw new Error("Index type FBS encoding unsupported");
@@ -162,3 +167,79 @@ export function encodeMatrixFBS(df) {
builder.finish(root);
return builder.asUint8Array();
}
+
+function promoteTypedArray(o) {
+ /*
+ Decide what internal data type to use for the data returned from
+ the server.
+
+ TODO - future optimization: not all int32/uint32 data series require
+ promotion to float64. We COULD simply look at the data to decide.
+ */
+ if (isFpTypedArray(o) || Array.isArray(o)) return o;
+
+ let TyepdArrayCtor;
+ switch (o.constructor) {
+ case Int8Array:
+ case Uint8Array:
+ case Uint8ClampedArray:
+ case Int16Array:
+ case Uint16Array:
+ TyepdArrayCtor = Float32Array;
+ break;
+
+ case Int32Array:
+ case Uint32Array:
+ TyepdArrayCtor = Float64Array;
+ break;
+
+ default:
+ throw new Error("Unexpected data type returned from server.");
+ }
+ if (o.constructor === TyepdArrayCtor) return o;
+ return new TyepdArrayCtor(o);
+}
+
+export function matrixFBSToDataframe(arrayBuffers) {
+ /*
+ Convert array of Matrix FBS to a Dataframe.
+
+ The application has strong assumptions that all scalar data will be
+ stored as a float32 or float64 (regardless of underlying data types).
+ For example, clipping of value ranges (eg, user-selected percentiles)
+ depends on the ability to use NaN in any numeric type.
+
+ All float data from the server is left as is. All non-float is promoted
+ to an appropriate float.
+ */
+ if (!Array.isArray(arrayBuffers)) {
+ arrayBuffers = [arrayBuffers];
+ }
+ if (arrayBuffers.length === 0) {
+ return Dataframe.Dataframe.empty();
+ }
+
+ const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
+ /* check that all FBS have same row dimensionality */
+ const { nRows } = fbs[0];
+ fbs.forEach((b) => {
+ if (b.nRows !== nRows)
+ throw new Error("FBS with inconsistent dimensionality");
+ });
+ const columns = fbs
+ .map((fb) =>
+ fb.columns.map((c) => {
+ if (isFpTypedArray(c) || Array.isArray(c)) return c;
+ return promoteTypedArray(c);
+ })
+ )
+ .flat();
+ // colIdx may be TypedArray or Array
+ const colIdx = fbs
+ .map((b) => (Array.isArray(b.colIdx) ? b.colIdx : Array.from(b.colIdx)))
+ .flat();
+ const nCols = columns.length;
+
+ const df = new Dataframe([nRows, nCols], columns, null, new KeyIndex(colIdx));
+ return df;
+}
diff --git a/client/src/util/stateManager/universe.js b/client/src/util/stateManager/universe.js
index c9f5d06e..1b23b85d 100644
--- a/client/src/util/stateManager/universe.js
+++ b/client/src/util/stateManager/universe.js
@@ -40,84 +40,6 @@ These functions are used exclusively by the actions and reducers to
build an internal POJO for use by the rendering components.
*/
-function promoteTypedArray(o) {
- /*
- Decide what internal data type to use for the data returned from
- the server.
-
- TODO - future optimization: not all int32/uint32 data series require
- promotion to float64. We COULD simply look at the data to decide.
- */
- if (isFpTypedArray(o) || Array.isArray(o)) return o;
-
- let TyepdArrayCtor;
- switch (o.constructor) {
- case Int8Array:
- case Uint8Array:
- case Uint8ClampedArray:
- case Int16Array:
- case Uint16Array:
- TyepdArrayCtor = Float32Array;
- break;
-
- case Int32Array:
- case Uint32Array:
- TyepdArrayCtor = Float64Array;
- break;
-
- default:
- throw new Error("Unexpected data type returned from server.");
- }
- if (o.constructor === TyepdArrayCtor) return o;
- return new TyepdArrayCtor(o);
-}
-
-export function matrixFBSToDataframe(arrayBuffers) {
- /*
- Convert array of Matrix FBS to a Dataframe.
-
- The application has strong assumptions that all scalar data will be
- stored as a float32 or float64 (regardless of underlying data types).
- For example, clipping of value ranges (eg, user-selected percentiles)
- depends on the ability to use NaN in any numeric type.
-
- All float data from the server is left as is. All non-float is promoted
- to an appropriate float.
- */
- if (!Array.isArray(arrayBuffers)) {
- arrayBuffers = [arrayBuffers];
- }
- if (arrayBuffers.length === 0) {
- return Dataframe.Dataframe.empty();
- }
-
- const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
- /* check that all FBS have same row dimensionality */
- const { nRows } = fbs[0];
- fbs.forEach((b) => {
- if (b.nRows !== nRows)
- throw new Error("FBS with inconsistent dimensionality");
- });
- const columns = fbs
- .map((fb) =>
- fb.columns.map((c) => {
- if (isFpTypedArray(c) || Array.isArray(c)) return c;
- return promoteTypedArray(c);
- })
- )
- .flat();
- const colIdx = fbs.map((b) => b.colIdx).flat();
- const nCols = columns.length;
-
- const df = new Dataframe.Dataframe(
- [nRows, nCols],
- columns,
- null,
- new Dataframe.KeyIndex(colIdx)
- );
- return df;
-}
-
export function createUniverseFromResponse(configResponse, schemaResponse) {
/*
build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -178,7 +100,7 @@ export function addObsAnnotations(universe, df) {
// for all of the new data, reconcile with schema and sort categories.
const dfs = Array.isArray(df) ? df : [df];
- const keys = dfs.map((d) => d.colIndex.keys()).flat();
+ const keys = dfs.map((d) => d.colIndex.labels()).flat();
const { schema } = universe;
keys.forEach((k) => {
const colSchema = schema.annotations.obsByName[k];
diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js
index 8c69c836..da219e0f 100644
--- a/client/src/util/stateManager/world.js
+++ b/client/src/util/stateManager/world.js
@@ -99,7 +99,7 @@ function clipDataframe(
if (upperQuantile > 1) upperQuantile = 1;
if (lowerQuantile === 0 && upperQuantile === 1) return df;
- const keys = df.colIndex.keys();
+ const keys = df.colIndex.labels();
return df.mapColumns((col, colIdx) => {
const colLabel = keys[colIdx];
if (!clipPredicate(df, colIdx, colLabel)) return col;
@@ -277,7 +277,7 @@ export function addObsDimensions(crossfilter, world) {
but not yet in the crossfilter
*/
const schema = world.schema.annotations.obsByName;
- const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+ const dimsWeNeed = world.obsAnnotations.colIndex.labels();
crossfilter = dimsWeNeed.reduce((xfltr, name) => {
const dimName = obsAnnoDimensionName(name);
if (xfltr.hasDimension(dimName)) return xfltr;
@@ -321,7 +321,7 @@ export function getSelectedByIndex(crossfilter) {
return array of obsIndex, containing all selected obs/cells.
*/
const selected = crossfilter.allSelectedMask(); // array of bool-ish
- const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+ const keys = crossfilter.data.rowIndex.labels(); // row keys, aka universe rowIndex
const set = new Int32Array(selected.length);
let numElems = 0;
diff --git a/client/src/util/typedCrossfilter/crossfilter.js b/client/src/util/typedCrossfilter/crossfilter.js
index 3ee99386..c1ccaa41 100644
--- a/client/src/util/typedCrossfilter/crossfilter.js
+++ b/client/src/util/typedCrossfilter/crossfilter.js
@@ -60,9 +60,7 @@ export default class ImmutableTypedCrossfilter {
}
setData(data) {
- const { selectionCache } = this;
- this.selectionCache = {};
- return new ImmutableTypedCrossfilter(data, this.dimensions, selectionCache);
+ return new ImmutableTypedCrossfilter(data, this.dimensions);
}
dimensionNames() {
diff --git a/dev_docs/developer_guidelines.md b/dev_docs/developer_guidelines.md
index f6a9a50c..93c03d63 100644
--- a/dev_docs/developer_guidelines.md
+++ b/dev_docs/developer_guidelines.md
@@ -51,11 +51,17 @@ JEST_ENV=prod make pydist install-dist dev-env smoke-test
## Server dev
### Install
+
+To install from the source tree
* Build the client and put static files in place: `make build-for-server-dev`
* Install from local files: `make install-dev`
+To install from a candidate python distribution
+* Make the distribution: `make pydist`
+* Install it: `make install-dist`
+
### Launch
-* `cellxgene launch [options]
`
+* `cellxgene launch [options] ` or `make start-server`
### Reloading
If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
@@ -78,13 +84,13 @@ If you would like to run the server tests individually, follow the steps below
## Client dev
### Install
1. Install prereqs for client: `make dev-env`
-2. Install cellxgene server: `make install-dev` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
+2. Install cellxgene server as described in the [server install](#install) instructions above.
### Launch
-To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
-1. Launch server (the client relies on the REST API being available): `cellxgene launch [options] `
-2. Launch client: in `client/` directory run `npm run start`
-3. Client will be served on localhost:3000
+To launch with hot reloading, you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made to source files.
+1. Launch server (the client relies on the REST API being available): `cellxgene launch --debug [other_options] ` or `make start-server`
+2. Launch client: in `client/` directory run `make start-frontend`
+3. Client will be served on `localhost:3000`
### Build
To build only the client: `make build-client`
@@ -95,10 +101,13 @@ We use `eslint` to lint the code and `prettier` as our code formatter.
### Test
If you would like to run the client tests individually, follow the steps below in the `client` directory
-1. For unit tests run `npm run unit-test` or `make unit-test`
-1. For the smoke test run `npm run smoke-test` or `make smoke-test`
+1. For unit tests run `make unit-test`
+1. For the smoke test run `make smoke-test` for the standard smoke test suite and `make smoke-test-annotations` for the annotations test suite.
+
+If you would like to run the smoke tests against a hot-reloaded version of the client:
+1. Start the hot-reloading servers as described in the [Client dev section](#client-dev). If you plan to run the standard test suite (without annotations), you'll have to start the backend server with annotations disabled (e.g. `CXG_OPTIONS='--debug --disable-annotations' make start-server`).
+1. From the project root, `cd client`
+1. Run either the standard E2E test suite with `CXG_CLIENT_PORT=3000 make e2e` or the annotations test suite with `CXG_CLIENT_PORT=3000 make e2e-annotations`
### Tips
* You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) with the `scripts/backend_dev` script.
-
-
diff --git a/docs/_site/index.html b/docs/_site/index.html
index ce1a61e9..51bd2b56 100644
--- a/docs/_site/index.html
+++ b/docs/_site/index.html
@@ -5,7 +5,7 @@
-
+
Index | cellxgene
@@ -16,10 +16,10 @@
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html
index af27f5f8..aff4fa54 100644
--- a/docs/_site/posts/annotations.html
+++ b/docs/_site/posts/annotations.html
@@ -5,7 +5,7 @@
-
+
annotations | cellxgene
@@ -16,10 +16,10 @@
+{"description":"Creating annotations","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html
index 18d25b9b..dd696127 100644
--- a/docs/_site/posts/contact.html
+++ b/docs/_site/posts/contact.html
@@ -5,7 +5,7 @@
-
+
Contact | cellxgene
@@ -16,10 +16,10 @@
+{"description":"Contact","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html
index 19cb1827..587af5cb 100644
--- a/docs/_site/posts/contribute.html
+++ b/docs/_site/posts/contribute.html
@@ -5,7 +5,7 @@
-
+
Code of conduct | cellxgene
@@ -16,10 +16,10 @@
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html
index 8334380c..967fbf9e 100644
--- a/docs/_site/posts/demo-data.html
+++ b/docs/_site/posts/demo-data.html
@@ -5,7 +5,7 @@
-
+
demo-data | cellxgene
@@ -16,10 +16,10 @@
+{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html
index 21271f02..87e458be 100644
--- a/docs/_site/posts/gallery.html
+++ b/docs/_site/posts/gallery.html
@@ -5,7 +5,7 @@
-
+
Gallery | cellxgene
@@ -16,10 +16,10 @@
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html
index 0a2a56c9..fa8a8e0f 100644
--- a/docs/_site/posts/hosted.html
+++ b/docs/_site/posts/hosted.html
@@ -5,7 +5,7 @@
-
+
Hosting cellxgene on the web | cellxgene
@@ -16,10 +16,10 @@
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html
index f86de462..147a82af 100644
--- a/docs/_site/posts/install.html
+++ b/docs/_site/posts/install.html
@@ -5,7 +5,7 @@
-
+
Install | cellxgene
@@ -16,10 +16,10 @@
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html
index 55d57112..63b5d415 100644
--- a/docs/_site/posts/launch.html
+++ b/docs/_site/posts/launch.html
@@ -5,7 +5,7 @@
-
+
demo-data | cellxgene
@@ -16,10 +16,10 @@
+{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html
index 57a2b355..a69bdeec 100644
--- a/docs/_site/posts/methods.html
+++ b/docs/_site/posts/methods.html
@@ -5,7 +5,7 @@
-
+
Methods | cellxgene
@@ -16,10 +16,10 @@
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html
index 114c812c..05f88230 100644
--- a/docs/_site/posts/prepare.html
+++ b/docs/_site/posts/prepare.html
@@ -5,7 +5,7 @@
-
+
prepare | cellxgene
@@ -16,10 +16,10 @@
+{"description":"Preparing your data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html
index 14e96e5b..1d053152 100644
--- a/docs/_site/posts/roadmap.html
+++ b/docs/_site/posts/roadmap.html
@@ -5,7 +5,7 @@
-
+
roadmap | cellxgene
@@ -16,10 +16,10 @@
+{"description":"Roadmap","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"http://schema.org"}
-
+
diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html
index 0430afc8..f0586b56 100644
--- a/docs/_site/posts/troubleshooting.html
+++ b/docs/_site/posts/troubleshooting.html
@@ -5,7 +5,7 @@
-
+
Troubleshooting | cellxgene
@@ -16,10 +16,10 @@
+{"description":"Troubleshooting","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"http://schema.org"}
-
+
diff --git a/docs/posts/cellxgene_cziscience_com.md b/docs/posts/cellxgene_cziscience_com.md
index 0a1bc9f7..51fa3c25 100644
--- a/docs/posts/cellxgene_cziscience_com.md
+++ b/docs/posts/cellxgene_cziscience_com.md
@@ -125,13 +125,71 @@ with a link to embed on your own site, please drop us a note at
- | A single-cell atlas of the peripheral immune response to severe COVID-19
- |
+ A single-cell atlas of the peripheral immune response to severe COVID-19 |
Blish Lab,
medRxiv preprint
|
-
+
+ | Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Ileum |
+
+ Single Cell Portal
+ |
+
+
+ | Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Lung |
+
+ Single Cell Portal
+ |
+
+
+ | Allergic inflammatory memory in human respiratory epithelial progenitor cells - epithelial cells |
+
+ Single Cell Portal
+ |
+
+
+ | Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal scrapings |
+
+ Single Cell Portal
+ |
+
+
+ | Allergic inflammatory memory in human respiratory epithelial progenitor cells - surgical |
+
+ Single Cell Portal
+ |
+
+
+ | Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal SSS |
+
+ Single Cell Portal
+ |
+
+
+ | ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum - epithelial cells |
+
+ Single Cell Portal
+ |
+
+
+ | ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum |
+
+ Single Cell Portal
+ |
+
+
+ | Human Lung HIV-TB Co-infection ACE2+ Cells |
+
+ Single Cell Portal
+ |
+
+
+ | Epithelial Cells in NHP mTB Granuloma and Uninvolved Lung |
+
+ Single Cell Portal
+ |
+
diff --git a/server/app/app.py b/server/app/app.py
index 80aefc7f..911ddf6e 100644
--- a/server/app/app.py
+++ b/server/app/app.py
@@ -58,7 +58,7 @@ def cache_control_always(**cache_kwargs):
@webbp.route("/", methods=["GET"])
@cache_control(public=True, max_age=ONE_WEEK)
-def dataset_index(dataset=None):
+def dataset_index(url_dataroot=None, dataset=None):
config = current_app.app_config
if dataset is None:
if config.single_dataset__datapath:
@@ -66,7 +66,10 @@ def dataset_index(dataset=None):
else:
return dataroot_index()
else:
- location = path_join(config.multi_dataset__dataroot, dataset)
+ dataroot = config.multi_dataset__dataroot.get(url_dataroot)
+ if dataroot is None:
+ abort(HTTPStatus.NOT_FOUND)
+ location = path_join(dataroot, dataset)
scripts = config.server__scripts
inline_scripts = config.server__inline_scripts
@@ -91,18 +94,21 @@ def health():
return health_check(config)
-def get_data_adaptor(dataset=None):
+def get_data_adaptor(url_dataroot=None, dataset=None):
config = current_app.app_config
if dataset is None:
datapath = config.single_dataset__datapath
else:
- datapath = path_join(config.multi_dataset__dataroot, dataset)
+ dataroot = config.multi_dataset__dataroot.get(url_dataroot)
+ if dataroot is None:
+ raise DatasetAccessError(f"Invalid dataset {url_dataroot}/{dataset}")
+ datapath = path_join(dataroot, dataset)
# path_join returns a normalized path. Therefore it is
# sufficient to check that the datapath starts with the
# dataroot to determine that the datapath is under the dataroot.
- if not datapath.startswith(config.multi_dataset__dataroot):
- raise DatasetAccessError("Invalid dataset {dataset}")
+ if not datapath.startswith(dataroot):
+ raise DatasetAccessError("Invalid dataset {url_dataroot}/{dataset}")
if datapath is None:
return common_rest.abort_and_log(HTTPStatus.BAD_REQUEST, "Invalid dataset NONE", loglevel=logging.INFO)
@@ -115,7 +121,7 @@ def rest_get_data_adaptor(func):
@wraps(func)
def wrapped_function(self, dataset=None):
try:
- with get_data_adaptor(dataset) as data_adaptor:
+ with get_data_adaptor(self.url_dataroot, dataset) as data_adaptor:
return func(self, data_adaptor)
except DatasetAccessError:
return common_rest.abort_and_log(
@@ -132,22 +138,23 @@ def dataroot_test_index():
data += "Welcome to cellxgene
"
config = current_app.app_config
- locator = DataLocator(config.multi_dataset__dataroot, region_name=config.data_locator__s3__region_name)
datasets = []
- for fname in locator.ls():
- location = path_join(config.multi_dataset__dataroot, fname)
- try:
- MatrixDataLoader(location, app_config=config)
- datasets.append(fname)
- except DatasetAccessError:
- # skip over invalid datasets
- pass
+ for url_dataroot, dataroot in config.multi_dataset__dataroot.items():
+ locator = DataLocator(dataroot, region_name=config.data_locator__s3__region_name)
+ for fname in locator.ls():
+ location = path_join(dataroot, fname)
+ try:
+ MatrixDataLoader(location, app_config=config)
+ datasets.append((url_dataroot, fname))
+ except DatasetAccessError:
+ # skip over invalid datasets
+ pass
data += "
Select one of these datasets...
"
data += ""
datasets.sort()
- for dataset in datasets:
- data += f"- {dataset}
"
+ for url_dataroot, dataset in datasets:
+ data += f"- {dataset}
"
data += "
"
data += "