fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-24 07:48:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve type handling for non-string annotation data (#465)

Browse Source 
* improve type handling for non-string annotation data

* improve clarity of code
This commit is contained in:
Bruce Martin
2018-11-26 10:32:29 -08:00
committed by Charlotte Weaver
parent 2b5094665f
commit 138d30909a
8 changed files with 273 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/components/categorical/categorical.js
+6 -45
View File
@@ -5,34 +5,13 @@ import { connect } from "react-redux";
import * as globals from "../../globals";
import Category from "./category";
/* Cap the max number of displayed categories */
const truncateCategories = options => {
const numOptions = _.size(options);
if (numOptions <= globals.maxCategoricalOptionsToDisplay) {
return options;
}
return _(options)
.map((v, k) => ({ name: k, val: v }))
.sortBy("val")
.slice(numOptions - globals.maxCategoricalOptionsToDisplay)
.transform((r, v) => {
r[v.name] = v.val;
}, {})
.value();
};
@connect(state => ({
ranges: _.get(state.controls.world, "summary.obs", null),
categorySelectionLimit: _.get(
state.config,
"parameters.max-category-items",
globals.configDefaults.parameters["max-category-items"]
)
categoricalSelectionState: state.controls.categoricalSelectionState
}))
class Categories extends React.Component {
render() {
const { ranges, categorySelectionLimit } = this.props;
if (!ranges) return null;
const { categoricalSelectionState } = this.props;
if (!categoricalSelectionState) return null;
return (
<div
@@ -47,27 +26,9 @@ class Categories extends React.Component {
>
Categorical Metadata
</p>
{_.map(ranges, (value, key) => {
const isColorField = key.includes("color") || key.includes("Color");
const isSelectableCategory =
value.options &&
!isColorField &&
key !== "name" &&
value.numOptions < categorySelectionLimit;
if (isSelectableCategory) {
const categoryOptions = truncateCategories(value.options);
return (
<Category
key={key}
metadataField={key}
values={categoryOptions}
isTruncated={categoryOptions !== value.options}
/>
);
}
return undefined;
})}
{_.map(categoricalSelectionState, (catState, catName) => (
<Category key={catName} metadataField={catName} />
))}
</div>
);
}
client/src/components/categorical/category.js
+34 -38
View File
@@ -2,29 +2,15 @@ import React from "react";
import _ from "lodash";
import { connect } from "react-redux";
import { FaChevronRight, FaChevronDown } from "react-icons/fa";
import memoize from "memoize-one";
import { Button, Tooltip, Position } from "@blueprintjs/core";
import { Button, Tooltip } from "@blueprintjs/core";
import * as globals from "../../globals";
import Value from "./value";
import alphabeticallySortedValues from "./util";
const countCategories = (values, optsAsBools) =>
_.reduce(
values,
(r, v, k) => {
r.total += 1;
if (optsAsBools[k]) {
r.on += 1;
}
return r;
},
{ total: 0, on: 0 }
);
@connect(state => ({
colorAccessor: state.controls.colorAccessor,
categoricalAsBooleansMap: state.controls.categoricalAsBooleansMap
categoricalSelectionState: state.controls.categoricalSelectionState
}))
class Category extends React.Component {
constructor(props) {
@@ -33,24 +19,30 @@ class Category extends React.Component {
isChecked: true,
isExpanded: false
};
this.countCategories = memoize((values, optsAsBools) =>
countCategories(values, optsAsBools)
);
}
componentDidUpdate() {
const { categoricalAsBooleansMap, metadataField, values } = this.props;
const categoryCount = this.countCategories(
values,
categoricalAsBooleansMap[metadataField]
);
if (categoryCount.on === categoryCount.total) {
const { categoricalSelectionState, metadataField } = this.props;
const cat = categoricalSelectionState[metadataField];
const categoryCount = {
// total number of options in this category
totalOptionCount: cat.numOptions,
// number of selected options in this category
selectedOptionCount: _.reduce(
cat.optionSelected,
(res, cond) => (cond ? res + 1 : res),
0
)
};
if (categoryCount.selectedOptionCount === categoryCount.totalOptionCount) {
/* everything is on, so not indeterminate */
this.checkbox.indeterminate = false;
} else if (categoryCount.on === 0) {
} else if (categoryCount.selectedOptionCount === 0) {
/* nothing is on, so no */
this.checkbox.indeterminate = false;
} else if (categoryCount.on < categoryCount.total) {
} else if (
categoryCount.selectedOptionCount < categoryCount.totalOptionCount
) {
/* to be explicit... */
this.checkbox.indeterminate = true;
}
@@ -74,11 +66,10 @@ class Category extends React.Component {
}
toggleNone() {
const { dispatch, metadataField, value } = this.props;
const { dispatch, metadataField } = this.props;
dispatch({
type: "categorical metadata filter none of these",
metadataField,
value
metadataField
});
this.setState({ isChecked: false });
}
@@ -94,13 +85,15 @@ class Category extends React.Component {
}
renderCategoryItems() {
const { values, metadataField } = this.props;
return _.map(alphabeticallySortedValues(values), (v, i) => (
const { categoricalSelectionState, metadataField } = this.props;
const cat = categoricalSelectionState[metadataField];
const optTuples = alphabeticallySortedValues([...cat.optionIndex]);
return _.map(optTuples, (tuple, i) => (
<Value
key={v}
key={tuple[1]}
metadataField={metadataField}
count={values[v]}
value={v}
optionIndex={tuple[1]}
i={i}
/>
));
@@ -108,12 +101,15 @@ class Category extends React.Component {
render() {
const { isExpanded, isChecked } = this.state;
const { metadataField, colorAccessor, isTruncated } = this.props;
const {
metadataField,
colorAccessor,
categoricalSelectionState
} = this.props;
const { isTruncated } = categoricalSelectionState[metadataField];
return (
<div
style={{
// display: "flex",
// alignItems: "baseline",
maxWidth: globals.maxControlsWidth
}}
>
client/src/components/categorical/util.js
+7 -3
View File
@@ -1,7 +1,11 @@
// jshint esversion: 6
// values is [ [optVal, optIdx], ...]
// index is range array
// return sorted index
export default values =>
Object.keys(values).sort((a, b) => {
const textA = a.toUpperCase();
const textB = b.toUpperCase();
values.sort((a, b) => {
const textA = String(a[0]).toUpperCase();
const textB = String(b[0]).toUpperCase();
return textA < textB ? -1 : textA > textB ? 1 : 0;
});
client/src/components/categorical/value.js
+15 -11
View File
@@ -4,45 +4,49 @@ import React from "react";
import _ from "lodash";
@connect(state => ({
categoricalAsBooleansMap: state.controls.categoricalAsBooleansMap,
categoricalSelectionState: state.controls.categoricalSelectionState,
colorScale: state.controls.colorScale,
colorAccessor: state.controls.colorAccessor,
schema: _.get(state.controls.world, "schema", null)
}))
class CategoryValue extends React.Component {
toggleOff() {
const { dispatch, metadataField, value } = this.props;
const { dispatch, metadataField, optionIndex } = this.props;
dispatch({
type: "categorical metadata filter deselect",
metadataField,
value
optionIndex
});
}
toggleOn() {
const { dispatch, metadataField, value } = this.props;
const { dispatch, metadataField, optionIndex } = this.props;
dispatch({
type: "categorical metadata filter select",
metadataField,
value
optionIndex
});
}
render() {
const {
categoricalAsBooleansMap,
categoricalSelectionState,
metadataField,
count,
value,
optionIndex,
colorAccessor,
colorScale,
i,
schema
} = this.props;
if (!categoricalAsBooleansMap) return null;
if (!categoricalSelectionState) return null;
const category = categoricalSelectionState[metadataField];
const selected = category.optionSelected[optionIndex];
const count = category.optionCount[optionIndex];
const value = category.optionValue[optionIndex];
const displayString = String(category.optionValue[optionIndex]).valueOf();
const selected = categoricalAsBooleansMap[metadataField][value];
/* this is the color scale, so add swatches below */
const c = metadataField === colorAccessor;
let categories = null;
@@ -78,7 +82,7 @@ class CategoryValue extends React.Component {
type="checkbox"
/>
<span className="bp3-control-indicator" />
{value}
{displayString}
</label>
</div>
<span>
client/src/reducers/controls.js
Vendored
+151 -55
View File
@@ -12,34 +12,109 @@ import {
diffexpDimensionName,
makeContinuousDimensionName
} from "../util/nameCreators";
import { fillRange } from "../util/typedCrossfilter/util";
function createCategoricalAsBooleansMap(world) {
/*
Selection state for categoricals are tracked in an Object that
has two main components for each category:
1. mapping of option value to an index
2. array of bool selection state by index
Remember that option values can be ANY js type, except undefined/null.
{
_category_name_1: {
// map of option value to index
optionIndex: Map([
optval1: index,
...
])
// index->selection true/false state
optionSelected: [ true/false, true/false, ... ]
// number of options
numOptions: number,
// isTruncated - true if the options for selection has
// been truncated (ie, was too large to implement)
}
}
*/
function topNoptions(summary) {
const counts = _.map(summary.categories, cat => summary.options[cat]);
const sortIndex = fillRange(new Array(summary.numOptions)).sort(
(a, b) => counts[b] - counts[a]
);
const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
const sortedCounts = _.map(sortIndex, i => counts[i]);
const N = globals.maxCategoricalOptionsToDisplay;
if (sortedCategories.length < N) {
return [sortedCategories, sortedCounts];
}
return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
}
function createCategoricalSelectionState(state, world) {
const res = {};
_.each(world.summary.obs, (value, key) => {
if (value.options && key !== "name") {
const optionsAsBooleans = {};
_.each(value.options, (_value, _key) => {
optionsAsBooleans[_key] = true;
});
res[key] = optionsAsBooleans;
_.forEach(world.summary.obs, (value, key) => {
if (value.categories) {
const isColorField = key.includes("color") || key.includes("Color");
const isSelectableCategory =
!isColorField &&
key !== "name" &&
value.categories.length < state.maxCategoryItems;
if (isSelectableCategory) {
const [optionValue, optionCount] = topNoptions(value);
// const optionCount = Object.values(value.options);
const optionIndex = new Map(optionValue.map((v, i) => [v, i]));
const numOptions = optionIndex.size;
const optionSelected = new Array(numOptions).fill(true);
const isTruncated = optionValue.length < value.numOptions;
res[key] = {
optionValue, // array: of natively typed option values
optionIndex, // map: option value (native type) -> option index
optionSelected, // array: t/f selection state
numOptions, // number: of options
isTruncated, // bool: true if list was truncated
optionCount // array: cardinality of each option
};
}
}
});
return res;
}
/*
given a categoricalSelectionState, return the list of all option values
where selection state is true (ie, they are selected).
*/
function selectedValuesForCategory(categorySelectionState) {
const selectedValues = _([...categorySelectionState.optionIndex])
.filter(tuple => categorySelectionState.optionSelected[tuple[1]])
.map(tuple => tuple[0])
.value();
return selectedValues;
}
const Controls = (
state = {
// data loading flag
loading: false,
error: null,
// configuration
maxCategoryItems: globals.configDefaults.parameters["max-category-items"],
// the whole big bang
universe: null,
// all of the data + selection state
world: null,
colorName: null,
colorRGB: null,
categoricalAsBooleansMap: null,
categoricalSelectionState: null,
crossfilter: null,
dimensionMap: null,
userDefinedGenes: [],
@@ -72,6 +147,17 @@ const Controls = (
Initialization, World/Universe management
and data loading.
******************************************************/
case "configuration load complete": {
// there are a couple of configuration items we need to retain
return {
...state,
maxCategoryItems: _.get(
state.config,
"parameters.max-category-items",
globals.configDefaults.parameters["max-category-items"]
)
};
}
case "initial data load start": {
return { ...state, loading: true };
}
@@ -83,7 +169,10 @@ const Controls = (
const world = World.createWorldFromEntireUniverse(universe);
const colorName = new Array(universe.nObs).fill(globals.defaultCellColor);
const colorRGB = _.map(colorName, c => parseRGB(c));
const categoricalAsBooleansMap = createCategoricalAsBooleansMap(world);
const categoricalSelectionState = createCategoricalSelectionState(
state,
world
);
const crossfilter = Crossfilter(world.obsAnnotations);
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
@@ -135,7 +224,7 @@ const Controls = (
world,
colorName,
colorRGB,
categoricalAsBooleansMap,
categoricalSelectionState,
crossfilter,
dimensionMap,
colorAccessor: null
@@ -152,7 +241,10 @@ const Controls = (
);
const colorName = new Array(world.nObs).fill(globals.defaultCellColor);
const colorRGB = _.map(colorName, c => parseRGB(c));
const categoricalAsBooleansMap = createCategoricalAsBooleansMap(world);
const categoricalSelectionState = createCategoricalSelectionState(
state,
world
);
const crossfilter = Crossfilter(world.obsAnnotations);
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
@@ -197,7 +289,7 @@ const Controls = (
world,
colorName,
colorRGB,
categoricalAsBooleansMap,
categoricalSelectionState,
crossfilter,
dimensionMap,
colorAccessor: null
@@ -422,83 +514,87 @@ const Controls = (
Categorical metadata
*******************************/
case "categorical metadata filter select": {
const newCategoricalAsBooleansMap = {
...state.categoricalAsBooleansMap,
const newOptionSelected = Array.from(
state.categoricalSelectionState[action.metadataField].optionSelected
);
newOptionSelected[action.optionIndex] = true;
const newCategoricalSelectionState = {
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalAsBooleansMap[action.metadataField],
[action.value]: true
...state.categoricalSelectionState[action.metadataField],
optionSelected: newOptionSelected
}
};
// update the filter for the one category that changed state
// update the filter to match all selected options
const cat = newCategoricalSelectionState[action.metadataField];
state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
_.filter(
_.map(
newCategoricalAsBooleansMap[action.metadataField],
(val, key) => (val ? key : false)
)
)
selectedValuesForCategory(cat)
);
return {
...state,
categoricalAsBooleansMap: newCategoricalAsBooleansMap
categoricalSelectionState: newCategoricalSelectionState
};
}
case "categorical metadata filter deselect": {
const newCategoricalAsBooleansMap = {
...state.categoricalAsBooleansMap,
const newOptionSelected = Array.from(
state.categoricalSelectionState[action.metadataField].optionSelected
);
newOptionSelected[action.optionIndex] = false;
const newCategoricalSelectionState = {
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalAsBooleansMap[action.metadataField],
[action.value]: false
...state.categoricalSelectionState[action.metadataField],
optionSelected: newOptionSelected
}
};
// update the filter for the one category that changed state
// update the filter to match all selected options
const cat = newCategoricalSelectionState[action.metadataField];
state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
_.filter(
_.map(
newCategoricalAsBooleansMap[action.metadataField],
(val, key) => (val ? key : false)
)
)
selectedValuesForCategory(cat)
);
return {
...state,
categoricalAsBooleansMap: newCategoricalAsBooleansMap
categoricalSelectionState: newCategoricalSelectionState
};
}
case "categorical metadata filter none of these": {
const newCategoricalAsBooleansMap = {
...state.categoricalAsBooleansMap
};
_.forEach(
newCategoricalAsBooleansMap[action.metadataField],
(v, k, c) => {
c[k] = false;
const newCategoricalSelectionState = {
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalSelectionState[action.metadataField],
optionSelected: Array.from(
state.categoricalSelectionState[action.metadataField].optionSelected
).fill(false)
}
);
};
state.dimensionMap[
obsAnnoDimensionName(action.metadataField)
].filterNone();
return {
...state,
categoricalAsBooleansMap: newCategoricalAsBooleansMap
categoricalSelectionState: newCategoricalSelectionState
};
}
case "categorical metadata filter all of these": {
const newCategoricalAsBooleansMap = {
...state.categoricalAsBooleansMap
};
_.forEach(
newCategoricalAsBooleansMap[action.metadataField],
(v, k, c) => {
c[k] = true;
const newCategoricalSelectionState = {
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalSelectionState[action.metadataField],
optionSelected: Array.from(
state.categoricalSelectionState[action.metadataField].optionSelected
).fill(true)
}
);
};
state.dimensionMap[
obsAnnoDimensionName(action.metadataField)
].filterAll();
return {
...state,
categoricalAsBooleansMap: newCategoricalAsBooleansMap
categoricalSelectionState: newCategoricalSelectionState
};
}
client/src/util/stateManager/summarizeAnnotations.js
+9 -1
View File
@@ -48,6 +48,12 @@ Example:
NOTE: will not summarize the required 'name' annotation, as that is
specified as unique per element.
TODO: XXX - this data structure coerces all metadata categories into a string
(ie, stores values as an Object property in the `options` field). This looses
information (eg, type) for category types which are not strings. Consider an
alterative data structure that does not use the object property for non-string
data types (and does not use _.countBy to summarize).
*/
function summarizeDimension(schema, annotations) {
return _(schema)
@@ -58,11 +64,13 @@ function summarizeDimension(schema, annotations) {
const continuous = type === "int32" || type === "float32";
if (!continuous) {
const categories = _.uniq(_.flatMap(annotations, name));
const options = _.countBy(annotations, name);
const numOptions = _.size(options);
return {
numOptions,
options
options,
categories
};
}
client/src/util/stateManager/universe.js
+27
View File
@@ -161,6 +161,32 @@ function RESTv02LayoutResponseToInternal(response) {
return layout;
}
function reconcileSchemaCategoriesWithSummary(universe) {
/*
where we treat types as (essentially) categorical metadata, update
the schema with data-derived categories (in addition to those in
the server declared schema).
For example, boolean defined fields in the schema do not contain
explicit declaration of categories (nor do string fields). In these
cases, add a 'categories' field to the schema so it is accessible.
*/
_.forEach(universe.schema.annotations.obs, s => {
if (
s.type === "string" ||
s.type === "boolean" ||
s.type === "categorical"
) {
const categories = _.union(
_.get(s, "categories", []),
_.get(universe.summary.obs[s.name], "categories", [])
);
s.categories = categories;
}
});
}
export function createUniverseFromRestV02Response(
configResponse,
schemaResponse,
@@ -199,6 +225,7 @@ export function createUniverseFromRestV02Response(
universe.varAnnotations
);
reconcileSchemaCategoriesWithSummary(universe);
return finalize(universe);
}
server/app/scanpy_engine/scanpy_engine.py
+24 -6
View File
@@ -65,6 +65,22 @@ class ScanpyEngine(CXGDriver):
else:
raise KeyError(f"Annotation name {name}, specified in --{ax_name}_name does not exist.")
@staticmethod
def _can_cast_to_float32(ann):
if ann.dtype.kind == "f" and np.can_cast(ann.dtype, np.float32):
return True
return False
@staticmethod
def _can_cast_to_int32(ann):
if ann.dtype.kind in ["i", "u"]:
if np.can_cast(ann.dtype, np.int32):
return True
ii32 = np.iinfo(np.int32)
if ann.min() >= ii32.min and ann.max() <= ii32.max:
return True
return False
def _create_schema(self):
self.schema = {
"dataframe": {
@@ -81,16 +97,18 @@ class ScanpyEngine(CXGDriver):
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
ann_schema = {"name": ann}
data_kind = curr_axis[ann].dtype.kind
if data_kind == "f":
dtype = curr_axis[ann].dtype
data_kind = dtype.kind
if self._can_cast_to_float32(curr_axis[ann]):
ann_schema["type"] = "float32"
elif data_kind in ["i", "u"]:
elif self._can_cast_to_int32(curr_axis[ann]):
ann_schema["type"] = "int32"
elif data_kind == "?":
elif dtype == np.bool_:
ann_schema["type"] = "boolean"
elif data_kind == "O" and curr_axis[ann].dtype == "object":
elif data_kind == "O" and dtype == "object":
ann_schema["type"] = "string"
elif data_kind == "O" and curr_axis[ann].dtype == "category":
elif data_kind == "O" and dtype == "category":
ann_schema["type"] = "categorical"
ann_schema["categories"] = curr_axis[ann].dtype.categories.tolist()
else: