Bkmartinjr/state mgr tests (#298)

Jest tests for part of util/stateManager

* reorganize test directories

* lint

* add Jest config to eslint

* add Jest config to eslint

* lint

* fix babel configuration for tests

* start at kvCache tests

* reorganize test directories

* lint

* add Jest config to eslint

* add Jest config to eslint

* lint

* fix babel configuration for tests

* start at kvCache tests

* fix cache flush bug manifest by new test cases

* new kvCache tests

* skeletal universe test

* fix race in test case

* more universe test cases
This commit is contained in:
Bruce Martin
2018-10-04 17:41:26 -07:00
committed by GitHub
parent 8674ad1f7f
commit 19f2728a18
10 changed files with 3194 additions and 2652 deletions
@@ -0,0 +1,249 @@
import _ from "lodash";
import * as kvCache from "../../../src/util/stateManager/keyvalcache";
/*
This is PRIVATE to keyvalcache and must be kept in sync with
any changs ot that module. Need to Know - to enable error handling test
*/
const cachePrivateKey = "__kvcachekey__";
/*
helper function - promisify setTimeout()
*/
function timeout(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
describe("kvcache API", () => {
/*
test the happy path create/set/get API
*/
test("simple create", () => {
/* with defaults */
const kvc = kvCache.create();
expect(kvc).toBeDefined();
expect(kvc).toEqual(expect.objectContaining({}));
expect(kvCache.get(kvc, "test")).toBeUndefined();
/* with params */
const kvc1 = kvCache.create(/* lowWatermark */ 99, /* minTTL */ 0);
expect(kvc1).toBeDefined();
expect(kvc1).toEqual(expect.objectContaining({}));
});
test("set/get", () => {
/*
- check basic get/set functionality
- check set does not mutate source cache
*/
const keyName = "foo";
const kvc1 = kvCache.create();
expect(kvc1).toBeDefined();
expect(kvCache.get(kvc1, keyName)).toBeUndefined();
const val2 = [2];
const kvc2 = kvCache.set(kvc1, keyName, val2);
expect(kvc2).toBeDefined();
expect(kvc2).not.toBe(kvc1);
expect(kvCache.get(kvc1, keyName)).toBeUndefined();
expect(kvCache.get(kvc2, keyName)).toBe(val2);
const val3 = [3];
const kvc3 = kvCache.set(kvc2, keyName, val3);
expect(kvc3).toBeDefined();
expect(kvc3).not.toBe(kvc1);
expect(kvc3).not.toBe(kvc2);
expect(kvCache.get(kvc1, keyName)).toBeUndefined();
expect(kvCache.get(kvc2, keyName)).toBe(val2);
expect(kvCache.get(kvc3, keyName)).toBe(val3);
});
});
describe("common error handling", () => {
/*
Test common error handlers
*/
test("set() protection from namespace pollution", () => {
/*
Test that set() will not allow use of the private cache key
*/
const kvc = kvCache.create();
expect(() => {
kvCache.set(kvc, cachePrivateKey, {});
}).toThrow();
});
test("create() does not accept bogus config", () => {
expect(() => {
kvCache.create([], {});
}).toThrow();
expect(() => {
kvCache.create(-99, 0);
}).toThrow();
expect(() => {
kvCache.create(100, -1);
}).toThrow();
expect(() => {
kvCache.create(1000, "foobar");
}).toThrow();
expect(() => {
kvCache.create(null, 8);
}).toThrow();
});
});
describe("map", () => {
/*
Test kvCache.map() - create new cache that is a transformation of an
existing cache
*/
test("map of empty cache", () => {
const kvc = kvCache.create();
const callback = jest.fn();
const kvcMapped = kvCache.map(kvc, callback);
expect(callback).not.toHaveBeenCalled();
expect(kvcMapped).toBeDefined();
expect(kvcMapped).not.toBe(kvc); // immutable operation
expect(kvcMapped).toEqual(kvc);
});
test("map of non-empty cache", () => {
const key = "aKey";
const val = [0, 1, 2];
let kvc = kvCache.create();
kvc = kvCache.set(kvc, key, val);
const mockCB = jest.fn().mockImplementation(v => [...v]);
const kvcMapped = kvCache.map(kvc, mockCB);
expect(kvcMapped).toBeDefined();
expect(kvcMapped).not.toBe(kvc); // immutable operation
expect(_.isEqual(kvc, kvcMapped)).toBe(true);
expect(mockCB).toHaveBeenCalledTimes(1);
expect(mockCB).toHaveBeenLastCalledWith(val, key);
});
});
describe("flush", () => {
/*
test various cache flush behavior
*/
test("flush - lowWatermark, disable minTTL", () => {
/*
verify lowWatermark functions correctly
*/
// set lowWatermark to 2, set three times - only the final two
// should remain.
let kvc = kvCache.create(2, 0);
["a", "b", "c"].forEach(k => {
kvc = kvCache.set(kvc, k, []);
});
expect(kvc).toEqual(
expect.objectContaining({
b: expect.arrayContaining([]),
c: expect.arrayContaining([])
})
);
expect(kvc).toEqual(
expect.not.objectContaining({
a: expect.arrayContaining([])
})
);
});
test("flush - minTTL, disable lowWatermark", async () => {
/*
verify minTTL functions correctly
*/
// set minTTL to 1 ms
let kvc = kvCache.create(0, 10);
kvc = kvCache.set(kvc, "a", []);
await timeout(20);
["b", "c"].forEach(k => {
kvc = kvCache.set(kvc, k, []);
});
expect(kvc).toEqual(
expect.objectContaining({
b: expect.arrayContaining([]),
c: expect.arrayContaining([])
})
);
expect(kvc).toEqual(
expect.not.objectContaining({
a: expect.arrayContaining([])
})
);
});
test("flush - minTTL and lowWatermark", async () => {
/*
verify minTTL functions correctly
*/
// set lowwatermark to 3, minTTL to 1 ms
let kvc = kvCache.create(3, 10);
kvc = kvCache.set(kvc, "a", []);
// delay
await timeout(20);
["b", "c"].forEach(k => {
kvc = kvCache.set(kvc, k, []);
});
expect(kvc).toEqual(
expect.objectContaining({
a: expect.arrayContaining([]),
b: expect.arrayContaining([]),
c: expect.arrayContaining([])
})
);
kvc = kvCache.set(kvc, "d", []);
expect(kvc).toEqual(
expect.objectContaining({
b: expect.arrayContaining([]),
c: expect.arrayContaining([]),
d: expect.arrayContaining([])
})
);
expect(kvc).toEqual(
expect.not.objectContaining({
a: expect.arrayContaining([])
})
);
});
test("manual flush", async () => {
let kvc = kvCache.create(1, 10);
["a", "b", "c", "d"].forEach(k => {
kvc = kvCache.set(kvc, k, []);
});
// Before TTL has expired, should have all values in cache.
expect(kvc).toEqual(
expect.objectContaining({
a: expect.arrayContaining([]),
b: expect.arrayContaining([]),
c: expect.arrayContaining([])
})
);
// let TTL expire
await timeout(10);
// manually flush
const postFlushKvc = kvCache.flush(kvc);
expect(postFlushKvc).toBeDefined();
expect(postFlushKvc).not.toBe(kvc);
expect(postFlushKvc).toEqual(
expect.objectContaining({
d: expect.arrayContaining([])
})
);
});
});
@@ -0,0 +1,230 @@
/* eslint no-bitwise: "off" */
import _ from "lodash";
import * as Universe from "../../../src/util/stateManager/universe";
/*
test data mocking REST 0.2 API responses
*/
const nObs = 10;
const nVar = 32;
const field4Categories = [83, true, "foo", 2.222222];
const fieldDCategories = [99, false, "mumble", 3.1415];
const aConfigResponse = {
config: {
features: [
{ method: "POST", path: "/cluster/", available: false },
{ method: "POST", path: "/layout/", available: false },
{ method: "POST", path: "/diffexp/", available: false },
{ method: "POST", path: "/saveLocal/", available: false }
],
displayNames: {
engine: "the little engine that could",
dataset: "all your zeros are mine"
}
}
};
const aSchemaResponse = {
schema: {
dataframe: {
nObs,
nVar,
type: "float32"
},
annotations: {
obs: [
{ name: "name", type: "string" },
{ name: "field1", type: "int32" },
{ name: "field2", type: "float32" },
{ name: "field3", type: "boolean" },
{
name: "field4",
type: "categorical",
categories: field4Categories
}
],
var: [
{ name: "name", type: "string" },
{ name: "fieldA", type: "int32" },
{ name: "fieldB", type: "float32" },
{ name: "fieldC", type: "boolean" },
{
name: "fieldD",
type: "categorical",
categories: fieldDCategories
}
]
}
}
};
const anAnnotationsObsResponse = {
names: ["name", "field1", "field2", "field3", "field4"],
data: _()
.range(nObs)
.map(idx => [
idx,
`obs${idx}`,
2 * idx,
idx + 0.0133,
idx & 1,
field4Categories[idx % field4Categories.length]
])
.value()
};
const anAnnotationsVarResponse = {
names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
data: _()
.range(nVar)
.map(idx => [
idx,
10 * idx,
idx + 2.90143,
idx & 1,
fieldDCategories[idx % fieldDCategories.length],
`var${idx}`
])
.value()
};
const aLayoutResponse = {
layout: {
ndims: 2,
coordinates: _()
.range(nObs)
.map(idx => [idx, Math.random(), Math.random()])
.value()
}
};
const aDataObsResponse = {
var: [2, 4, 29],
obs: _()
.range(nObs)
.map(idx => [idx, Math.random(), Math.random(), Math.random()])
.value()
};
describe("createUniverseFromRestV02Response", () => {
/*
test createUniverseFromRestV02Response - this function converts
a set of REST 0.2 responses into a "new" Universe.
createUniverseFromRestV02Response(
configResponse,
schemaResponse,
annotationsObsResponse,
annotationsVarResponse,
layoutObsResponse
) --> Universe
where:
configResponse: GET /.../config
schemaResponse: GET /.../schema
annotationsObsResponse: GET /.../annotations/obs
annotationsVarResponse: GET /.../annotations/var
layoutObsResponse: GET /.../layout/obs
See spec in docs/REST_API.md.
*/
test("create from test data", () => {
/*
create a universe from sample data nad validate its shape & contents
*/
const universe = Universe.createUniverseFromRestV02Response(
aConfigResponse,
aSchemaResponse,
anAnnotationsObsResponse,
anAnnotationsVarResponse,
aLayoutResponse
);
expect(universe).toBeDefined();
expect(universe).toMatchObject(
expect.objectContaining({
api: "0.2",
nObs,
nVar,
schema: aSchemaResponse.schema,
obsAnnotations: expect.any(Array),
varAnnotations: expect.any(Array),
obsNameToIndexMap: expect.any(Object),
varNameToIndexMap: expect.any(Object),
obsLayout: expect.objectContaining({
X: expect.any(Float32Array),
Y: expect.any(Float32Array)
}),
varDataCache: expect.any(Object)
})
);
expect(_.size(universe.obsAnnotations)).toBe(nObs);
expect(_.size(universe.obsNameToIndexMap)).toBe(nObs);
expect(_.size(universe.obsLayout.X)).toBe(nObs);
expect(_.size(universe.obsLayout.Y)).toBe(nObs);
expect(_.size(universe.varAnnotations)).toBe(nVar);
expect(_.size(universe.varNameToIndexMap)).toBe(nVar);
});
});
describe("convertExpressionRESTv02ToObject", () => {
/*
test convertExpressionRESTv02ToObject
convertExpressionRESTv02ToObject(
universe,
response) --> { geneName: Float32Array, geneName: Float32Array, ... }
reponse is a /data/obs response:
{
var: [ varIndices fetched ],
obs: [
[ obsIndex, evalue, ... ],
...
]
}
*/
test("create from response data", () => {
const universe = Universe.createUniverseFromRestV02Response(
aConfigResponse,
aSchemaResponse,
anAnnotationsObsResponse,
anAnnotationsVarResponse,
aLayoutResponse
);
const expression = Universe.convertExpressionRESTv02ToObject(
universe,
aDataObsResponse
);
/* Check that the expected keys are present */
const expectedGeneNames = _.map(
aDataObsResponse.var,
v => anAnnotationsVarResponse.data[v][5]
);
expect(Object.keys(expression)).toEqual(
expect.arrayContaining(expectedGeneNames)
);
const expectedExpressionValues = _.map(
_.unzip(aDataObsResponse.obs),
a => new Float32Array(a)
);
// console.log(aDataObsResponse);
// console.log(expression);
// console.log(_.unzip(aDataObsResponse.obs));
_.forEach(aDataObsResponse.var, (varIdx, idx) => {
const varName = universe.varAnnotations[varIdx].name;
expect(varName).toBeDefined();
expect(varIdx).toBe(universe.varNameToIndexMap[varName]);
expect(expression[varName]).toEqual(expectedExpressionValues[idx + 1]);
});
});
});
@@ -1,6 +1,6 @@
// jshint esversion: 6
import BitArray from "../../src/util/typedCrossfilter/bitArray";
import BitArray from "../../../src/util/typedCrossfilter/bitArray";
const defaultTestLength = 8;
@@ -9,18 +9,18 @@ describe("default select state", () => {
const ba = new BitArray(defaultTestLength);
expect(ba).toBeDefined();
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
const dim = ba.allocDimension();
expect(dim).toBeDefined();
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
ba.freeDimension(dim);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
});
@@ -34,44 +34,44 @@ describe("select and deselect", () => {
expect(dim1).toBeDefined();
ba.selectAll(dim1);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(true);
}
const dim2 = ba.allocDimension();
expect(dim2).toBeDefined();
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
ba.selectAll(dim2);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(true);
}
ba.deselectAll(dim1);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
ba.deselectAll(dim2);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
ba.selectAll(dim1);
ba.selectAll(dim2);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(true);
}
ba.freeDimension(dim1);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(true);
}
ba.freeDimension(dim2);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
});
@@ -84,26 +84,26 @@ describe("select and deselect", () => {
ba.selectOne(dim, 0);
expect(ba.isSelected(0)).toEqual(true);
for (let i = 1; i < defaultTestLength; i++) {
for (let i = 1; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
ba.deselectOne(dim, 0);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
ba.selectOne(dim, 1);
expect(ba.isSelected(1)).toEqual(true);
expect(ba.isSelected(0)).toEqual(false);
for (let i = 2; i < defaultTestLength; i++) {
for (let i = 2; i < defaultTestLength; i += 1) {
expect(ba.isSelected(i)).toEqual(false);
}
ba.selectAll(dim);
ba.deselectOne(dim, defaultTestLength - 1);
expect(ba.isSelected(defaultTestLength - 1)).toEqual(false);
for (let i = 0; i < defaultTestLength - 1; i++) {
for (let i = 0; i < defaultTestLength - 1; i += 1) {
expect(ba.isSelected(i)).toEqual(true);
}
});
@@ -124,7 +124,7 @@ describe("selectionCount", () => {
ba.selectAll(dim2);
expect(ba.selectionCount).toEqual(defaultTestLength);
for (let i = 0; i < defaultTestLength; i++) {
for (let i = 0; i < defaultTestLength; i += 1) {
ba.deselectOne(dim1, i);
expect(ba.selectionCount).toEqual(defaultTestLength - i - 1);
expect(ba.selectionCount).toEqual(ba.countAllOnes());
@@ -1,7 +1,7 @@
// jshint esversion: 6
// const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
import PositiveIntervals from "../../src/util/typedCrossfilter/positiveIntervals";
import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
describe("canonicalize", () => {
test("empty", () => {
@@ -1,6 +1,6 @@
// jshint esversion: 6
import _ from "lodash";
import crossfilter from "../../src/util/typedCrossfilter";
import crossfilter from "../../../src/util/typedCrossfilter";
const someData = [
{
@@ -161,7 +161,7 @@ describe("typedCrossfilter", () => {
expect(type).toBeDefined();
// initially, all should be filtered
expect(payments.allFiltered().length).toEqual(payments.size());
expect(payments.allFiltered()).toHaveLength(payments.size());
expect(payments.allFiltered()).toEqual(payments.all());
expect(payments.countFiltered()).toEqual(someData.length);
+5 -1
View File
@@ -5,5 +5,9 @@ module.exports = {
"stage-0",
"react"
],
plugins: ["istanbul"]
plugins: [
"babel-plugin-transform-react-constant-elements",
"babel-plugin-transform-decorators-legacy",
"babel-plugin-transform-runtime"
]
};
+19 -1
View File
@@ -34,5 +34,23 @@ module.exports = {
"react/prop-types": [0],
"space-before-function-paren": "off",
"function-paren-newline": "off"
}
},
overrides: [
{
files: ["**/*.test.js"],
env: {
jest: true // now **/*.test.js files' env has both es6 *and* jest
},
// Can't extend in overrides: https://github.com/eslint/eslint/issues/8813
// "extends": ["plugin:jest/recommended"]
plugins: ["jest"],
rules: {
"jest/no-disabled-tests": "warn",
"jest/no-focused-tests": "error",
"jest/no-identical-title": "error",
"jest/prefer-to-have-length": "warn",
"jest/valid-expect": "error"
}
}
]
};
+2619 -2613
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -80,6 +80,7 @@
"eslint-loader": "^1.5.0",
"eslint-plugin-filenames": "^1.3.2",
"eslint-plugin-import": "^2.14.0",
"eslint-plugin-jest": "^21.24.1",
"eslint-plugin-jsx-a11y": "^6.1.1",
"eslint-plugin-react": "^7.11.1",
"express": "^4.14.0",
+51 -17
View File
@@ -2,7 +2,8 @@
import _ from "lodash";
/*
Very simple key/value cache for use by World & Universe.
Very simple key/value cache for use by World & Universe. Cache keys must
be a string, and values are any JS non-primitive value.
* constructor(lowWatermark, minTTL):
- lowWatermark defines the number of cache elements below which
@@ -18,8 +19,21 @@ Very simple key/value cache for use by World & Universe.
*/
const cachePrivateKey = "__kvcachekey__";
const defaultLowWatermark = 32;
const defaultMinTTL = 1000;
function create(lowWatermark = defaultLowWatermark, minTTL = defaultMinTTL) {
if (typeof minTTL !== "number" || typeof lowWatermark !== "number") {
throw new TypeError(
"minTTL and lowWatermark parameters must be a primitive number"
);
}
if (lowWatermark < 0 || minTTL < 0) {
throw new RangeError(
"minTTL and lowWatermark parameters must be number greater than zero"
);
}
function create(lowWatermark = 32, minTTL = 1000) {
return {
[cachePrivateKey]: {
lowWatermark,
@@ -29,6 +43,10 @@ function create(lowWatermark = 32, minTTL = 1000) {
}
function get(kvcache, key) {
if (key === cachePrivateKey) {
throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
}
const val = kvcache[key];
if (val) {
val[cachePrivateKey] = Date.now();
@@ -37,10 +55,20 @@ function get(kvcache, key) {
}
function set(kvcache, key, val) {
if (key === cachePrivateKey) {
throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
}
const newKvCache = { ...kvcache };
newKvCache[key] = val;
val[cachePrivateKey] = Date.now();
flush(newKvCache, newKvCache[cachePrivateKey].minTTL);
flushInPlace(newKvCache);
return newKvCache;
}
function flush(kvcache) {
const newKvCache = { ...kvcache };
flushInPlace(newKvCache);
return newKvCache;
}
@@ -48,24 +76,24 @@ function set(kvcache, key, val) {
Flush elements from cache IF cache size is greater than lowWatermark, and
those elements are older than minAgeMS
*/
function flush(kvcache, minAgeMs = 0) {
if (minAgeMs < 0) return kvcache;
const eol = Date.now() - minAgeMs;
const { lowWatermark } = kvcache[cachePrivateKey];
const keys = _(kvcache)
function flushInPlace(kvCache) {
const { lowWatermark, minTTL } = kvCache[cachePrivateKey];
const eol = Date.now() - minTTL;
const allKeys = _(kvCache)
.keys()
.filter(k => k !== cachePrivateKey)
.filter(k => kvcache[k][cachePrivateKey] < eol)
.sortBy([k => kvcache[k][cachePrivateKey]])
.sortBy([k => kvCache[k][cachePrivateKey]])
.value();
if (keys.length > lowWatermark) {
const numKeysToDelete = keys.length - lowWatermark;
const keysToDelete = _.slice(keys, 0, numKeysToDelete);
_.forEach(keysToDelete, k => delete kvcache[k]);
if (allKeys.length > lowWatermark) {
const keysToDelete = _(allKeys)
.slice(0, allKeys.length - lowWatermark)
.filter(k => kvCache[k][cachePrivateKey] <= eol)
.value();
_.forEach(keysToDelete, k => delete kvCache[k]);
}
return kvcache;
return kvCache;
}
/*
@@ -76,9 +104,15 @@ function map(srcKvCache, cb, createOptions) {
.keys()
.filter(k => k !== cachePrivateKey)
.value();
const newKvCache = create(createOptions.lowWatermark, createOptions.minTTL);
const lowWatermark = _.get(
createOptions,
"lowWatermark",
defaultLowWatermark
);
const minTTL = _.get(createOptions, "minTTL", defaultMinTTL);
const newKvCache = create(lowWatermark, minTTL);
_.forEach(keysInSrcKvCache, key => {
const val = cb(get(srcKvCache, key));
const val = cb(get(srcKvCache, key), key);
newKvCache[key] = val;
val[cachePrivateKey] = Date.now();
});