TS Revert (1) (#2402)

* revert all commits to before Typescript migration * update compat workflow to match latest deps (#2335) * update compat workflow to match latest deps * attempt to debug * attempt to debug * remove debugging code * typo * update deps to match desktop (#2340) * fix: don't run lint with `--fix` on push tests (#2273) * fix: don't run lint with `--fix` on push tests * npx Co-authored-by: maniarathi <mani.arathi@gmail.com> Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com> * rename X_approx_distribution to X_approximate_distribution (#2337) * Correctly handle non-finite numbers in heuristic determination of X distribution (#2342) * handle non-finites explicitly * improve and test edge case handling for distribution estimation * revert debugging changes * code readability * clean up type inferencing (#2332) * unit tests for 64 bit conversion * clean up type handling * type inference tests * more type inference fixes * use schema to determine user intent for data typing * stop using deprecated API * fbs type encoding test * add missing test * add more tests * correctly infer X type for CXG adaptor * lint * fix typo * ts migration * cleanup from PR review * lint * PR review changes * remove unused packages from client (#2359) * remove unused packages from client * add missing peer dep * fix: disable FE auth testing on compatibility tests (#2377) * update: release process (#2277) Co-authored-by: maniarathi <mani.arathi@gmail.com> * fix: remove spaces in param setup (#2380) * delete deploy workflow (#2396) * undo reformatting which now does not pass lint * fix snapshots which changed due to npm dep changes * add missing quoting to snapshot * another snapshot typo fix * TS Revert (2) - replay PR #2347 and #2354 (#2403) * replay edits from PR 2347 * TS Revert (3) - replay edits in PR #2327 (#2404) * replay edits in PR 2327 * TS Revert (4) - replay PR #2355 (#2405) * replay edits in PR 2355 * add additional babel config * reformat with new prettier config Co-authored-by: Severiano Badajoz <sbadajoz@chanzuckerberg.com> Co-authored-by: maniarathi <mani.arathi@gmail.com> Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
2026-09-25 08:08:11 +08:00 · 2021-08-23 15:01:36 -07:00
parent 295590a7c6
commit eaae6df5e3
253 changed files with 11175 additions and 22271 deletions
@@ -0,0 +1,346 @@
+import { doBinaryRequest, doFetch } from "./fetchHelpers";
+import { matrixFBSToDataframe } from "../util/stateManager/matrix";
+import { _getColumnSchema } from "./schema";
+import {
+  addObsAnnoColumn,
+  removeObsAnnoColumn,
+  addObsAnnoCategory,
+  removeObsAnnoCategory,
+  addObsLayout,
+} from "../util/stateManager/schemaHelpers";
+import { isArrayOrTypedArray } from "../util/typeHelpers";
+import { _whereCacheCreate } from "./whereCache";
+import AnnoMatrix from "./annoMatrix";
+import PromiseLimit from "../util/promiseLimit";
+import {
+  _expectSimpleQuery,
+  _expectComplexQuery,
+  _urlEncodeLabelQuery,
+  _urlEncodeComplexQuery,
+  _hashStringValues,
+} from "./query";
+import {
+  normalizeResponse,
+  normalizeWritableCategoricalSchema,
+} from "./normalize";
+
+const promiseThrottle = new PromiseLimit(5);
+
+export default class AnnoMatrixLoader extends AnnoMatrix {
+  /*
+  AnnoMatrix implementation which proxies to HTTP server using the CXG REST API.
+  Used as the base (non-view) instance.
+
+  Public API is same as AnnoMatrix class (refer there for API description),
+  with the addition of the constructor which bootstraps:
+
+    new AnnoMatrixLoader(serverBaseURL, schema) -> instance
+
+  */
+  constructor(baseURL, schema) {
+    const { nObs, nVar } = schema.dataframe;
+    super(schema, nObs, nVar);
+
+    if (baseURL[baseURL.length - 1] !== "/") {
+      // must have trailing slash
+      baseURL += "/";
+    }
+    this.baseURL = baseURL;
+    Object.seal(this);
+  }
+
+  /**
+   ** Public.  API described in base class.
+   **/
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category (aka label) to the schema for an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, category);
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a single "category" (aka "label") from the data & schema of an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = await this.resetObsColumnValues(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = removeObsAnnoCategory(
+      newAnnoMatrix.schema,
+      col,
+      category
+    );
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    /*
+		drop column from field
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = removeObsAnnoColumn(this.schema, col);
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+		add a column to field, initializing with value.  Value may 
+    be one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, it must be of same size as nObs and same type as Ctor
+		*/
+    colSchema.writable = true;
+    const colName = colSchema.name;
+    if (
+      _getColumnSchema(this.schema, "obs", colName) ||
+      this._cache.obs.hasCol(colName)
+    ) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    let data;
+    if (isArrayOrTypedArray(value)) {
+      if (value.constructor !== Ctor)
+        throw new Error("Mismatched value array type");
+      if (value.length !== this.nObs)
+        throw new Error("Value array has incorrect length");
+      data = value.slice();
+    } else {
+      data = new Ctor(this.nObs).fill(value);
+    }
+    newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
+    normalizeWritableCategoricalSchema(
+      colSchema,
+      newAnnoMatrix._cache.obs.col(colName)
+    );
+    newAnnoMatrix.schema = addObsAnnoColumn(this.schema, colName, colSchema);
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs oldColName to newColName.  oldCol must be writable.
+    */
+    const oldColSchema = _getColumnSchema(this.schema, "obs", oldCol);
+    _writableCheck(oldColSchema); // throws on error
+
+    const value = this._cache.obs.hasCol(oldCol)
+      ? this._cache.obs.col(oldCol).asArray()
+      : undefined;
+    return this.dropObsColumn(oldCol).addObsColumn(
+      {
+        ...oldColSchema,
+        name: newCol,
+      },
+      value.constructor,
+      value
+    );
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    /*
+		Set all rows identified by rowLabels to value.
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const rowIndices = this.rowIndex.getOffsets(rowLabels);
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, len = rowIndices.length; i < len; i += 1) {
+      const idx = rowIndices[i];
+      if (idx === undefined) throw new Error("Unknown row label");
+      data[idx] = value;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(value)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, value);
+    }
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set all rows with value 'oldValue' to 'newValue'.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    if (!colSchema.categories.includes(oldValue)) {
+      throw new Error("unknown category");
+    }
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, l = data.length; i < l; i += 1) {
+      if (data[i] === oldValue) data[i] = newValue;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(newValue)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, newValue);
+    }
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    /*
+    add new layout to the obs embeddings
+    */
+    const { name: colName } = colSchema;
+    if (_getColumnSchema(this.schema, "emb", colName)) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsLayout(this.schema, colSchema);
+    return newAnnoMatrix;
+  }
+
+  /**
+   ** Private below
+   **/
+  async _doLoad(field, query) {
+    /*
+    _doLoad - evaluates the query against the field. Returns:
+      * whereCache update: column query map mapping the query to the column labels
+      * Dataframe containing the new columns (one per dimension)
+    */
+    let doRequest;
+    let priority = 10; // default fetch priority
+
+    switch (field) {
+      case "obs":
+      case "var": {
+        doRequest = _obsOrVarLoader(this.baseURL, field, query);
+        break;
+      }
+      case "X": {
+        doRequest = _XLoader(this.baseURL, field, query);
+        break;
+      }
+      case "emb": {
+        doRequest = _embLoader(this.baseURL, field, query);
+        priority = 0; // high prio load for embeddings
+        break;
+      }
+      default:
+        throw new Error("Unknown field name");
+    }
+
+    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
+    let result = matrixFBSToDataframe(buffer);
+    if (!result || result.isEmpty()) throw Error("Unknown field/col");
+
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      result.colIndex.labels()
+    );
+
+    result = normalizeResponse(field, query, this.schema, result);
+
+    return [whereCacheUpdate, result];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _writableCheck(colSchema) {
+  if (!colSchema?.writable) {
+    throw new Error("Unknown or readonly obs column");
+  }
+}
+
+function _writableCategoryTypeCheck(colSchema) {
+  _writableCheck(colSchema);
+  if (colSchema.type !== "categorical") {
+    throw new Error("column must be categorical");
+  }
+}
+
+function _embLoader(baseURL, _field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}layout/obs`;
+  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _obsOrVarLoader(baseURL, field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}annotations/${field}`;
+  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _XLoader(baseURL, field, query) {
+  _expectComplexQuery(query);
+
+  if (query.where) {
+    const urlBase = `${baseURL}data/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+    const url = `${urlBase}?${urlQuery}`;
+    return () => doBinaryRequest(url);
+  }
+
+  if (query.summarize) {
+    const urlBase = `${baseURL}summarize/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+
+    if (urlBase.length + urlQuery.length < 2000) {
+      const url = `${urlBase}?${urlQuery}`;
+      return () => doBinaryRequest(url);
+    }
+
+    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
+    return async () => {
+      const res = await doFetch(url, {
+        method: "POST",
+        body: urlQuery,
+        headers: new Headers({
+          Accept: "application/octet-stream",
+          "Content-Type": "application/x-www-form-urlencoded",
+        }),
+      });
+      return res.arrayBuffer();
+    };
+  }
+
+  throw new Error("Unknown query structure");
+}