Bump version to 0.16.4

* Fix frontend mishandling of null userinfo

If the authentication is disabled, the userinfo endpoint returns null.
This case needs to be handled.

 #1780

* Small fix for handling refesh tokens in auth

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 0.16.4
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.github/workflows/push_tests.yml

@@ -73,6 +73,7 @@ jobs:
 
   smoke-tests:
     runs-on: macos-latest
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
@@ -102,6 +103,7 @@ jobs:
 
   smoke-tests-annotations:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7

Dockerfile

@@ -4,7 +4,8 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 RUN apt-get update && \
-    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

Procfile

@@ -1 +0,0 @@
-web: gunicorn --chdir server/eb app:application --log-file -

app.json

@@ -1,29 +0,0 @@
-{
-  "name": "cellxgene",
-  "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene",
-  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
-  "keywords": [
-    "scientific",
-    "visualization",
-    "scrna-seq",
-    "transcriptomics",
-    "dataviz"
-  ],
-  "buildpacks": [
-    {
-      "url": "heroku/nodejs"
-    },
-    {
-      "url": "heroku/python"
-    }
-  ],
-  "stack": "heroku-18",
-  "env": {
-    "DATASET": {
-      "description": "Link to dataset",
-      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
-      "required": "true"
-    }
-  }
-}

client/configuration/eslint/eslint.js

@@ -64,6 +64,12 @@ module.exports = {
       "LabeledStatement",
       "WithStatement",
     ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
   },
   overrides: [
     {

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -75,7 +75,7 @@
           src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
           style="width: 80px; height: 80px;"
         />
-        <div>Edge ≥ 15</div>
+        <div>Edge ≥ 79</div>
       </a>
     </div>
   </div>

client/configuration/webpack/webpack.config.dev.js

@@ -31,7 +31,11 @@ const devConfig = {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
         include: [nodeModules, fonts],
-        query: { name: "static/assets/[name].[ext]" },
+        query: {
+          name: "static/assets/[name].[ext]",
+          // (thuang): This is needed to make sure @font url path is '/static/assets/'
+          publicPath: "/",
+        },
       },
     ],
   },

client/configuration/webpack/webpack.config.prod.js

@@ -45,7 +45,11 @@ const prodConfig = {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
         include: [nodeModules, fonts],
-        query: { name: "static/assets/[name]-[contenthash].[ext]" },
+        query: {
+          name: "static/assets/[name]-[contenthash].[ext]",
+          // (thuang): This is needed to make sure @font url path is '../static/assets/'
+          publicPath: "static/",
+        },
       },
     ],
   },

client/configuration/webpack/webpack.config.shared.js

@@ -7,7 +7,7 @@ const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
 
-const publicPath = "/";
+const publicPath = "";
 
 const rawObsoleteHTMLTemplate = fs.readFileSync(
   `${__dirname}/obsoleteHTMLTemplate.html`,

client/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.0",
+  "version": "0.16.4",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -13787,9 +13787,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.19",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz",
-      "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ=="
+      "version": "4.17.20",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
     },
     "lodash._reinterpolate": {
       "version": "3.0.0",

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.0",
+  "version": "0.16.4",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -34,7 +34,7 @@
     "Safari >= 10.1",
     "iOS >= 10.3",
     "Firefox >= 60",
-    "Edge >= 15",
+    "Edge >= 79",
     "not Explorer > 0"
   ],
   "dependencies": {
@@ -52,7 +52,7 @@
     "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.19",
+    "lodash": "^4.17.20",
     "memoize-one": "^5.1.1",
     "react": "^16.13.1",
     "react-async": "^10.0.1",

client/server/development.js

@@ -1,20 +1,19 @@
-/* eslint-disable */
-// jshint esversion: 6
-var path = require("path");
-var historyApiFallback = require("connect-history-api-fallback");
-var chalk = require("chalk");
-var express = require("express");
-var favicon = require("serve-favicon");
-var webpack = require("webpack");
-var config = require("../configuration/webpack/webpack.config.dev");
-var utils = require("./utils");
+const path = require("path");
+const historyApiFallback = require("connect-history-api-fallback");
+const chalk = require("chalk");
+const express = require("express");
+const favicon = require("serve-favicon");
+const webpack = require("webpack");
+const devMiddleware = require("webpack-dev-middleware");
+const config = require("../configuration/webpack/webpack.config.dev");
+const utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
 const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
 
 // Set up compiler
-var compiler = webpack(config);
+const compiler = webpack(config);
 
 compiler.plugin("invalid", () => {
   utils.clearConsole();
@@ -26,12 +25,12 @@ compiler.plugin("done", (stats) => {
 });
 
 // Launch server
-var app = express();
+const app = express();
 
 app.use(historyApiFallback({ verbose: false }));
 
 app.use(
-  require("webpack-dev-middleware")(compiler, {
+  devMiddleware(compiler, {
     logLevel: "warn",
     publicPath: config.output.publicPath,
   })

client/src/actions/index.js

@@ -41,6 +41,17 @@ async function configFetch(dispatch) {
   });
 }
 
+async function userInfoFetch(dispatch) {
+  return fetchJson("userinfo").then((response) => {
+    const { userinfo } = response || {};
+    dispatch({
+      type: "userinfo load complete",
+      userinfo,
+    });
+    return userinfo;
+  });
+}
+
 function prefetchEmbeddings(annoMatrix) {
   /*
   prefetch requests for all embeddings
@@ -62,6 +73,7 @@ const doInitialDataLoad = () =>
         configFetch(dispatch),
         schemaFetch(dispatch),
         userColorsFetchAndLoad(dispatch),
+        userInfoFetch(dispatch),
       ]);
 
       const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;

client/src/components/autosave/filenameDialog.js

@@ -14,6 +14,7 @@ import {
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
   auth: state.config?.authentication,
+  userinfo: state.userinfo,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
 }))
 class FilenameDialog extends React.Component {
@@ -91,13 +92,19 @@ class FilenameDialog extends React.Component {
   };
 
   render() {
-    const { writableCategoriesEnabled, annotations, idhash, auth } = this.props;
+    const {
+      writableCategoriesEnabled,
+      annotations,
+      idhash,
+      userinfo,
+    } = this.props;
     const { filenameText } = this.state;
 
     return writableCategoriesEnabled &&
+      annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
       !annotations.dataCollectionName &&
-      auth.is_authenticated ? (
+      userinfo.is_authenticated ? (
       <Dialog
         icon="tag"
         title="Annotations Collection"

client/src/components/categorical/index.js

@@ -1,6 +1,6 @@
 // jshint esversion: 6
 import React from "react";
-import { Button } from "@blueprintjs/core";
+import { AnchorButton, Tooltip, Position } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
@@ -15,6 +15,7 @@ import actions from "../../actions";
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   schema: state.annoMatrix?.schema,
   ontology: state.ontology,
+  userinfo: state.userinfo,
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -127,7 +128,12 @@ class Categories extends React.Component {
       newCategoryText,
       expandedCats,
     } = this.state;
-    const { writableCategoriesEnabled, schema, ontology } = this.props;
+    const {
+      writableCategoriesEnabled,
+      schema,
+      ontology,
+      userinfo,
+    } = this.props;
     const ontologyEnabled = ontology?.enabled ?? false;
     /* all names, sorted in display order.  Will be rendered in this order */
     const allCategoryNames = ControlsHelpers.selectableCategoryNames(
@@ -203,15 +209,30 @@ class Categories extends React.Component {
         )}
 
         {writableCategoriesEnabled ? (
-          <div>
-            <Button
+          <Tooltip
+            content={
+              userinfo.is_authenticated
+                ? "Create a new category"
+                : "You must be logged in to create new categorical fields"
+            }
+            position={Position.RIGHT}
+            boundary="viewport"
+            hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            modifiers={{
+              preventOverflow: { enabled: false },
+              hide: { enabled: false },
+            }}
+          >
+            <AnchorButton
+              type="button"
               data-testid="open-annotation-dialog"
               onClick={this.handleEnableAnnoMode}
               intent="primary"
+              disabled={!userinfo.is_authenticated}
             >
               Create new category
-            </Button>
-          </div>
+            </AnchorButton>
+          </Tooltip>
         ) : null}
       </div>
     );

client/src/components/menubar/authButtons.js

@@ -4,7 +4,7 @@ import * as globals from "../../globals";
 import styles from "./menubar.css";
 
 const Auth = React.memo((props) => {
-  const { auth } = props;
+  const { auth, userinfo } = props;
 
   if (!auth || (auth && !auth.requires_client_login)) return null;
 
@@ -19,10 +19,10 @@ const Auth = React.memo((props) => {
           type="button"
           data-testid="auth-button"
           disabled={false}
-          icon={!auth.is_authenticated ? "log-in" : "log-out"}
-          href={!auth.is_authenticated ? auth.login : auth.logout}
+          icon={!userinfo.is_authenticated ? "log-in" : "log-out"}
+          href={!userinfo.is_authenticated ? auth.login : auth.logout}
         >
-          {!auth.is_authenticated ? "Log In" : "Log Out"}
+          {!userinfo.is_authenticated ? "Log In" : "Log Out"}
         </AnchorButton>
       </Tooltip>
     </div>

client/src/components/menubar/index.js

@@ -42,6 +42,7 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     celllist2: state.differential.celllist2,
     libraryVersions: state.config?.["library_versions"],
     auth: state.config?.authentication,
+    userinfo: state.userinfo,
     undoDisabled: state["@@undoable/past"].length === 0,
     redoDisabled: state["@@undoable/future"].length === 0,
     aboutLink: state.config?.links?.["about-dataset"],
@@ -221,6 +222,7 @@ class MenuBar extends React.PureComponent {
       subsetResetPossible,
       enableReembedding,
       auth,
+      userinfo,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
 
@@ -246,7 +248,7 @@ class MenuBar extends React.PureComponent {
           zIndex: 3,
         }}
       >
-        <AuthButtons auth={auth} />
+        <AuthButtons auth={auth} userinfo={userinfo} />
         <InformationMenu
           libraryVersions={libraryVersions}
           aboutLink={aboutLink}

client/src/components/scatterplot/scatterplot.js

@@ -528,8 +528,8 @@ class Scatterplot extends React.PureComponent {
                 return (
                   <ScatterplotAxis
                     minimized={minimized}
-                    scatterplotYYaccessor={scatterplotXXaccessor}
-                    scatterplotXXaccessor={scatterplotYYaccessor}
+                    scatterplotYYaccessor={scatterplotYYaccessor}
+                    scatterplotXXaccessor={scatterplotXXaccessor}
                     xScale={asyncProps.xScale}
                     yScale={asyncProps.yScale}
                   />

client/src/reducers/annotations.js

@@ -26,6 +26,7 @@ const Annotations = (
     categoryBeingEdited: null,
     categoryAddingNewLabel: null,
     labelEditable: { category: null, label: null },
+    promptForFilename: true,
   },
   action
 ) => {
@@ -37,10 +38,13 @@ const Annotations = (
         action.config.parameters?.[
           "annotations-data-collection-name-is-read-only"
         ] ?? false;
+      const promptForFilename =
+        action.config.parameters?.["user_annotation_collection_name_enabled"];
       return {
         ...state,
         dataCollectionNameIsReadOnly,
         dataCollectionName,
+        promptForFilename,
       };
     }
 

client/src/reducers/index.js

@@ -4,6 +4,7 @@ import thunk from "redux-thunk";
 import cascadeReducers from "./cascade";
 import undoable from "./undoable";
 import config from "./config";
+import userinfo from "./userinfo";
 import annoMatrix from "./annoMatrix";
 import obsCrossfilter from "./obsCrossfilter";
 import categoricalSelection from "./categoricalSelection";
@@ -41,6 +42,7 @@ const Reducer = undoable(
     ["pointDilation", pointDialation],
     ["reembedController", reembedController],
     ["autosave", autosave],
+    ["userinfo", userinfo],
   ]),
   [
     "annoMatrix",

client/src/reducers/userinfo.js

@@ -0,0 +1,27 @@
+// jshint esversion: 6
+const UserInfo = (state = {}, action) => {
+  switch (action.type) {
+    case "initial data load start":
+      return {
+        ...state,
+        loading: true,
+        error: null,
+      };
+    case "userinfo load complete":
+      return {
+        ...state,
+        loading: false,
+        error: null,
+        ...action.userinfo,
+      };
+    case "initial data load error":
+      return {
+        ...state,
+        error: action.error,
+      };
+    default:
+      return state;
+  }
+};
+
+export default UserInfo;

client/src/util/stateManager/colorHelpers.js

@@ -94,15 +94,26 @@ export const createColorTable = memoize(_createColorTable);
 export function loadUserColorConfig(userColors) {
   const convertedUserColors = {};
   Object.keys(userColors).forEach((category) => {
-    const [colors, scaleMap] = Object.keys(userColors[category]).reduce(
-      (acc, label, i) => {
-        const color = parseRGB(userColors[category][label]);
-        acc[0][label] = color;
-        acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
-        return acc;
-      },
-      [{}, {}]
-    );
+    // We cannot iterate over keys without sorting
+    // because we handle categorical values in alphabetical order __ignoring case__
+    //  while Object.keys() _usually_ is ordered alphabetically where all upper characters are less than lowercase (A, B, C, a, b, c)
+    const [colors, scaleMap] = Object.keys(userColors[category])
+      .sort((a, b) => {
+        a = a.toLowerCase();
+        b = b.toLowerCase();
+        if (a === b) return 0;
+        if (a > b) return 1;
+        return -1;
+      })
+      .reduce(
+        (acc, label, i) => {
+          const color = parseRGB(userColors[category][label]);
+          acc[0][label] = color;
+          acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
+          return acc;
+        },
+        [{}, {}]
+      );
     const scale = (i) => scaleMap[i];
     convertedUserColors[category] = { colors, scale };
   });

docs/posts/hosted.md

@@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list.
 
 # Deploying cellxgene with Heroku
 
-## Quickstart
+## Heroku Support
 
-Clicking on the following button will forward you to Heroku to begin the deployment process:
+The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.
 
-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
- <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
-</a>
+While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml).
 
-If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
+You may have to tweak the `Dockerfile` like so:
 
-Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
+```Dockerfile
+FROM ubuntu:bionic
 
-### Default settings
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-- `App name`: the unique name for your deployment
-- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
-- `App owner`: Who will own this app. Either you personally or an organization/team
-- `Region`: Location of the server where the app will be deployed (EU or US)
+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    pip3 install cellxgene
 
-### Configuration
+# ENTRYPOINT ["cellxgene"]  # Heroku doesn't work well with ENTRYPOINT
+```
 
-- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view
-- This defaults to pbm3k.h5ad
+and provide a `heroku.yml` file similar to this:
 
-After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
+```yml
+build:
+  docker:
+    web: Dockerfile
+run:
+  web:
+    command:
+      - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
+```
 
 ## What is Heroku?
 

experiments/heroku/Dockerfile

@@ -1,7 +0,0 @@
-FROM python:3.7
-
-WORKDIR /usr/src/app
-
-RUN pip3 install cellxgene
-
-expose 5005

experiments/heroku/README.md

@@ -1,58 +0,0 @@
-# cellxgene cloud deployment with Heroku
-
-## Quickstart
-
-Clicking on the following button will forward you to Heroku to begin the deployment process:
-
-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene/tree/main">
- <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
-</a>
-
-If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
-
-Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
-
-#### Default settings
-
-- `App name`: the unique name for your deployment
-- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
-- `App owner`: Who will own this app. Either you personally or an organization/team
-- `Region`: Location of the server where the app will be deployed (EU or US)
-
-#### Configuration
-
-- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view
-- This defaults to pbm3k.h5ad
-
-After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
-
-## What is Heroku?
-
-Heroku is a quick and easy way to host applications on the cloud.
-
-A Heroku deployment of cellxgene means that the app is not running on your local machine. Instead, the app is installed, configured, and ran on the Heroku servers (read: cloud).
-
-On Heroku's servers, applications run on a [dyno](https://www.heroku.com/dynos) which are Heroku's implementation and abstraction of containers.
-
-Heroku is one of many options available for hosting instances of cellxgene on the web.
-Some other options include: Amazon Web Services, Google Cloud Platform, Digital Ocean, and Microsoft Azure.
-
-## Why use Heroku to deploy cellxgene?
-
-What Heroku enables is a quick, non-technical method of setting up a cellxgene instance. No command line knowledge needed. This also allows machines to access the instance via the internet, so sharing a visualized dataset is as simple as sharing a link.
-
-Because cellxgene currently heavily relies on its Python backend for providing the viewer with the necessary data and tooling, it is currently not possible to host cellxgene as a static webpage.
-
-This is a good option if you want to quickly deploy an instance of cellxgene to the web. Heroku deployments are free for small datasets up to around 250MBs in size. See below regarding larger datasets.
-
-## When should I not deploy with Heroku?
-
-- The default free dyno offered by Heroku is limited in memory to 512 MBs
-  - The amount of memory needed for the dyno is roughly the same size as the h5ad file
-  - Heroku offers tiered paid dynos. More can be found [here](https://www.heroku.com/pricing)
-  - Note that this can get _very_ expensive for larger datasets (\$25+ a month)
-- On the free dyno, after 30 minutes of inactivity, Heroku will put your app into a hibernation mode. On the next access, Heroku will need time to boot the dyno back online.
-- Having multiple simultaneous users requires more memory. This means that the free container size is easily overwhelmed by multiple users, even with small datasets; this can be addressed by purchasing a larger container size
-- For this facilitated Heroku deployment to work, your dataset must be hosted on a publicly accessible URL
-- By default, Heroku publically shares your instance to anyone with the URL.
-  - There are many ways of securing your instance. One quick and simple way is by installing [wwwhisper](https://elements.heroku.com/addons/wwwhisper), a Heroku addon

heroku.yml

@@ -1,5 +0,0 @@
-build:
-  docker:
-    web: experiments/heroku/Dockerfile
-run:
-  web: cellxgene launch $DATASET --host 0.0.0.0 --port $PORT

server/__init__.py

@@ -1,8 +1,9 @@
-from server.common.utils import import_plugins
 import logging
 import sys
 
-__version__ = "0.16.0"
+from server.common.utils.utils import import_plugins
+
+__version__ = "0.16.4"
 display_version = "cellxgene v" + __version__
 
 try:

server/app/app.py

@@ -1,22 +1,20 @@
 import datetime
 import logging
+from functools import wraps
+from http import HTTPStatus
 
-from flask import Flask, redirect, current_app, make_response, render_template, abort
-from flask import Blueprint, request
+from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \
+    send_from_directory
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
-from http import HTTPStatus
-
 import server.common.rest as common_rest
-from server.common.errors import DatasetAccessError, RequestException
-from server.common.utils import path_join, Float32JSONEncoder
 from server.common.data_locator import DataLocator
+from server.common.errors import DatasetAccessError, RequestException
 from server.common.health import health_check
+from server.common.utils.utils import path_join, Float32JSONEncoder
 from server.data_common.matrix_loader import MatrixDataLoader
 
-from functools import wraps
-
 webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
 
 ONE_WEEK = 7 * 24 * 60 * 60
@@ -243,8 +241,15 @@ class ConfigAPI(DatasetResource):
         return common_rest.config_get(current_app.app_config, data_adaptor)
 
 
+class UserInfoAPI(DatasetResource):
+    @cache_control_always(no_store=True)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.userinfo_get(current_app.app_config, data_adaptor)
+
+
 class AnnotationsObsAPI(DatasetResource):
-    @cache_control(public=True, max_age=ONE_WEEK)
+    @cache_control(public=True, no_store=True)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.annotations_obs_get(request, data_adaptor)
@@ -311,6 +316,7 @@ def get_api_resources(bp_api, url_dataroot=None):
     # Initialization routes
     add_resource(SchemaAPI, "/schema")
     add_resource(ConfigAPI, "/config")
+    add_resource(UserInfoAPI, "/userinfo")
     # Data routes
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
@@ -330,7 +336,7 @@ class Server:
         pass
 
     def __init__(self, app_config):
-        self.app = Flask(__name__, static_folder="../common/web/static")
+        self.app = Flask(__name__, static_folder=None)
         self._before_adding_routes(self.app, app_config)
         self.app.json_encoder = Float32JSONEncoder
         server_config = app_config.server_config
@@ -364,11 +370,23 @@ class Server:
                     lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
                     methods=["GET"],
                 )
+                self.app.add_url_rule(
+                    f"/{url_dataroot}/<dataset>/static/<path:filename>",
+                    f"static_assets_{url_dataroot}",
+                    view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
+                    methods=["GET"]
+                )
 
         else:
             bp_api = Blueprint("api", __name__, url_prefix=api_version)
             resources = get_api_resources(bp_api)
             self.app.register_blueprint(resources.blueprint)
+            self.app.add_url_rule(
+                "/static/<path:filename>",
+                "static_assets",
+                view_func=lambda filename: send_from_directory("../common/web/static", filename),
+                methods=["GET"]
+            )
 
         self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
         self.app.app_config = app_config

server/auth/auth_oauth.py

@@ -2,8 +2,9 @@ from flask import session, request, redirect, current_app, after_this_request, h
 from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from server.common.errors import AuthenticationError, ConfigurationError
 from urllib.parse import urlencode
-from urllib.request import urlopen
 import json
+import requests
+import base64
 
 # It is not required to have authlib or jose.
 # However, it is a configuration error to use this auth type if they are not installed.
@@ -15,14 +16,29 @@ except ModuleNotFoundError:
 
 try:
     from jose import jwt
+    from jose.exceptions import ExpiredSignatureError, JWTError, JWTClaimsError
 except ModuleNotFoundError:
     missingimport.append("jose")
 
 
+class Tokens:
+    """Simple class to represent the tokens that are saved/restored from the cookie"""
+
+    def __init__(self, access_token, id_token, refresh_token, expires_at):
+        self.access_token = access_token
+        self.id_token = id_token
+        self.refresh_token = refresh_token
+        self.expires_at = expires_at
+
+        # expires_at may be None after a token refresh, and so it is not checked here
+        if not (access_token and id_token and refresh_token):
+            raise KeyError(str(self.__dict__))
+
+
 class AuthTypeOAuth(AuthTypeClientBase):
     """An authentication type for oauth2 logins."""
 
-    CXG_ID_TOKEN = "id_token"
+    CXG_TOKENS = "auth_tokens"
 
     def __init__(self, server_config):
         super().__init__()
@@ -45,8 +61,8 @@ class AuthTypeOAuth(AuthTypeClientBase):
         # any JSON Web Token (JWT) issued by the authorization server and signed using the RS256
         try:
             jwksloc = f"{self.api_base_url}/.well-known/jwks.json"
-            jwksurl = urlopen(jwksloc)
-            self.jwks = json.loads(jwksurl.read())
+            jwksurl = requests.get(jwksloc)
+            self.jwks = jwksurl.json()
         except Exception:
             raise ConfigurationError(f"error in oauth, api_url_base: {self.api_base_url}, cannot access {jwksloc}")
 
@@ -86,48 +102,43 @@ class AuthTypeOAuth(AuthTypeClientBase):
             self.callback_base_url = f"http://{server_config.app__host}:{server_config.app__port}"
 
         self.client = self.oauth.register(
-            "oauth",
+            "auth0",
             client_id=self.client_id,
             client_secret=self.client_secret,
             api_base_url=self.api_base_url,
+            refresh_token_url=f"{self.api_base_url}/oauth/token",
             access_token_url=f"{self.api_base_url}/oauth/token",
             authorize_url=f"{self.api_base_url}/authorize",
-            client_kwargs={
-                "scope" : "openid profile email",
-            }
+            client_kwargs={"scope": "openid profile email offline_access"},
         )
 
     def is_user_authenticated(self):
-        try:
-            payload = self.get_jwt_payload()
-            return payload is not None
-        except AuthenticationError:
-            return False
+        payload = self.get_userinfo()
+        return payload is not None
 
     def get_user_id(self):
-        payload = self.get_jwt_payload()
+        payload = self.get_userinfo()
         if payload and payload.get("sub"):
             return payload.get("sub")
         return None
 
     def get_user_name(self):
-        payload = self.get_jwt_payload()
+        payload = self.get_userinfo()
         if payload and payload.get("name"):
             return payload.get("name")
         return None
 
     def get_user_email(self):
-        payload = self.get_jwt_payload()
+        payload = self.get_userinfo()
         if payload and payload.get("email"):
             return payload.get("email")
         return None
 
     def update_response(self, response):
-        response.cache_control.update(
-            dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
+        response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
 
     def login(self):
-        callbackurl = f'{self.callback_base_url}/oauth2/callback'
+        callbackurl = f"{self.callback_base_url}/oauth2/callback"
         return_path = request.args.get("dataset", "")
         return_to = f"{self.callback_base_url}/{return_path}"
         # save the return path in the session cookie, accessed in the callback function
@@ -137,48 +148,88 @@ class AuthTypeOAuth(AuthTypeClientBase):
         return response
 
     def logout(self):
+        self.remove_tokens()
+        params = {"returnTo": self.callback_base_url, "client_id": self.client_id}
+        response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
+        self.update_response(response)
+        return response
+
+    def callback(self):
+        data = self.client.authorize_access_token()
+        tokens = Tokens(
+            access_token=data.get("access_token"),
+            id_token=data.get("id_token"),
+            refresh_token=data.get("refresh_token"),
+            expires_at=data.get("expires_at"),
+        )
+        self.save_tokens(tokens)
+        oauth_callback_redirect = session.pop("oauth_callback_redirect", "/")
+        response = redirect(oauth_callback_redirect)
+        self.update_response(response)
+        return response
+
+    def get_tokens(self):
+        """Extract the tokens from the cookie, and store them in the flask global context"""
+        if "tokens" in g:
+            return g.tokens
+
+        try:
+            if self.session_cookie:
+                tokensdict = session.get(self.CXG_TOKENS)
+                if tokensdict:
+                    g.tokens = Tokens(**tokensdict)
+                else:
+                    return None
+            else:
+                value = request.cookies.get(self.cookie_params["key"])
+                value = base64.b64decode(value)
+                try:
+                    tokensdict = json.loads(value)
+                    g.tokens = Tokens(**tokensdict)
+                except (TypeError, KeyError, json.decoder.JSONDecodeError):
+                    g.pop("tokens", None)
+                    return None
+
+        except (TypeError, KeyError):
+            g.pop("tokens", None)
+            return None
+
+        return g.tokens
+
+    def save_tokens(self, tokens):
+        g.tokens = tokens
         if self.session_cookie:
-            if self.CXG_ID_TOKEN in session:
-                del session[self.CXG_ID_TOKEN]
+            session[self.CXG_TOKENS] = tokens.__dict__
         else:
+
+            @after_this_request
+            def set_cookie(response):
+                args = self.cookie_params.copy()
+                value = base64.b64encode(json.dumps(tokens.__dict__).encode("utf-8"))
+                del args["key"]
+                try:
+                    response.set_cookie(self.cookie_params["key"], value, **args)
+                except Exception as e:
+                    raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e
+                return response
+
+    def remove_tokens(self):
+        g.pop("tokens", None)
+        if self.session_cookie:
+            if self.CXG_TOKENS in session:
+                del session[self.CXG_TOKENS]
+        else:
+
             @after_this_request
             def remove_cookie(response):
                 response.set_cookie(self.cookie_params["key"], "", expires=0)
                 self.update_response(response)
                 return response
 
-        params = {'returnTo' : self.callback_base_url, 'client_id' : self.client_id}
-        response = redirect(self.client.api_base_url + '/v2/logout?' + urlencode(params))
-        self.update_response(response)
-        return response
-
-    def callback(self):
-        token = self.client.authorize_access_token()
-        id_token = token.get("id_token")
-        oauth_callback_redirect = session.pop("oauth_callback_redirect", "/")
-        resp = redirect(oauth_callback_redirect)
-
-        if self.session_cookie:
-            session[self.CXG_ID_TOKEN] = id_token
-        else:
-            args = self.cookie_params.copy()
-            del args["key"]
-            try:
-                resp.set_cookie(
-                    self.cookie_params["key"],
-                    id_token,
-                    **args)
-                g.token = id_token
-            except Exception as e:
-                raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e
-
-        self.update_response(resp)
-        return resp
-
     def get_login_url(self, data_adaptor):
         """Return the url for the login route"""
         if current_app.app_config.is_multi_dataset():
-            return f"/login?dataset={data_adaptor.uri_path}"
+            return f"/login?dataset={data_adaptor.uri_path}/"
         else:
             return "/login"
 
@@ -186,55 +237,104 @@ class AuthTypeOAuth(AuthTypeClientBase):
         """Return the url for the logout route"""
         return "/logout"
 
-    def get_token(self):
-        """Function to return the token"""
-        if "token" in g:
-            return g.token
-        if self.session_cookie:
-            g.token = session.get(self.CXG_ID_TOKEN)
-        else:
-            g.token = request.cookies.get(self.cookie_params["key"])
-
-        return g.token
-
-    def get_jwt_payload(self):
-        if not has_request_context():
+    def check_jwt_payload(self, id_token):
+        try:
+            unverified_header = jwt.get_unverified_header(id_token)
+        except JWTError:
             return None
 
-        token = self.get_token()
-        if token is None:
-            return None
-
-        unverified_header = jwt.get_unverified_header(token)
         rsa_key = {}
-        for key in self.jwks['keys']:
-            if key['kid'] == unverified_header['kid']:
+        for key in self.jwks["keys"]:
+            if key["kid"] == unverified_header["kid"]:
                 rsa_key = {
-                    'kty': key['kty'],
-                    'kid': key['kid'],
-                    'use': key['use'],
-                    'n': key['n'],
-                    'e': key['e']
+                    "kty": key["kty"],
+                    "kid": key["kid"],
+                    "use": key["use"],
+                    "n": key.get("n"),
+                    "e": key.get("e"),
                 }
         if rsa_key:
+            options = {}
+            if not rsa_key["n"] or not rsa_key["e"]:
+                # this is a mock auth server, do not validate
+                options = {"verify_signature": False, "verify_iss": False}
             try:
                 payload = jwt.decode(
-                    token,
+                    id_token,
                     rsa_key,
                     algorithms=self.algorithms,
                     audience=self.audience,
-                    issuer=self.api_base_url + "/"
+                    issuer=self.api_base_url + "/",
+                    options=options,
                 )
                 return payload
 
-            except jwt.JWTError as e:
-                raise AuthenticationError(f"invalid signature: {str(e)}")
-            except jwt.ExpiredSignatureError as e:
-                raise AuthenticationError(f"token expired: {str(e)}")
-            except jwt.JWTClaimsError as e:
-                raise AuthenticationError(f"invalid claims {str(e)}")
+            except ExpiredSignatureError:
+                # This exception is handled in get_userinfo
+                raise
+            except JWTClaimsError as e:
+                raise AuthenticationError(f"invalid claims {str(e)}") from e
+            except JWTError as e:
+                raise AuthenticationError(f"invalid signature: {str(e)}") from e
 
         raise AuthenticationError("Unable to find the appropriate key")
 
+    def get_userinfo(self):
+        if not has_request_context():
+            return None
+
+        # check if the userinfo has been retrieved already in this request
+        if "userinfo" in g:
+            return g.get("userinfo")
+
+        # if there is no id_token, return None (user is not authenticated)
+        tokens = self.get_tokens()
+        if tokens is None or tokens.id_token is None:
+            return None
+
+        try:
+            # check the jwt payload.  This raises an AuthenticationError if the token is not valid.
+            # It the token has expired, we attempt to refresh the token
+            g.userinfo = self.check_jwt_payload(tokens.id_token)
+            return g.userinfo
+
+        except ExpiredSignatureError:
+            tokens = self.refresh_expired_token(tokens.refresh_token)
+            if tokens is None or tokens.id_token is None:
+                return None
+            else:
+                try:
+                    g.userinfo = self.check_jwt_payload(tokens.id_token)
+                    return g.userinfo
+                except JWTError as e:
+                    raise AuthenticationError(f"error during token refresh: {str(e)}") from e
+
+        except AuthenticationError:
+            self.remove_tokens()
+            raise
+
+    def refresh_expired_token(self, refresh_token):
+        params = {
+            "grant_type": "refresh_token",
+            "client_id": self.client_id,
+            "refresh_token": refresh_token,
+            "client_secret": self.client_secret,
+        }
+        headers = {"content-type": "application/x-www-form-urlencoded"}
+        request = requests.post(f"{self.api_base_url}/oauth/token", urlencode(params), headers=headers)
+        if request.status_code != 200:
+            # unable to refresh the token, log the user out
+            self.remove_tokens()
+            return None
+        data = request.json()
+        tokens = Tokens(
+            access_token=data.get("access_token"),
+            id_token=data.get("id_token"),
+            refresh_token=data.get("refresh_token", refresh_token),
+            expires_at=data.get("expires_at"),
+        )
+        self.save_tokens(tokens)
+        return tokens
+
 
 AuthTypeFactory.register("oauth", AuthTypeOAuth)

server/cli/cli.py

@@ -1,9 +1,10 @@
 import click
 
-from .. import __version__
+from .convert_to_cxg import convert_to_cxg
 from .launch import launch
 from .prepare import prepare
 from .upgrade import log_upgrade_check
+from .. import __version__
 
 
 @click.group(
@@ -29,3 +30,4 @@ def cli(upgrade_check):
 
 cli.add_command(launch)
 cli.add_command(prepare)
+cli.add_command(convert_to_cxg)

server/cli/convert_to_cxg.py

@@ -0,0 +1,129 @@
+from os import path
+
+import click
+
+from server.converters.h5ad_data_file import H5ADDataFile
+
+
+@click.command(
+    name="convert",
+    short_help="Converts an H5AD dataset to the CXG format.",
+    help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
+         "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
+         "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
+         "usually with the generated CXG file.",
+)
+@click.argument(
+    "input-file",
+    nargs=1,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "-o",
+    "--output-directory",
+    help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
+         "CXG extension.",
+)
+@click.option(
+    "-b",
+    "--backed",
+    help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
+         "but will use less memory.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "-t",
+    "--title",
+    help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
+         "the dataset title will be the filename.",
+)
+@click.option(
+    "-a",
+    "--about",
+    help="A fully qualified URL that provides more information about the dataset and will be included as "
+         "metadata about the CXG file.",
+)
+@click.option(
+    "-s",
+    "--sparse-threshold",
+    help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
+         "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
+         "convert to dense array.",
+    default=0.0,
+    show_default=True,
+)
+@click.option("--obs-names",
+              help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
+                   "the one designated by the dataframe generated-index.")
+@click.option("--var-names",
+              help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+                   "the one designated by the dataframe generated-index.")
+@click.option(
+    "--disable-custom-colors",
+    help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "--disable-corpora-schema",
+    help="When set, conversion process will neither extract nor store Corpora schema information. See "
+         "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
+         "information.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "--overwrite",
+    help="When set to true, will overwrite the output file if the output file already exists.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+def convert_to_cxg(
+        input_file,
+        output_directory,
+        backed,
+        title,
+        about,
+        sparse_threshold,
+        obs_names,
+        var_names,
+        disable_custom_colors,
+        disable_corpora_schema,
+        overwrite,
+):
+    """
+    Convert a dataset file into CXG.
+    """
+
+    h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names,
+                                  use_corpora_schema=not disable_corpora_schema)
+
+    # Get the directory that will hold all the CXG files
+    cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
+
+    h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold,
+                          convert_anndata_colors_to_cxg_colors=not disable_custom_colors)
+
+
+def get_output_directory(input_filename, output_directory, should_overwrite):
+    """
+    Get the name of the CXG output directory to be created/populated during the dataset conversion.
+    """
+
+    if output_directory and (not path.isdir(output_directory) or (path.isdir(output_directory) and should_overwrite)):
+        if output_directory.endswith(".cxg"):
+            return output_directory
+        return output_directory + ".cxg"
+    if output_directory and path.isdir(output_directory) and not should_overwrite:
+        raise click.BadParameter(
+            f"Output directory {output_directory} already exists. If you'd like to overwrite, then run the command "
+            f"with the --overwrite flag."
+        )
+
+    return path.splitext(input_filename)[0] + ".cxg"

server/cli/launch.py

@@ -1,19 +1,19 @@
 import errno
 import functools
 import logging
-from os import devnull
 import sys
 import webbrowser
+from os import devnull
 
 import click
 from flask_compress import Compress
 from flask_cors import CORS
 
-from server.common.utils import sort_options
-from server.common.errors import DatasetAccessError, ConfigurationError
+from server.app.app import Server
 from server.common.app_config import AppConfig
 from server.common.default_config import default_config
-from server.app.app import Server
+from server.common.errors import DatasetAccessError, ConfigurationError
+from server.common.utils.utils import sort_options
 
 DEFAULT_CONFIG = AppConfig()
 
@@ -33,7 +33,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<path>",
         help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-        "Incompatible with --annotations-dir.",
+             "Incompatible with --annotations-dir.",
     )
     @click.option(
         "--annotations-dir",
@@ -42,7 +42,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<directory path>",
         help="Directory of where to save output annotations; filename will be specified in the application. "
-        "Incompatible with --annotations-file.",
+             "Incompatible with --annotations-file.",
     )
     @click.option(
         "--experimental-annotations-ontology",
@@ -170,7 +170,7 @@ def server_args(func):
         default=DEFAULT_CONFIG.server_config.app__debug,
         show_default=True,
         help="Run in debug mode. This is helpful for cellxgene developers, "
-        "or when you want more information about an error condition.",
+             "or when you want more information about an error condition.",
     )
     @click.option(
         "--verbose",
@@ -203,7 +203,7 @@ def server_args(func):
         multiple=True,
         metavar="<text>",
         help="Additional script files to include in HTML page. If not specified, "
-        "no additional script files will be included.",
+             "no additional script files will be included.",
         show_default=False,
     )
     @functools.wraps(func)
@@ -223,7 +223,7 @@ def launch_args(func):
         default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
         metavar="<data directory>",
         help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
-        " to folder containing H5AD and/or CXG datasets.",
+             " to folder containing H5AD and/or CXG datasets.",
         hidden=True,
     )  # TODO, unhide when dataroot is supported)
     @click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +307,32 @@ class CliLaunchServer(Server):
 )
 @launch_args
 def launch(
-    datapath,
-    dataroot,
-    verbose,
-    debug,
-    open_browser,
-    port,
-    host,
-    embedding,
-    obs_names,
-    var_names,
-    max_category_items,
-    disable_custom_colors,
-    diffexp_lfc_cutoff,
-    title,
-    scripts,
-    about,
-    disable_annotations,
-    annotations_file,
-    annotations_dir,
-    backed,
-    disable_diffexp,
-    experimental_annotations_ontology,
-    experimental_annotations_ontology_obo,
-    experimental_enable_reembedding,
-    config_file,
-    dump_default_config,
+        datapath,
+        dataroot,
+        verbose,
+        debug,
+        open_browser,
+        port,
+        host,
+        embedding,
+        obs_names,
+        var_names,
+        max_category_items,
+        disable_custom_colors,
+        diffexp_lfc_cutoff,
+        title,
+        scripts,
+        about,
+        disable_annotations,
+        annotations_file,
+        annotations_dir,
+        backed,
+        disable_diffexp,
+        experimental_annotations_ontology,
+        experimental_annotations_ontology_obo,
+        experimental_enable_reembedding,
+        config_file,
+        dump_default_config,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.

server/cli/prepare.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
-from server.common.utils import sort_options
+from server.common.utils.utils import sort_options
 
 
 @sort_options
@@ -37,7 +37,7 @@ from server.common.utils import sort_options
     default=False,
     is_flag=True,
     help="Do not run quality control metrics. By default cellxgene runs them "
-    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+         "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
 )
 @click.option(
     "--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils import sort_options
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def prepare(
-    data,
-    embedding,
-    recipe,
-    output,
-    plotting,
-    sparse,
-    overwrite,
-    set_obs_names,
-    set_var_names,
-    skip_qc,
-    make_obs_names_unique,
-    make_var_names_unique,
+        data,
+        embedding,
+        recipe,
+        output,
+        plotting,
+        sparse,
+        overwrite,
+        set_obs_names,
+        set_var_names,
+        skip_qc,
+        make_obs_names_unique,
+        make_var_names_unique,
 ):
     """
     Preprocess data for use with cellxgene.

server/common/annotations/annotations.py

@@ -0,0 +1,70 @@
+from abc import ABCMeta, abstractmethod
+
+import fastobo
+import fsspec
+
+from server.common.errors import OntologyLoadFailure
+from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+
+
+class Annotations(metaclass=ABCMeta):
+    """ baseclass for annotations, including ontologies"""
+
+    """ our default ontology is the PURL for the Cell Ontology.
+    See http://www.obofoundry.org/ontology/cl.html """
+    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
+
+    def __init__(self):
+        self.ontology_data = None
+
+    def load_ontology(self, path):
+        """Load and parse ontologies - currently support OBO files only."""
+        if path is None:
+            path = self.DefaultOnotology
+
+        try:
+            with fsspec.open(path) as f:
+                obo = fastobo.iter(f)
+                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
+                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
+                self.ontology_data = names
+
+        except FileNotFoundError as e:
+            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
+
+        except SyntaxError as e:
+            raise OntologyLoadFailure("Syntax error loading OBO ontology") from e
+
+        except Exception as e:
+            raise OntologyLoadFailure("Error loading OBO file") from e
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    @abstractmethod
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        pass
+
+    @abstractmethod
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        pass
+
+    @abstractmethod
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        pass
+
+    @abstractmethod
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        pass

server/common/annotations/hosted_tiledb.py

@@ -0,0 +1,154 @@
+import json
+import os
+import re
+import time
+
+import pandas as pd
+import tiledb
+from flask import current_app
+
+from server.common.annotations.annotations import Annotations
+from server.common.errors import AnnotationCategoryNameError
+from server.common.utils.sanitization_utils import sanitize_values_in_list
+from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from server.db.cellxgene_orm import Annotation
+
+
+class AnnotationsHostedTileDB(Annotations):
+    CXG_ANNO_COLLECTION = "cxg_anno_collection"
+
+    def __init__(self, directory_path, db):
+        super().__init__()
+        self.db = db
+        if directory_path[-1] == "/":
+            self.directory_path = directory_path
+        else:
+            self.directory_path = directory_path + "/"
+
+    def check_category_names(self, df):
+        original_category_names = df.keys().to_list()
+        sanitized_category_names = set(sanitize_values_in_list(original_category_names).values())
+        unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
+        if unsanitary_original_category_names:
+            raise AnnotationCategoryNameError(
+                f"{unsanitary_original_category_names} are not valid category names, please resubmit")
+
+    def is_safe_collection_name(self, name):
+        """
+        return true if this is a safe collection name
+        this is ultra conservative. If we want to allow full legal file name syntax,
+        we could look at modules like `pathvalidate`
+        """
+        if name is None:
+            return False
+        return re.match(r"^[\w\-]+$", name) is not None
+
+    def set_collection(self, name):
+        self.CXG_ANNO_COLLECTION = name
+
+    def read_labels(self, data_adaptor):
+        user_id = current_app.auth.get_user_id()
+        if user_id is None:
+            return
+        dataset_name = data_adaptor.get_location()
+        dataset_id = self.db.get_or_create_dataset(dataset_name)
+
+        annotation_object = self.db.query_for_most_recent(
+            Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
+        )
+        if annotation_object:
+            df = tiledb.open(annotation_object.tiledb_uri)
+            pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
+            return pandas_df
+        else:
+            return None
+
+    def convert_to_pandas_df(self, tileDBArray, schema_hints):
+        repr_meta = None
+        index_dims = None
+        schema_hints = json.loads(schema_hints)
+
+        if '__pandas_attribute_repr' in tileDBArray.meta:
+            # backwards compatibility... unsure if necessary at this point
+            repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
+        if '__pandas_index_dims' in tileDBArray.meta:
+            index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
+
+        data = tileDBArray[:]
+        indexes = list()
+
+        for col_name, col_val in data.items():
+            # If the column values are byte literals, decode them
+            if isinstance(col_val[0], bytes):
+                col_val = [value.decode('utf-8') for value in col_val]
+
+            if schema_hints and col_name in schema_hints:
+                type = schema_hints.get(col_name).get("type")
+                if type and type == "categorical":
+                    new_col = pd.Series(col_val, dtype='category')
+                    data[col_name] = new_col
+            elif repr_meta and col_name in repr_meta:
+                new_col = pd.Series(col_val, dtype=repr_meta[col_name])
+                data[col_name] = new_col
+            elif index_dims and col_name in index_dims:
+                new_col = pd.Series(col_val, dtype=index_dims[col_name])
+                data[col_name] = new_col
+                indexes.append(col_name)
+
+        new_df = pd.DataFrame.from_dict(data)
+        if len(indexes) > 0:
+            new_df.set_index(indexes, inplace=True)
+
+        return new_df
+
+    def write_labels(self, df, data_adaptor):
+        auth_user_id = current_app.auth.get_user_id()
+        user_name = current_app.auth.get_user_name()
+        timestamp = time.time()
+        dataset_location = data_adaptor.get_location()
+        dataset_id = self.db.get_or_create_dataset(dataset_location)
+        dataset_name = data_adaptor.get_title()
+        user_id = self.db.get_or_create_user(auth_user_id)
+        """
+        NOTE: The uri contains the dataset name, user name and a timestamp as a convenience for debugging purposes.
+        People may have the same name and time.time() can be server dependent.
+        See - https://docs.python.org/2/library/time.html#time.time
+
+        The annotations objects in the database should be used as the source of truth about who an annotation belongs
+        to (for authorization purposes) and what time it was created (for garbage collection).
+        """
+        uri = f"{self.directory_path}{dataset_name}/{user_name}/{timestamp}"
+        if uri.startswith("s3://"):
+            pass
+        else:
+            os.makedirs(uri, exist_ok=True)
+        _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
+        annotation = Annotation(
+            tiledb_uri=uri,
+            user_id=user_id,
+            dataset_id=str(dataset_id),
+            schema_hints=json.dumps(dataframe_schema_type_hints)
+        )
+        if not df.empty:
+            self.check_category_names(df)
+            # convert to tiledb datatypes
+
+            for col in df:
+                df[col] = df[col].astype(get_dtype_of_array(df[col]))
+            tiledb.from_pandas(uri, df)
+
+        self.db.session.add(annotation)
+        self.db.session.commit()
+
+    def update_parameters(self, parameters, data_adaptor):
+        params = {}
+        params["annotations"] = True
+        params["user_annotation_collection_name_enabled"] = False
+
+        if self.ontology_data:
+            params["annotations_cell_ontology_enabled"] = True
+            params["annotations_cell_ontology_terms"] = self.ontology_data
+        else:
+            params["annotations_cell_ontology_enabled"] = False
+
+        parameters.update(params)

server/common/annotations/local_file_csv.py

@@ -1,90 +1,19 @@
-import json
-import uuid
-import time
-from datetime import datetime
-import re
-import os
-import pandas as pd
-from hashlib import blake2b
 import base64
-from server import __version__ as cellxgene_version
+import os
+import re
 import threading
-from server.common.errors import AnnotationsError, OntologyLoadFailure
-from server.common.utils import series_to_schema
-import fsspec
-import fastobo
-from flask import session, current_app, has_request_context
-from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from hashlib import blake2b
 
-from server.db.cellxgene_orm import CellxGeneDataset, Annotation
-from server.db.db_utils import DbUtils
+import pandas as pd
+from flask import session, has_request_context, current_app
 
-
-class Annotations(metaclass=ABCMeta):
-    """ baseclass for annotations, including ontologies"""
-
-    """ our default ontology is the PURL for the Cell Ontology.
-    See http://www.obofoundry.org/ontology/cl.html """
-    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
-
-    def __init__(self):
-        self.ontology_data = None
-
-    def load_ontology(self, path):
-        """Load and parse ontologies - currently support OBO files only."""
-        if path is None:
-            path = self.DefaultOnotology
-
-        try:
-            with fsspec.open(path) as f:
-                obo = fastobo.iter(f)
-                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
-                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
-                self.ontology_data = names
-
-        except FileNotFoundError as e:
-            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
-
-        except SyntaxError as e:
-            raise OntologyLoadFailure("Syntax error loading OBO ontology") from e
-
-        except Exception as e:
-            raise OntologyLoadFailure("Error loading OBO file") from e
-
-    def get_schema(self, data_adaptor):
-        schema = []
-        labels = self.read_labels(data_adaptor)
-        if labels is not None and not labels.empty:
-            for col in labels.columns:
-                col_schema = dict(name=col, writable=True)
-                col_schema.update(series_to_schema(labels[col]))
-                schema.append(col_schema)
-
-        return schema
-
-    @abstractmethod
-    def set_collection(self, name):
-        """set or create a new annotation collection"""
-        pass
-
-    @abstractmethod
-    def read_labels(self, data_adaptor):
-        """Return the labels as a pandas.DataFrame"""
-        pass
-
-    @abstractmethod
-    def write_labels(self, df, data_adaptor):
-        """Write the labels (df) to a persistent storage such that it can later be read"""
-        pass
-
-    @abstractmethod
-    def update_parameters(self, parameters, data_adaptor):
-        """Update configuration parameters that describe information about the annotations feature"""
-        pass
+from server import __version__ as cellxgene_version
+from server.common.annotations.annotations import Annotations
+from server.common.errors import AnnotationsError
 
 
 class AnnotationsLocalFile(Annotations):
-
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
     def __init__(self, output_dir, output_file):
@@ -101,7 +30,6 @@ class AnnotationsLocalFile(Annotations):
     def is_safe_collection_name(self, name):
         """
         return true if this is a safe collection name
-
         this is ultra conservative. If we want to allow full legal file name syntax,
         we could look at modules like `pathvalidate`
         """
@@ -243,6 +171,7 @@ class AnnotationsLocalFile(Annotations):
     def update_parameters(self, parameters, data_adaptor):
         params = {}
         params["annotations"] = True
+        params["user_annotation_collection_name_enabled"] = True
 
         if self.ontology_data:
             params["annotations_cell_ontology_enabled"] = True
@@ -265,55 +194,3 @@ class AnnotationsLocalFile(Annotations):
                 params["annotations-data-collection-name"] = collection
 
         parameters.update(params)
-
-
-class AnnotationsHostedTileDB(Annotations):
-    def __init__(self, directory_path: str, db: DbUtils):
-        super().__init__()
-        self.db = db
-        self.directory_path = directory_path
-
-    def set_collection(self, name):
-        pass
-
-    def read_labels(self, data_adaptor):
-        uid = current_app.auth.get_user_id()
-        dataset_name = data_adaptor.get_location()
-        dataset = self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])
-        # Todo @madison retrieve latest based on timestamp
-        annotation_object = self.db.query_for_most_recent(  # noqa F841
-            Annotation, [Annotation.user_id == uid, Annotation.dataset == dataset]
-        )
-        # Todo in future pr, retrieve dataframe from tiledb uri
-
-    def write_labels(self, df, data_adaptor):
-        uid = current_app.auth.get_user_id()
-        timestamp = time.time()
-        dataset_name = data_adaptor.get_location()
-        try:
-            dataset_id = self.db.query(
-                table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
-            )[0].id
-        except IndexError:
-            dataset_id = uuid.uuid4()
-            dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
-            self.db.session.add(dataset)
-
-        uri = f"{self.directory_path}/{dataset_name}/{uid}/{timestamp}"
-        if "s3" in uri:
-            pass
-        else:
-            os.makedirs(uri, exist_ok=True)
-        schema_hints = {}
-        annotation = Annotation(
-            tiledb_uri=uri,
-            user_id=uid,
-            dataset_id=str(dataset_id),
-            schema_hints=json.dumps(schema_hints)
-        )
-        # todo in future pr -- write df to tiledb, store at uri
-        self.db.session.add(annotation)
-        self.db.session.commit()
-
-    def update_parameters(self, parameters, data_adaptor):
-        pass

server/common/app_config.py

@@ -1,22 +1,24 @@
-from server import display_version as cellxgene_display_version
-from flatten_dict import flatten, unflatten
-import os
-from os.path import splitext, basename, isdir
-import sys
-from urllib.parse import urlparse, quote_plus
-import yaml
 import copy
+import os
+import sys
+import warnings
+from os.path import splitext, basename, isdir
+from urllib.parse import urlparse, quote_plus
 
+import yaml
+from flatten_dict import flatten, unflatten
+
+import server.compute.diffexp_cxg as diffexp_tiledb
+from server import display_version as cellxgene_display_version
+from server.auth.auth import AuthTypeFactory
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.data_locator import discover_s3_region_name
 from server.common.default_config import get_default_config
 from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
+from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
-from server.common.utils import find_available_port, is_port_available
-import warnings
-from server.common.annotations import AnnotationsLocalFile
-from server.common.utils import custom_format_warning
-import server.compute.diffexp_cxg as diffexp_tiledb
-from server.common.data_locator import discover_s3_region_name
-from server.auth.auth import AuthTypeFactory
+from server.db.db_utils import DbUtils
 
 DEFAULT_SERVER_PORT = 5005
 # anything bigger than this will generate a special message
@@ -148,7 +150,6 @@ class AppConfig(object):
         parameters is done"""
 
         if messagefn is None:
-
             def noop(message):
                 pass
 
@@ -274,18 +275,39 @@ class AppConfig(object):
 
         if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
             config["authentication"] = {
-                "is_authenticated": auth.is_user_authenticated(),
                 "requires_client_login": auth.requires_client_login(),
-                "username": auth.get_user_name(),
             }
             if auth.requires_client_login():
                 config["authentication"].update({
                     "login": auth.get_login_url(data_adaptor),
-                    "logout" : auth.get_logout_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
                 })
 
         return c
 
+    def get_client_userinfo(self, data_adaptor):
+        """
+        Return the userinfo as required by the /userinfo REST route
+        """
+
+        server_config = self.server_config
+        dataset_config = data_adaptor.dataset_config
+        auth = server_config.auth
+
+        # make sure the configuration has been checked.
+        self.check_config()
+
+        if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+            userinfo = {}
+            userinfo["userinfo"] = {
+                "is_authenticated": auth.is_user_authenticated(),
+                "username": auth.get_user_name(),
+                "user_id": auth.get_user_id()
+            }
+            return userinfo
+        else:
+            return None
+
 
 class BaseConfig(object):
     """This class handles the mechanics of updating and checking attributes.
@@ -456,6 +478,7 @@ class ServerConfig(BaseConfig):
 
     def complete_config(self, context):
         self.handle_app(context)
+        self.handle_data_source(context)
         self.handle_authentication(context)
         self.handle_data_locator(context)
         self.handle_adaptor(context)  # may depend on data_locator
@@ -569,12 +592,9 @@ class ServerConfig(BaseConfig):
                 region_name = None
             self.data_locator__s3__region_name = region_name
 
-    def handle_single_dataset(self, context):
+    def handle_data_source(self, context):
         self.check_attr("single_dataset__datapath", (str, type(None)))
-        self.check_attr("single_dataset__title", (str, type(None)))
-        self.check_attr("single_dataset__about", (str, type(None)))
-        self.check_attr("single_dataset__obs_names", (str, type(None)))
-        self.check_attr("single_dataset__var_names", (str, type(None)))
+        self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
 
         if self.single_dataset__datapath is None:
             if self.multi_dataset__dataroot is None:
@@ -585,6 +605,16 @@ class ServerConfig(BaseConfig):
             if self.multi_dataset__dataroot is not None:
                 raise ConfigurationError("must supply only one of datapath or dataroot")
 
+    def handle_single_dataset(self, context):
+        self.check_attr("single_dataset__datapath", (str, type(None)))
+        self.check_attr("single_dataset__title", (str, type(None)))
+        self.check_attr("single_dataset__about", (str, type(None)))
+        self.check_attr("single_dataset__obs_names", (str, type(None)))
+        self.check_attr("single_dataset__var_names", (str, type(None)))
+
+        if self.single_dataset__datapath is None:
+            return
+
         # create the matrix data cache manager:
         if self.matrix_data_cache_manager is None:
             self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
@@ -736,6 +766,9 @@ class DatasetConfig(BaseConfig):
             self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
             self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
             self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
+            self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
+            self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
+                dc["user_annotations"]["hosted_tiledb_array"]["hosted_file_directory"]  # noqa E501
 
             self.embeddings__names = dc["embeddings"]["names"]
             self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
@@ -786,6 +819,8 @@ class DatasetConfig(BaseConfig):
         self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
         self.check_attr("user_annotations__ontology__enable", bool)
         self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
+        self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
+        self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
 
         if self.user_annotations__enable:
             server_config = self.app_config.server_config
@@ -797,43 +832,49 @@ class DatasetConfig(BaseConfig):
 
             # TODO, replace this with a factory pattern once we have more than one way
             # to do annotations.  currently only local_file_csv
-            if self.user_annotations__type != "local_file_csv":
-                raise ConfigurationError('The only annotation type support is "local_file_csv"')
+            if self.user_annotations__type == "local_file_csv":
+                dirname = self.user_annotations__local_file_csv__directory
+                filename = self.user_annotations__local_file_csv__file
 
-            dirname = self.user_annotations__local_file_csv__directory
-            filename = self.user_annotations__local_file_csv__file
+                if filename is not None and dirname is not None:
+                    raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
 
-            if filename is not None and dirname is not None:
-                raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+                if filename is not None:
+                    lf_name, lf_ext = splitext(filename)
+                    if lf_ext and lf_ext != ".csv":
+                        raise ConfigurationError(f"annotation file type must be .csv: {filename}")
 
-            if filename is not None:
-                lf_name, lf_ext = splitext(filename)
-                if lf_ext and lf_ext != ".csv":
-                    raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+                if dirname is not None and not isdir(dirname):
+                    try:
+                        os.mkdir(dirname)
+                    except OSError:
+                        raise ConfigurationError("Unable to create directory specified by --annotations-dir")
 
-            if dirname is not None and not isdir(dirname):
-                try:
-                    os.mkdir(dirname)
-                except OSError:
-                    raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+                self.user_annotations = AnnotationsLocalFile(dirname, filename)
 
-            self.user_annotations = AnnotationsLocalFile(dirname, filename)
-
-            # if the user has specified a fixed label file, go ahead and validate it
-            # so that we can remove errors early in the process.
-            server_config = self.app_config.server_config
-            if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
-                with server_config.matrix_data_cache_manager.data_adaptor(
-                    self.tag, server_config.single_dataset__datapath, self.app_config
-                ) as data_adaptor:
-                    data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
-
-            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
-                try:
-                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
-                except OntologyLoadFailure as e:
-                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+                # if the user has specified a fixed label file, go ahead and validate it
+                # so that we can remove errors early in the process.
+                server_config = self.app_config.server_config
+                if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+                    with server_config.matrix_data_cache_manager.data_adaptor(
+                            self.tag, server_config.single_dataset__datapath, self.app_config
+                    ) as data_adaptor:
+                        data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
 
+                if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
+                    try:
+                        self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
+                    except OntologyLoadFailure as e:
+                        raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+            elif self.user_annotations__type == "hosted_tiledb_array":
+                self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
+                self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
+                self.user_annotations = AnnotationsHostedTileDB(
+                    directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
+                    db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
+                )
+            else:
+                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
         else:
             if self.user_annotations__type == "local_file_csv":
                 dirname = self.user_annotations__local_file_csv__directory
@@ -875,7 +916,7 @@ class DatasetConfig(BaseConfig):
         server_config = self.app_config.server_config
         if server_config.single_dataset__datapath:
             with server_config.matrix_data_cache_manager.data_adaptor(
-                self.tag, server_config.single_dataset__datapath, self.app_config
+                    self.tag, server_config.single_dataset__datapath, self.app_config
             ) as data_adaptor:
                 if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
                     context["messagefn"](

server/common/aws_secret_utils.py

@@ -0,0 +1,84 @@
+import logging
+import os
+import sys
+
+import boto3
+from flask import json
+
+from server.common.data_locator import discover_s3_region_name
+from server.common.errors import SecretKeyRetrievalError
+
+
+def handle_config_from_secret(app_config):
+    """Update configuration from the secret manager"""
+    secret_name = os.getenv("CXG_AWS_SECRET_NAME")
+    if not secret_name:
+        return
+
+    # need to find the secret manager region.
+    #  1. from CXG_AWS_SECRET_REGION_NAME
+    #  2. discover from dataroot location (if on s3)
+    #  3. discover from config file location (if on s3)
+    secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
+    if secret_region_name is None:
+        secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
+        if not secret_region_name:
+            from server.eb.app import config_file
+            secret_region_name = discover_s3_region_name(config_file)
+    if not secret_region_name:
+        logging.error("Could not determine the AWS Secret Manager region")
+        sys.exit(1)
+
+    secrets = get_secret_key(secret_region_name, secret_name)
+
+    if not secrets:
+        return
+
+    server_attrs = (
+        ("flask_secret_key", "app__flask_secret_key"),
+        ("oauth_client_secret", "authentication__params_oauth__client_secret"),
+    )
+    default_dataset_attrs = (
+        ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
+    )
+
+    # update server configuration attributes
+    for key, attr in server_attrs:
+        cur_val = getattr(app_config.server_config, attr)
+        if cur_val:
+            continue
+
+        # replace the attr with the secret if it is not set
+        val = secrets.get(key)
+        if val:
+            logging.info(f"set {attr} from secret")
+            app_config.update_server_config(**{attr : val})
+
+    # update default dataset configuration attributes
+    for key, attr in default_dataset_attrs:
+        cur_val = getattr(app_config.default_dataset_config, attr)
+        if cur_val:
+            continue
+
+        # replace the attr with the secret if it is not set
+        val = secrets.get(key)
+        if val:
+            logging.info(f"set {attr} from secret")
+            app_config.update_default_dataset_config(**{attr : val})
+
+
+def get_secret_key(region_name, secret_name):
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager", region_name=region_name)
+
+    try:
+        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+        if "SecretString" in get_secret_value_response:
+            var = get_secret_value_response["SecretString"]
+            secret = json.loads(var)
+            return secret
+    except Exception as e:
+        logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
+        raise SecretKeyRetrievalError
+
+    return None

server/common/corpora.py

@@ -9,6 +9,7 @@ import collections
 import json
 
 from server.cli.upgrade import validate_version_str
+from server.common.utils.corpora_constants import CorporaConstants
 
 
 def corpora_get_versions_from_anndata(adata):
@@ -56,26 +57,13 @@ def corpora_get_props_from_anndata(adata):
     if not version_is_supported:
         raise ValueError("Unsupported Corpora schema version")
 
-    required_simple_fields = [
-        "version",
-        "title",
-        "layer_descriptions",
-        "organism",
-        "organism_ontology_term_id",
-        "project_name",
-        "project_description",
-    ]
-    # Spec says some values encoded as JSON due to the inability of AnnData to store complex types.
-    required_json_fields = ["contributors", "project_links"]
-    optional_simple_fields = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]
-
     corpora_props = {}
-    for key in required_simple_fields:
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
         if key not in adata.uns:
             raise KeyError(f"missing Corpora schema field {key}")
         corpora_props[key] = adata.uns[key]
 
-    for key in required_json_fields:
+    for key in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
         if key not in adata.uns:
             raise KeyError(f"missing Corpora schema field {key}")
         try:
@@ -83,7 +71,7 @@ def corpora_get_props_from_anndata(adata):
         except json.JSONDecodeError:
             raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
 
-    for key in optional_simple_fields:
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
         if key in adata.uns:
             corpora_props[key] = adata.uns[key]
 

server/common/default_config.py

@@ -171,6 +171,9 @@ dataset:
   user_annotations:
     enable: true
     type: local_file_csv
+    hosted_tiledb_array:
+        db_uri: null
+        hosted_file_directory: null
     local_file_csv:
       directory: null
       file: null

server/common/errors.py

@@ -46,6 +46,12 @@ define_request_exception(
     "Raised when there is an authentication error",
     default_status_code=HTTPStatus.UNAUTHORIZED)
 
+define_request_exception(
+    "AnnotationCategoryNameError",
+    "Raised when an annotation category name cant be saved",
+    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+
 define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
+define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")

server/common/rest.py

@@ -121,6 +121,11 @@ def config_get(app_config, data_adaptor):
     return make_response(jsonify(config), HTTPStatus.OK)
 
 
+def userinfo_get(app_config, data_adaptor):
+    config = app_config.get_client_userinfo(data_adaptor)
+    return make_response(jsonify(config), HTTPStatus.OK)
+
+
 def annotations_obs_get(request, data_adaptor):
     fields = request.args.getlist("annotation-name", None)
     num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)

server/common/utils/corpora_constants.py

@@ -0,0 +1,16 @@
+class CorporaConstants(object):
+    REQUIRED_SIMPLE_METADATA_FIELDS = [
+        "version",
+        "title",
+        "layer_descriptions",
+        "organism",
+        "organism_ontology_term_id",
+        "project_name",
+        "project_description",
+    ]
+
+    # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
+    # types.
+    REQUIRED_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+
+    OPTIONAL_SIMPLE_METADATA_FIELDS = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]

server/common/utils/cxg_constants.py

@@ -0,0 +1,4 @@
+class CxgConstants(object):
+    # The CXG container version number.  Must be a semver string (major.minor.patch)
+    # DO NOT UPDATE THIS WITHOUT ALSO UPDATING CXG SPECIFICATION.
+    CXG_VERSION = "0.2.0"

server/common/utils/cxg_generation_utils.py

@@ -0,0 +1,178 @@
+import json
+
+import numpy as np
+import tiledb
+
+from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+
+
+def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
+    """
+    Saves the contents of the dictionary to the CXG output directory specified.
+
+    This function is primarily used to save metadata about a dataset to the CXG directory. At some point, tiledb will
+    have support for metadata on groups at which point the utility of this function should be revisited. Until such
+    feature exists, this function create an empty array and annotate that array.
+
+    For more information, visit https://github.com/TileDB-Inc/TileDB-Py/issues/254.
+    """
+
+    array_name = f"{cxg_container}/{group_metadata_name}"
+
+    # Because TileDB does not allow one to attach metadata directly to a CXG group, we need to have a workaround
+    # where we create an empty array and attached the metadata onto to this empty array. Below we construct this empty
+    # array.
+    tiledb.from_numpy(array_name, np.zeros((1,)))
+
+    with tiledb.DenseArray(array_name, mode="w") as metadata_array:
+        for key, value in metadata_dict.items():
+            metadata_array.meta[key] = value
+
+
+def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, index_column_name, ctx):
+    """
+    Saves the contents of the dataframe to the CXG output directory specified.
+
+    Current access patterns are oriented toward reading very large slices of the dataframe, one attribute at a time.
+    Attribute data also tends to be (often) repetitive (bools, categories, strings). Given this, we use a large tile
+    size (1000) and very aggressive compression levels.
+    """
+
+    def create_dataframe_array(array_name, dataframe):
+        tiledb_filter = tiledb.FilterList(
+            [
+                # Attempt aggressive compression as many of these dataframes are very repetitive strings, bools and
+                # other non-float data.
+                tiledb.ZstdFilter(level=22),
+            ]
+        )
+        attrs = [
+            tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
+            for column in dataframe
+        ]
+        domain = tiledb.Domain(
+            tiledb.Dim(domain=(0, dataframe.shape[0] - 1), tile=min(dataframe.shape[0], 1000), dtype=np.uint32)
+        )
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
+        )
+        tiledb.DenseArray.create(array_name, schema)
+
+    array_name = f"{cxg_container}/{dataframe_name}"
+
+    create_dataframe_array(array_name, dataframe)
+
+    with tiledb.DenseArray(array_name, mode="w", ctx=ctx) as array:
+        value = {}
+        schema_hints = {}
+        for column_name, column_values in dataframe.items():
+            dtype, hints = get_dtype_and_schema_of_array(column_values)
+            value[column_name] = column_values.to_numpy(dtype=dtype)
+            if hints:
+                schema_hints.update({column_name: hints})
+
+        schema_hints.update({"index": index_column_name})
+        array[:] = value
+        array.meta["cxg_schema"] = json.dumps(schema_hints)
+
+    tiledb.consolidate(array_name, ctx=ctx)
+
+
+def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
+    """
+    Saves contents of ndarray to the CXG output directory specified.
+
+    Generally this function is used to convert dataset embeddings. Because embeddings are typically accessed with
+    very large slices (or all of the embedding), they do not benefit from overly aggressive compression due to their
+    format.  Given this, we use a large tile size (1000) but only default compression level.
+    """
+
+    def create_ndarray_array(ndarray_name, ndarray):
+        filters = tiledb.FilterList([tiledb.ZstdFilter()])
+        attrs = [tiledb.Attr(dtype=ndarray.dtype, filters=filters)]
+        dimensions = [
+            tiledb.Dim(
+                domain=(0, ndarray.shape[dimension] - 1), tile=min(ndarray.shape[dimension], 1000), dtype=np.uint32
+            )
+            for dimension in range(ndarray.ndim)
+        ]
+        domain = tiledb.Domain(*dimensions)
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
+        )
+        tiledb.DenseArray.create(ndarray_name, schema)
+
+    create_ndarray_array(ndarray_name, ndarray)
+
+    with tiledb.DenseArray(ndarray_name, mode="w", ctx=ctx) as array:
+        array[:] = ndarray
+
+    tiledb.consolidate(ndarray_name, ctx=ctx)
+
+
+def convert_matrix_to_cxg_array(
+        matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
+):
+    """
+    Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
+    is true or not. Note that when the matrix is encoded as a SparseArray, it only writes the values that are
+    nonzero. This means that if you count the number of elements in the SparseArray, it will not equal the total
+    number of elements in the matrix, only the number of nonzero elements.
+
+    Furthermore, if the `column_shift_for_sparse_encoding` matrix is not None, this function will subtract the sparse
+    encoding from the original given matrix and as previously stated, only write the nonzero values to the TileDB
+    SparseArray.
+    """
+
+    def create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array):
+        filters = tiledb.FilterList([tiledb.ZstdFilter()])
+        attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
+        if encode_as_sparse_array:
+            domain = tiledb.Domain(
+                tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 512), dtype=np.uint32),
+                tiledb.Dim(
+                    name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 2048), dtype=np.uint32
+                ),
+            )
+        else:
+            domain = tiledb.Domain(
+                tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 50), dtype=np.uint32),
+                tiledb.Dim(
+                    name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 100), dtype=np.uint32
+                ),
+            )
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=encode_as_sparse_array, attrs=attrs, cell_order="row-major", tile_order="col-major"
+        )
+        if encode_as_sparse_array:
+            tiledb.SparseArray.create(matrix_name, schema)
+        else:
+            tiledb.DenseArray.create(matrix_name, schema)
+
+    number_of_rows = matrix.shape[0]
+    number_of_columns = matrix.shape[1]
+    stride = min(int(np.power(10, np.around(np.log10(1e9 / number_of_columns)))), 10_000)
+
+    create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array)
+
+    if encode_as_sparse_array:
+        with tiledb.SparseArray(matrix_name, mode="w", ctx=ctx) as array:
+            for start_row_index in range(0, number_of_rows, stride):
+                end_row_index = min(start_row_index + stride, number_of_rows)
+                matrix_subset = matrix[start_row_index:end_row_index, :]
+                if not isinstance(matrix_subset, np.ndarray):
+                    matrix_subset = matrix_subset.toarray()
+                if column_shift_for_sparse_encoding is not None:
+                    matrix_subset = matrix_subset - column_shift_for_sparse_encoding
+                indices = np.nonzero(matrix_subset)
+                trow = indices[0] + start_row_index
+                array[trow, indices[1]] = matrix_subset[indices[0], indices[1]]
+
+    else:
+        with tiledb.DenseArray(matrix_name, mode="w", ctx=ctx) as array:
+            for start_row_index in range(0, number_of_rows, stride):
+                end_row_index = min(start_row_index + stride, number_of_rows)
+                matrix_subset = matrix[start_row_index:end_row_index, :]
+                if not isinstance(matrix_subset, np.ndarray):
+                    matrix_subset = matrix_subset.toarray()
+                array[start_row_index:end_row_index, :] = matrix_subset

server/common/utils/matrix_utils.py

@@ -0,0 +1,112 @@
+import logging
+
+import numpy as np
+from scipy.stats import mode
+
+
+def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
+    """
+    Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
+    a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
+    elements evaluated.  This function may return before evaluating the whole matrix if it can be determined that matrix
+    is not sparse enough.
+    """
+
+    if sparse_threshold == 100.0:
+        return True
+    if sparse_threshold == 0.0:
+        return False
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    # For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
+    # maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
+    # quit early for large dense matrices.
+    row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_row_index in range(0, total_number_of_rows, row_stride):
+        end_row_index = min(start_row_index + row_stride, total_number_of_rows)
+
+        matrix_subset = matrix[start_row_index:end_row_index, :]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        number_of_non_zero_elements += np.count_nonzero(matrix_subset)
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_row_index != total_number_of_rows:
+                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
+                    end_row_index * total_number_of_columns)
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
+                    f"{percentage_of_non_zero_elements:6.2f}")
+            else:
+                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (exact): "
+                    f"{percentage_of_non_zero_elements:6.2f}")
+            return False
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return is_sparse
+
+
+def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
+    """
+    Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
+    shift encoding works by taking the most common value in each column, then subtracting that value from each  element
+    of the column.  If each column mostly contains its most common value, then the resulting matrix can be very sparse.
+
+    This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
+    sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
+    the number of non-zeros encountered and number of elements evaluated.  This function may return before evaluating
+    the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
+    """
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    stride = max(1, 128_000_000 // total_number_of_rows)
+    column_shift = np.zeros(total_number_of_columns)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_column_index in range(0, total_number_of_columns, stride):
+        end_column_index = min(start_column_index + stride, total_number_of_columns)
+
+        matrix_subset = matrix[:, start_column_index:end_column_index]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        matrix_subset_mode = mode(matrix_subset)
+
+        column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
+        number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
+            matrix_subset_mode.count
+        )
+
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_column_index != total_number_of_columns:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
+                    % (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
+                )
+            else:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
+                    % (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
+                )
+            return None
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return column_shift if is_sparse else None

server/common/utils/sanitization_utils.py

@@ -0,0 +1,40 @@
+import re
+
+
+def sanitize_values_in_list(list_of_keys: list):
+    """
+    Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
+    safe and unique.
+    """
+
+    if not all([isinstance(key, str) for key in list_of_keys]):
+        raise Exception("List of keys to sanitize must contain all strings.")
+
+    # Mask out [~/.] and anything outside the ASCII range.
+    mask = re.compile(r"[^ -\-0-\[\]-\}]")
+    clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
+
+    # Dedupe the clean keys list
+    deduped_clean_keys_list = []
+    for index, clean_key in enumerate(clean_keys_list):
+        total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
+        total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
+        deduped_clean_keys_list.append(
+            clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
+            if total_occurrences_of_clean_key > 1
+            else clean_key
+        )
+
+    return dict(zip(list_of_keys, deduped_clean_keys_list))
+
+
+def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
+    """
+    Clean and dedupe the keys in the given dictionary.
+    """
+
+    clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
+    for original_key, sanitized_key in clean_keys.items():
+        if original_key != sanitized_key:
+            dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
+            del dict_to_sanitize[original_key]

server/common/utils/type_conversion_utils.py

@@ -0,0 +1,147 @@
+import logging
+
+import numpy as np
+import pandas as pd
+
+
+def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
+    dtypes_by_column_name = {}
+    schema_type_hints_by_column_name = {}
+
+    for column_name, column_values in dataframe.items():
+        dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \
+            get_dtype_and_schema_of_array(column_values)
+
+    return dtypes_by_column_name, schema_type_hints_by_column_name
+
+
+def get_dtype_of_array(array: pd.Series):
+    return get_dtype_and_schema_of_array(array)[0]
+
+
+def get_schema_type_hint_of_array(array: pd.Series):
+    return get_dtype_and_schema_of_array(array)[1]
+
+
+def get_dtype_and_schema_of_array(array: pd.Series):
+    return (get_dtype_from_dtype(array.dtype, array_values=array),
+            get_schema_type_hint_from_dtype(array.dtype, array_values=array))
+
+
+def get_dtype_from_dtype(dtype, array_values=None):
+    """
+    Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
+    for 64 bit values which will get downcast to 32 bit.
+    """
+
+    dtype_name = dtype.name
+    dtype_kind = dtype.kind
+
+    if dtype_name == "bool":
+        return np.uint8
+    if dtype_name == "object" and dtype_kind == "O":
+        return np.unicode
+    if dtype_name == "category":
+        return get_dtype_from_dtype(dtype.categories.dtype, array_values)
+
+    if can_cast_to_int32(dtype, array_values):
+        return np.int32
+    if can_cast_to_float32(dtype, array_values):
+        return np.float32
+    if not can_cast_to_float32(dtype, array_values):
+        return np.float64
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def get_schema_type_hint_from_dtype(dtype, array_values=None):
+    """
+    Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
+    and will be downcast to 32 bit.
+    """
+
+    dtype_name = dtype.name
+    dtype_kind = dtype.kind
+
+    if dtype == np.float32 or dtype == np.int32:
+        return {"type": dtype_name}
+    if dtype_name == "bool":
+        return {"type": "boolean"}
+    if dtype_name == "object" and dtype_kind == "O":
+        return {"type": "string"}
+    if dtype_name == "category":
+        return {"type": "categorical", "categories": dtype.categories.tolist()}
+
+    if can_cast_to_int32(dtype, array_values):
+        return {"type": "int32"}
+    if can_cast_to_float32(dtype, array_values):
+        return {"type": "float32"}
+    if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
+        return {"type": "float64"}
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def can_cast_to_float32(dtype, array_values):
+    """
+    A dtype can be cast to float32 if it is a float type and converting it to float32 presents the same output as the
+    original values. Note that NaNs fail equality (i.e. np.NaN != np.NaN) so we use np.testing.assert_equal to ensure
+    that the arrays are equal minus NaNs.
+
+    We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
+    Since NaNs are floating points in numpy, we upcast the integer array to float32.
+    """
+
+    if dtype.kind == "f":
+        # Try to convert the array to float32
+        converted_float32_values = array_values.to_numpy(np.float32)
+        original_values = array_values.to_numpy()
+
+        # Verify that the two arrays are equal except for NaNs (which will equate to be unequal).
+        if not ((converted_float32_values != original_values) == np.isnan(original_values)).all():
+            return False
+
+        if dtype != np.float32:
+            logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
+        return True
+
+    if dtype.kind == "O" and array_values.hasnans:
+        return True
+
+    return False
+
+
+def can_cast_to_int32(dtype, array_values=None):
+    """
+    A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
+    the higher precision type has values that are entirely within the range of the downcast type.
+    """
+
+    # Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
+    # return False.
+    if array_values.hasnans:
+        return False
+
+    # If the array is categorical, then we need to order the array values so that functions min and max that occur
+    # later, can function. They do not function on unordered categories.
+    ordered_array_values = array_values
+    if array_values.dtype.name == "category" and not array_values.cat.ordered:
+        ordered_array_values = array_values.cat.as_ordered()
+
+    if dtype.kind in ["i", "u"]:
+        if np.can_cast(dtype, np.int32):
+            return True
+        ii32 = np.iinfo(np.int32)
+        if not ordered_array_values.empty and (
+                ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \
+                ordered_array_values.empty:
+            return True
+    return False
+
+
+def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
+    if series_to_convert.hasnans and dtype == np.int32:
+        logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
+
+    return series_to_convert.to_numpy(dtype)

server/common/utils/utils.py

@@ -5,12 +5,11 @@ import logging
 import os
 import pkgutil
 import socket
-import warnings
-
-from flask import json
 from urllib.parse import urlsplit, urljoin
+
 import numpy as np
-import pandas as pd
+from flask import json
+
 from server.common.errors import ConfigurationError
 
 
@@ -94,61 +93,6 @@ def jsonify_numpy(data):
     return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
 
 
-def dtype_to_schema(dtype):
-    schema = {}
-    if dtype == np.float32:
-        schema["type"] = "float32"
-    elif dtype == np.int32:
-        schema["type"] = "int32"
-    elif dtype == np.bool_:
-        schema["type"] = "boolean"
-    elif dtype == np.str:
-        schema["type"] = "string"
-    elif dtype == "category":
-        schema["type"] = "categorical"
-        schema["categories"] = dtype.categories.tolist()
-    else:
-        raise TypeError(f"Annotations of type {dtype} are unsupported.")
-    return schema
-
-
-def can_cast_to_float32(array):
-    if array.dtype.kind == "f":
-        if not np.can_cast(array.dtype, np.float32):
-            warnings.warn(f"Annotation {array.name} will be converted to 32 bit float and may lose precision.")
-        return True
-    return False
-
-
-def can_cast_to_int32(array):
-    if array.dtype.kind in ["i", "u"]:
-        if np.can_cast(array.dtype, np.int32):
-            return True
-        ii32 = np.iinfo(np.int32)
-        if array.min() >= ii32.min and array.max() <= ii32.max:
-            return True
-    return False
-
-
-def series_to_schema(array):
-    assert type(array) == pd.Series
-    try:
-        return dtype_to_schema(array.dtype)
-    except TypeError:
-        dtype = array.dtype
-        data_kind = dtype.kind
-        schema = {}
-        if can_cast_to_float32(array):
-            schema["type"] = "float32"
-        elif can_cast_to_int32(array):
-            schema["type"] = "int32"
-        elif data_kind == "O" and dtype == "object":
-            schema["type"] = "string"
-        else:
-            raise TypeError(f"Annotations of type {dtype} are unsupported.")
-        return schema
-
-
 def import_plugins(plugin_module):
     """
     Load optional plugin modules from server.common.plugins

server/compute/scanpy.py

@@ -43,7 +43,7 @@ def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap
     for k in list(adata.uns.keys()):
         del adata.uns[k]
 
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_obs - 1, 50), **pca_options)
+    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
     sc.pp.neighbors(adata, **neighbors_options)
     sc.tl.umap(adata, **umap_options)
 

server/converters/cxgtool.py

@@ -1,661 +0,0 @@
-"""
-This program converts an [AnnData H5AD](https://anndata.readthedocs.io/en/stable/)
-into a cellxgene TileDB structure, aka a [CXG](../../dev_docs/cxg.md).
-
-IF YOU UPDATE THIS FILE, IN ANY WAY THAT MODIFIES THE CXG FORMAT or CONTENTS,
-YOU MUST UPDATE THE CXG SPECIFICATION and VERSION NUMBER.
-"""
-import re
-import anndata
-import tiledb
-import argparse
-import numpy as np
-from os.path import splitext, basename
-import json
-from scipy.stats import mode
-
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.errors import ColorFormatException
-from server.common.corpora import (
-    corpora_get_props_from_anndata,
-    corpora_get_versions_from_anndata,
-    corpora_is_version_supported,
-)
-
-
-# the CXG container version number.  Must be a semver string (major.minor.patch)
-# DO NOT UPDATE THIS WITHOUT ALSO UPDATING THE CXG SPECIFICATION.
-CXG_VERSION = "0.2.0"
-
-# log_level must have a default
-log_level = 3
-
-
-def log(level, *args):
-    global log_level
-    if log_level and level <= log_level:
-        print(*args)
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("h5ad", nargs="?", help="H5AD file name")
-    parser.add_argument(
-        "--backed", action="store_true", help="loaded in file backed mode. Will be slower, but use less memory."
-    )
-    parser.add_argument(
-        "--disable-custom-colors",
-        action="store_true",
-        default=False,
-        help="Do not extract scanpy-compatible category colors from h5ad file.",
-    )
-    parser.add_argument(
-        "--obs-names", help="Name of annotation to use for observations. If not specified, will use the obs index."
-    )
-    parser.add_argument(
-        "--var-names", help="Name of annotation to use for variables. If not specified, will use the var index."
-    )
-    parser.add_argument("--verbose", "-v", action="count", default=0, help="verbose output")
-    parser.add_argument("--title", help="Human readable dataset title.  If omitted, will use filename")
-    parser.add_argument(
-        "--about",
-        metavar="<URL>",
-        help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
-    )
-    parser.add_argument("--out", "--output", "-o", help="output CXG file name")
-    parser.add_argument(
-        "--sparse-threshold",
-        "-s",
-        type=float,
-        default=0.0,  # force dense by default
-        help="The X array will be sparse if the percent of non-zeros falls below this value",
-    )
-    parser.add_argument(
-        "--disable-corpora",
-        action="store_true",
-        default=False,
-        help="Disable extraction and storing of Corpora schema information.",
-    )
-    args = parser.parse_args()
-
-    global log_level
-    log_level = args.verbose
-
-    adata = anndata.read_h5ad(args.h5ad, backed="r" if args.backed else None)
-    log(1, f"{basename(args.h5ad)} loaded...")
-
-    basefname = splitext(basename(args.h5ad))[0]
-    out = args.out if args.out is not None else basefname
-    container = out if splitext(out)[1] == ".cxg" else out + ".cxg"
-
-    corpora_props = load_corpora_props(args, adata) if not args.disable_corpora else None
-    cxg_group_metadata = create_cxg_group_metadata(
-        adata,
-        basefname,
-        title=args.title,
-        about=args.about,
-        corpora_props=corpora_props,
-        extract_colors=not args.disable_custom_colors,
-    )
-
-    write_cxg(
-        adata,
-        container,
-        cxg_group_metadata=cxg_group_metadata,
-        var_names=args.var_names,
-        obs_names=args.obs_names,
-        sparse_threshold=args.sparse_threshold,
-    )
-
-    log(1, "done")
-
-
-def write_cxg(adata, container, cxg_group_metadata, var_names=None, obs_names=None, sparse_threshold=5.0):
-    if not adata.var.index.is_unique:
-        raise ValueError("Variable index is not unique - unable to convert.")
-    if not adata.obs.index.is_unique:
-        raise ValueError("Observation index is not unique - unable to convert.")
-
-    """
-    TileDB bug TileDB-Inc/TileDB#1575 requires that we sanitize all column names
-    prior to saving.  This can be reverted when the bug is fixed.
-    """
-    log(0, "Warning: sanitizing all dataframe column names.")
-    clean_all_column_names(adata)
-
-    ctx = tiledb.Ctx(
-        {
-            "sm.num_reader_threads": 32,
-            "sm.num_writer_threads": 32,
-            "sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
-        }
-    )
-
-    tiledb.group_create(container, ctx=ctx)
-    log(1, f"\t...group created, with name {container}")
-
-    # dataset metadata
-    save_metadata(container, cxg_group_metadata)
-    log(1, "\t...dataset metadata saved")
-
-    # var/gene dataframe
-    save_dataframe(container, "var", adata.var, var_names, ctx=ctx)
-    log(1, "\t...var dataframe created")
-
-    # obs/cell dataframe
-    save_dataframe(container, "obs", adata.obs, obs_names, ctx=ctx)
-    log(1, "\t...obs dataframe created")
-
-    # embeddings
-    e_container = f"{container}/emb"
-    tiledb.group_create(e_container, ctx=ctx)
-    save_embeddings(e_container, adata, ctx)
-    log(1, "\t...embeddings created")
-
-    # X matrix
-    save_X(container, adata.X, ctx, sparse_threshold)
-    log(1, "\t...X created")
-
-
-"""
-TODO: the code used to handle type inferencing should not be duplicated between
-this tool and the server/common/utils code. When this tool is merged into
-the cellxgene CLI, consolidate.
-"""
-
-
-def dtype_to_schema(dtype):
-    if dtype == np.float32:
-        return (np.float32, {})
-    elif dtype == np.int32:
-        return (np.int32, {})
-    elif dtype == np.bool_:
-        return (np.uint8, {"type": "boolean"})
-    elif dtype == np.str:
-        return (np.unicode, {"type": "string"})
-    elif dtype == "category":
-        typ, hint = cxg_type(dtype.categories)
-        return (typ, {"type": "categorical", "categories": dtype.categories.tolist()})
-    else:
-        raise TypeError(f"Annotations of type {dtype} are unsupported.")
-
-
-def _can_cast_to_float32(array):
-    if array.dtype.kind == "f":
-        # force downcast for all floats
-        return True
-    return False
-
-
-def _can_cast_to_int32(array):
-    if array.dtype.kind in ["i", "u"]:
-        if np.can_cast(array.dtype, np.int32):
-            return True
-        ii32 = np.iinfo(np.int32)
-        if array.min() >= ii32.min and array.max() <= ii32.max:
-            return True
-    return False
-
-
-def cxg_type(array):
-    try:
-        return dtype_to_schema(array.dtype)
-    except TypeError:
-        dtype = array.dtype
-        data_kind = dtype.kind
-        if _can_cast_to_float32(array):
-            return (np.float32, {})
-        elif _can_cast_to_int32(array):
-            return (np.int32, {})
-        elif data_kind == "O" and dtype == "object":
-            return (np.unicode, {"type": "string"})
-        else:
-            raise TypeError(f"Annotations of type {dtype} are unsupported.")
-
-
-def cxg_dtype(array):
-    return cxg_type(array)[0]
-
-
-def create_dataframe(name, df, ctx):
-    """
-    Current access patterns are oriented toward reading very large slices of
-    the dataframe, one attribute at a time.  Attribute data also tends to be
-    (often) repetitive (bools, categories, strings).
-    Given this, we use:
-    * a large tile size (1000)
-    * very aggressive compression levels
-    """
-    filter = tiledb.FilterList(
-        [
-            # attempt aggressive compression as many of these dataframes are very repetitive
-            # strings, bools and other non-float data.
-            tiledb.ZstdFilter(level=22),
-        ]
-    )
-    attrs = [tiledb.Attr(name=col, dtype=cxg_dtype(df[col]), filters=filter) for col in df]
-    domain = tiledb.Domain(tiledb.Dim(domain=(0, df.shape[0] - 1), tile=min(df.shape[0], 1000), dtype=np.uint32))
-    schema = tiledb.ArraySchema(
-        domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
-    )
-    tiledb.DenseArray.create(name, schema)
-
-
-def create_unique_column_name(df_cols, col_name_prefix):
-    """
-    given the columns of a dataframe, and a name prefix, return a column name which
-    does not exist in the dataframe, AND which is prefixed by `prefix`
-
-    The approach is to append a numeric suffix, starting at zero and increasing by
-    one, until an unused name is found (eg, prefix_0, prefix_1, ...).
-    """
-    suffix = 0
-    while f"{col_name_prefix}{suffix}" in df_cols:
-        suffix += 1
-    return f"{col_name_prefix}{suffix}"
-
-
-def alias_index_col(df, df_name, index_col_name):
-    """
-    We rely in the existance of a unique, human-readable index for
-    any dataframe (eg, var is typically gene name, obs the cell name).
-    The user can specify these via the --obs-names and --var-names config.
-    If they are not specified, use the existing index to create them, giving
-    the resulting column a unique name (eg, "name").
-
-    In both cases, enforce that the result is unique, and communicate the
-    index column name via the 'index' field in the schema hints.
-    """
-    if index_col_name is None:
-        if not df.index.is_unique:
-            raise KeyError(
-                f"Values in {df_name}.index must be unique. "
-                "Please prepare data to contain unique index values, or specify an "
-                "alternative with --{ax_name}-name."
-            )
-        index_col_name = create_unique_column_name(df.columns, "name_")
-        # turn the index into a normal column
-        df.rename_axis(index_col_name, inplace=True)
-        df.reset_index(inplace=True)
-
-    elif index_col_name in df.columns:
-        # User has specified alternative column for unique names, and it exists
-        if not df[index_col_name].is_unique:
-            raise KeyError(
-                f"Values in {df_name}.{index_col_name} must be unique. Please prepare data to contain unique values."
-            )
-
-    else:
-        raise KeyError(f"Annotation {index_col_name}, specified in --{df_name}-name, does not exist.")
-
-    return (df, index_col_name)
-
-
-def save_dataframe(container, name, df, index_col_name, ctx):
-    A_name = f"{container}/{name}"
-    (df, index_col_name) = alias_index_col(df, name, index_col_name)
-    create_dataframe(A_name, df, ctx=ctx)
-    with tiledb.DenseArray(A_name, mode="w", ctx=ctx) as A:
-        value = {}
-        schema_hints = {}
-        for k, v in df.items():
-            dtype, hints = cxg_type(v)
-            value[k] = v.to_numpy(dtype=dtype)
-            if hints:
-                schema_hints.update({k: hints})
-
-        schema_hints.update({"index": index_col_name})
-        A[:] = value
-        A.meta["cxg_schema"] = json.dumps(schema_hints)
-
-    tiledb.consolidate(A_name, ctx=ctx)
-
-
-def create_emb(e_name, emb):
-    """
-    Embeddings are typically accessed with very large slices (or all of the embedding),
-    and do not benefit from overly aggressive compression due to their format.  Given
-    this, we use:
-    * large tile size (1000)
-    * default compression level
-    """
-    filters = tiledb.FilterList([tiledb.ZstdFilter()])
-    attrs = [tiledb.Attr(dtype=emb.dtype, filters=filters)]
-    dims = []
-    for d in range(emb.ndim):
-        shape = emb.shape
-        dims.append(tiledb.Dim(domain=(0, shape[d] - 1), tile=min(shape[d], 1000), dtype=np.uint32))
-    domain = tiledb.Domain(*dims)
-    schema = tiledb.ArraySchema(
-        domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
-    )
-    tiledb.DenseArray.create(e_name, schema)
-
-
-def is_valid_embedding(adata, name, arr):
-    """ return True if this layout data is a valid array for front-end presentation:
-        * ndarray, with shape (n_obs, >= 2), dtype float/int/uint
-        * follows ScanPy embedding naming conventions
-        * with all values finite or NaN (no +Inf or -Inf)
-    """
-    is_valid = type(name) == str and name.startswith("X_") and len(name) > 2
-    is_valid = is_valid and type(arr) == np.ndarray and arr.dtype.kind in "fiu"
-    is_valid = is_valid and arr.shape[0] == adata.n_obs and arr.shape[1] >= 2
-    is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
-    return is_valid
-
-
-def save_embeddings(container, adata, ctx):
-    for (name, value) in adata.obsm.items():
-        if is_valid_embedding(adata, name, value):
-            e_name = f"{container}/{name[2:]}"
-            create_emb(e_name, value)
-            with tiledb.DenseArray(e_name, mode="w", ctx=ctx) as A:
-                A[:] = value
-            tiledb.consolidate(e_name, ctx=ctx)
-            log(1, f"\t\t...{name} embedding created")
-
-
-def create_X(X_name, shape, is_sparse):
-    """
-    The X matrix is accessed in both row and column oriented patterns, depending on the
-    particular operation.  Because of the data type, default compression works best.
-    The tile size, (50, 100) for dense, and (512,2048) for sparse,
-    and global layout (row/col) was chosen empirically, by benchmarking
-    the current cellxgene backend.
-    """
-    filters = tiledb.FilterList([tiledb.ZstdFilter()])
-    attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
-    if is_sparse:
-        domain = tiledb.Domain(
-            tiledb.Dim(name="obs", domain=(0, shape[0] - 1), tile=min(shape[0], 512), dtype=np.uint32),
-            tiledb.Dim(name="var", domain=(0, shape[1] - 1), tile=min(shape[1], 2048), dtype=np.uint32),
-        )
-    else:
-        domain = tiledb.Domain(
-            tiledb.Dim(name="obs", domain=(0, shape[0] - 1), tile=min(shape[0], 50), dtype=np.uint32),
-            tiledb.Dim(name="var", domain=(0, shape[1] - 1), tile=min(shape[1], 100), dtype=np.uint32),
-        )
-    schema = tiledb.ArraySchema(
-        domain=domain, sparse=is_sparse, attrs=attrs, cell_order="row-major", tile_order="col-major"
-    )
-    if is_sparse:
-        tiledb.SparseArray.create(X_name, schema)
-    else:
-        tiledb.DenseArray.create(X_name, schema)
-
-
-def evaluate_for_sparse_encoding(xdata, sparse_threshold):
-    """
-    This function determines if the X matrix has a sparsity below the sparse_threshold.
-    This function also returns the number of non-zeros encountered and number
-    of elements evaluated.  This function may return before evaluating the whole X matrix
-    if it can be determined that X is not sparse enough.
-    """
-    shape = xdata.shape
-    stride = min(int(np.power(10, np.around(np.log10(1e9 / shape[1])))), 10_000)
-    nnz = 0
-    maxnnz = int(shape[0] * shape[1] * sparse_threshold / 100)
-    for row in range(0, shape[0], stride):
-        lim = min(row + stride, shape[0])
-        a = xdata[row:lim, :]
-        if type(a) is not np.ndarray:
-            a = a.toarray()
-        nnz += np.count_nonzero(a)
-        if nnz > maxnnz:
-            return (False, nnz, lim * shape[1])
-        log(2, "\t...rows", lim, "of", shape[0], "nnz", nnz, "nnz percent %5.2f%%" % (100 * nnz / (lim * shape[1])))
-
-    is_sparse = (100.0 * nnz / (shape[0] * shape[1])) < sparse_threshold
-    return (is_sparse, nnz, shape[0] * shape[1])
-
-
-def evaluate_for_sparse_column_shift_encoding(xdata, sparse_threshold):
-    """Column shift encoding works by taking the most common value in each column, then
-    subtracting that value from each element of the column.  If each column mostly contains
-    its most common value, then the resulting matrix can be very sparse.
-
-    This function determines if column shift encoding can be used to transform
-    the X matrix into a sparse matrix with a sparsity below the sparse_threshold.
-    If so, return the col_shift array that stores this encoding.
-    This function also returns the number of non-zeros encountered and number
-    of elements evaluated.  This function may return before evaluating the whole X matrix
-    if it can be determined that X cannot benefit from column shift encoding.
-    """
-    shape = xdata.shape
-    stride = max(1, 128_000_000 // shape[0])
-    col_shift = np.zeros(shape[1])
-    nnz = 0
-    maxnnz = int(shape[0] * shape[1] * sparse_threshold / 100)
-    for col in range(0, shape[1], stride):
-        lim = min(col + stride, shape[1])
-        a = xdata[:, col:lim]
-        if type(a) is not np.ndarray:
-            a = a.toarray()
-        m = mode(a)
-        col_shift[col:lim] = m.mode
-        nnz += shape[0] * (lim - col) - np.sum(m.count)
-        if nnz > maxnnz:
-            return (None, nnz, shape[0] * lim)
-        log(2, "\t...cols", lim, "of", shape[1], "nnz", nnz, "nnz percent %5.2f%%" % (100 * nnz / (lim * shape[0])))
-
-    is_sparse = (100.0 * nnz / (shape[0] * shape[1])) < sparse_threshold
-    return (col_shift if is_sparse else None, nnz, shape[0] * shape[1])
-
-
-def save_X(container, xdata, ctx, sparse_threshold, expect_sparse=False):
-    # Save X count matrix
-    X_name = f"{container}/X"
-
-    shape = xdata.shape
-    log(1, "\t...shape:", str(shape))
-
-    col_shift = None
-    if sparse_threshold == 100:
-        is_sparse = True
-    elif sparse_threshold == 0:
-        is_sparse = False
-    else:
-        is_sparse, nnz, nelem = evaluate_for_sparse_encoding(xdata, sparse_threshold)
-        percent = 100.0 * nnz / nelem
-        if nelem != shape[0] * shape[1]:
-            log(1, "\t...sparse=", is_sparse, "non-zeros percent (estimate): %6.2f" % percent)
-        else:
-            log(1, "\t...sparse=", is_sparse, "non-zeros:", nnz, "percent: %6.2f" % percent)
-
-        is_sparse = percent < sparse_threshold
-        if not is_sparse:
-            col_shift, nnz, nelem = evaluate_for_sparse_column_shift_encoding(xdata, sparse_threshold)
-            is_sparse = col_shift is not None
-            percent = 100.0 * nnz / nelem
-            if nelem != shape[0] * shape[1]:
-                log(1, "\t...sparse=", is_sparse, "col shift non-zeros percent (estimate): %6.2f" % percent)
-            else:
-                log(1, "\t...sparse=", is_sparse, "col shift non-zeros:", nnz, "percent: %6.2f" % percent)
-
-    if expect_sparse is True and is_sparse is False:
-        return False
-
-    create_X(X_name, shape, is_sparse)
-    stride = min(int(np.power(10, np.around(np.log10(1e9 / shape[1])))), 10_000)
-    if is_sparse:
-        if col_shift is not None:
-            log(1, "\t...output X as sparse matrix with column shift encoding")
-            X_col_shift_name = f"{container}/X_col_shift"
-            filters = tiledb.FilterList([tiledb.ZstdFilter()])
-            attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
-            domain = tiledb.Domain(tiledb.Dim(domain=(0, shape[1] - 1), tile=min(shape[1], 5000), dtype=np.uint32))
-            schema = tiledb.ArraySchema(domain=domain, attrs=attrs)
-            tiledb.DenseArray.create(X_col_shift_name, schema)
-            with tiledb.DenseArray(X_col_shift_name, mode="w", ctx=ctx) as X_col_shift:
-                X_col_shift[:] = col_shift
-            tiledb.consolidate(X_col_shift_name, ctx=ctx)
-        else:
-            log(1, "\t...output X as sparse matrix")
-
-        with tiledb.SparseArray(X_name, mode="w", ctx=ctx) as X:
-            nnz = 0
-            for row in range(0, shape[0], stride):
-                lim = min(row + stride, shape[0])
-                a = xdata[row:lim, :]
-                if type(a) is not np.ndarray:
-                    a = a.toarray()
-                if col_shift is not None:
-                    a = a - col_shift
-                indices = np.nonzero(a)
-                trow = indices[0] + row
-                nnz += indices[0].shape[0]
-                X[trow, indices[1]] = a[indices[0], indices[1]]
-                log(2, "\t...rows", lim, "of", shape[0], "nnz", nnz, "sparse", nnz / (lim * shape[1]))
-
-    else:
-        log(1, "\t...output X as dense matrix")
-        with tiledb.DenseArray(X_name, mode="w", ctx=ctx) as X:
-            for row in range(0, shape[0], stride):
-                lim = min(row + stride, shape[0])
-                a = xdata[row:lim, :]
-                if type(a) is not np.ndarray:
-                    a = a.toarray()
-                X[row:lim, :] = a
-                log(2, "\t...rows", row, "to", lim)
-
-    tiledb.consolidate(X_name, ctx=ctx)
-    if hasattr(tiledb, "vacuum"):
-        tiledb.vacuum(X_name)
-
-    return is_sparse
-
-
-def save_metadata(container, metadata_dict):
-    """
-    Save all dataset-wide metadata.   This includes:
-    * CXG version
-    * dataset metadata, such as title and about link.
-
-    Longer term, tiledb will have support for metadata on groups. Until
-    such feature exists, create an empty array and annotate that array.
-
-    https://github.com/TileDB-Inc/TileDB-Py/issues/254
-    """
-    a_name = f"{container}/cxg_group_metadata"
-    with tiledb.from_numpy(a_name, np.zeros((1,))) as A:
-        pass
-    with tiledb.DenseArray(a_name, mode="w") as A:
-        for k, v in metadata_dict.items():
-            A.meta[k] = v
-
-
-def load_corpora_props(args, adata):
-    versions = corpora_get_versions_from_anndata(adata)
-    if versions is None:
-        return None
-
-    [corpora_schema_version, corpora_encoding_version] = versions
-    corpora_props = corpora_get_props_from_anndata(adata)
-    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
-    if not version_is_supported or not corpora_props:
-        log(0, "ERROR: Unknown source file schema version is unsupported")
-        raise ValueError("Unsupported Corpora schema version")
-
-    log(1, "FYI, file appears to be encoded using Corpora schema standards...")
-    if args.title is not None or args.about is not None:
-        log(0, "Warning: explicit specification of --title or --about will override Corpora schema fields.")
-
-    return corpora_props
-
-
-def create_cxg_group_metadata(adata, basefname, title=None, about=None, corpora_props=None, extract_colors=True):
-
-    if corpora_props is not None:
-        # clobber encoding version to be OUR version, not the source H5AD encoding
-        corpora_props["version"].update({"corpora_encoding_version": CXG_VERSION})
-        corpora_project_links = corpora_props.get("project_links", [])
-        corpora_about_link = next(
-            (link for link in corpora_project_links if (link.get("link_type", None) == "SUMMARY")), {}
-        )
-    else:
-        corpora_about_link = {}
-
-    title = title or corpora_about_link.get("link_name", basefname)
-    about = about or corpora_about_link.get("link_url")
-
-    cxg_group_metadata = {"cxg_version": CXG_VERSION, "cxg_properties": json.dumps({"title": title, "about": about})}
-    if corpora_props is not None:
-        cxg_group_metadata.update({"corpora": json.dumps(corpora_props)})
-
-    if extract_colors:
-        try:
-            cxg_group_metadata["cxg_category_colors"] = json.dumps(
-                convert_anndata_category_colors_to_cxg_category_colors(adata)
-            )
-        except ColorFormatException:
-            log(
-                0,
-                "Warning: failed to extract colors from h5ad file! "
-                "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.",
-            )
-
-    return cxg_group_metadata
-
-
-def sanitize_keys(keys):
-    """
-    We need names to be safe to use as attribute names in tiledb.  See:
-        TileDB-Inc/TileDB#1575
-        TileDB-Inc/TileDB-Py#294
-    This can be entirely removed once they add proper escaping.
-
-    Args: list of keys
-    Returns: dict of {old_key: new_key, ...}
-
-    Returned new keys will be both safe and unique.
-
-    Masking out [~/.] and anything outside the ASCII range.
-    """
-    p = re.compile(r"[^ -\.0-\[\]-\}]")
-    clean_keys = {k: p.sub("_", k) for k in keys}
-
-    used_keys = set()
-    clean_unique_keys = {}
-    for k, v in clean_keys.items():
-        if v not in used_keys:
-            used_keys.add(v)
-            clean_unique_keys[k] = v
-            continue
-
-        # else, needs deduping.
-        counter = 1
-        while True:
-            candidate_name = v + "-" + str(counter)
-            if candidate_name not in used_keys:
-                used_keys.add(candidate_name)
-                clean_unique_keys[k] = candidate_name
-                break
-            counter += 1
-
-    for k, v, in clean_unique_keys.items():
-        if k != v:
-            log(1, f"Renaming {k} to {v}")
-    return clean_unique_keys
-
-
-def sanitize_df(df):
-    df.rename(columns=sanitize_keys(df.keys().tolist()), inplace=True)
-
-
-def sanitize_mapping(mapping):
-    clean_keys = sanitize_keys([k for k in mapping.keys()])
-    for old_key, new_key in clean_keys.items():
-        if old_key != new_key:
-            mapping[new_key] = mapping[old_key]
-            del mapping[old_key]
-
-
-def clean_all_column_names(adata):
-    sanitize_df(adata.obs)
-    sanitize_df(adata.var)
-    sanitize_mapping(adata.obsm)
-
-
-if __name__ == "__main__":
-    main()

server/converters/h5ad_data_file.py

@@ -0,0 +1,250 @@
+import json
+import logging
+from os import path
+
+import anndata
+import numpy as np
+import tiledb
+
+from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from server.common.corpora import corpora_get_props_from_anndata
+from server.common.errors import ColorFormatException
+from server.common.utils.cxg_constants import CxgConstants
+from server.common.utils.cxg_generation_utils import (
+    convert_dictionary_to_cxg_group,
+    convert_dataframe_to_cxg_array,
+    convert_ndarray_to_cxg_dense_array,
+    convert_matrix_to_cxg_array,
+)
+from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+
+
+class H5ADDataFile:
+    """ Class encapsulating required information about an H5AD datafile that ultimately will be transformed into
+    another format (currently just CXG is supported). """
+
+    def __init__(
+            self,
+            input_filename,
+            backed=False,
+            dataset_title=None,
+            dataset_about=None,
+            obs_index_column_name=None,
+            vars_index_column_name=None,
+            use_corpora_schema=True,
+    ):
+        self.input_filename = input_filename
+        self.backed = backed
+        self.dataset_title = dataset_title
+        self.dataset_about = dataset_about
+        self.obs_index_column_name = obs_index_column_name
+        self.vars_index_column_name = vars_index_column_name
+
+        self.use_corpora_schema = use_corpora_schema
+
+        self.validate_input_file_type()
+
+        self.extract_anndata_elements_from_file()
+        self.extract_metadata_about_dataset()
+
+        self.validate_anndata()
+
+    def to_cxg(self, output_cxg_directory, sparse_threshold, convert_anndata_colors_to_cxg_colors=True):
+        """
+        Writes the following attributes of the anndata to CXG: 1) the metadata as metadata attached to an empty
+        DenseArray, 2) the obs DataFrame as a DenseArray, 3) the var DataFrame as a DenseArray, 4) all valid
+        embeddings stored in obsm, each one as a DenseArray, 5) the main X matrix of the anndata as either a
+        SparseArray or DenseArray based on the `sparse_threshold`, and optionally 6) the column shift of the main X
+        matrix that might turn an otherwise Dense matrix into a Sparse matrix.
+        """
+
+        logging.info("Beginning writing to CXG.")
+        ctx = tiledb.Ctx(
+            {
+                "sm.num_reader_threads": 32,
+                "sm.num_writer_threads": 32,
+                "sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
+            }
+        )
+
+        tiledb.group_create(output_cxg_directory, ctx=ctx)
+        logging.info(f"\t...group created, with name {output_cxg_directory}")
+
+        convert_dictionary_to_cxg_group(
+            output_cxg_directory, self.generate_cxg_metadata(convert_anndata_colors_to_cxg_colors)
+        )
+        logging.info("\t...dataset metadata saved")
+
+        convert_dataframe_to_cxg_array(output_cxg_directory, "obs", self.obs, self.obs_index_column_name, ctx)
+        logging.info("\t...dataset obs dataframe saved")
+
+        convert_dataframe_to_cxg_array(output_cxg_directory, "var", self.var, self.var_index_column_name, ctx)
+        logging.info("\t...dataset var dataframe saved")
+
+        self.write_anndata_embeddings_to_cxg(output_cxg_directory, ctx)
+        logging.info("\t...dataset embeddings saved")
+
+        self.write_anndata_x_matrix_to_cxg(output_cxg_directory, ctx, sparse_threshold)
+        logging.info("\t...dataset X matrix saved")
+
+        logging.info("Completed writing to CXG.")
+
+    def write_anndata_x_matrix_to_cxg(self, output_cxg_directory, ctx, sparse_threshold):
+        matrix_container = f"{output_cxg_directory}/X"
+
+        x_matrix_data = self.anndata.X
+        is_sparse = is_matrix_sparse(x_matrix_data, sparse_threshold)
+        if not is_sparse:
+            col_shift = get_column_shift_encode_for_matrix(x_matrix_data, sparse_threshold)
+            is_sparse = col_shift is not None
+        else:
+            col_shift = None
+
+        if col_shift is not None:
+            logging.info("Converting matrix X as sparse matrix with column shift encoding")
+            x_col_shift_name = f"{output_cxg_directory}/X_col_shift"
+            convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
+
+        convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
+
+        tiledb.consolidate(matrix_container, ctx=ctx)
+        if hasattr(tiledb, "vacuum"):
+            tiledb.vacuum(matrix_container)
+
+    def write_anndata_embeddings_to_cxg(self, output_cxg_directory, ctx):
+        def is_valid_embedding(adata, embedding_name, embedding_array):
+            """
+            Returns true if this layout data is a valid array for front-end presentation with the following criteria:
+                * ndarray, with shape (n_obs, >= 2), dtype float/int/uint
+                * follows ScanPy embedding naming conventions
+                * with all values finite or NaN (no +Inf or -Inf)
+            """
+
+            is_valid = isinstance(embedding_name, str) and embedding_name.startswith("X_") and len(embedding_name) > 2
+            is_valid = is_valid and isinstance(embedding_array, np.ndarray) and embedding_array.dtype.kind in "fiu"
+            is_valid = is_valid and embedding_array.shape[0] == adata.n_obs and embedding_array.shape[1] >= 2
+            is_valid = is_valid and not np.any(np.isinf(embedding_array)) and not np.all(np.isnan(embedding_array))
+            return is_valid
+
+        embedding_container = f"{output_cxg_directory}/emb"
+        tiledb.group_create(embedding_container, ctx=ctx)
+
+        for embedding_name, embedding_values in self.anndata.obsm.items():
+            if is_valid_embedding(self.anndata, embedding_name, embedding_values):
+                embedding_name = f"{embedding_container}/{embedding_name[2:]}"
+                convert_ndarray_to_cxg_dense_array(embedding_name, embedding_values, ctx)
+                logging.info(f"\t\t...{embedding_name} embedding created")
+
+    def generate_cxg_metadata(self, convert_anndata_colors_to_cxg_colors):
+        """
+        Return a dictionary containing metadata about CXG dataset. This include data about the version as well as
+        Corpora schema properties if they exist, among other pieces of metadata.
+        """
+
+        cxg_group_metadata = {
+            "cxg_version": CxgConstants.CXG_VERSION,
+            "cxg_properties": json.dumps({"title": self.dataset_title, "about": self.dataset_about}),
+        }
+        if self.corpora_properties is not None:
+            cxg_group_metadata["corpora"] = json.dumps(self.corpora_properties)
+
+        if convert_anndata_colors_to_cxg_colors:
+            try:
+                cxg_group_metadata["cxg_category_colors"] = json.dumps(
+                    convert_anndata_category_colors_to_cxg_category_colors(self.anndata)
+                )
+            except ColorFormatException:
+                logging.warning(
+                    "Failed to extract colors from H5AD file! Fix the H5AD file or rerun with "
+                    "--disable-custom-colors. See help for more details."
+                )
+
+        return cxg_group_metadata
+
+    def validate_input_file_type(self):
+        """
+        Validate that the input file is of a type that we can handle. Currently the only valid file type is `.h5ad`.
+        """
+
+        if not self.input_filename.endswith(".h5ad"):
+            raise Exception(f"Cannot process input file {self.input_filename}. File must be an H5AD.")
+
+        if self.dataset_title or self.dataset_about:
+            logging.warning(
+                "If you convert this dataset into CXG and you explicit specify values for the dataset title metadata "
+                "or the dataset about metadata, it will override any metadata that is extracted as part of the "
+                "Corpora schema fields."
+            )
+
+    def validate_anndata(self):
+        if not self.var.index.is_unique:
+            raise ValueError("Variable index in AnnData object is not unique.")
+        if not self.obs.index.is_unique:
+            raise ValueError("Observation index in AnnData object is not unique.")
+
+    def extract_anndata_elements_from_file(self):
+        logging.info(f"Reading in AnnData dataset: {path.basename(self.input_filename)}")
+        self.anndata = anndata.read_h5ad(self.input_filename, backed="r" if self.backed else None)
+        logging.info("Completed reading in AnnData dataset!")
+
+        self.obs = self.transform_dataframe_index_into_column(self.anndata.obs, "obs", self.obs_index_column_name)
+        self.var = self.transform_dataframe_index_into_column(self.anndata.var, "var", self.vars_index_column_name)
+
+    def extract_metadata_about_dataset(self):
+        """
+        Extract metadata information about the dataset that upon conversion will be saved as group metadata with the
+        CXG that is generated. This metadata information includes Corpora schema properties, the dataset title and
+        a link that details more information about the dataset.
+        """
+
+        self.corpora_properties = corpora_get_props_from_anndata(self.anndata) if self.use_corpora_schema else None
+        if self.corpora_properties is None and self.use_corpora_schema:
+            # If the return value is None, this means that we were not able to figure out what version of the Corpora
+            # schema the object is using and therefore cannot extract any properties.
+            raise ValueError("Unknown source file schema version is unsupported.")
+
+        # The title and about properties of the dataset are set by the following order: if they are explicitly defined
+        # then use the explicit value. If the dataset is a Corpora-schema based schema, then extract the title and about
+        # from the corpora_properties. Otherwise, use the input filename (only for title, about will be blank).
+        if self.corpora_properties:
+            corpora_project_links = self.corpora_properties.get("project_links", [])
+            corpora_about_link = next(
+                (link for link in corpora_project_links if (link.get("link_type", None) == "SUMMARY")), {}
+            )
+        else:
+            corpora_about_link = {}
+
+        filename = path.splitext(path.basename(self.input_filename))[0]
+
+        self.dataset_title = self.dataset_title if self.dataset_title else corpora_about_link.get("link_name", filename)
+        self.dataset_about = self.dataset_about if self.dataset_about else corpora_about_link.get("link_url")
+
+    def transform_dataframe_index_into_column(self, dataframe, dataframe_name, index_column_name):
+        """
+        Convert the dataframe's index into another column in the dataframe. If an index_column_name is specified,
+        use that column as the index instead.
+        """
+
+        if index_column_name is None:
+            # Create a unique column name for the index.
+            suffix = 0
+            while f"name_{suffix}" in dataframe.columns:
+                suffix += 1
+            index_column_name = f"name_{suffix}"
+
+            # Turn the index into a normal column
+            dataframe.rename_axis(index_column_name, inplace=True)
+            dataframe.reset_index(inplace=True)
+
+        elif index_column_name in dataframe.columns:
+            # User has specified alternative column for unique names, and it exists
+            if not dataframe[index_column_name].is_unique:
+                raise KeyError(
+                    f"Values in {dataframe_name}.{index_column_name} must be unique. Please prepare data to contain "
+                    f"unique values."
+                )
+        else:
+            raise KeyError(f"Column {index_column_name} does not exist.")
+
+        setattr(self, f"{dataframe_name}_index_column_name", index_column_name)
+        return dataframe

server/converters/to_sparse.py

@@ -2,12 +2,15 @@
 Script to create a sparse dataset in CXG format based on an input dataset in CXG format.
 The input dataset is not modified.
 """
+import argparse
 import os
 import shutil
-import tiledb
-import argparse
 import sys
-import server.converters.cxgtool as cxgtool
+
+import tiledb
+
+from server.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array
+from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 def main():
@@ -49,9 +52,25 @@ def main():
     )
 
     with tiledb.DenseArray(os.path.join(args.input, "X"), mode="r", ctx=ctx) as X_in:
-        is_sparse = cxgtool.save_X(args.output, X_in, ctx, args.sparse_threshold, expect_sparse=True)
+        x_matrix_data = X_in[:, :]
+        matrix_container = args.output
 
-    if is_sparse is False:
+        is_sparse = is_matrix_sparse(x_matrix_data, args.sparse_threshold)
+        if not is_sparse:
+            col_shift = get_column_shift_encode_for_matrix(x_matrix_data, args.sparse_threshold)
+            is_sparse = col_shift is not None
+        else:
+            col_shift = None
+
+        if col_shift is not None:
+            x_col_shift_name = f"{args.output}/X_col_shift"
+            convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
+            tiledb.consolidate(matrix_container, ctx=ctx)
+        if is_sparse:
+            convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
+            tiledb.consolidate(matrix_container, ctx=ctx)
+
+    if not is_sparse:
         print("The array is not sparse, cleaning up, abort.")
         shutil.rmtree(args.output)
         sys.exit(1)

server/data_anndata/anndata_adaptor.py

@@ -1,22 +1,22 @@
 import warnings
-
-import numpy as np
-from pandas.core.dtypes.dtypes import CategoricalDtype
-import anndata
-from scipy import sparse
-from packaging import version
 from datetime import datetime
+
+import anndata
+import numpy as np
+from packaging import version
+from pandas.core.dtypes.dtypes import CategoricalDtype
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-from server.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
-from server.common.utils import series_to_schema
+import server.compute.diffexp_generic as diffexp_generic
 from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from server.common.constants import Axis, MAX_LAYOUTS
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.compute.scanpy import scanpy_umap
-import server.compute.diffexp_generic as diffexp_generic
 from server.common.corpora import corpora_get_props_from_anndata
+from server.common.errors import PrepareError, DatasetAccessError, FilterError
+from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from server.compute.scanpy import scanpy_umap
+from server.data_common.data_adaptor import DataAdaptor
+from server.data_common.fbs.matrix import encode_matrix_fbs
 
 anndata_version = version.parse(str(anndata.__version__)).release
 
@@ -137,7 +137,7 @@ class AnndataAdaptor(DataAdaptor):
             curr_axis = getattr(self.data, str(ax))
             for ann in curr_axis:
                 ann_schema = {"name": ann, "writable": False}
-                ann_schema.update(series_to_schema(curr_axis[ann]))
+                ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann]))
                 self.schema["annotations"][ax]["columns"].append(ann_schema)
 
         for layout in self.get_embedding_names():

server/data_common/data_adaptor.py

@@ -1,14 +1,15 @@
 from abc import ABCMeta, abstractmethod
-from server_timing import Timing as ServerTiming
-import numpy as np
-import pandas as pd
 from os.path import basename, splitext
 
-from server.data_common.fbs.matrix import encode_matrix_fbs
+import numpy as np
+import pandas as pd
+from server_timing import Timing as ServerTiming
+
+from server.common.app_config import AppFeature, AppConfig
 from server.common.constants import Axis
 from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
-from server.common.utils import jsonify_numpy
-from server.common.app_config import AppFeature, AppConfig
+from server.common.utils.utils import jsonify_numpy
+from server.data_common.fbs.matrix import encode_matrix_fbs
 
 
 class DataAdaptor(metaclass=ABCMeta):
@@ -172,7 +173,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0] : i[1]] = True
+                mask[i[0]: i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -313,7 +314,7 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 

server/data_common/fbs/matrix.py

@@ -85,7 +85,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
 def column_encoding(arr):
     column_encoding_type_map = {
         # array protocol string:  ( array_type, as_type )
-        np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64),
         np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),

server/data_cxg/cxg_adaptor.py

@@ -1,24 +1,25 @@
-import os
 import json
 import logging
-from server.common.utils import dtype_to_schema
-from server.common.errors import DatasetAccessError, ConfigurationError
-from server.common.utils import path_join
+import os
+import threading
+
+import numpy as np
+import pandas as pd
+import tiledb
+from server_timing import Timing as ServerTiming
+
+import server.compute.diffexp_cxg as diffexp_cxg
 from server.common.constants import Axis
+from server.common.errors import DatasetAccessError, ConfigurationError
+from server.common.immutable_kvcache import ImmutableKVCache
+from server.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
+from server.common.utils.utils import path_join
 from server.data_common.data_adaptor import DataAdaptor
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_cxg.cxg_util import pack_selector_from_mask
-import server.compute.diffexp_cxg as diffexp_cxg
-from server.common.immutable_kvcache import ImmutableKVCache
-import tiledb
-import numpy as np
-import pandas as pd
-from server_timing import Timing as ServerTiming
-import threading
 
 
 class CxgAdaptor(DataAdaptor):
-
     # TODO:  The tiledb context parameters should be a configuration option
     tiledb_ctx = tiledb.Ctx(
         {"sm.tile_cache_size": 8 * 1024 * 1024 * 1024, "sm.num_reader_threads": 32, "vfs.s3.region": "us-east-1"}
@@ -50,8 +51,14 @@ class CxgAdaptor(DataAdaptor):
         """Set the tiledb context.  This should be set before any instances of CxgAdaptor are created"""
         try:
             CxgAdaptor.tiledb_ctx = tiledb.Ctx(context_params)
+            tiledb.default_ctx(context_params)
+
         except tiledb.libtiledb.TileDBError as e:
-            raise ConfigurationError(f"Invalid tiledb context: {str(e)}")
+            if e.message == "Global context already initialized!":
+                if tiledb.default_ctx().config().dict() != CxgAdaptor.tiledb_ctx.config().dict():
+                    raise ConfigurationError("Cannot change tiledb configuration once it is set")
+            else:
+                raise ConfigurationError(f"Invalid tiledb context: {str(e)}")
 
     @staticmethod
     def pre_load_validation(data_locator):
@@ -337,32 +344,6 @@ class CxgAdaptor(DataAdaptor):
             raise DatasetAccessError("cxg matrix missing embeddings")
         return embeddings
 
-    @staticmethod
-    def _get_col_type(attr, schema_hints={}):
-        type_hint = schema_hints.get(attr.name, {})
-        dtype = attr.dtype
-        schema = {}
-        # type hints take precedence
-        if "type" in type_hint:
-            schema["type"] = type_hint["type"]
-        elif dtype == np.float32:
-            schema["type"] = "float32"
-        elif dtype == np.int32:
-            schema["type"] = "int32"
-        elif dtype == np.bool_:
-            schema["type"] = "boolean"
-        elif dtype == np.str:
-            schema["type"] = "string"
-        elif dtype == "category":
-            schema["type"] = "categorical"
-            schema["categories"] = dtype.categories.tolist()
-        else:
-            raise TypeError(f"Annotations of type {dtype} are unsupported.")
-
-        if schema["type"] == "categorical" and "categories" in schema_hints:
-            schema["categories"] = schema_hints["categories"]
-        return schema
-
     def _get_schema(self):
         if self.schema:
             return self.schema
@@ -389,7 +370,7 @@ class CxgAdaptor(DataAdaptor):
                     if schema["type"] == "categorical" and "categories" in type_hint:
                         schema["categories"] = type_hint["categories"]
                 else:
-                    schema.update(dtype_to_schema(attr.dtype))
+                    schema.update(get_schema_type_hint_from_dtype(attr.dtype))
                 cols.append(schema)
 
             annotations[ax] = dict(columns=cols)

server/db/db_utils.py

@@ -1,9 +1,10 @@
 import typing
+import uuid
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from server.db.cellxgene_orm import Base
+from server.db.cellxgene_orm import Base, CellxGeneDataset, CellxGeneUser
 
 
 class DbUtils:
@@ -34,7 +35,33 @@ class DbUtils:
         )
 
     def query_for_most_recent(self, table: Base, filter_args: typing.List[bool] = None) -> Base:
-        return self.session.query(table).filter(*filter_args).order_by(table.created_at.desc()).limit(1).all()[0]
+        try:
+            return self.session.query(table).filter(*filter_args).order_by(table.created_at.desc()).limit(1).all()[0]
+        except IndexError:
+            return None
+
+    def get_or_create_dataset(self, dataset_name):
+        try:
+            dataset_id = self.query(
+                table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
+            )[0].id
+        except IndexError:
+            dataset_id = uuid.uuid4()
+            dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
+            self.session.add(dataset)
+            self.session.commit()
+        return str(dataset_id)
+
+    def get_or_create_user(self, user_id):
+        try:
+            user_id = self.query(
+                table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id]
+            )[0].id
+        except IndexError:
+            user = CellxGeneUser(id=user_id)
+            self.session.add(user)
+            self.session.commit()
+        return str(user_id)
 
 
 class DBSessionMaker:

server/eb/README.md

@@ -203,7 +203,7 @@ $ EB_INSTANCE=m5.large
 $ CXG_DATAROOT=<location to your S3 bucket>
 $ CXG_CONFIG_FILE=<location to your config file>
 
-# Potentially also set envvars for the sercret key.
+# Potentially also set envvars for the secret key.
 
 $ eb create $EB_ENV --instance-type $EB_INSTANCE \
    --envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE

server/eb/app.py

@@ -7,7 +7,9 @@ import base64
 from flask import json
 import logging
 from flask_talisman import Talisman
-import boto3
+
+from server.common.aws_secret_utils import handle_config_from_secret
+from server.common.errors import SecretKeyRetrievalError
 
 
 if os.path.isdir("/opt/python/log"):
@@ -31,63 +33,6 @@ except Exception:
     sys.exit(1)
 
 
-def get_secret_key(region_name, secret_name):
-    session = boto3.session.Session()
-    client = session.client(service_name="secretsmanager", region_name=region_name)
-
-    try:
-        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
-        if "SecretString" in get_secret_value_response:
-            var = get_secret_value_response["SecretString"]
-            secret = json.loads(var)
-            return secret
-    except Exception:
-        logging.critical("Caught exception during get_secret_key", exc_info=True)
-        sys.exit(1)
-
-    return None
-
-
-def handle_config_from_secret(app_config):
-    """Update configuration from the secret manager"""
-    secret_name = os.getenv("CXG_AWS_SECRET_NAME")
-    if not secret_name:
-        return
-
-    # need to find the secret manager region.
-    #  1. from CXG_AWS_SECRET_REGION_NAME
-    #  2. discover from dataroot location (if on s3)
-    #  3. discover from config file location (if on s3)
-    secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
-    if secret_region_name is None:
-        secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
-        if not secret_region_name:
-            secret_region_name = discover_s3_region_name(config_file)
-    if not secret_region_name:
-        logging.error("Could not determine the AWS Secret Manager region")
-        sys.exit(1)
-
-    secrets = get_secret_key(secret_region_name, secret_name)
-    if not secrets:
-        return
-
-    keyattrs = (
-        ("flask_secret_key", "app__flask_secret_key"),
-        ("oauth_client_secret", "authentication__params_oauth__client_secret")
-    )
-
-    for key, attr in keyattrs:
-        curval = getattr(app_config.server_config, attr)
-        if curval:
-            continue
-
-        # replace the attr with the secret if it is not set
-        val = secrets.get(key)
-        if val:
-            logging.error(f"set {attr} from secret")
-            app_config.update_server_config(**{attr : val})
-
-
 class WSGIServer(Server):
     def __init__(self, app_config):
         super().__init__(app_config)
@@ -98,14 +43,19 @@ class WSGIServer(Server):
         server_config = app_config.server_config
         # This hash should be in sync with the script within
         # `client/configuration/webpack/obsoleteHTMLTemplate.html`
-        obsolete_browser_script_hash = ["'SHA25-0028D52E332C015C3ED9929926F4000BB4020B8CB85C1F5769D6AA3BA711F58E'"]
+
+        # It is _very_ difficult to generate the correct hash manually,
+        # consider forcing CSP to fail on the local server by intercepting the response via Requestly
+        # this should print the failing script's hash to console.
+        # See more here: https://github.com/chanzuckerberg/cellxgene/pull/1745
+        obsolete_browser_script_hash = ["'sha256-/rmgOi/skq9MpiZxPv6lPb1PNSN+Uf4NaUHO/IjyfwM='"]
         csp = {
             "default-src": ["'self'"],
             "connect-src": ["'self'"],
-            "script-src": ["'self'", "'unsafe-eval'", "'unsafe-inline'"]
+            "script-src": ["'self'", "'unsafe-eval'"]
             + obsolete_browser_script_hash + script_hashes,
             "style-src": ["'self'", "'unsafe-inline'"],
-            "img-src": ["'self'", "'https://cellxgene.cziscience.com'", "data:"],
+            "img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"],
             "object-src": ["'none'"],
             "base-uri": ["'none'"],
             "frame-ancestors": ["'none'"],
@@ -201,11 +151,14 @@ try:
         app_config.update_server_config(multi_dataset__dataroot=dataroot)
 
     # update from secret manager
-    handle_config_from_secret(app_config)
+    try:
+        handle_config_from_secret(app_config)
+    except SecretKeyRetrievalError:
+        sys.exit(1)
 
     # features are unsupported in the current hosted server
     app_config.update_default_dataset_config(
-        user_annotations__enable=False, embeddings__enable_reembedding=False,
+        embeddings__enable_reembedding=False,
     )
     app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
     app_config.complete_config(logging.info)

server/requirements-dev.txt

@@ -1,8 +1,11 @@
+Authlib>=0.14.3
 black
 bumpversion>=0.5
-parameterized>=0.7.0
-pytest>=3.6.3
-twine>=1.12.1
 codecov>=2.0.15
+parameterized>=0.7.0
+psycopg2==2.7.7
+pytest>=3.6.3
+python-jose>=3.2.0
 scanpy>=1.4.6
+twine>=1.12.1
 -r requirements.txt

server/requirements.txt

@@ -15,11 +15,10 @@ numba>=0.49.1
 numpy>=1.16.0
 packaging>=20.0
 pandas>=0.24.2
-psycopg2==2.7.7
 PyYAML>=5.3
 scipy>=1.3.0
 requests>=2.22.0
 sqlalchemy>=1.3.18
 tiledb>=0.5.9,>=0.6.2
-s3fs>=0.4.2
+s3fs==0.4.2
 gunicorn>=20.0.4

server/test/__init__.py

@@ -1,28 +1,60 @@
+import os
 import random
 import shutil
 import string
 import tempfile
-import requests
 import time
-import os
-from subprocess import Popen
-from os import path, popen
 from contextlib import contextmanager
+from os import path, popen
+from subprocess import Popen
 
 import pandas as pd
+import requests
 
-from server.common.annotations import AnnotationsLocalFile
-from server.common.data_locator import DataLocator
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
 from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
-from server.common.utils import find_available_port
+from server.common.data_locator import DataLocator
+from server.common.utils.utils import find_available_port
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
-
+from server.db.db_utils import DbUtils
 
 PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
 FIXTURES_ROOT = PROJECT_ROOT + "/server/test/fixtures"
 
 
+def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
+    tmp_dir = tempfile.mkdtemp()
+    fname = {
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+        MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
+    }[ext]
+    data_locator = DataLocator(fname)
+    config = AppConfig()
+    config.update_server_config(
+        multi_dataset__dataroot=data_locator.path,
+        authentication__type="test",
+    )
+    config.update_default_dataset_config(
+        embeddings__names=["umap"],
+        presentation__max_categories=100,
+        diffexp__lfc_cutoff=0.01,
+        user_annotations__type="hosted_tiledb_array",
+        user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
+        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir
+    )
+
+    config.complete_config()
+
+    data = MatrixDataLoader(data_locator.abspath()).open(config)
+    annotations = AnnotationsHostedTileDB(
+        tmp_dir,
+        DbUtils("postgresql://postgres:test_pw@localhost:5432"),
+    )
+    return data, tmp_dir, annotations
+
+
 def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
     tmp_dir = tempfile.mkdtemp()
     annotations_file = path.join(tmp_dir, "test_annotations.csv")
@@ -40,6 +72,7 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
     config.update_default_dataset_config(
         embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
     )
+
     config.complete_config()
     data = MatrixDataLoader(data_locator.abspath()).open(config)
     annotations = AnnotationsLocalFile(None, annotations_file)
@@ -104,7 +137,7 @@ def start_test_server(command_line_args=[], app_config=None):
     yaml config file, which this server will read and parse.
     """
 
-    start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1)
+    start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
     port = int(os.environ.get("CXG_SERVER_PORT", start))
     port = find_available_port("localhost", port)
     command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args

server/test/unit/auth/test_auth.py

@@ -21,9 +21,10 @@ class AuthTest(unittest.TestCase):
 
         with test_server(app_config=c) as server:
             session = requests.Session()
-            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
-            assert "authentication" not in data_config["config"]
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertNotIn("authentication", config["config"])
+            self.assertIsNone(userinfo)
 
     def test_auth_session(self):
         c = AppConfig()
@@ -35,11 +36,12 @@ class AuthTest(unittest.TestCase):
 
         with test_server(app_config=c) as server:
             session = requests.Session()
-            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
-            assert data_config["config"]["authentication"]["is_authenticated"]
-            assert not data_config["config"]["authentication"]["requires_client_login"]
-            assert data_config["config"]["authentication"]["username"] == "anonymous"
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+
+            self.assertFalse(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
 
     def test_auth_test(self):
         c = AppConfig()
@@ -61,45 +63,46 @@ class AuthTest(unittest.TestCase):
             session = requests.Session()
 
             # auth datasets
-            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
-            assert not data_config["config"]["authentication"]["is_authenticated"]
-            assert data_config["config"]["authentication"]["requires_client_login"]
-            assert data_config["config"]["authentication"]["username"] is None
-            assert data_config["config"]["parameters"]["annotations"]
+            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
 
-            login_uri = data_config["config"]["authentication"]["login"]
-            logout_uri = data_config["config"]["authentication"]["logout"]
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
 
-            assert login_uri == "/login?dataset=auth/pbmc3k.cxg"
-            assert logout_uri == "/logout?dataset=auth/pbmc3k.cxg"
+            login_uri = config["config"]["authentication"]["login"]
+            logout_uri = config["config"]["authentication"]["logout"]
+
+            self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
+            self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
 
             r = session.get(f"{server}/{login_uri}")
             # check that the login redirect worked
-            assert r.history[0].status_code == 302
-            assert r.url == f"{server}/auth/pbmc3k.cxg/"
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")
 
-            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
-            assert data_config["config"]["authentication"]["is_authenticated"]
-            assert data_config["config"]["authentication"]["username"] == "test_account"
-            assert data_config["config"]["parameters"]["annotations"]
+            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "test_account")
+            self.assertTrue(config["config"]["parameters"]["annotations"])
 
             r = session.get(f"{server}/{logout_uri}")
             # check that the logout redirect worked
-            assert r.history[0].status_code == 302
-            assert r.url == f"{server}/auth/pbmc3k.cxg/"
-            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
-            assert not data_config["config"]["authentication"]["is_authenticated"]
-            assert data_config["config"]["authentication"]["username"] is None
-            assert data_config["config"]["parameters"]["annotations"]
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")
+            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
 
             # no-auth datasets
-            r = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
-            assert "authentication" not in data_config["config"]
-            assert not data_config["config"]["parameters"]["annotations"]
+            config = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertIsNone(userinfo)
+            self.assertFalse(config["config"]["parameters"]["annotations"])
 
     def test_auth_test_single(self):
         c = AppConfig()
@@ -111,37 +114,36 @@ class AuthTest(unittest.TestCase):
 
         with test_server(app_config=c) as server:
             session = requests.Session()
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
 
-            r = session.get(f"{server}/api/v0.2/config")
-            data_config = r.json()
-            assert not data_config["config"]["authentication"]["is_authenticated"]
-            assert data_config["config"]["authentication"]["requires_client_login"]
-            assert data_config["config"]["authentication"]["username"] is None
-            assert data_config["config"]["parameters"]["annotations"]
+            login_uri = config["config"]["authentication"]["login"]
+            logout_uri = config["config"]["authentication"]["logout"]
 
-            login_uri = data_config["config"]["authentication"]["login"]
-            logout_uri = data_config["config"]["authentication"]["logout"]
-
-            assert login_uri == "/login"
-            assert logout_uri == "/logout"
+            self.assertEqual(login_uri, "/login")
+            self.assertEqual(logout_uri, "/logout")
 
             r = session.get(f"{server}/{login_uri}")
             # check that the login redirect worked
-            assert r.history[0].status_code == 302
-            assert r.url == f"{server}/"
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/")
 
-            r = session.get(f"{server}/api/v0.2/config")
-            data_config = r.json()
-            assert data_config["config"]["authentication"]["is_authenticated"]
-            assert data_config["config"]["authentication"]["username"] == "test_account"
-            assert data_config["config"]["parameters"]["annotations"]
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "test_account")
+            self.assertTrue(config["config"]["parameters"]["annotations"])
 
             r = session.get(f"{server}/{logout_uri}")
             # check that the logout redirect worked
-            assert r.history[0].status_code == 302
-            assert r.url == f"{server}/"
-            r = session.get(f"{server}/api/v0.2/config")
-            data_config = r.json()
-            assert not data_config["config"]["authentication"]["is_authenticated"]
-            assert data_config["config"]["authentication"]["username"] is None
-            assert data_config["config"]["parameters"]["annotations"]
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/")
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])

server/test/unit/auth/test_oauth.py

@@ -0,0 +1,176 @@
+import unittest
+import random
+import time
+import base64
+import json
+import requests
+
+from flask import Flask, jsonify, make_response, request, redirect
+from multiprocessing import Process
+
+import jose
+from server.common.app_config import AppConfig
+from server.test import FIXTURES_ROOT, test_server
+
+# This tests the oauth authentication type.
+# This test starts a cellxgene server and a mock oauth server.
+# API requests to login and logout and get the userinfo are made
+# to the cellxgene server, which then sends requests to the mock
+# oauth server.
+
+# number of seconds that the oauth token is valid
+TOKEN_EXPIRES = 5
+
+# Create a mocked out oauth token, which servers all the endpoints needed by the oauth type.
+mock_oauth_app = Flask("mock_oauth_app")
+
+
+@mock_oauth_app.route("/authorize")
+def authorize():
+    callback = request.args.get("redirect_uri")
+    state = request.args.get("state")
+    return redirect(callback + f"?code=fakecode&state={state}")
+
+
+@mock_oauth_app.route("/oauth/token", methods=["POST"])
+def token():
+    headers = dict(alg="RS256", kid="fake_kid")
+    payload = dict(name="fake_user", sub="fake_id", email="fake_user@email.com", email_verified=True)
+    jwt = jose.jwt.encode(claims=payload, key="mysecret", algorithm="HS256", headers=headers)
+    r = {
+        "access_token": f"access-{time.time()}",
+        "id_token": jwt,
+        "refresh_token": f"random-{time.time()}",
+        "scope": "openid profile email",
+        "expires_in": TOKEN_EXPIRES,
+        "token_type": "Bearer",
+        "expires_at": time.time() + TOKEN_EXPIRES,
+    }
+    return make_response(jsonify(r))
+
+
+@mock_oauth_app.route("/v2/logout")
+def logout():
+    return_to = request.args.get("returnTo")
+    return redirect(return_to)
+
+
+@mock_oauth_app.route("/.well-known/jwks.json")
+def jwks():
+    data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
+    return make_response(jsonify(dict(keys=[data])))
+
+
+# The port that the mock oauth server will listen on
+PORT = random.randint(10000, 12000)
+
+
+# function to launch the mock oauth server
+def launch_mock_oauth():
+    mock_oauth_app.run(port=PORT)
+
+
+class AuthTest(unittest.TestCase):
+    def setUp(self):
+        self.dataset_dataroot = FIXTURES_ROOT
+        self.mock_oauth_process = Process(target=launch_mock_oauth)
+        self.mock_oauth_process.start()
+
+    def tearDown(self):
+        self.mock_oauth_process.terminate()
+
+    def auth_flow(self, app_config, cookie_key=None):
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+
+            # auth datasets
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            login_uri = config["config"]["authentication"]["login"]
+            logout_uri = config["config"]["authentication"]["logout"]
+
+            self.assertEqual(login_uri, "/login?dataset=d/pbmc3k.cxg/")
+            self.assertEqual(logout_uri, "/logout")
+
+            r = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            if cookie_key:
+                cookie = session.cookies.get(cookie_key)
+                token = json.loads(base64.b64decode(cookie))
+                access_token_before = token.get("access_token")
+                expires_at_before = token.get("expires_at")
+
+                # let the token expire
+                time.sleep(TOKEN_EXPIRES + 1)
+
+                # check that refresh works
+                session.get(f"{server}/{login_uri}")
+                userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+                self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+                self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
+
+                cookie = session.cookies.get(cookie_key)
+                token = json.loads(base64.b64decode(cookie))
+                access_token_after = token.get("access_token")
+                expires_at_after = token.get("expires_at")
+
+                self.assertNotEqual(access_token_before, access_token_after)
+                self.assertTrue(expires_at_after - expires_at_before > TOKEN_EXPIRES)
+
+            r = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}")
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+    def test_auth_oauth_session(self):
+        # test with session cookies
+        app_config = AppConfig()
+        app_config.update_server_config(
+            authentication__type="oauth",
+            authentication__params_oauth__api_base_url=f"http://localhost:{PORT}",
+            authentication__params_oauth__client_id="mock_client_id",
+            authentication__params_oauth__client_secret="mock_client_secret",
+            authentication__params_oauth__session_cookie=True,
+        )
+
+        app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
+        app_config.complete_config()
+
+        self.auth_flow(app_config)
+
+    def test_auth_oauth_cookie(self):
+        # test with specified cookie
+        app_config = AppConfig()
+        app_config.update_server_config(
+            authentication__type="oauth",
+            authentication__params_oauth__api_base_url=f"http://localhost:{PORT}",
+            authentication__params_oauth__client_id="mock_client_id",
+            authentication__params_oauth__client_secret="mock_client_secret",
+            authentication__params_oauth__session_cookie=False,
+            authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60),
+        )
+
+        app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
+        app_config.complete_config()
+
+        self.auth_flow(app_config, "test_cxguser")

server/test/unit/common/test_api.py

@@ -423,11 +423,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
             MatrixDataType.H5AD, annotations_fixture=True
         )
-        cls._setupClass(cls, [
-            "--annotations-file",
-            cls.annotations.output_file,
-            cls.data.get_location(),
-        ])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ])
 
     @classmethod
     def tearDownClass(cls):

server/test/unit/common/test_app_config.py

@@ -1,12 +1,21 @@
+import os
 import unittest
+from unittest import mock
+from unittest.mock import patch
+
+import requests
+
 from server.common.app_config import AppConfig
 from server.common.errors import ConfigurationError
 from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
-import requests
+
 
 # NOTE, there are more tests that should be written for AppConfig.
 # this is just a start.
 
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
 
 class AppConfigTest(unittest.TestCase):
     def test_update(self):
@@ -98,3 +107,29 @@ class AppConfigTest(unittest.TestCase):
 
             r = session.get(f"{server}/health")
             assert r.json()["status"] == "pass"
+
+    @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
+    @patch('server.common.aws_secret_utils.get_secret_key')
+    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "flask_secret_key": "mock_flask_secret",
+            "oauth_client_secret": "mock_oauth_secret",
+            "db_uri": "mock_db_uri"
+        }
+
+        config = AppConfig()
+
+        with self.assertLogs(level="INFO") as logger:
+            from server.common.aws_secret_utils import handle_config_from_secret
+            # should not throw error
+            # "AttributeError: 'XConfig' object has no attribute 'x'"
+            handle_config_from_secret(config)
+
+            # should log 3 lines (one for each var set from a secret)
+            self.assertEqual(len(logger.output), 3)
+            self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0])
+            self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1])
+            self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2])
+            self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
+            self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
+            self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")

server/test/unit/common/test_corpora.py

@@ -1,9 +1,10 @@
-import unittest
-import anndata
 import json
-import tempfile
 import shutil
+import tempfile
+import unittest
 from http import HTTPStatus
+
+import anndata
 import requests
 
 from server.common.corpora import (
@@ -104,7 +105,7 @@ class CorporaRESTAPITest(unittest.TestCase):
             "project_links": json.dumps([
                 {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
             ]),
-            "default_embedding": "X_tsne"
+            "default_embedding": "X_tsne",
         }
         adata.uns.update(corpora_props)
         adata.write(path)

server/test/unit/common/test_writable_annotation.py

@@ -1,15 +1,148 @@
 import json
-from os import path, listdir
-import unittest
-import server.test.unit.decode_fbs as decode_fbs
 import shutil
+import unittest
+from os import path, listdir
+from unittest.mock import MagicMock, patch
 
 import numpy as np
 import pandas as pd
+import tiledb
+from flask import Flask
 
+import server.test.unit.decode_fbs as decode_fbs
+from server.common.errors import AnnotationCategoryNameError
 from server.common.rest import schema_get_helper, annotations_put_fbs_helper
-from server.test import data_with_tmp_annotations, make_fbs
 from server.data_common.matrix_loader import MatrixDataType
+from server.db.cellxgene_orm import CellxGeneDataset, Annotation
+from server.test import data_with_tmp_annotations, make_fbs, data_with_tmp_tiledb_annotations
+
+
+class auth(object):
+    def get_user_id():
+        return "1234"
+
+    def get_user_name():
+        return "person name"
+
+
+class WritableTileDBStoredAnnotationTest(unittest.TestCase):
+    def setUp(self):
+        self.user_id = '1234'
+        self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
+        self.data.dataset_config.user_annotations = self.annotations
+        self.db = self.annotations.db
+        self.n_rows = self.data.get_shape()[0]
+        self.test_dict = {
+            "cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
+            "cat_B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
+        }
+        self.fbs = make_fbs(self.test_dict)
+        self.df = pd.DataFrame(self.test_dict)
+        self.app = Flask('fake_app')
+        self.app.__setattr__("auth", auth)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def annotation_put_fbs(self, fbs):
+        annotations_put_fbs_helper(self.data, fbs)
+        res = json.dumps({"status": "OK"})
+        return res
+
+    def test_category_name_throws_errors_for_categories_that_cant_be_converted_to_filenames(self):
+        with self.app.test_request_context():
+            bad_category_names = make_fbs(
+                {
+                    "cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
+                    "cat/B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
+                }
+            )
+            with self.assertRaises(AnnotationCategoryNameError):
+                self.annotation_put_fbs(bad_category_names)
+
+    def test_convert_to_pandas__converts_tiledb_to_pandas_df(self):
+        with self.app.test_request_context():
+            self.annotations.write_labels(self.df, self.data)
+            dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
+            annotation = self.db.query_for_most_recent(
+                Annotation,
+                [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
+            )
+            # retrieve tiledb array
+            df = tiledb.open(annotation.tiledb_uri)
+            self.assertEqual(type(df), tiledb.array.SparseArray)
+
+            # convert to pandas df
+            pandas_df = self.annotations.convert_to_pandas_df(df, annotation.schema_hints)
+            self.assertEqual(type(pandas_df), pd.DataFrame)
+
+    def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
+        with self.app.test_request_context():
+            new_name = 'new_dataset/location'
+            self.data.get_location = MagicMock(return_value=new_name)
+            num_datasets = len(self.db.query([CellxGeneDataset]))
+            self.annotation_put_fbs(self.fbs)
+            more_datasets = len(self.db.query([CellxGeneDataset]))
+            self.assertGreater(more_datasets, num_datasets)
+
+            self.assertGreater(len(self.db.query([CellxGeneDataset], [CellxGeneDataset.name == new_name])), 0)
+
+    def test_write_labels_links_to_existing_dataset(self):
+        with self.app.test_request_context():
+            # add dataset to to db
+            self.annotation_put_fbs(self.fbs)
+
+            num_datasets = len(self.db.query([CellxGeneDataset]))
+
+            # create another annotation with the same dataset
+            self.annotation_put_fbs(self.fbs)
+
+            same_num_datasets = len(self.db.query([CellxGeneDataset]))
+
+            self.assertEqual(num_datasets, same_num_datasets)
+
+    def test_read_labels_returns_pandas_df(self):
+        with self.app.test_request_context():
+            self.annotation_put_fbs(self.fbs)
+            pandas_df = self.annotations.read_labels(self.data)
+            self.assertEqual(type(pandas_df), pd.DataFrame)
+
+    def test_read_labels_returns_df_matching_original(self):
+        with self.app.test_request_context():
+            self.annotation_put_fbs(self.fbs)
+            pandas_df = self.annotations.read_labels(self.data)
+
+            self.assertEqual(pandas_df.shape, (self.n_rows, 2))
+            self.assertEqual(set(pandas_df.columns), {"cat_A", "cat_B"})
+
+            self.assertTrue(self.data.original_obs_index.equals(pandas_df.index))
+
+            self.assertTrue(np.all(pandas_df["cat_A"] == ["label_A"] * self.n_rows))
+            self.assertTrue(np.all(pandas_df["cat_B"] == ["label_B"] * self.n_rows))
+
+    def test_error_checks(self):
+        # verify that the expected errors are generated
+        with self.app.test_request_context():
+            n_rows = self.data.get_shape()[0]
+            fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
+
+            # ensure we catch attempt to overwrite non-writable data
+            with self.assertRaises(KeyError):
+                self.annotation_put_fbs(fbs_bad)
+
+    @patch('server.common.annotations.hosted_tiledb.current_app')
+    def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
+        mock_user_id.auth.get_user_id.return_value = '1234'
+        self.annotations.write_labels(self.df, self.data)
+        # get uri
+        dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
+        annotation = self.db.query_for_most_recent(
+            Annotation,
+            [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)]
+        )
+
+        df = tiledb.open(annotation.tiledb_uri)
+        self.assertEqual(type(df), tiledb.array.SparseArray)
 
 
 class WritableAnnotationTest(unittest.TestCase):

server/test/unit/common/utils/test_cxg_generation_utils.py

@@ -0,0 +1,149 @@
+import json
+import unittest
+from os import popen, path, mkdir
+from shutil import rmtree
+from uuid import uuid4
+
+import numpy as np
+import tiledb
+from pandas import Series, DataFrame
+
+from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
+                                                      convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
+
+PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
+
+
+class TestCxgGenerationUtils(unittest.TestCase):
+    def setUp(self):
+        self.testing_cxg_temp_directory = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}"
+        mkdir(self.testing_cxg_temp_directory)
+
+    def tearDown(self):
+        if path.isdir(self.testing_cxg_temp_directory):
+            rmtree(self.testing_cxg_temp_directory)
+
+    def test__convert_dictionary_to_cxg_group__writes_successfully(self):
+        random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
+        dictionary_name = "favorite_desserts"
+        expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
+
+        convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
+                                        group_metadata_name=dictionary_name)
+
+        array = tiledb.open(expected_array_directory)
+        actual_stored_metadata = dict(array.meta.items())
+
+        self.assertTrue(path.isdir(expected_array_directory))
+        self.assertTrue(isinstance(array, tiledb.DenseArray))
+        self.assertEqual(random_dictionary, actual_stored_metadata)
+
+    def test__convert_dataframe_to_cxg_array__writes_successfully(self):
+        random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
+        random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
+        random_dataframe_name = f"random_dataframe_{uuid4()}"
+        random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
+
+        convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
+                                       "int_category", tiledb.Ctx())
+
+        expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
+        expected_array_metadata = {
+            "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
+                                      "index": "int_category"})}
+
+        actual_stored_dataframe_array = tiledb.open(expected_array_directory)
+        actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
+
+        self.assertTrue(path.isdir(expected_array_directory))
+        self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
+        self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
+        self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
+        self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
+
+    def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
+        ndarray = np.random.rand(3, 2)
+        ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
+
+        convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(ndarray_name)
+
+        self.assertTrue(path.isdir(ndarray_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
+        self.assertTrue((actual_stored_array[:, :] == ndarray).all())
+
+    def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
+        matrix = np.float32(np.random.rand(3, 2))
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
+        self.assertTrue((actual_stored_array[:, :] == matrix).all())
+
+    def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
+        matrix = np.zeros([3, 2])
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
+        matrix = np.zeros([3, 3])
+        matrix[0, 0] = 1
+        matrix[1, 1] = 1
+        matrix[2, 2] = 2
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[0, 0][''] == 1)
+        self.assertTrue(actual_stored_array[1, 1][''] == 1)
+        self.assertTrue(actual_stored_array[2, 2][''] == 2)
+        self.assertTrue(actual_stored_array[:, :][''].size == 3)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
+        matrix = np.ones((3, 2))
+        # The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
+        # a matrix of zeros which is sparse.
+        column_shift = np.ones((3, 2))
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
+                                    column_shift_for_sparse_encoding=column_shift)
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
+        matrix = np.ones((2, 2))
+        # Only column shift the first column of ones.
+        column_shift = np.array([[1, 0], [1, 0]])
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
+                                    column_shift_for_sparse_encoding=column_shift)
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[0, 1][''] == 1)
+        self.assertTrue(actual_stored_array[1, 1][''] == 1)
+        self.assertTrue(actual_stored_array[:, :][''].size == 2)

server/test/unit/common/utils/test_matrix_utils.py

@@ -0,0 +1,67 @@
+import unittest
+
+import numpy as np
+
+from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+
+
+class TestMatrixUtils(unittest.TestCase):
+
+    def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
+        matrix = np.array([1, 2, 3])
+
+        self.assertFalse(is_matrix_sparse(matrix, 0))
+        self.assertTrue(is_matrix_sparse(matrix, 100))
+
+    def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
+        matrix = np.zeros([3, 4])
+        matrix[2][3] = 1.0
+        matrix[1][1] = 2.2
+
+        self.assertTrue(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+        matrix[0][1] = 2.2
+        matrix[1][1] = 3.7
+
+        self.assertFalse(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
+        matrix = np.ones([20000, 20])
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
+
+    def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+
+        self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
+        matrix = np.ones([2, 2])
+        expected_column_shift = [1, 1]
+
+        actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
+        self.assertTrue((expected_column_shift == actual_column_shift).all())
+
+    def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
+        matrix = np.array([[1, 2], [3, 4]])
+
+        self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
+        matrix = np.random.rand(20000, 20)
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])

server/test/unit/common/utils/test_sanitization_utils.py

@@ -0,0 +1,56 @@
+import unittest
+
+from server.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
+
+
+class TestSanitizationUtils(unittest.TestCase):
+
+    def test__sanitize_values_in_list__not_strings_raises_exception(self):
+        keys_to_sanitize = [1, 2, 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
+        keys_to_sanitize = ["1", "2", 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
+        keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
+        expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
+
+        actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
+
+    def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
+        dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
+
+    def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
+        dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_keys_in_dictionary(dictionary_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
+        dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)

server/test/unit/common/utils/test_type_conversion_utils.py

@@ -0,0 +1,205 @@
+import unittest
+from time import time
+from unittest.mock import patch
+
+import numpy as np
+from pandas import Series, DataFrame
+
+from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
+    get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, convert_pandas_series_to_numpy
+
+
+class TestTypeConversionUtils(unittest.TestCase):
+
+    def test__can_cast_to_float32__string_is_false(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_float32__float64_is_true_warning_outputted(self):
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
+
+        with self.assertLogs(level="WARN") as logger:
+            can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
+            self.assertIn("may lose precision", logger.output[0])
+
+        self.assertTrue(can_cast)
+
+    @patch("logging.warning")
+    def test__can_cast_to_float32__float32_is_false(self, mock_log_warning):
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32))
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+        assert not mock_log_warning.called
+
+    def test__can_cast_to_float32__categorical_float64_is_false(self):
+        array_to_convert = Series(data=[1.1, 2.2, 3.3], dtype="category")
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_float32__categorical_int64_with_nans_is_true(self):
+        array_to_convert = Series(data=[1, 2, np.NaN], dtype="category")
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_float_32__float_32_with_nans_is_true(self):
+        array_to_convert = Series(data=[1, 2, np.NaN], dtype=np.dtype(np.float32))
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_int32__string_is_false(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_int32__int64_is_true(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_int32__int16_is_true(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_int32__int64_with_large_value_is_false(self):
+        array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_int32__int64_with_nans_is_false(self):
+        array_to_convert = Series(data=[np.NaN, "2", "3"], dtype="category")
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
+        types = [np.float32, np.int32, np.bool_, str]
+        expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
+
+    def test__get_dtype_of_array__categories_return_as_expected(self):
+        array = Series(data=["a", "b", "c"], dtype="category")
+        expected_dtype = np.unicode
+
+        actual_dtype = get_dtype_of_array(array)
+
+        self.assertEqual(expected_dtype, actual_dtype)
+
+    def test__get_dtype_of_array__unordered_integer_categories_return_as_expected(self):
+        array = Series(data=[2, 3, 1, 3, 1, 2], dtype="category")
+        expected_dtype = np.int32
+
+        actual_dtype = get_dtype_of_array(array)
+
+        self.assertEqual(expected_dtype, actual_dtype)
+
+    def test__get_dtype_of_array__castable_dtypes_return_as_expected(self):
+        types = [np.float64, np.int64]
+        expected_dtypes = [np.float32, np.int32]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
+
+    def test__get_dtype_of_array__unsupported_type_raises_exception(self):
+        unsupported_array = Series(list([time() for _ in range(2)]), dtype="datetime64[ns]")
+
+        with self.assertRaises(TypeError) as exception_context:
+            get_dtype_of_array(unsupported_array)
+
+        self.assertIn("unsupported", str(exception_context.exception))
+
+    def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self):
+        types = [np.float32, np.int32, np.bool_, str]
+        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
+
+    def test__get_schema_type_hint_of_array__categories_return_as_expected(self):
+        array = Series(data=["a", "b", "b"], dtype="category")
+        expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]}
+
+        actual_schema_hint = get_schema_type_hint_of_array(array)
+
+        self.assertEqual(expected_schema_hint, actual_schema_hint)
+
+    def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self):
+        types = [np.float64, np.int64]
+        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(
+                    f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
+                    i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
+
+    def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
+        float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
+        category_array = Series(data=["a", "b", "b"], dtype="category")
+        dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
+
+        expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
+        expected_schema_type_hints_dict = {"float_array": {"type": "float32"},
+                                           "category_array": {"type": "categorical", "categories": ["a", "b"]}}
+
+        actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
+
+        self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
+        self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)
+
+    def test__convert_pandas_series_to_numpy__categorical_float64_to_float64_with_nans(self):
+        expected_float_array = np.array([1.1, 2.2, np.NaN], dtype=np.float64)
+        float_series = Series(data=[1.1, 2.2, np.NaN], dtype="category")
+
+        actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
+
+        np.testing.assert_equal(expected_float_array, actual_float_array)
+
+    def test__convert_pandas_series_to_numpy__float64_to_float64(self):
+        expected_float_array = np.array([1.1, 2.2], dtype=np.float64)
+        float_series = Series(data=[1.1, 2.2], dtype=np.dtype(np.float64))
+
+        actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
+
+        np.testing.assert_equal(expected_float_array, actual_float_array)
+
+    def test__convert_pandas_series_to_numpy__int64_to_int32_with_nans_throws_error(self):
+        int_series = Series(data=[1, 2, np.NaN], dtype="category")
+
+        with self.assertLogs(level="ERROR") as logger:
+            convert_pandas_series_to_numpy(int_series, np.int32)
+
+            self.assertIn("Cannot convert a pandas Series object to an integer dtype if it contains NaNs",
+                          logger.output[0])

server/test/unit/common/utils/test_utils.py

@@ -2,7 +2,7 @@ import os
 import shutil
 import unittest
 
-from server.common.utils import import_plugins
+from server.common.utils.utils import import_plugins
 from server.test import PROJECT_ROOT, random_string
 
 

server/test/unit/compute/test_diffexp_cxg.py

@@ -1,14 +1,16 @@
+import os
+import tempfile
 import unittest
-from server.data_common.matrix_loader import MatrixDataLoader
-from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT
+
+import numpy as np
+
 import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
-from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
-from server.test.performance.create_test_matrix import create_test_h5ad
+from server.converters.h5ad_data_file import H5ADDataFile
 from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
-import numpy as np
-import tempfile
-import os
+from server.data_common.matrix_loader import MatrixDataLoader
+from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT
+from server.test.performance.create_test_matrix import create_test_h5ad
 
 
 class DiffExpTest(unittest.TestCase):
@@ -98,21 +100,22 @@ class DiffExpTest(unittest.TestCase):
     def sparse_diffexp(self, apply_col_shift):
         with tempfile.TemporaryDirectory() as dirname:
             # create a sparse matrix
-            h5adfile = os.path.join(dirname, "sparse.h5ad")
-            create_test_h5ad(h5adfile, 2000, 2000, 10, apply_col_shift)
-            adaptor_anndata = self.load_dataset(h5adfile, extra_dataset_config=dict(embeddings__names=[]))
-            adata = adaptor_anndata.data
+            h5adfile_path = os.path.join(dirname, "sparse.h5ad")
+            create_test_h5ad(h5adfile_path, 2000, 2000, 10, apply_col_shift)
+
+            h5ad_file_to_convert = H5ADDataFile(h5adfile_path, use_corpora_schema=False)
 
             sparsename = os.path.join(dirname, "sparse.cxg")
-            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="sparse.h5ad", title="sparse",)
-            write_cxg(adata=adata, container=sparsename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=11)
+            h5ad_file_to_convert.to_cxg(sparsename, 11, True)
+
+            adaptor_anndata = self.load_dataset(h5adfile_path, extra_dataset_config=dict(embeddings__names=[]))
+
             adaptor_sparse = self.load_dataset(sparsename)
             assert adaptor_sparse.open_array("X").schema.sparse
             assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
 
             densename = os.path.join(dirname, "dense.cxg")
-            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="dense.h5ad", title="dense",)
-            write_cxg(adata=adata, container=densename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=0)
+            h5ad_file_to_convert.to_cxg(densename, True, 0)
             adaptor_dense = self.load_dataset(densename)
             assert not adaptor_dense.open_array("X").schema.sparse
             assert not adaptor_dense.has_array("X_col_shift")

server/test/unit/converters/test_cxgtool.py

@@ -1,41 +0,0 @@
-import shutil
-import unittest
-
-import anndata
-
-from server.common.data_locator import DataLocator
-from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
-from server.data_cxg.cxg_adaptor import CxgAdaptor
-from server.test import PROJECT_ROOT, app_config, random_string
-from server.test.fixtures.fixtures import pbmc3k_colors
-
-
-class TestCxgAdaptor(unittest.TestCase):
-    def setUp(self) -> None:
-        self.fixtures = []
-
-    def tearDown(self) -> None:
-        try:
-            for data_locator in self.fixtures:
-                print("REMOVING ", data_locator)
-                shutil.rmtree(data_locator)
-        except FileNotFoundError:
-            pass
-
-    def test_cxg_category_colors(self):
-        data = self.convert_pbmc3k(extract_colors=True)
-        self.assertEqual(data.get_colors(), pbmc3k_colors)
-        data = self.convert_pbmc3k(extract_colors=False)
-        self.assertEqual(data.get_colors(), {})
-
-    def convert_pbmc3k(self, **kwargs):
-        rand_str = random_string(8)
-        data_locator = f"/tmp/test_{rand_str}.cxg"
-        self.fixtures.append(data_locator)
-        source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
-        cxg_group_metadata = create_cxg_group_metadata(
-            adata=source_h5ad, basefname="pbmc3k.h5ad", title="pbmc3k", **kwargs
-        )
-        write_cxg(adata=source_h5ad, container=data_locator, cxg_group_metadata=cxg_group_metadata)
-        config = app_config(data_locator)
-        return CxgAdaptor(DataLocator(data_locator), config)

server/test/unit/converters/test_h5ad_data_file.py

@@ -0,0 +1,235 @@
+import json
+import unittest
+from glob import glob
+from os import popen, remove, path
+from shutil import rmtree
+from uuid import uuid4
+
+import anndata
+import numpy as np
+from pandas import Series, DataFrame
+
+from server.common.utils.corpora_constants import CorporaConstants
+from server.converters.h5ad_data_file import H5ADDataFile
+
+PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
+
+
+class TestH5ADDataFile(unittest.TestCase):
+
+    def setUp(self):
+        self.sample_anndata = self._create_sample_anndata_dataset()
+        self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
+
+        self.sample_output_directory = path.splitext(self.sample_h5ad_filename)[0] + ".cxg"
+
+    def tearDown(self):
+        if self.sample_h5ad_filename:
+            remove(self.sample_h5ad_filename)
+
+        if path.isdir(self.sample_output_directory):
+            rmtree(self.sample_output_directory)
+
+    def test__create_h5ad_data_file__non_h5ad_raises_exception(self):
+        non_h5ad_filename = "my_fancy_dataset.csv"
+
+        with self.assertRaises(Exception) as exception_context:
+            H5ADDataFile(non_h5ad_filename)
+
+        self.assertIn("File must be an H5AD", str(exception_context.exception))
+
+    def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
+        with self.assertLogs(level="WARN") as logger:
+            H5ADDataFile(self.sample_h5ad_filename, dataset_title="My Awesome Dataset",
+                         dataset_about="http://www.awesomedataset.com", use_corpora_schema=False)
+
+        self.assertIn("will override any metadata that is extracted", logger.output[0])
+
+    def test__create_h5ad_data_file__reads_anndata_successfully(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+
+        self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
+        self.assertEqual(h5ad_file.anndata.obs.sort_index(inplace=True),
+                         self.sample_anndata.obs.sort_index(inplace=True))
+        self.assertEqual(h5ad_file.anndata.var.sort_index(inplace=True),
+                         self.sample_anndata.var.sort_index(inplace=True))
+
+        for key in h5ad_file.anndata.obsm.keys():
+            self.assertIn(key, self.sample_anndata.obsm.keys())
+            self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
+
+        for key in self.sample_anndata.obsm.keys():
+            self.assertIn(key, h5ad_file.anndata.obsm.keys())
+            self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
+
+    def test__create_h5ad_data_file__copies_index_of_obs_and_var_to_column(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+
+        # The automatic name chosen for the index should be "name_0"
+        self.assertNotIn("name_0", self.sample_anndata.obs.columns)
+        self.assertIn("name_0", h5ad_file.obs.columns)
+
+        self.assertNotIn("name_0", self.sample_anndata.var.columns)
+        self.assertIn("name_0", h5ad_file.var.columns)
+
+    def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
+                                 obs_index_column_name="float_category", vars_index_column_name="int_category")
+
+        self.assertNotIn("name_0", h5ad_file.obs.columns)
+        self.assertNotIn("name_0", h5ad_file.var.columns)
+
+    def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
+
+        with self.assertRaises(Exception) as exception_context:
+            H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
+                         obs_index_column_name="float_category", vars_index_column_name="bool_category")
+
+        self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
+
+    def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
+        with self.assertRaises(Exception) as exception_context:
+            H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
+                         obs_index_column_name="unknown_category", vars_index_column_name="i_dont_exist")
+
+        self.assertIn("does not exist", str(exception_context.exception))
+
+    def test__create_h5ad_data_file__extract_about_and_title_from_dataset(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
+
+        self.assertEqual(h5ad_file.dataset_title, "random_link_name")
+        self.assertEqual(h5ad_file.dataset_about, "www.link.com")
+
+    def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, dataset_about="override_about",
+                                 dataset_title="override_title")
+
+        self.assertEqual(h5ad_file.dataset_title, "override_title")
+        self.assertEqual(h5ad_file.dataset_about, "override_about")
+
+    def test__to_cxg__simple_anndata_no_corpora_and_sparse(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+        h5ad_file.to_cxg(self.sample_output_directory, 100)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__simple_anndata_with_corpora_and_sparse(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
+        h5ad_file.to_cxg(self.sample_output_directory, 100)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__simple_anndata_no_corpora_and_dense(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+        h5ad_file.to_cxg(self.sample_output_directory, 0)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__simple_anndata_with_corpora_and_dense(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
+        h5ad_file.to_cxg(self.sample_output_directory, 0)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__with_sparse_column_encoding(self):
+        anndata = self._create_sample_anndata_dataset()
+        anndata.X = np.ones((3, 4))
+        sparse_with_column_shift_filename = self._write_anndata_to_file(anndata)
+
+        h5ad_file = H5ADDataFile(sparse_with_column_shift_filename)
+        h5ad_file.to_cxg(self.sample_output_directory, 50)
+
+        self._validate_expected_generated_list_of_tiledb_files(has_column_encoding=True)
+
+        # Clean up
+        remove(sparse_with_column_shift_filename)
+
+    def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False):
+        expected_directories, expected_obs_files, expected_var_files = \
+            self._get_expected_generated_list_of_tiledb_files()
+
+        for directory in expected_directories:
+            self.assertTrue(path.isdir(directory))
+
+        for obs_file in expected_obs_files:
+            expected_location_of_obs_file = f"{self.sample_output_directory}/obs/*/{obs_file}"
+            self.assertTrue(path.isfile(glob(expected_location_of_obs_file)[0]))
+
+        for var_file in expected_var_files:
+            expected_location_of_var_file = f"{self.sample_output_directory}/var/*/{var_file}"
+            self.assertTrue(path.isfile(glob(expected_location_of_var_file)[0]))
+
+        if has_column_encoding:
+            self.assertTrue(path.isdir(f"{self.sample_output_directory}/X_col_shift"))
+
+    def _get_expected_generated_list_of_tiledb_files(self):
+
+        # Expected directories
+        metadata_directory = f"{self.sample_output_directory}/cxg_group_metadata"
+        main_x_directory = f"{self.sample_output_directory}/X"
+        overall_embedding_directory = f"{self.sample_output_directory}/emb"
+        specific_embedding_directory = f"{self.sample_output_directory}/emb/awesome_embedding"
+        obs_directory = f"{self.sample_output_directory}/obs"
+        var_directory = f"{self.sample_output_directory}/var"
+
+        # Obs files
+        obs_files = []
+        obs_files.append("name_0.tdb")
+        obs_files.append("name_0_var.tdb")
+        obs_files.append("string_category.tdb")
+        obs_files.append("string_category_var.tdb")
+        obs_files.append("float_category.tdb")
+
+        # Var files
+        var_files = []
+        var_files.append("name_0.tdb")
+        var_files.append("name_0_var.tdb")
+        var_files.append("bool_category.tdb")
+        var_files.append("int_category.tdb")
+
+        return [metadata_directory, main_x_directory, overall_embedding_directory, specific_embedding_directory,
+                obs_directory, var_directory], obs_files, var_files
+
+    def _write_anndata_to_file(self, anndata):
+        temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad"
+        anndata.write(temporary_filename)
+
+        return temporary_filename
+
+    def _create_sample_anndata_dataset(self):
+        # Create X
+        X = np.random.rand(3, 4)
+
+        # Create obs
+        random_string_category = Series(data=["a", "b", "b"], dtype="category")
+        random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
+        obs_dataframe = DataFrame(
+            data={"string_category": random_string_category, "float_category": random_float_category})
+        obs = obs_dataframe
+
+        # Create vars
+        random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int32)
+        random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
+        var_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
+        var = var_dataframe
+
+        # Create embeddings
+        random_embedding = np.random.rand(3, 2)
+        obsm = {"X_awesome_embedding": random_embedding}
+
+        # Create uns corpora metadata
+        uns = {}
+        for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
+            uns[metadata_field] = "random"
+
+        for metadata_field in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
+            uns[metadata_field] = json.dumps({"random_key": "random_value"})
+
+        # Need to carefully set the corpora schema versions in order for tests to pass.
+        uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+
+        # Set project links to be a dictionary
+        uns["project_links"] = json.dumps(
+            [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}])
+
+        return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)

server/test/unit/data_anndata/test_nan_anndata_adaptor.py

@@ -22,8 +22,9 @@ class NaNTest(unittest.TestCase):
             self.data._create_schema()
 
     def test_load(self):
-        with self.assertWarns(UserWarning):
+        with self.assertLogs(level="WARN") as logger:
             self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.assertTrue(logger.output)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)

setup.py

@@ -11,7 +11,7 @@ with open("server/requirements-prepare.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="0.16.0",
+    version="0.16.4",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",