initial support for corpora schema conventions (#1676)

* initial support for corpora schema conventions

* remove debugging print

* add corpora util module

* tests

* lint

* PR review edits

* PR changes

* more PR changes

* more PR chnages

* PR fixes

* formatting

* PR updates

* lint

* PR review

server/app/app.py

@@ -130,7 +130,7 @@ def get_data_adaptor(url_dataroot=None, dataset=None):
         # sufficient to check that the datapath starts with the
         # dataroot to determine that the datapath is under the dataroot.
         if not datapath.startswith(dataroot):
-            raise DatasetAccessError("Invalid dataset {url_dataroot}/{dataset}")
+            raise DatasetAccessError(f"Invalid dataset {url_dataroot}/{dataset}")
 
     if datapath is None:
         return common_rest.abort_and_log(HTTPStatus.BAD_REQUEST, "Invalid dataset NONE", loglevel=logging.INFO)

server/common/app_config.py

@@ -242,6 +242,12 @@ class AppConfig(object):
             "about_legal_privacy": dataset_config.app__about_legal_privacy,
         }
 
+        # dataset_props
+        # TODO/Note: putting info from the dataset into the /config is not ideal.
+        # However, it is definitely not part of /schema, and we do not have a top-level
+        # route for data properties.  Consider creating one at some point.
+        corpora_props = data_adaptor.get_corpora_props()
+
         data_adaptor.update_parameters(parameters)
         if annotation:
             annotation.update_parameters(parameters, data_adaptor)
@@ -254,6 +260,7 @@ class AppConfig(object):
         config["library_versions"] = library_versions
         config["links"] = links
         config["parameters"] = parameters
+        config["corpora_props"] = corpora_props
         config["limits"] = {
             "column_request_max": server_config.limits__column_request_max,
             "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
@@ -825,7 +832,8 @@ class DatasetConfig(BaseConfig):
         if server_config.single_dataset__datapath:
             if self.embeddings__enable_reembedding:
                 matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config)
+                    server_config.single_dataset__datapath, app_config=self.app_config
+                )
                 if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
                     raise ConfigurationError("'enable-reembedding is only supported with H5AD files.")
                 if server_config.adaptor__anndata_adaptor__backed:

server/common/corpora.py

@@ -0,0 +1,90 @@
+"""
+Corpora schema conventions support.  Helper functions for reading.
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
+"""
+import collections
+import json
+
+from server.cli.upgrade import validate_version_str
+
+
+def corpora_get_versions_from_anndata(adata):
+    """
+    Given an AnnData object, return:
+        * None - if not a Corpora object
+        * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
+
+    Implements the identification protocol defined in the specification.
+    """
+
+    # per Corpora AnnData spec, this is a corpora file if the following is true
+    if "version" not in adata.uns_keys():
+        return None
+    version = adata.uns["version"]
+    if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
+        return None
+
+    corpora_schema_version = version.get("corpora_schema_version")
+    corpora_encoding_version = version.get("corpora_encoding_version")
+
+    # TODO: spec says these must be SEMVER values, so check.
+    if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
+        return [corpora_schema_version, corpora_encoding_version]
+
+
+def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
+    return (
+        corpora_schema_version
+        and corpora_encoding_version
+        and corpora_schema_version.startswith("1.")
+        and corpora_encoding_version.startswith("0.1.")
+    )
+
+
+def corpora_get_props_from_anndata(adata):
+    """
+    Get Corpora dataset properties from an AnnData
+    """
+    versions = corpora_get_versions_from_anndata(adata)
+    if versions is None:
+        return None
+    [corpora_schema_version, corpora_encoding_version] = versions
+    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
+    if not version_is_supported:
+        raise ValueError("Unsupported Corpora schema version")
+
+    required_simple_fields = [
+        "version",
+        "title",
+        "layer_descriptions",
+        "organism",
+        "organism_ontology_term_id",
+        "project_name",
+        "project_description",
+    ]
+    # Spec says some values encoded as JSON due to the inability of AnnData to store complex types.
+    required_json_fields = ["contributors", "project_links"]
+    optional_simple_fields = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]
+
+    corpora_props = {}
+    for key in required_simple_fields:
+        if key not in adata.uns:
+            raise KeyError(f"missing Corpora schema field {key}")
+        corpora_props[key] = adata.uns[key]
+
+    for key in required_json_fields:
+        if key not in adata.uns:
+            raise KeyError(f"missing Corpora schema field {key}")
+        try:
+            corpora_props[key] = json.loads(adata.uns[key])
+        except json.JSONDecodeError:
+            raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
+
+    for key in optional_simple_fields:
+        if key in adata.uns:
+            corpora_props[key] = adata.uns[key]
+
+    return corpora_props

server/converters/cxgtool.py

@@ -1,70 +1,9 @@
 """
 This program converts an [AnnData H5AD](https://anndata.readthedocs.io/en/stable/)
-into a cellxgene TileDB structure, aka a 'CXG'.
-
-The organization of the TileDB structure is:
-
-    the.cxg                         TileDB Group
-    ├─ obs                          TileDB array containing cell (row) attributes, one attribute per
-    │                               dataframe column, shape (n_obs,)
-    ├─ var                          TileDB array containing gene (column) attributes, with one attribute per
-    │                               dataframe column, shape (n_var,)
-    ├─ X                            Main count matrix as a 2D TileDB array, single unnamed numeric attribute
-    ├─ X_col_shift                  TilebDB Array used in column shift encoding, shape (n_var,), dtype = X.dtype.
-    │                               Single unnamed numeric attribute.  If this array is sparse, and X_col_shift exists,
-    │                               then all values in the i'th column were subtracted by X_col_shift[i].
-    ├─ emb                          TileDB group, storing optional embeddings (group may be empty)
-    │  └─ <name1>                   TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
-    └─ cxg_group_metadata           Empty array used only to stash metadata about the overall object.
-       └─ cxg_category_colors       CXG colors object as described below:
-                                        {
-                                             "<category_name>": {
-                                                 "<label_name>": "<color_hex_code>",
-                                                 ...
-                                             },
-                                             ...
-                                        }
-    ...
-
-All arrays are defined to have a uint32 domain, zero based.  All X counts and embedding
-coordinates are coerced to float32, which is ample precision for visualization purposes.
-Dataframe (metadata) types are generally preserved, or where that is not possible,
-converted to something with equal representative value in the cellxgene application
-(eg, categorical types are converted to string, bools to uint8, etc).
-
-The following objects are also decorated with auxiliary metadata using TileDB
-array metadata:
-
-* cxg_group_metadata: minimally, will contain a 'cxg_version' field, which
-  is a semver string identifying the version number of the CXG layout.
-  It may also contain 'cxg_parameters', a JSON-encoded parameter list
-  describing CXG-wide dataset parameters.
-
-* obs, var: both contain an optional 'cxg_schema' field that is a json string,
-  containing per-column (attribute) schema hinting.  This is used where the TileDB
-  native typing information is insufficient to reconstruct useful information
-  such as categorical typing from Pandas DataFrames, and to communicate which column
-  is the preferred human-readable index for obs & var.
-
-This file also embodies a number of empirically derived tiledb schema parameters,
-including the global data layout, spatial tile size, and the like. The CXG is
-self-describing in these areas, and the actual values (eg, tile size) are empirically
-derived from benchmarking. They may change in the future.
-
-cxgtool.py will extract color information stored in arrays in the 'uns' anndata
-property with the key "{category_name}_colors". For this to work, the following
-command must result in a mapping from category names to matplotlib-compatible colors:
-
-```
-dict(zip(adata.obs[cat].cat.categories, adata.uns[f"{cat}_colors"]))
-```
-
----
-
-TODO/ISSUES:
-* add sub-command structure to argparse, for future sub-commands
-* Possible future work: accept Loom files
+into a cellxgene TileDB structure, aka a [CXG](../../dev_docs/cxg.md).
 
+IF YOU UPDATE THIS FILE, IN ANY WAY THAT MODIFIES THE CXG FORMAT or CONTENTS,
+YOU MUST UPDATE THE CXG SPECIFICATION and VERSION NUMBER.
 """
 import re
 import anndata
@@ -77,10 +16,16 @@ from scipy.stats import mode
 
 from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from server.common.errors import ColorFormatException
+from server.common.corpora import (
+    corpora_get_props_from_anndata,
+    corpora_get_versions_from_anndata,
+    corpora_is_version_supported,
+)
 
 
-# the CXG container version number.  Must be a semver string.
-CXG_VERSION = "0.1"
+# the CXG container version number.  Must be a semver string (major.minor.patch)
+# DO NOT UPDATE THIS WITHOUT ALSO UPDATING THE CXG SPECIFICATION.
+CXG_VERSION = "0.2.0"
 
 # log_level must have a default
 log_level = 3
@@ -125,6 +70,12 @@ def main():
         default=0.0,  # force dense by default
         help="The X array will be sparse if the percent of non-zeros falls below this value",
     )
+    parser.add_argument(
+        "--disable-corpora",
+        action="store_true",
+        default=False,
+        help="Disable extraction and storing of Corpora schema information.",
+    )
     args = parser.parse_args()
 
     global log_level
@@ -136,25 +87,30 @@ def main():
     basefname = splitext(basename(args.h5ad))[0]
     out = args.out if args.out is not None else basefname
     container = out if splitext(out)[1] == ".cxg" else out + ".cxg"
-    title = args.title if args.title is not None else basefname
+
+    corpora_props = load_corpora_props(args, adata) if not args.disable_corpora else None
+    cxg_group_metadata = create_cxg_group_metadata(
+        adata,
+        basefname,
+        title=args.title,
+        about=args.about,
+        corpora_props=corpora_props,
+        extract_colors=not args.disable_custom_colors,
+    )
 
     write_cxg(
         adata,
         container,
-        title,
+        cxg_group_metadata=cxg_group_metadata,
         var_names=args.var_names,
         obs_names=args.obs_names,
-        about=args.about,
-        extract_colors=not args.disable_custom_colors,
         sparse_threshold=args.sparse_threshold,
     )
 
     log(1, "done")
 
 
-def write_cxg(
-    adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False, sparse_threshold=5.0
-):
+def write_cxg(adata, container, cxg_group_metadata, var_names=None, obs_names=None, sparse_threshold=5.0):
     if not adata.var.index.is_unique:
         raise ValueError("Variable index is not unique - unable to convert.")
     if not adata.obs.index.is_unique:
@@ -179,19 +135,7 @@ def write_cxg(
     log(1, f"\t...group created, with name {container}")
 
     # dataset metadata
-    metadata_dict = dict(cxg_version=CXG_VERSION, cxg_properties=json.dumps({"title": title, "about": about}))
-    if extract_colors:
-        try:
-            metadata_dict["cxg_category_colors"] = json.dumps(
-                convert_anndata_category_colors_to_cxg_category_colors(adata)
-            )
-        except ColorFormatException:
-            log(
-                0,
-                "Warning: failed to extract colors from h5ad file! "
-                "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.",
-            )
-    save_metadata(container, metadata_dict)
+    save_metadata(container, cxg_group_metadata)
     log(1, "\t...dataset metadata saved")
 
     # var/gene dataframe
@@ -601,6 +545,59 @@ def save_metadata(container, metadata_dict):
             A.meta[k] = v
 
 
+def load_corpora_props(args, adata):
+    versions = corpora_get_versions_from_anndata(adata)
+    if versions is None:
+        return None
+
+    [corpora_schema_version, corpora_encoding_version] = versions
+    corpora_props = corpora_get_props_from_anndata(adata)
+    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
+    if not version_is_supported or not corpora_props:
+        log(0, "ERROR: Unknown source file schema version is unsupported")
+        raise ValueError("Unsupported Corpora schema version")
+
+    log(1, "FYI, file appears to be encoded using Corpora schema standards...")
+    if args.title is not None or args.about is not None:
+        log(0, "Warning: explicit specification of --title or --about will override Corpora schema fields.")
+
+    return corpora_props
+
+
+def create_cxg_group_metadata(adata, basefname, title=None, about=None, corpora_props=None, extract_colors=True):
+
+    if corpora_props is not None:
+        # clobber encoding version to be OUR version, not the source H5AD encoding
+        corpora_props["version"].update({"corpora_encoding_version": CXG_VERSION})
+        corpora_project_links = corpora_props.get("project_links", [])
+        corpora_about_link = next(
+            (link for link in corpora_project_links if (link.get("link_type", None) == "SUMMARY")), {}
+        )
+    else:
+        corpora_about_link = {}
+
+    title = title or corpora_about_link.get("link_name", basefname)
+    about = about or corpora_about_link.get("link_url")
+
+    cxg_group_metadata = {"cxg_version": CXG_VERSION, "cxg_properties": json.dumps({"title": title, "about": about})}
+    if corpora_props is not None:
+        cxg_group_metadata.update({"corpora": json.dumps(corpora_props)})
+
+    if extract_colors:
+        try:
+            cxg_group_metadata["cxg_category_colors"] = json.dumps(
+                convert_anndata_category_colors_to_cxg_category_colors(adata)
+            )
+        except ColorFormatException:
+            log(
+                0,
+                "Warning: failed to extract colors from h5ad file! "
+                "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.",
+            )
+
+    return cxg_group_metadata
+
+
 def sanitize_keys(keys):
     """
     We need names to be safe to use as attribute names in tiledb.  See:

server/data_anndata/anndata_adaptor.py

@@ -17,6 +17,7 @@ from server.common.constants import Axis, MAX_LAYOUTS
 from server.common.errors import PrepareError, DatasetAccessError, FilterError
 from server.compute.scanpy import scanpy_umap
 import server.compute.diffexp_generic as diffexp_generic
+from server.common.corpora import corpora_get_props_from_anndata
 
 anndata_version = version.parse(str(anndata.__version__)).release
 
@@ -57,6 +58,9 @@ class AnndataAdaptor(DataAdaptor):
     def open(data_locator, app_config, dataset_config=None):
         return AnndataAdaptor(data_locator, app_config, dataset_config)
 
+    def get_corpora_props(self):
+        return corpora_get_props_from_anndata(self.data)
+
     def get_name(self):
         return "cellxgene anndata adaptor version"
 

server/data_common/data_adaptor.py

@@ -135,6 +135,9 @@ class DataAdaptor(metaclass=ABCMeta):
             location = location[:-1]
         return splitext(basename(location))[0]
 
+    def get_corpora_props(self):
+        return None
+
     @abstractmethod
     def get_schema(self):
         """

server/data_cxg/cxg_adaptor.py

@@ -74,6 +74,9 @@ class CxgAdaptor(DataAdaptor):
     def get_title(self):
         return self.title if self.title else super().get_title()
 
+    def get_corpora_props(self):
+        return self.corpora_props if self.corpora_props else super().get_corpora_props()
+
     def get_name(self):
         return "cellxgene cxg adaptor version"
 
@@ -144,26 +147,31 @@ class CxgAdaptor(DataAdaptor):
         * version 0.1 -- metadata attache to cxg_group_metadata array.
           Same as 0, except it adds group metadata.
         """
+        title = None
+        about = None
+        corpora_props = None
         if self.has_array("cxg_group_metadata"):
             # version >0
             gmd = self.open_array("cxg_group_metadata")
             cxg_version = gmd.meta["cxg_version"]
-            if cxg_version == "0.1":
+            # version 0.1 used a malformed/shorthand semver string.
+            if cxg_version == "0.1" or cxg_version == "0.2.0":
                 cxg_properties = json.loads(gmd.meta["cxg_properties"])
                 title = cxg_properties.get("title", None)
                 about = cxg_properties.get("about", None)
+            if cxg_version == "0.2.0":
+                corpora_props = json.loads(gmd.meta["corpora"]) if "corpora" in gmd.meta else None
         else:
             # version 0
             cxg_version = "0.0"
-            title = None
-            about = None
 
-        if cxg_version not in ["0.0", "0.1"]:
+        if cxg_version not in ["0.0", "0.1", "0.2.0"]:
             raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
 
         self.title = title
         self.about = about
         self.cxg_version = cxg_version
+        self.corpora_props = corpora_props
 
     @staticmethod
     def _open_array(uri, tiledb_ctx):

server/test/test_anndata_adaptor_data_load.py

@@ -43,13 +43,10 @@ class DataLocatorAdaptorTest(unittest.TestCase):
     def get_basic_config(self):
         config = AppConfig()
         config.update_server_config(
-            single_dataset__obs_names=None,
-            single_dataset__var_names=None,
+            single_dataset__obs_names=None, single_dataset__var_names=None,
         )
         config.update_default_dataset_config(
-            embeddings__names=["umap"],
-            presentation__max_categories=100,
-            diffexp__lfc_cutoff=0.01,
+            embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
         )
         return config
 

server/test/test_corpora.py

@@ -0,0 +1,73 @@
+import unittest
+import anndata
+import json
+
+from server.common.corpora import (
+    corpora_get_versions_from_anndata,
+    corpora_is_version_supported,
+    corpora_get_props_from_anndata,
+)
+from server.test import PROJECT_ROOT
+
+
+class CorporaAPITest(unittest.TestCase):
+    def test_corpora_get_versions_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something legit
+        adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
+        self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
+
+    def test_corpora_is_version_supported(self):
+        self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
+
+    def test_corpora_get_props_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # unsupported version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
+        with self.assertRaises(ValueError):
+            corpora_get_props_from_anndata(adata)
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        some_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+            "project_name": "project_name",
+            "project_description": "project_description",
+            "contributors": json.dumps([{"contributors": "contributors"}]),
+            "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
+        }
+        for k in some_fields:
+            adata.uns[k] = some_fields[k]
+        some_fields["contributors"] = json.loads(some_fields["contributors"])
+        some_fields["project_links"] = json.loads(some_fields["project_links"])
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

server/test/test_cxgtool.py

@@ -4,7 +4,7 @@ import unittest
 import anndata
 
 from server.common.data_locator import DataLocator
-from server.converters.cxgtool import write_cxg
+from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
 from server.data_cxg.cxg_adaptor import CxgAdaptor
 from server.test import PROJECT_ROOT, app_config, random_string
 from server.test.test_datasets.fixtures import pbmc3k_colors
@@ -33,6 +33,9 @@ class TestCxgAdaptor(unittest.TestCase):
         data_locator = f"/tmp/test_{rand_str}.cxg"
         self.fixtures.append(data_locator)
         source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
-        write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs)
+        cxg_group_metadata = create_cxg_group_metadata(
+            adata=source_h5ad, basefname="pbmc3k.h5ad", title="pbmc3k", **kwargs
+        )
+        write_cxg(adata=source_h5ad, container=data_locator, cxg_group_metadata=cxg_group_metadata)
         config = app_config(data_locator)
         return CxgAdaptor(DataLocator(data_locator), config)

server/test/test_diffexp.py

@@ -3,7 +3,7 @@ from server.data_common.matrix_loader import MatrixDataLoader
 from server.test import PROJECT_ROOT, app_config
 import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
-from server.converters.cxgtool import write_cxg
+from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
 from server.test.create_test_matrix import create_test_h5ad
 from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
 import numpy as np
@@ -16,8 +16,7 @@ class DiffExpTest(unittest.TestCase):
     adaptor types and different algorithms."""
 
     def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
-        config = app_config(path, extra_server_config=extra_server_config,
-                            extra_dataset_config=extra_dataset_config)
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
         loader = MatrixDataLoader(path)
         adaptor = loader.open(config)
         return adaptor
@@ -105,13 +104,15 @@ class DiffExpTest(unittest.TestCase):
             adata = adaptor_anndata.data
 
             sparsename = os.path.join(dirname, "sparse.cxg")
-            write_cxg(adata=adata, container=sparsename, title="sparse", sparse_threshold=11)
+            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="sparse.h5ad", title="sparse",)
+            write_cxg(adata=adata, container=sparsename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=11)
             adaptor_sparse = self.load_dataset(sparsename)
             assert adaptor_sparse.open_array("X").schema.sparse
             assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
 
             densename = os.path.join(dirname, "dense.cxg")
-            write_cxg(adata=adata, container=densename, title="dense", sparse_threshold=0)
+            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="dense.h5ad", title="dense",)
+            write_cxg(adata=adata, container=densename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=0)
             adaptor_dense = self.load_dataset(densename)
             assert not adaptor_dense.open_array("X").schema.sparse
             assert not adaptor_dense.has_array("X_col_shift")