fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-05 20:58:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Introduce a config file to cellxgene (#1264)

Browse Source 
* Introduce a config file to cellxgene

The config file format is in yaml.  The default config is located
in server/common/default_config.py.  A user may create a yaml file
that contains a subset of these fields.  It can be used during cellxgene
launch, or for hosted cellxgene.

The code has also been refactored.  Much of the logic to check arguments
has moved from launch to app config.

It is now possible to set the tiledb context parameters using the config
file.  Other feature will soon be handled in a similar way.
This commit is contained in:
bmccandless
2020-03-22 09:34:11 -07:00
committed by GitHub
parent 1351c8f724
commit 8180be83b8
19 changed files with 691 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/data_cxg/cxg_adaptor.py
+20 -8
View File
@@ -1,7 +1,7 @@
import os
import json
from server.common.utils import dtype_to_schema
from server.common.errors import DatasetAccessError
from server.common.errors import DatasetAccessError, ConfigurationError
from server.common.utils import path_join
from server.common.constants import Axis
from server.data_common.data_adaptor import DataAdaptor
@@ -16,7 +16,11 @@ import threading
class CxgAdaptor(DataAdaptor):
# TODO: The tiledb context parameters should be a configuration option
tiledb_ctx = tiledb.Ctx({"sm.tile_cache_size": 8 * 1024 * 1024 * 1024, "sm.num_reader_threads": 32})
tiledb_ctx = tiledb.Ctx({
"sm.tile_cache_size": 8 * 1024 * 1024 * 1024,
"sm.num_reader_threads": 32,
"vfs.s3.region": "us-east-1"
})
def __init__(self, data_locator, config=None):
super().__init__(config)
@@ -36,6 +40,14 @@ class CxgAdaptor(DataAdaptor):
array.close()
self.arrays.clear()
@staticmethod
def set_tiledb_context(context_params):
"""Set the tiledb context. This should be set before any instances of CxgAdaptor are created"""
try:
CxgAdaptor.tiledb_ctx = tiledb.Ctx(context_params)
except tiledb.libtiledb.TileDBError as e:
raise ConfigurationError(f"Invalid tiledb context: {str(e)}")
@staticmethod
def pre_load_validation(data_locator):
location = data_locator.uri_or_path
@@ -100,15 +112,15 @@ class CxgAdaptor(DataAdaptor):
Return True if this looks like a valid CXG, False if not. Just a quick/cheap
test, not to be fully trusted.
"""
if not tiledb.object_type(url) == "group":
if not tiledb.object_type(url, ctx=CxgAdaptor.tiledb_ctx) == "group":
return False
if not tiledb.object_type(path_join(url, "obs")) == "array":
if not tiledb.object_type(path_join(url, "obs"), ctx=CxgAdaptor.tiledb_ctx) == "array":
return False
if not tiledb.object_type(path_join(url, "var")) == "array":
if not tiledb.object_type(path_join(url, "var"), ctx=CxgAdaptor.tiledb_ctx) == "array":
return False
if not tiledb.object_type(path_join(url, "X")) == "array":
if not tiledb.object_type(path_join(url, "X"), ctx=CxgAdaptor.tiledb_ctx) == "array":
return False
if not tiledb.object_type(path_join(url, "emb")) == "group":
if not tiledb.object_type(path_join(url, "emb"), ctx=CxgAdaptor.tiledb_ctx) == "group":
return False
return True
@@ -126,7 +138,7 @@ class CxgAdaptor(DataAdaptor):
* version 0.1 -- metadata attache to cxg_group_metadata array.
Same as 0, except it adds group metadata.
"""
a_type = tiledb.object_type(path_join(self.url, "cxg_group_metadata"))
a_type = tiledb.object_type(path_join(self.url, "cxg_group_metadata"), ctx=self.tiledb_ctx)
if a_type is None:
# version 0
cxg_version = "0.0"