fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-26 19:38:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Introduce a config file to cellxgene (#1264)

Browse Source 
* Introduce a config file to cellxgene

The config file format is in yaml.  The default config is located
in server/common/default_config.py.  A user may create a yaml file
that contains a subset of these fields.  It can be used during cellxgene
launch, or for hosted cellxgene.

The code has also been refactored.  Much of the logic to check arguments
has moved from launch to app config.

It is now possible to set the tiledb context parameters using the config
file.  Other feature will soon be handled in a similar way.
This commit is contained in:
bmccandless
2020-03-22 09:34:11 -07:00
committed by GitHub
parent 1351c8f724
commit 8180be83b8
19 changed files with 691 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/app.py
+14 -14
View File
@@ -26,14 +26,14 @@ webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
def dataset_index(dataset=None):
config = current_app.app_config
if dataset is None:
if config.datapath:
location = config.datapath
if config.single_dataset__datapath:
location = config.single_dataset__datapath
else:
return dataroot_index()
else:
location = path_join(config.dataroot, dataset)
location = path_join(config.multi_dataset__dataroot, dataset)
scripts = config.scripts
scripts = config.server__scripts
try:
cache_manager = current_app.matrix_data_cache_manager
@@ -59,13 +59,13 @@ def get_data_adaptor(dataset=None):
config = current_app.app_config
if dataset is None:
datapath = config.datapath
datapath = config.single_dataset__datapath
else:
datapath = path_join(config.dataroot, dataset)
datapath = path_join(config.multi_dataset__dataroot, dataset)
# path_join returns a normalized path. Therefore it is
# sufficient to check that the datapath starts with the
# dataroot to determine that the datapath is under the dataroot.
if not datapath.startswith(config.dataroot):
if not datapath.startswith(config.multi_dataset__dataroot):
raise DatasetAccessError("Invalid dataset {dataset}")
if datapath is None:
@@ -100,10 +100,10 @@ def dataroot_test_index():
try:
config = current_app.app_config
locator = DataLocator(config.dataroot)
locator = DataLocator(config.multi_dataset__dataroot)
datasets = []
for fname in locator.ls():
location = path_join(config.dataroot, fname)
location = path_join(config.multi_dataset__dataroot, fname)
try:
MatrixDataLoader(location, app_config=config)
datasets.append(fname)
@@ -118,7 +118,7 @@ def dataroot_test_index():
data += f"<li><a href={dataset}>{dataset}</a></li>"
data += "</ul>"
except Exception as e:
data += f"<br/>Unable to locate datasets from {config.dataroot}: {str(e)}"
data += f'<br/>Unable to locate datasets from {config.multi_dataset__dataroot}: {str(e)}'
data += "</body></html>"
return make_response(data)
@@ -127,12 +127,12 @@ def dataroot_test_index():
def dataroot_index():
# Handle the base url for the cellxgene server when running in multi dataset mode
config = current_app.app_config
if not config.multi_dataset_index:
if not config.multi_dataset__index:
abort(404)
elif config.multi_dataset_index is True:
elif config.multi_dataset__index is True:
return dataroot_test_index()
else:
return redirect(config.dataroot_index)
return redirect(config.multi_dataset__index)
class SchemaAPI(Resource):
@@ -222,7 +222,7 @@ class Server:
self.app.register_blueprint(webbp)
api_version = "/api/v0.2"
if app_config.datapath:
if app_config.single_dataset__datapath:
bp_api = Blueprint("api", __name__, url_prefix=api_version)
resources = get_api_resources(bp_api)
self.app.register_blueprint(resources.blueprint)
server/cli/launch.py
+117 -181
View File
@@ -1,40 +1,32 @@
import errno
import functools
import logging
from os import devnull, mkdir, environ
from os.path import splitext, basename, isdir
from os import devnull
import sys
import warnings
import webbrowser
from urllib.parse import urlparse
import click
from server.common.utils import custom_format_warning
from server.common.utils import find_available_port, is_port_available, sort_options
from server.common.errors import DatasetAccessError
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
from server.common.annotations import AnnotationsLocalFile
from server.common.utils import sort_options
from server.common.errors import DatasetAccessError, ConfigurationError
from server.data_common.matrix_loader import MatrixDataCacheManager
from server.common.app_config import AppConfig
from server.common.default_config import default_config
from server.common.errors import OntologyLoadFailure
# anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
DEFAULT_SERVER_PORT = int(environ.get("CXG_SERVER_PORT", "5005"))
DEFAULT_CONFIG = AppConfig()
def annotation_args(func):
@click.option(
"--disable-annotations",
is_flag=True,
default=False,
default=not DEFAULT_CONFIG.user_annotations__enable,
show_default=True,
help="Disable user annotation of data.",
)
@click.option(
"--annotations-file",
default=None,
default=DEFAULT_CONFIG.user_annotations__local_file_csv__file,
show_default=True,
multiple=False,
metavar="<path>",
@@ -43,7 +35,7 @@ def annotation_args(func):
)
@click.option(
"--annotations-dir",
default=None,
default=DEFAULT_CONFIG.user_annotations__local_file_csv__directory,
show_default=False,
multiple=False,
metavar="<directory path>",
@@ -53,13 +45,13 @@ def annotation_args(func):
@click.option(
"--experimental-annotations-ontology",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.user_annotations__ontology__enable,
show_default=True,
help="When creating annotations, optionally autocomplete names from ontology terms.",
)
@click.option(
"--experimental-annotations-ontology-obo",
default=None,
default=DEFAULT_CONFIG.user_annotations__ontology__obo_location,
show_default=True,
metavar="<path or url>",
help="Location of OBO file defining cell annotation autosuggest terms.",
@@ -74,7 +66,7 @@ def annotation_args(func):
def config_args(func):
@click.option(
"--max-category-items",
default=1000,
default=DEFAULT_CONFIG.presentation__max_categories,
metavar="<integer>",
show_default=True,
help="Will not display categories with more distinct values than specified.",
@@ -82,7 +74,7 @@ def config_args(func):
@click.option(
"--diffexp-lfc-cutoff",
"-de",
default=0.01,
default=DEFAULT_CONFIG.diffexp__lfc_cutoff,
show_default=True,
metavar="<float>",
help="Minimum log fold change threshold for differential expression.",
@@ -90,14 +82,14 @@ def config_args(func):
@click.option(
"--disable-diffexp",
is_flag=True,
default=False,
default=not DEFAULT_CONFIG.diffexp__enable,
show_default=False,
help="Disable on-demand differential expression.",
)
@click.option(
"--embedding",
"-e",
default=[],
default=DEFAULT_CONFIG.embeddings__names,
multiple=True,
show_default=False,
metavar="<text>",
@@ -106,7 +98,7 @@ def config_args(func):
@click.option(
"--experimental-enable-reembedding",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.embeddings__enable_reembedding,
show_default=False,
hidden=True,
help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
@@ -122,14 +114,14 @@ def dataset_args(func):
@click.option(
"--obs-names",
"-obs",
default=None,
default=DEFAULT_CONFIG.single_dataset__obs_names,
metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
)
@click.option(
"--var-names",
"-var",
default=None,
default=DEFAULT_CONFIG.single_dataset__var_names,
metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
)
@@ -137,13 +129,20 @@ def dataset_args(func):
"--backed",
"-b",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.adaptor__anndata_adaptor__backed,
show_default=False,
help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
)
@click.option("--title", "-t", metavar="<text>", help="Title to display. If omitted will use file name.")
@click.option(
"--title",
"-t",
default=DEFAULT_CONFIG.single_dataset__title,
metavar="<text>",
help="Title to display. If omitted will use file name."
)
@click.option(
"--about",
default=DEFAULT_CONFIG.single_dataset__about,
metavar="<URL>",
help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).",
)
@@ -159,7 +158,7 @@ def server_args(func):
"--debug",
"-d",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.server__debug,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",
@@ -168,7 +167,7 @@ def server_args(func):
"--verbose",
"-v",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.server__verbose,
show_default=True,
help="Provide verbose output, including warnings and all server requests.",
)
@@ -176,21 +175,22 @@ def server_args(func):
"--port",
"-p",
metavar="<port>",
default=None,
default=DEFAULT_CONFIG.server__port,
type=int,
show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",
)
@click.option(
"--host",
metavar="<IP address>",
default="127.0.0.1",
default=DEFAULT_CONFIG.server__host,
show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
)
@click.option(
"--scripts",
"-s",
default=[],
default=DEFAULT_CONFIG.server__scripts,
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
@@ -211,7 +211,7 @@ def launch_args(func):
@server_args
@click.option(
"--dataroot",
default=None,
default=DEFAULT_CONFIG.multi_dataset__dataroot,
metavar="<data directory>",
help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
" to folder containing H5AD and/or CXG datasets.",
@@ -223,10 +223,27 @@ def launch_args(func):
"-o",
"open_browser",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.server__open_browser,
show_default=True,
help="Open web browser after launch.",
)
@click.option(
"--config-file",
"-c",
"config_file",
is_flag=True,
default=None,
show_default=True,
help="Location to yaml file with configuration settings",
)
@click.option(
"--dump-default-config",
"dump_default_config",
is_flag=True,
default=False,
show_default=True,
help="Print default configuration settings and exit",
)
@click.help_option("--help", "-h", help="Show this message and exit.")
@functools.wraps(func)
def wrapper(*args, **kwargs):
@@ -252,11 +269,6 @@ def handle_scripts(scripts):
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
def handle_verbose(verbose):
if not verbose:
sys.tracebacklimit = 0
@sort_options
@click.command(
short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
@@ -287,6 +299,8 @@ def launch(
experimental_annotations_ontology,
experimental_annotations_ontology_obo,
experimental_enable_reembedding,
config_file,
dump_default_config
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -307,162 +321,79 @@ def launch(
#
# > cellxgene launch --dataroot <url>
if dump_default_config:
print(default_config)
sys.exit(0)
# Startup message
click.echo("[cellxgene] Starting the CLI...")
if datapath is None and dataroot is None:
# TODO: change the error message once dataroot is fully supported
raise click.ClickException('Missing argument "<path to data file>."')
# raise click.ClickException("must supply either <path to data file> or --dataroot")
if datapath is not None and dataroot is not None:
raise click.ClickException("must supply only one of <path to data file> or --dataroot")
if datapath:
# preload this data set
matrix_data_loader = MatrixDataLoader(datapath)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise click.ClickException(str(e))
if experimental_enable_reembedding:
if matrix_data_loader.matrix_data_type() != MatrixDataType.H5AD:
raise click.ClickException("--experimental-enable-reembedding is only supported with H5AD files.")
if backed:
raise click.ClickException(
"--experimental-enable-reembedding is not supported when run in --backed mode."
)
file_size = matrix_data_loader.file_size()
if file_size > BIG_FILE_SIZE_THRESHOLD:
click.echo(f"[cellxgene] Loading data from {basename(datapath)}, this may take a while...")
else:
click.echo(f"[cellxgene] Loading data from {basename(datapath)}.")
if debug:
verbose = True
open_browser = False
else:
warnings.formatwarning = custom_format_warning
handle_verbose(verbose)
handle_scripts(scripts)
if port:
if debug:
raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).")
if not is_port_available(host, int(port)):
raise click.ClickException(
f"The port selected {port} is in use, please specify an open port using the --port flag."
)
else:
port = find_available_port(host, DEFAULT_SERVER_PORT)
if disable_annotations:
if annotations_file is not None:
click.echo("Warning: --annotations-file ignored as annotations are disabled.")
if annotations_dir is not None:
click.echo("Warning: --annotations-dir ignored as annotations are disabled.")
if experimental_annotations_ontology:
click.echo("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
if experimental_annotations_ontology_obo is not None:
click.echo("Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled.")
else:
if annotations_file is not None and annotations_dir is not None:
raise click.ClickException(
"--annotations-file and --annotations-dir " "may not be used together."
)
if annotations_file is not None:
lf_name, lf_ext = splitext(annotations_file)
if lf_ext and lf_ext != ".csv":
raise click.FileError(basename(annotations_file), hint="annotation file type must be .csv")
if annotations_dir is not None and not isdir(annotations_dir):
try:
mkdir(annotations_dir)
except OSError:
raise click.ClickException(
"Unable to create directory specified by " "--annotations-dir"
)
if about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(about):
raise click.ClickException("Must provide an absolute URL for --about. (Example format: http://example.com)")
# Setup app
cellxgene_url = f"http://{host}:{port}"
# app config
app_config = AppConfig(
datapath=datapath,
dataroot=dataroot,
title=title,
about=about,
scripts=scripts,
layout=embedding,
max_category_items=max_category_items,
diffexp_lfc_cutoff=diffexp_lfc_cutoff,
obs_names=obs_names,
var_names=var_names,
anndata_backed=backed,
disable_diffexp=disable_diffexp,
enable_reembedding=experimental_enable_reembedding,
)
app_config = AppConfig()
matrix_data_cache_manager = MatrixDataCacheManager()
data_adaptor = None
if datapath:
try:
with matrix_data_cache_manager.data_adaptor(datapath, app_config) as data_adaptor:
if not disable_diffexp and data_adaptor.parameters.get("diffexp_may_be_slow", False):
click.echo(
f"[cellxgene] CAUTION: due to the size of your dataset, "
f"running differential expression may take longer or fail."
)
except Exception as e:
raise click.ClickException(str(e))
try:
if config_file:
app_config.update_from_config_file(config_file)
# create an annotations object. Only AnnotationsLocalFile is used (for now)
annotations = None
app_config.update(
server__verbose=verbose,
server__debug=debug,
server__host=host,
server__port=port,
server__scripts=scripts,
server__open_browser=open_browser,
if not disable_annotations:
annotations = AnnotationsLocalFile(annotations_dir, annotations_file)
single_dataset__datapath=datapath,
single_dataset__title=title,
single_dataset__about=about,
single_dataset__obs_names=obs_names,
single_dataset__var_names=var_names,
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
multi_dataset__dataroot=dataroot,
if annotations_file and data_adaptor:
data_adaptor.check_new_labels(annotations.read_labels(data_adaptor))
user_annotations__enable=not disable_annotations,
user_annotations__local_file_csv__file=annotations_file,
user_annotations__local_file_csv__directory=annotations_dir,
user_annotations__ontology__enable=experimental_annotations_ontology,
user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
if experimental_annotations_ontology or bool(experimental_annotations_ontology_obo):
try:
annotations.load_ontology(experimental_annotations_ontology_obo)
except OntologyLoadFailure as e:
raise click.ClickException("Unable to load ontology terms\n" + str(e))
presentation__max_categories=max_category_items,
embeddings__names=embedding,
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
adaptor__anndata_adaptor__backed=backed,
)
# process the configuration
# any errors will be thrown as an exception.
# any info messages will be passed to the messagefn function.
matrix_data_cache_manager = MatrixDataCacheManager()
def messagefn(message):
click.echo("[cellxgene] " + message)
app_config.complete_config(matrix_data_cache_manager, messagefn)
except (ConfigurationError, DatasetAccessError) as e:
raise click.ClickException(e)
handle_scripts(scripts)
user_annotations = app_config.user_annotations
# create the server
from server.app.app import Server
server = Server(matrix_data_cache_manager, user_annotations, app_config)
server = Server(matrix_data_cache_manager, annotations, app_config)
if not verbose:
if not app_config.server__verbose:
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
if open_browser:
cellxgene_url = f"http://{app_config.server__host}:{app_config.server__port}"
if app_config.server__open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
webbrowser.open(cellxgene_url)
else:
@@ -470,12 +401,17 @@ def launch(
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not verbose:
if not app_config.server__verbose:
f = open(devnull, "w")
sys.stdout = f
try:
server.app.run(host=host, debug=debug, port=port, threaded=False if debug else True, use_debugger=False)
server.app.run(
host=app_config.server__host,
debug=app_config.server__debug,
port=app_config.server__port,
threaded=not app_config.server__debug,
use_debugger=False)
except OSError as e:
if e.errno == errno.EADDRINUSE:
raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
server/common/app_config.py
+338 -63
View File
@@ -1,6 +1,25 @@
# -*- coding: utf-8 -*-
from server import __version__ as cellxgene_version
from flatten_dict import flatten
from os import mkdir, environ
from os.path import splitext, basename, isdir
import sys
from urllib.parse import urlparse
import yaml
from server.common.default_config import get_default_config
from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
from server.common.utils import find_available_port, is_port_available
import warnings
from server.common.annotations import AnnotationsLocalFile
from server.common.utils import custom_format_warning
DEFAULT_SERVER_PORT = int(environ.get("CXG_SERVER_PORT", "5005"))
# anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
class AppFeature(object):
@@ -19,78 +38,334 @@ class AppFeature(object):
class AppConfig(object):
def __init__(self, **kw):
super().__init__()
def __init__(self):
# app inputs
self.datapath = None
self.dataroot = None
self.title = ""
self.about = None
self.scripts = []
self.layout = []
self.max_category_items = 100
self.diffexp_lfc_cutoff = 0.01
self.disable_diffexp = False
self.enable_reembedding = False
self.anndata_backed = False
self.default_config = get_default_config()
# The index page when in multi-dataset mode:
# False or None: this returns a 404 code
# True: loads a test index page, which links to the datasets that are available in the dataroot
# string/URL: redirect to this URL: flask.redirect(config.multi_dataset_index)
self.multi_dataset_index = False
dc = self.default_config
try:
self.server__verbose = dc["server"]["verbose"]
self.server__debug = dc["server"]["debug"]
self.server__host = dc["server"]["host"]
self.server__port = dc["server"]["port"]
self.server__scripts = dc["server"]["scripts"]
self.server__open_browser = dc["server"]["open_browser"]
self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
self.multi_dataset__index = dc["multi_dataset"]["index"]
self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
self.single_dataset__datapath = dc["single_dataset"]["datapath"]
self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
self.single_dataset__var_names = dc["single_dataset"]["var_names"]
self.single_dataset__about = dc["single_dataset"]["about"]
self.single_dataset__title = dc["single_dataset"]["title"]
self.user_annotations__enable = dc["user_annotations"]["enable"]
self.user_annotations__type = dc["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
self.presentation__max_categories = dc["presentation"]["max_categories"]
self.embeddings__names = dc["embeddings"]["names"]
self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
self.diffexp__enable = dc["diffexp"]["enable"]
self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# A list of allowed matrix types. If an empty list, then all matrix types are allowed
self.multi_dataset_allowed_matrix_type = []
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
# TODO these options may not apply to all datasets in the multi dataset.
# may need to invent a way to associate these config parameters with
# specific datasets.
self.obs_names = None
self.var_names = None
# Set to true when config_completed is called
self.is_completed = False
# parameters
self.diffexp_may_be_slow = False
def update_from_config_file(self, config_file):
with open(config_file) as fyaml:
config = yaml.load(fyaml, Loader=yaml.FullLoader)
inputs = [
"datapath",
"dataroot",
"title",
"about",
"scripts",
"layout",
"max_category_items",
"diffexp_lfc_cutoff",
"obs_names",
"var_names",
"anndata_backed",
"disable_diffexp",
"enable_reembedding",
"multi_dataset_index",
"multi_dataset_allowed_matrix_type",
]
# special case for tiledb_ctx whose value is a dict, and cannot
# be handled by the flattening below
if config.get("adaptor", {}).get("cxg_adaptor", {}).get("tiledb_ctx"):
value = config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__cxg_adaptor__tiledb_ctx = value
del config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.update(inputs, kw)
flat_config = flatten(config)
for key, value in flat_config.items():
# name of the attribute
attr = "__".join(key)
if not hasattr(self, attr):
raise ConfigurationError(f"Unknown key from config file: {key}")
try:
setattr(self, attr, value)
except KeyError:
raise ConfigurationError(f"Unable to set config attribute: {key}")
def update(self, inputs, kw):
for k, v in kw.items():
if k in inputs:
setattr(self, k, v)
else:
raise RuntimeError(f"unknown config parameter {k}.")
self.is_completed = False
def update(self, **kw):
for key, value in kw.items():
if not hasattr(self, key):
raise ConfigurationError(f"unknown config parameter {key}.")
try:
setattr(self, key, value)
except KeyError:
raise ConfigurationError(f"Unable to set config parameter {key}.")
self.is_completed = False
def complete_config(self, matrix_data_cache_manager=None, messagefn=None):
"""The configure options are checked, and any additional setup based on the config
parameters is done"""
if matrix_data_cache_manager is None:
matrix_data_cache_manager = MatrixDataCacheManager()
if messagefn is None:
def noop(message):
pass
messagefn = noop
# TODO: to give better error messages we can add a mapping between where each config
# attribute originated (e.g. command line argument or config file), then in the error
# messages we can give correct context for attributes with bad value.
context = dict(matrix_cache=matrix_data_cache_manager, messagefn=messagefn)
self.handle_server(context)
self.handle_single_dataset(context)
self.handle_multi_dataset(context)
self.handle_user_annotations(context)
self.handle_embeddings(context)
self.handle_diffexp(context)
self.handle_adaptor(context)
self.is_completed = True
def __check_attr(self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
def handle_server(self, context):
self.__check_attr("server__verbose", bool)
self.__check_attr("server__debug", bool)
self.__check_attr("server__host", str)
self.__check_attr("server__port", (type(None), int))
self.__check_attr("server__scripts", (list, tuple))
self.__check_attr("server__open_browser", bool)
if self.server__port:
if self.server__debug:
raise ConfigurationError(
"'port' and 'debug' may not be used together (try 'verbose' for error logging)."
)
if not is_port_available(self.server__host, self.server__port):
raise ConfigurationError(
f"The port selected {self.server__port} is in use, please configure an open port."
)
else:
self.server__port = find_available_port(self.server__host, DEFAULT_SERVER_PORT)
if self.server__debug:
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
self.server__verbose = True
self.server__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.server__verbose:
sys.tracebacklimit = 0
def handle_presentation(self, context):
self.__check_attr("presentation__max_categories", int)
def handle_single_dataset(self, context):
self.__check_attr("single_dataset__datapath", (str, type(None)))
self.__check_attr("single_dataset__title", (str, type(None)))
self.__check_attr("single_dataset__about", (str, type(None)))
self.__check_attr("single_dataset__obs_names", (str, type(None)))
self.__check_attr("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
if self.multi_dataset__dataroot is None:
# TODO: change the error message once dataroot is fully supported
raise ConfigurationError("missing datapath")
return
else:
if self.multi_dataset__dataroot is not None:
raise ConfigurationError("must supply only one of datapath or dataroot")
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_multi_dataset(self, context):
self.__check_attr("multi_dataset__dataroot", (type(None), str))
self.__check_attr("multi_dataset__index", (type(None), bool, str))
self.__check_attr("multi_dataset__allowed_matrix_types", (list))
if self.multi_dataset__dataroot is None:
return
# error checking
for mtype in self.multi_dataset__allowed_matrix_types:
try:
MatrixDataType(mtype)
except ValueError:
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
def handle_user_annotations(self, context):
self.__check_attr("user_annotations__enable", bool)
self.__check_attr("user_annotations__type", str)
self.__check_attr("user_annotations__local_file_csv__directory", (type(None), str))
self.__check_attr("user_annotations__local_file_csv__file", (type(None), str))
self.__check_attr("user_annotations__ontology__enable", bool)
self.__check_attr("user_annotations__ontology__obo_location", (type(None), str))
if self.user_annotations__enable:
# TODO, replace this with a factory pattern once we have more than one way
# to do annotations. currently only local_file_csv
if self.user_annotations__type != "local_file_csv":
raise ConfigurationError('The only annotation type support is "local_file_csv"')
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
mkdir(dirname)
except OSError:
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
if self.single_dataset__datapath and self.user_annotations__local_file_csv__file:
with context["matrix_cache"].data_adaptor(self.single_dataset__datapath, self) as data_adaptor:
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
else:
if self.user_annotations__type == "local_file_csv":
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None:
context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
if dirname is not None:
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
if self.user_annotations__ontology__enable:
context["messagefn"](
"Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
)
if self.user_annotations__ontology__obo_location is not None:
context["messagefn"](
"Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
)
def handle_embeddings(self, context):
self.__check_attr("embeddings__names", (list, tuple))
self.__check_attr("embeddings__enable_reembedding", bool)
if self.single_dataset__datapath:
if self.embeddings__enable_reembedding:
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath)
if matrix_data_loader.matrix_data_type() != MatrixDataType.H5AD:
raise ConfigurationError("'enable-reembedding is only supported with H5AD files.")
if self.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
def handle_diffexp(self, context):
self.__check_attr("diffexp__enable", bool)
self.__check_attr("diffexp__lfc_cutoff", float)
if self.single_dataset__datapath:
with context["matrix_cache"].data_adaptor(self.single_dataset__datapath, self) as data_adaptor:
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
context["messagefn"](
f"CAUTION: due to the size of your dataset, "
f"running differential expression may take longer or fail."
)
def handle_adaptor(self, context):
# cxg
self.__check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
from server.data_cxg.cxg_adaptor import CxgAdaptor
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
# anndata
self.__check_attr("adaptor__anndata_adaptor__backed", bool)
def get_title(self, data_adaptor):
return self.title if self.title else data_adaptor.get_title()
return self.single_dataset__title if self.single_dataset__title else data_adaptor.get_title()
def get_about(self, data_adaptor):
return self.about if self.about else data_adaptor.get_about()
return self.single_dataset__about if self.single_dataset__about else data_adaptor.get_about()
def get_config(self, data_adaptor, annotation=None):
def get_client_config(self, data_adaptor, annotation=None):
"""
Return the configuration as required by the /config REST route
"""
# FIXME The current set of config is not consistently presented:
# we have camalCase, hyphen-text, and underscore_text
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
# features
features = [f.todict() for f in data_adaptor.get_features(annotation)]
@@ -110,14 +385,14 @@ class AppConfig(object):
# parameters
parameters = {
"layout": self.layout,
"max-category-items": self.max_category_items,
"obs_names": self.obs_names,
"var_names": self.var_names,
"diffexp_lfc_cutoff": self.diffexp_lfc_cutoff,
"backed": self.anndata_backed,
"disable-diffexp": self.disable_diffexp,
"enable-reembedding": self.enable_reembedding,
"layout": self.embeddings__names,
"max-category-items": self.presentation__max_categories,
"obs_names": self.single_dataset__obs_names,
"var_names": self.single_dataset__var_names,
"diffexp_lfc_cutoff": self.diffexp__lfc_cutoff,
"backed": self.adaptor__anndata_adaptor__backed,
"disable-diffexp": not self.diffexp__enable,
"enable-reembedding": self.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
server/common/default_config.py
+67
View File
@@ -0,0 +1,67 @@
import yaml
default_config = """
# cellxgene configuration
server:
verbose: false
debug: false
host: "127.0.0.1"
port : null
scripts : []
open_browser: false
presentation:
max_categories: 1000
multi_dataset:
dataroot: null
# The index page when in multi-dataset mode:
# false or null: this returns a 404 code
# true: loads a test index page, which links to the datasets that are available in the dataroot
# string/URL: redirect to this URL: flask.redirect(config.multi_dataset__index)
index: false
# A list of allowed matrix types. If an empty list, then all matrix types are allowed
allowed_matrix_types: []
single_dataset:
datapath: null
obs_names: null
var_names: null
about: null
title: null
user_annotations:
enable: true
type: local_file_csv
local_file_csv:
directory: null
file: null
ontology:
enable: false
obo_location: null
embeddings:
names : []
enable_reembedding: false
diffexp:
enable: true
lfc_cutoff: 0.01
adaptor:
cxg_adaptor:
tiledb_ctx:
sm.tile_cache_size: 8589934592
sm.num_reader_threads: 32
vfs.s3.region: us-east-1
anndata_adaptor:
backed: false
"""
def get_default_config():
return yaml.load(default_config, Loader=yaml.Loader)
server/common/errors.py
+8
View File
@@ -60,3 +60,11 @@ class OntologyLoadFailure(Exception):
"""
pass
class ConfigurationError(Exception):
"""
Raised when checking configuration errors
"""
pass
server/common/rest.py
+3 -3
View File
@@ -33,7 +33,7 @@ def schema_get(data_adaptor, annotations):
def config_get(app_config, data_adaptor, annotations):
config = app_config.get_config(data_adaptor, annotations)
config = app_config.get_client_config(data_adaptor, annotations)
return make_response(jsonify(config), HTTPStatus.OK)
@@ -131,7 +131,7 @@ def data_var_put(request, data_adaptor):
def diffexp_obs_post(request, data_adaptor):
if data_adaptor.config.disable_diffexp:
if not data_adaptor.config.diffexp__enable:
return make_response(f"diffexp not supported.", HTTPStatus.BAD_REQUEST)
args = request.get_json()
@@ -198,7 +198,7 @@ def layout_obs_put(request, data_adaptor):
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
if preferred_mimetype != "application/octet-stream":
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
if not data_adaptor.config.enable_reembedding:
if not data_adaptor.config.embedding__enable_reembedding:
return make_response(f"Computed embedding not supported.", HTTPStatus.BAD_REQUEST)
args = request.get_json()
server/data_anndata/anndata_adaptor.py
+11 -9
View File
@@ -95,7 +95,9 @@ class AnndataAdaptor(DataAdaptor):
"""
self.original_obs_index = self.data.obs.index
for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
config_name = f"single_dataset__{var_name}_names"
parameter_name = f"{var_name}_names"
name = getattr(self.config, config_name)
df_axis = getattr(self.data, str(ax_name))
if name is None:
@@ -107,7 +109,7 @@ class AnndataAdaptor(DataAdaptor):
"alternative with --{ax_name}-name."
)
name = self._create_unique_column_name(df_axis.columns, "name_")
self.parameters[config_name] = name
self.parameters[parameter_name] = name
# reset index to simple range; alias name to point at the
# previously specified index.
df_axis.rename_axis(name, inplace=True)
@@ -119,7 +121,7 @@ class AnndataAdaptor(DataAdaptor):
f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values."
)
df_axis.reset_index(drop=True, inplace=True)
self.parameters[config_name] = name
self.parameters[parameter_name] = name
else:
# user specified a non-existent column name
raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
@@ -157,7 +159,7 @@ class AnndataAdaptor(DataAdaptor):
with data_locator.local_handle() as lh:
# as of AnnData 0.6.19, backed mode performs initial load fast, but at the
# cost of significantly slower access to X data.
backed = "r" if self.config.anndata_backed else None
backed = "r" if self.config.adaptor__anndata_adaptor__backed else None
self.data = anndata.read_h5ad(lh, backed=backed)
except ValueError:
@@ -177,7 +179,7 @@ class AnndataAdaptor(DataAdaptor):
)
def _validate_and_initialize(self):
if anndata_version_is_pre_070() and self.config.anndata_backed:
if anndata_version_is_pre_070() and self.config.adaptor__anndata_adaptor__backed:
warnings.warn(
f"Use of --backed mode with anndata versions older than 0.7 will have serious "
"performance issues. Please update to at least anndata 0.7 or later."
@@ -195,7 +197,7 @@ class AnndataAdaptor(DataAdaptor):
# heuristic
n_values = self.data.shape[0] * self.data.shape[1]
if (n_values > 1e8 and self.config.anndata_backed is True) or (n_values > 5e8):
if (n_values > 1e8 and self.config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
self.parameters.update({"diffexp_may_be_slow": True})
def _is_valid_layout(self, arr):
@@ -242,7 +244,7 @@ class AnndataAdaptor(DataAdaptor):
)
if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.config.max_category_items:
if category_num > 500 and category_num > self.config.presentation__max_categories:
warnings.warn(
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the "
@@ -273,7 +275,7 @@ class AnndataAdaptor(DataAdaptor):
c) cap total list of layouts at global const MAX_LAYOUTS
"""
# load default layouts from the data.
layouts = self.config.layout
layouts = self.config.embeddings__names
if layouts is None or len(layouts) == 0:
layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
@@ -342,7 +344,7 @@ class AnndataAdaptor(DataAdaptor):
return getattr(self.data.obs, term_name)
def get_obs_index(self):
name = getattr(self.config, "obs_names")
name = self.config.single_dataset__obs_names
if name is None:
return self.original_obs_index
else:
server/data_common/data_adaptor.py
+5 -9
View File
@@ -16,12 +16,8 @@ class DataAdaptor(metaclass=ABCMeta):
"""Base class for loading and accessing matrix data"""
def __init__(self, config):
# config will normally be a type that inherits from AppConfig.
# the following is for backwards compatability with tests
if config is None:
config = AppConfig()
elif type(config) == dict:
config = AppConfig(**config)
if type(config) != AppConfig:
raise TypeError("config expected to be of type AppConfig")
# config is the application configuration
self.config = config
@@ -140,8 +136,8 @@ class DataAdaptor(metaclass=ABCMeta):
features = [
AppFeature("/cluster/", method="POST", available=False),
AppFeature("/layout/obs", method="GET", available=self.get_embedding_names() is not None),
AppFeature("/layout/obs", method="PUT", available=self.config.enable_reembedding),
AppFeature("/diffexp/", method="POST", available=not self.config.disable_diffexp),
AppFeature("/layout/obs", method="PUT", available=self.config.embeddings__enable_reembedding),
AppFeature("/diffexp/", method="POST", available=self.config.diffexp__enable),
AppFeature("/annotations/obs", method="PUT", available=annotations is not None),
]
return features
@@ -290,7 +286,7 @@ class DataAdaptor(metaclass=ABCMeta):
if top_n is None:
top_n = DEFAULT_TOP_N
result = diffexp_ttest(self, obs_mask_A, obs_mask_B, top_n, self.config.diffexp_lfc_cutoff)
result = diffexp_ttest(self, obs_mask_A, obs_mask_B, top_n, self.config.diffexp__lfc_cutoff)
try:
return jsonify_numpy(result)
server/data_common/matrix_loader.py
+3 -3
View File
@@ -174,12 +174,12 @@ class MatrixDataLoader(object):
if not app_config:
return True
if not app_config.dataroot:
if not app_config.multi_dataset__dataroot:
return True
if len(app_config.multi_dataset_allowed_matrix_type) == 0:
if len(app_config.multi_dataset__allowed_matrix_types) == 0:
return True
for val in app_config.multi_dataset_allowed_matrix_type:
for val in app_config.multi_dataset__allowed_matrix_types:
try:
if self.matrix_data_type == MatrixDataType(val):
return True
server/data_cxg/cxg_adaptor.py
+20 -8
View File
@@ -1,7 +1,7 @@
import os
import json
from server.common.utils import dtype_to_schema
from server.common.errors import DatasetAccessError
from server.common.errors import DatasetAccessError, ConfigurationError
from server.common.utils import path_join
from server.common.constants import Axis
from server.data_common.data_adaptor import DataAdaptor
@@ -16,7 +16,11 @@ import threading
class CxgAdaptor(DataAdaptor):
# TODO: The tiledb context parameters should be a configuration option
tiledb_ctx = tiledb.Ctx({"sm.tile_cache_size": 8 * 1024 * 1024 * 1024, "sm.num_reader_threads": 32})
tiledb_ctx = tiledb.Ctx({
"sm.tile_cache_size": 8 * 1024 * 1024 * 1024,
"sm.num_reader_threads": 32,
"vfs.s3.region": "us-east-1"
})
def __init__(self, data_locator, config=None):
super().__init__(config)
@@ -36,6 +40,14 @@ class CxgAdaptor(DataAdaptor):
array.close()
self.arrays.clear()
@staticmethod
def set_tiledb_context(context_params):
"""Set the tiledb context. This should be set before any instances of CxgAdaptor are created"""
try:
CxgAdaptor.tiledb_ctx = tiledb.Ctx(context_params)
except tiledb.libtiledb.TileDBError as e:
raise ConfigurationError(f"Invalid tiledb context: {str(e)}")
@staticmethod
def pre_load_validation(data_locator):
location = data_locator.uri_or_path
@@ -100,15 +112,15 @@ class CxgAdaptor(DataAdaptor):
Return True if this looks like a valid CXG, False if not. Just a quick/cheap
test, not to be fully trusted.
"""
if not tiledb.object_type(url) == "group":
if not tiledb.object_type(url, ctx=CxgAdaptor.tiledb_ctx) == "group":
return False
if not tiledb.object_type(path_join(url, "obs")) == "array":
if not tiledb.object_type(path_join(url, "obs"), ctx=CxgAdaptor.tiledb_ctx) == "array":
return False
if not tiledb.object_type(path_join(url, "var")) == "array":
if not tiledb.object_type(path_join(url, "var"), ctx=CxgAdaptor.tiledb_ctx) == "array":
return False
if not tiledb.object_type(path_join(url, "X")) == "array":
if not tiledb.object_type(path_join(url, "X"), ctx=CxgAdaptor.tiledb_ctx) == "array":
return False
if not tiledb.object_type(path_join(url, "emb")) == "group":
if not tiledb.object_type(path_join(url, "emb"), ctx=CxgAdaptor.tiledb_ctx) == "group":
return False
return True
@@ -126,7 +138,7 @@ class CxgAdaptor(DataAdaptor):
* version 0.1 -- metadata attache to cxg_group_metadata array.
Same as 0, except it adds group metadata.
"""
a_type = tiledb.object_type(path_join(self.url, "cxg_group_metadata"))
a_type = tiledb.object_type(path_join(self.url, "cxg_group_metadata"), ctx=self.tiledb_ctx)
if a_type is None:
# version 0
cxg_version = "0.0"
server/eb/Makefile
+3
View File
@@ -20,6 +20,9 @@ build: clean
cp app.py artifact.dir/application.py; \
cp ../requirements.txt artifact.dir; \
cp -r .ebextensions artifact.dir; \
if [ -f config.yaml ] ; then \
cp config.yaml artifact.dir; \
fi; \
(cd artifact.dir; \
cp -r server/common/web/static static; \
zip -r ../artifact.zip . --exclude server/test/\* server/eb/\* ; ); \
server/eb/README.md
+4
View File
@@ -40,6 +40,10 @@ There are many more options to these commands that may be important or necessary
3. Create the artifact.zip file for the application
If you have additional configuration for the application,
place that application in server/eb/config.yaml. This will
be included in the eb deployment.
```
make build
```
server/eb/app.py
+28 -24
View File
@@ -24,49 +24,53 @@ try:
from server.app.app import Server
from server.data_common.matrix_loader import MatrixDataCacheManager
except Exception:
logging.exception("Exception importing server modules")
logging.critical("Exception importing server modules", exc_info=True)
sys.exit(1)
try:
dataroot = os.getenv("CXG_DATAROOT")
if dataroot is None:
logging.error("CXG_DATAROOT environment variable must be set")
sys.exit(1)
app_config = AppConfig()
app_config = AppConfig(
datapath=None,
dataroot=dataroot,
title="",
about=None,
scripts=[],
layout=[],
max_category_items=100,
diffexp_lfc_cutoff=0.01,
obs_names=None,
var_names=None,
anndata_backed=False,
disable_diffexp=True,
multi_dataset_index=None,
multi_dataset_allowed_matrix_type=["cxg"],
config_file = "config.yaml"
if os.path.exists(config_file):
logging.info(f"Configuration from {config_file}")
app_config.update_from_config_file(config_file)
if dataroot:
logging.info(f"Configuration from CXG_DATAROOT")
app_config.update(
multi_dataset__dataroot=dataroot,
)
# features are unsupported in the current hosted server
app_config.update(
diffexp__enable=False,
user_annotations__enable=False,
embeddings__enable_reembedding=False,
multi_dataset__allowed_matrix_types=["cxg"],
)
matrix_data_cache_manager = MatrixDataCacheManager()
annotations = None
app_config.complete_config(matrix_data_cache_manager, logging.info)
user_annotations = app_config.user_annotations
server = Server(matrix_data_cache_manager, annotations, app_config)
server = Server(matrix_data_cache_manager, user_annotations, app_config)
debug = False
application = server.app
except Exception:
logging.exception("Caught exception during initialization")
logging.critical("Caught exception during initialization", exc_info=True)
sys.exit(1)
logging.info(f"starting server with CXG_DATAROOT={dataroot}")
if app_config.multi_dataset__dataroot:
logging.info(f"starting server with multi_dataset__dataroot={app_config.multi_dataset__dataroot}")
elif app_config.single_dataset__datapath:
logging.info(f"starting server with single_dataset__datapath={app_config.single_dataset__datapath}")
if __name__ == "__main__":
try:
application.run(debug=debug, threaded=not debug, use_debugger=False)
except Exception:
logging.exception("Caught exception during server run")
logging.critical("Caught exception during initialization", exc_info=True)
sys.exit(1)
server/requirements.txt
+2
View File
@@ -8,10 +8,12 @@ Flask-Cors>=3.0.6
Flask-RESTful>=0.3.6
flask-server-timing>=0.1.2
flatbuffers>=1.10.0
flatten-dict>=0.2.0
fsspec>=0.4.4
numpy>=1.16.0
packaging>=20.0
pandas>=0.24.2
PyYAML>=5.3
scipy>=1.3.0
requests>=2.22.0
tiledb>=0.5.3
server/test/__init__.py
+11 -6
View File
@@ -6,6 +6,7 @@ import pandas as pd
from server.common.annotations import AnnotationsLocalFile
from server.common.data_locator import DataLocator
from server.common.app_config import AppConfig
from server.data_common.fbs.matrix import encode_matrix_fbs
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
@@ -16,18 +17,22 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
if annotations_fixture:
shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file)
args = {
"layout": ["umap"],
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"embeddings__names": ["umap"],
"presentation__max_categories": 100,
"single_dataset__obs_names": None,
"single_dataset__var_names": None,
"diffexp__lfc_cutoff": 0.01,
}
fname = {
MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
data = MatrixDataLoader(data_locator.abspath()).open(args)
config = AppConfig()
config.update(**args)
config.update(single_dataset__datapath=data_locator.path)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsLocalFile(None, annotations_file)
return data, tmp_dir, annotations
server/test/test_anndata_adaptor.py
+15 -10
View File
@@ -13,6 +13,7 @@ import pandas as pd
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.errors import FilterError
from server.common.data_locator import DataLocator
from server.common.app_config import AppConfig
"""
Test the anndata adaptor using the pbmc3k data set.
@@ -33,14 +34,18 @@ Test the anndata adaptor using the pbmc3k data set.
class AdaptorTest(unittest.TestCase):
def setUp(self):
args = {
"layout": ["umap"],
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"anndata_backed": self.backed,
"embeddings__names": ["umap"],
"presentation__max_categories": 100,
"single_dataset__obs_names": None,
"single_dataset__var_names": None,
"diffexp__lfc_cutoff": 0.01,
"adaptor__anndata_adaptor__backed": self.backed,
"single_dataset__datapath" : self.data_locator
}
self.data = AnndataAdaptor(DataLocator(self.data_locator), args)
config = AppConfig()
config.update(**args)
config.complete_config()
self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
def test_init(self):
self.assertEqual(self.data.cell_count, 2638)
@@ -88,7 +93,7 @@ class AdaptorTest(unittest.TestCase):
def test_get_schema(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh:
schema = json.load(fh)
self.assertEqual(self.data.get_schema(), schema)
self.assertDictEqual(self.data.get_schema(), schema)
def test_schema_produces_error(self):
self.data.data.obs["time"] = pd.Series(
@@ -109,9 +114,9 @@ class AdaptorTest(unittest.TestCase):
self.assertEqual(len(feature), 1)
check_feature("POST", "/cluster/", False)
check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp)
check_feature("POST", "/diffexp/", self.data.config.diffexp__enable)
check_feature("GET", "/layout/obs", True)
check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding)
check_feature("PUT", "/layout/obs", self.data.config.embeddings__enable_reembedding)
check_feature("PUT", "/annotations/obs", False)
def test_layout(self):
server/test/test_anndata_adaptor_data_load.py
+21 -9
View File
@@ -3,6 +3,7 @@ import json
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.data_locator import DataLocator
from server.common.app_config import AppConfig
class DataLoadAdaptorTest(unittest.TestCase):
@@ -12,7 +13,10 @@ class DataLoadAdaptorTest(unittest.TestCase):
def setUp(self):
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data = AnndataAdaptor(self.data_file)
config = AppConfig()
config.update(single_dataset__datapath=self.data_file.path)
config.complete_config()
self.data = AnndataAdaptor(self.data_file, config)
def test_delayed_load_data(self):
self.data._create_schema()
@@ -37,11 +41,11 @@ class DataLocatorAdaptorTest(unittest.TestCase):
def setUp(self):
self.args = {
"layout": ["umap"],
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"embeddings__names": ["umap"],
"presentation__max_categories": 100,
"single_dataset__obs_names": None,
"single_dataset__var_names": None,
"diffexp__lfc_cutoff": 0.01,
}
def stdAsserts(self, data):
@@ -52,17 +56,25 @@ class DataLocatorAdaptorTest(unittest.TestCase):
def test_posix_file(self):
locator = DataLocator("../example-dataset/pbmc3k.h5ad")
data = AnndataAdaptor(locator, self.args)
config = AppConfig()
config.update(**self.args)
config.update(single_dataset__datapath=locator.path)
config.complete_config()
data = AnndataAdaptor(locator, config)
self.stdAsserts(data)
def test_url_https(self):
url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
locator = DataLocator(url)
data = AnndataAdaptor(locator, self.args)
config = AppConfig()
config.update(**self.args)
data = AnndataAdaptor(locator, config)
self.stdAsserts(data)
def test_url_http(self):
url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
locator = DataLocator(url)
data = AnndataAdaptor(locator, self.args)
config = AppConfig()
config.update(**self.args)
data = AnndataAdaptor(locator, config)
self.stdAsserts(data)
server/test/test_nan_anndata_adaptor.py
+19 -7
View File
@@ -8,25 +8,37 @@ import server.test.decode_fbs as decode_fbs
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.errors import FilterError
from server.common.data_locator import DataLocator
from server.common.app_config import AppConfig
class NaNTest(unittest.TestCase):
def setUp(self):
self.args = {
"layout": ["umap"],
"max_category_items": 100,
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"embeddings__names": ["umap"],
"presentation__max_categories": 100,
"single_dataset__obs_names": None,
"single_dataset__var_names": None,
"diffexp__lfc_cutoff": 0.01,
}
config = AppConfig()
config.update(**self.args)
locator = DataLocator("test/test_datasets/nan.h5ad")
config.update(single_dataset__datapath=locator.path)
config.complete_config()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
self.data = AnndataAdaptor(locator, config)
self.data._create_schema()
def test_load(self):
with self.assertWarns(UserWarning):
AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
config = AppConfig()
config.update(**self.args)
locator = DataLocator("test/test_datasets/nan.h5ad")
config.update(single_dataset__datapath=locator.path)
config.complete_config()
self.data = AnndataAdaptor(locator, config)
def test_init(self):
self.assertEqual(self.data.cell_count, 100)
server/test/test_writable_annotation.py
+2 -2
View File
@@ -149,7 +149,7 @@ class WritableAnnotationTest(unittest.TestCase):
self.assertEqual(len(feature), 1)
check_feature("POST", "/cluster/", False)
check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp)
check_feature("POST", "/diffexp/", self.data.config.diffexp__enable)
check_feature("GET", "/layout/obs", True)
check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding)
check_feature("PUT", "/layout/obs", self.data.config.embeddings__enable_reembedding)
check_feature("PUT", "/annotations/obs", True)