mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-29 09:58:11 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -1,70 +0,0 @@
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
import fastobo
|
||||
import fsspec
|
||||
|
||||
from server.common.errors import OntologyLoadFailure
|
||||
from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
|
||||
|
||||
|
||||
class Annotations(metaclass=ABCMeta):
|
||||
""" baseclass for annotations, including ontologies"""
|
||||
|
||||
""" our default ontology is the PURL for the Cell Ontology.
|
||||
See http://www.obofoundry.org/ontology/cl.html """
|
||||
DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
|
||||
|
||||
def __init__(self):
|
||||
self.ontology_data = None
|
||||
|
||||
def load_ontology(self, path):
|
||||
"""Load and parse ontologies - currently support OBO files only."""
|
||||
if path is None:
|
||||
path = self.DefaultOnotology
|
||||
|
||||
try:
|
||||
with fsspec.open(path) as f:
|
||||
obo = fastobo.iter(f)
|
||||
terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
|
||||
names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
|
||||
self.ontology_data = names
|
||||
|
||||
except FileNotFoundError as e:
|
||||
raise OntologyLoadFailure("Unable to find OBO ontology path") from e
|
||||
|
||||
except SyntaxError as e:
|
||||
raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
|
||||
|
||||
except Exception as e:
|
||||
raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
|
||||
|
||||
def get_schema(self, data_adaptor):
|
||||
schema = []
|
||||
labels = self.read_labels(data_adaptor)
|
||||
if labels is not None and not labels.empty:
|
||||
for col in labels.columns:
|
||||
col_schema = dict(name=col, writable=True)
|
||||
col_schema.update(get_schema_type_hint_of_array(labels[col]))
|
||||
schema.append(col_schema)
|
||||
|
||||
return schema
|
||||
|
||||
@abstractmethod
|
||||
def set_collection(self, name):
|
||||
"""set or create a new annotation collection"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def read_labels(self, data_adaptor):
|
||||
"""Return the labels as a pandas.DataFrame"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def write_labels(self, df, data_adaptor):
|
||||
"""Write the labels (df) to a persistent storage such that it can later be read"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def update_parameters(self, parameters, data_adaptor):
|
||||
"""Update configuration parameters that describe information about the annotations feature"""
|
||||
pass
|
||||
@@ -1,171 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import pandas as pd
|
||||
import tiledb
|
||||
from flask import current_app
|
||||
|
||||
from server.common.annotations.annotations import Annotations
|
||||
from server.common.errors import AnnotationCategoryNameError
|
||||
from server.common.utils.sanitization_utils import sanitize_values_in_list
|
||||
from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
|
||||
from server.db.cellxgene_orm import Annotation
|
||||
|
||||
|
||||
class AnnotationsHostedTileDB(Annotations):
|
||||
CXG_ANNO_COLLECTION = "cxg_anno_collection"
|
||||
|
||||
def __init__(self, directory_path, db):
|
||||
super().__init__()
|
||||
self.db = db
|
||||
if directory_path[-1] == "/":
|
||||
self.directory_path = directory_path
|
||||
else:
|
||||
self.directory_path = directory_path + "/"
|
||||
|
||||
def check_category_names(self, df):
|
||||
original_category_names = df.keys().to_list()
|
||||
sanitized_category_names = set(sanitize_values_in_list(original_category_names).values())
|
||||
unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
|
||||
if unsanitary_original_category_names:
|
||||
raise AnnotationCategoryNameError(
|
||||
f"{unsanitary_original_category_names} are not valid category names, please resubmit"
|
||||
)
|
||||
|
||||
def get_user_name(self):
|
||||
return current_app.auth.get_user_name()
|
||||
|
||||
def get_user_id(self):
|
||||
return current_app.auth.get_user_id()
|
||||
|
||||
def is_safe_collection_name(self, name):
|
||||
"""
|
||||
return true if this is a safe collection name
|
||||
this is ultra conservative. If we want to allow full legal file name syntax,
|
||||
we could look at modules like `pathvalidate`
|
||||
"""
|
||||
if name is None:
|
||||
return False
|
||||
return re.match(r"^[\w\-]+$", name) is not None
|
||||
|
||||
def set_collection(self, name):
|
||||
self.CXG_ANNO_COLLECTION = name
|
||||
|
||||
def read_labels(self, data_adaptor):
|
||||
user_id = self.get_user_id()
|
||||
if user_id is None:
|
||||
return
|
||||
dataset_name = data_adaptor.get_location()
|
||||
dataset_id = self.db.get_or_create_dataset(dataset_name)
|
||||
|
||||
annotation_object = self.db.query_for_most_recent(
|
||||
Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
|
||||
)
|
||||
if annotation_object:
|
||||
if annotation_object.tiledb_uri == "":
|
||||
# this mean the user has removed all the categories.
|
||||
return None
|
||||
try:
|
||||
df = tiledb.open(annotation_object.tiledb_uri)
|
||||
except tiledb.TileDBError:
|
||||
# don't crash if the annotations file is missing or can't be read.
|
||||
current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
|
||||
return None
|
||||
pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
|
||||
return pandas_df
|
||||
else:
|
||||
return None
|
||||
|
||||
def convert_to_pandas_df(self, tileDBArray, schema_hints):
|
||||
repr_meta = None
|
||||
index_dims = None
|
||||
schema_hints = json.loads(schema_hints)
|
||||
|
||||
if "__pandas_attribute_repr" in tileDBArray.meta:
|
||||
# backwards compatibility... unsure if necessary at this point
|
||||
repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
|
||||
if "__pandas_index_dims" in tileDBArray.meta:
|
||||
index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
|
||||
|
||||
data = tileDBArray[:]
|
||||
indexes = list()
|
||||
|
||||
for col_name, col_val in data.items():
|
||||
# If the column values are byte literals, decode them
|
||||
if isinstance(col_val[0], bytes):
|
||||
col_val = [value.decode("utf-8") for value in col_val]
|
||||
|
||||
if schema_hints and col_name in schema_hints:
|
||||
type = schema_hints.get(col_name).get("type")
|
||||
if type and type == "categorical":
|
||||
new_col = pd.Series(col_val, dtype="category")
|
||||
data[col_name] = new_col
|
||||
elif repr_meta and col_name in repr_meta:
|
||||
new_col = pd.Series(col_val, dtype=repr_meta[col_name])
|
||||
data[col_name] = new_col
|
||||
elif index_dims and col_name in index_dims:
|
||||
new_col = pd.Series(col_val, dtype=index_dims[col_name])
|
||||
data[col_name] = new_col
|
||||
indexes.append(col_name)
|
||||
|
||||
new_df = pd.DataFrame.from_dict(data)
|
||||
if len(indexes) > 0:
|
||||
new_df.set_index(indexes, inplace=True)
|
||||
|
||||
return new_df
|
||||
|
||||
def write_labels(self, df, data_adaptor):
|
||||
auth_user_id = self.get_user_id()
|
||||
user_name = self.get_user_name()
|
||||
timestamp = time.time()
|
||||
dataset_location = data_adaptor.get_location()
|
||||
dataset_id = self.db.get_or_create_dataset(dataset_location)
|
||||
dataset_name = data_adaptor.get_title()
|
||||
user_id = self.db.get_or_create_user(auth_user_id)
|
||||
"""
|
||||
NOTE: The uri contains the dataset name, user name and a timestamp as a convenience for debugging purposes.
|
||||
People may have the same name and time.time() can be server dependent.
|
||||
See - https://docs.python.org/2/library/time.html#time.time
|
||||
|
||||
The annotations objects in the database should be used as the source of truth about who an annotation belongs
|
||||
to (for authorization purposes) and what time it was created (for garbage collection).
|
||||
"""
|
||||
uri = f"{self.directory_path}{dataset_name}/{user_name}/{timestamp}"
|
||||
if uri.startswith("s3://"):
|
||||
pass
|
||||
else:
|
||||
os.makedirs(uri, exist_ok=True)
|
||||
_, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
|
||||
if not df.empty:
|
||||
self.check_category_names(df)
|
||||
# convert to tiledb datatypes
|
||||
|
||||
for col in df:
|
||||
df[col] = df[col].astype(get_dtype_of_array(df[col]))
|
||||
tiledb.from_pandas(uri, df, sparse=True)
|
||||
else:
|
||||
uri = ""
|
||||
|
||||
annotation = Annotation(
|
||||
tiledb_uri=uri,
|
||||
user_id=user_id,
|
||||
dataset_id=str(dataset_id),
|
||||
schema_hints=json.dumps(dataframe_schema_type_hints),
|
||||
)
|
||||
self.db.session.add(annotation)
|
||||
self.db.session.commit()
|
||||
|
||||
def update_parameters(self, parameters, data_adaptor):
|
||||
params = {}
|
||||
params["annotations"] = True
|
||||
params["user_annotation_collection_name_enabled"] = False
|
||||
|
||||
if self.ontology_data:
|
||||
params["annotations_cell_ontology_enabled"] = True
|
||||
params["annotations_cell_ontology_terms"] = self.ontology_data
|
||||
else:
|
||||
params["annotations_cell_ontology_enabled"] = False
|
||||
|
||||
parameters.update(params)
|
||||
@@ -1,196 +0,0 @@
|
||||
import base64
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from hashlib import blake2b
|
||||
|
||||
import pandas as pd
|
||||
from flask import session, has_request_context, current_app
|
||||
|
||||
from server import __version__ as cellxgene_version
|
||||
from server.common.annotations.annotations import Annotations
|
||||
from server.common.errors import AnnotationsError
|
||||
|
||||
|
||||
class AnnotationsLocalFile(Annotations):
|
||||
CXG_ANNO_COLLECTION = "cxg_anno_collection"
|
||||
|
||||
def __init__(self, output_dir, output_file):
|
||||
super().__init__()
|
||||
self.output_dir = output_dir
|
||||
self.output_file = output_file
|
||||
# lock used to protect label file write ops
|
||||
self.label_lock = threading.RLock()
|
||||
|
||||
# cache the most recent annotations
|
||||
self.last_fname = None
|
||||
self.last_labels = None
|
||||
|
||||
def is_safe_collection_name(self, name):
|
||||
"""
|
||||
return true if this is a safe collection name
|
||||
this is ultra conservative. If we want to allow full legal file name syntax,
|
||||
we could look at modules like `pathvalidate`
|
||||
"""
|
||||
if name is None:
|
||||
return False
|
||||
return re.match(r"^[\w\-]+$", name) is not None
|
||||
|
||||
def set_collection(self, name):
|
||||
session[self.CXG_ANNO_COLLECTION] = name
|
||||
session.permanent = True
|
||||
|
||||
def get_collection(self):
|
||||
if session is None:
|
||||
return None
|
||||
return session.get(self.CXG_ANNO_COLLECTION)
|
||||
|
||||
def read_labels(self, data_adaptor):
|
||||
if has_request_context():
|
||||
if not current_app.auth.is_user_authenticated():
|
||||
return pd.DataFrame()
|
||||
|
||||
fname = self._get_filename(data_adaptor)
|
||||
with self.label_lock:
|
||||
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
|
||||
# returned the cached labels if possible, otherwise read them from the file
|
||||
if fname == self.last_fname:
|
||||
return self.last_labels
|
||||
else:
|
||||
labels = pd.read_csv(
|
||||
fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
|
||||
)
|
||||
# update the cache
|
||||
self.last_fname = fname
|
||||
self.last_labels = labels
|
||||
return labels
|
||||
else:
|
||||
return pd.DataFrame()
|
||||
|
||||
def write_labels(self, df, data_adaptor):
|
||||
# update our internal state and save it. Multi-threading often enabled,
|
||||
# so treat this as a critical section.
|
||||
with self.label_lock:
|
||||
lastmod = data_adaptor.get_last_mod_time()
|
||||
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
|
||||
header = (
|
||||
f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
|
||||
f"using cellxgene version {cellxgene_version}\n"
|
||||
f"# Input data file was {data_adaptor.get_location()}, "
|
||||
f"which was last modified on {lastmodstr}\n"
|
||||
)
|
||||
|
||||
fname = self._get_filename(data_adaptor)
|
||||
self._backup(fname)
|
||||
if not df.empty:
|
||||
with open(fname, "w", newline="") as f:
|
||||
if header is not None:
|
||||
f.write(header)
|
||||
df.to_csv(f)
|
||||
else:
|
||||
open(fname, "w").close()
|
||||
|
||||
# update the cache
|
||||
self.last_fname = fname
|
||||
self.last_labels = df
|
||||
|
||||
def _get_userdata_idhash(self, data_adaptor):
|
||||
"""
|
||||
Return a short hash that weakly identifies the user and dataset.
|
||||
Used to create safe annotations output file names.
|
||||
"""
|
||||
uid = current_app.auth.get_user_id()
|
||||
id = (uid + data_adaptor.get_location()).encode()
|
||||
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
|
||||
return idhash
|
||||
|
||||
def _get_output_dir(self):
|
||||
if self.output_dir:
|
||||
return self.output_dir
|
||||
|
||||
if self.output_file:
|
||||
return os.path.dirname(self.path.abspath(self.output_dir))
|
||||
|
||||
return os.getcwd()
|
||||
|
||||
def _get_filename(self, data_adaptor):
|
||||
""" return the current annotation file name """
|
||||
if self.output_file:
|
||||
return self.output_file
|
||||
|
||||
# we need to generate a file name, which we can only do if we have a UID and collection name
|
||||
if session is None:
|
||||
raise AnnotationsError("unable to determine file name for annotations")
|
||||
|
||||
collection = self.get_collection()
|
||||
if collection is None:
|
||||
return None
|
||||
|
||||
if data_adaptor is None:
|
||||
raise AnnotationsError("unable to determine file name for annotations")
|
||||
|
||||
idhash = self._get_userdata_idhash(data_adaptor)
|
||||
return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv")
|
||||
|
||||
def _backup(self, fname, max_backups=9):
|
||||
"""
|
||||
save N backups of file to backup_dir.
|
||||
1. fname -> backup_dir/fname-TIME
|
||||
2. delete excess files in backup_dir
|
||||
"""
|
||||
root, ext = os.path.splitext(fname)
|
||||
backup_dir = f"{root}-backups"
|
||||
|
||||
# Make sure there is work to do
|
||||
if not os.path.exists(fname):
|
||||
return
|
||||
|
||||
# Ensure backup_dir exists
|
||||
if not os.path.exists(backup_dir):
|
||||
os.mkdir(backup_dir)
|
||||
|
||||
# Save current file to backup_dir
|
||||
fname_base = os.path.basename(fname)
|
||||
fname_base_root, fname_base_ext = os.path.splitext(fname_base)
|
||||
# don't use ISO standard time format, as it contains characters illegal on some filesytems.
|
||||
nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
|
||||
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
|
||||
if os.path.exists(backup_fname):
|
||||
os.remove(backup_fname)
|
||||
os.rename(fname, backup_fname)
|
||||
|
||||
# prune the backup_dir to max number of backup files, keeping the most recent backups
|
||||
backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
|
||||
excess_count = len(backups) - max_backups
|
||||
if excess_count > 0:
|
||||
backups.sort()
|
||||
for bu in backups[0:excess_count]:
|
||||
os.remove(os.path.join(backup_dir, bu))
|
||||
|
||||
def update_parameters(self, parameters, data_adaptor):
|
||||
params = {}
|
||||
params["annotations"] = True
|
||||
params["user_annotation_collection_name_enabled"] = True
|
||||
|
||||
if self.ontology_data:
|
||||
params["annotations_cell_ontology_enabled"] = True
|
||||
params["annotations_cell_ontology_terms"] = self.ontology_data
|
||||
else:
|
||||
params["annotations_cell_ontology_enabled"] = False
|
||||
|
||||
if self.output_file is not None:
|
||||
# user has hard-wired the name of the annotation data collection
|
||||
fname = os.path.basename(self.output_file)
|
||||
collection_fname = os.path.splitext(fname)[0]
|
||||
params["annotations-data-collection-is-read-only"] = True
|
||||
params["annotations-data-collection-name"] = collection_fname
|
||||
|
||||
elif session is not None:
|
||||
collection = self.get_collection()
|
||||
if current_app.auth.is_user_authenticated():
|
||||
params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
|
||||
params["annotations-data-collection-is-read-only"] = False
|
||||
params["annotations-data-collection-name"] = collection
|
||||
|
||||
parameters.update(params)
|
||||
@@ -1,23 +0,0 @@
|
||||
import logging
|
||||
|
||||
import boto3
|
||||
from flask import json
|
||||
|
||||
from server.common.errors import SecretKeyRetrievalError
|
||||
|
||||
|
||||
def get_secret_key(region_name, secret_name):
|
||||
session = boto3.session.Session()
|
||||
client = session.client(service_name="secretsmanager", region_name=region_name)
|
||||
|
||||
try:
|
||||
get_secret_value_response = client.get_secret_value(SecretId=secret_name)
|
||||
if "SecretString" in get_secret_value_response:
|
||||
var = get_secret_value_response["SecretString"]
|
||||
secret = json.loads(var)
|
||||
return secret
|
||||
except Exception as e:
|
||||
logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
|
||||
raise SecretKeyRetrievalError(str(e))
|
||||
|
||||
return None
|
||||
@@ -1,233 +0,0 @@
|
||||
import re
|
||||
|
||||
from server.common.errors import ColorFormatException
|
||||
|
||||
HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
|
||||
|
||||
# https://www.w3.org/TR/css-color-4/#named-colors
|
||||
CSS4_NAMED_COLORS = dict(
|
||||
aliceblue="#f0f8ff",
|
||||
antiquewhite="#faebd7",
|
||||
aqua="#00ffff",
|
||||
aquamarine="#7fffd4",
|
||||
azure="#f0ffff",
|
||||
beige="#f5f5dc",
|
||||
bisque="#ffe4c4",
|
||||
black="#000000",
|
||||
blanchedalmond="#ffebcd",
|
||||
blue="#0000ff",
|
||||
blueviolet="#8a2be2",
|
||||
brown="#a52a2a",
|
||||
burlywood="#deb887",
|
||||
cadetblue="#5f9ea0",
|
||||
chartreuse="#7fff00",
|
||||
chocolate="#d2691e",
|
||||
coral="#ff7f50",
|
||||
cornflowerblue="#6495ed",
|
||||
cornsilk="#fff8dc",
|
||||
crimson="#dc143c",
|
||||
cyan="#00ffff",
|
||||
darkblue="#00008b",
|
||||
darkcyan="#008b8b",
|
||||
darkgoldenrod="#b8860b",
|
||||
darkgray="#a9a9a9",
|
||||
darkgreen="#006400",
|
||||
darkgrey="#a9a9a9",
|
||||
darkkhaki="#bdb76b",
|
||||
darkmagenta="#8b008b",
|
||||
darkolivegreen="#556b2f",
|
||||
darkorange="#ff8c00",
|
||||
darkorchid="#9932cc",
|
||||
darkred="#8b0000",
|
||||
darksalmon="#e9967a",
|
||||
darkseagreen="#8fbc8f",
|
||||
darkslateblue="#483d8b",
|
||||
darkslategray="#2f4f4f",
|
||||
darkslategrey="#2f4f4f",
|
||||
darkturquoise="#00ced1",
|
||||
darkviolet="#9400d3",
|
||||
deeppink="#ff1493",
|
||||
deepskyblue="#00bfff",
|
||||
dimgray="#696969",
|
||||
dimgrey="#696969",
|
||||
dodgerblue="#1e90ff",
|
||||
firebrick="#b22222",
|
||||
floralwhite="#fffaf0",
|
||||
forestgreen="#228b22",
|
||||
fuchsia="#ff00ff",
|
||||
gainsboro="#dcdcdc",
|
||||
ghostwhite="#f8f8ff",
|
||||
gold="#ffd700",
|
||||
goldenrod="#daa520",
|
||||
gray="#808080",
|
||||
green="#008000",
|
||||
greenyellow="#adff2f",
|
||||
grey="#808080",
|
||||
honeydew="#f0fff0",
|
||||
hotpink="#ff69b4",
|
||||
indianred="#cd5c5c",
|
||||
indigo="#4b0082",
|
||||
ivory="#fffff0",
|
||||
khaki="#f0e68c",
|
||||
lavender="#e6e6fa",
|
||||
lavenderblush="#fff0f5",
|
||||
lawngreen="#7cfc00",
|
||||
lemonchiffon="#fffacd",
|
||||
lightblue="#add8e6",
|
||||
lightcoral="#f08080",
|
||||
lightcyan="#e0ffff",
|
||||
lightgoldenrodyellow="#fafad2",
|
||||
lightgray="#d3d3d3",
|
||||
lightgreen="#90ee90",
|
||||
lightgrey="#d3d3d3",
|
||||
lightpink="#ffb6c1",
|
||||
lightsalmon="#ffa07a",
|
||||
lightseagreen="#20b2aa",
|
||||
lightskyblue="#87cefa",
|
||||
lightslategray="#778899",
|
||||
lightslategrey="#778899",
|
||||
lightsteelblue="#b0c4de",
|
||||
lightyellow="#ffffe0",
|
||||
lime="#00ff00",
|
||||
limegreen="#32cd32",
|
||||
linen="#faf0e6",
|
||||
magenta="#ff00ff",
|
||||
maroon="#800000",
|
||||
mediumaquamarine="#66cdaa",
|
||||
mediumblue="#0000cd",
|
||||
mediumorchid="#ba55d3",
|
||||
mediumpurple="#9370db",
|
||||
mediumseagreen="#3cb371",
|
||||
mediumslateblue="#7b68ee",
|
||||
mediumspringgreen="#00fa9a",
|
||||
mediumturquoise="#48d1cc",
|
||||
mediumvioletred="#c71585",
|
||||
midnightblue="#191970",
|
||||
mintcream="#f5fffa",
|
||||
mistyrose="#ffe4e1",
|
||||
moccasin="#ffe4b5",
|
||||
navajowhite="#ffdead",
|
||||
navy="#000080",
|
||||
oldlace="#fdf5e6",
|
||||
olive="#808000",
|
||||
olivedrab="#6b8e23",
|
||||
orange="#ffa500",
|
||||
orangered="#ff4500",
|
||||
orchid="#da70d6",
|
||||
palegoldenrod="#eee8aa",
|
||||
palegreen="#98fb98",
|
||||
paleturquoise="#afeeee",
|
||||
palevioletred="#db7093",
|
||||
papayawhip="#ffefd5",
|
||||
peachpuff="#ffdab9",
|
||||
peru="#cd853f",
|
||||
pink="#ffc0cb",
|
||||
plum="#dda0dd",
|
||||
powderblue="#b0e0e6",
|
||||
purple="#800080",
|
||||
rebeccapurple="#663399",
|
||||
red="#ff0000",
|
||||
rosybrown="#bc8f8f",
|
||||
royalblue="#4169e1",
|
||||
saddlebrown="#8b4513",
|
||||
salmon="#fa8072",
|
||||
sandybrown="#f4a460",
|
||||
seagreen="#2e8b57",
|
||||
seashell="#fff5ee",
|
||||
sienna="#a0522d",
|
||||
silver="#c0c0c0",
|
||||
skyblue="#87ceeb",
|
||||
slateblue="#6a5acd",
|
||||
slategray="#708090",
|
||||
slategrey="#708090",
|
||||
snow="#fffafa",
|
||||
springgreen="#00ff7f",
|
||||
steelblue="#4682b4",
|
||||
tan="#d2b48c",
|
||||
teal="#008080",
|
||||
thistle="#d8bfd8",
|
||||
tomato="#ff6347",
|
||||
turquoise="#40e0d0",
|
||||
violet="#ee82ee",
|
||||
wheat="#f5deb3",
|
||||
white="#ffffff",
|
||||
whitesmoke="#f5f5f5",
|
||||
yellow="#ffff00",
|
||||
yellowgreen="#9acd32",
|
||||
)
|
||||
|
||||
|
||||
def convert_color_to_hex_format(unknown):
|
||||
"""
|
||||
Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet.
|
||||
|
||||
The function accepts for the following formats:
|
||||
- A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html
|
||||
- RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0]
|
||||
- RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255]
|
||||
- Hex triplet string, as in "#08c0ff"
|
||||
|
||||
:param unknown: color info of unknown format
|
||||
:return: a hex triplet representing that color
|
||||
"""
|
||||
try:
|
||||
if type(unknown) in (list, tuple) and len(unknown) == 3:
|
||||
if all(0.0 <= ele <= 1.0 for ele in unknown):
|
||||
tup = tuple(int(ele * 255) for ele in unknown)
|
||||
elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown):
|
||||
tup = tuple(unknown)
|
||||
else:
|
||||
raise ColorFormatException("Unknown color iterable format!")
|
||||
return "#%02x%02x%02x" % tup
|
||||
elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS:
|
||||
return CSS4_NAMED_COLORS[unknown.lower()]
|
||||
elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown):
|
||||
return unknown.lower()
|
||||
else:
|
||||
raise ColorFormatException("Unknown color format type!")
|
||||
except Exception as e:
|
||||
raise ColorFormatException(e)
|
||||
|
||||
|
||||
def convert_anndata_category_colors_to_cxg_category_colors(data):
|
||||
"""
|
||||
Convert color information from anndata files to the cellxgene color data format as described below:
|
||||
{
|
||||
"<category_name>": {
|
||||
"<label_name>": "<color_hex_code>",
|
||||
...
|
||||
},
|
||||
...
|
||||
}
|
||||
|
||||
For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307.
|
||||
|
||||
For more on the anndata color data structure, see
|
||||
https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178.
|
||||
|
||||
Handling of malformed data:
|
||||
- For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot
|
||||
convert to a hex triplet string, a ColorFormatException is raised
|
||||
- No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no
|
||||
adata.obs[f"{category}"] key
|
||||
|
||||
:param data: the anndata file
|
||||
:return: cellxgene color data structure as described above
|
||||
"""
|
||||
cxg_colors = dict()
|
||||
color_key_suffix = "_colors"
|
||||
for uns_key in data.uns.keys():
|
||||
# find uns array that describes colors for a category
|
||||
if not uns_key.endswith(color_key_suffix):
|
||||
continue
|
||||
|
||||
# check to see if we actually have observations for that category
|
||||
category_name = uns_key[: -len(color_key_suffix)]
|
||||
if category_name not in data.obs.keys():
|
||||
continue
|
||||
|
||||
# create the cellxgene color entry for this category
|
||||
cxg_colors[category_name] = dict(
|
||||
zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
|
||||
)
|
||||
return cxg_colors
|
||||
@@ -1,4 +0,0 @@
|
||||
from server.common.aws_secret_utils import get_secret_key # noqa F504
|
||||
|
||||
DEFAULT_SERVER_PORT = 5005
|
||||
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
|
||||
@@ -1,247 +0,0 @@
|
||||
import yaml
|
||||
from flatten_dict import unflatten
|
||||
|
||||
from server.default_config import get_default_config
|
||||
from server.common.config.dataset_config import DatasetConfig
|
||||
from server.common.config.server_config import ServerConfig
|
||||
from server.common.config.external_config import ExternalConfig
|
||||
from server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
class AppConfig(object):
|
||||
"""
|
||||
AppConfig stores all the configuration for cellxgene.
|
||||
AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
|
||||
The server_config contains attributes that refer to the server process as a whole.
|
||||
The default_dataset_config refers to attributes that are associated with the features and
|
||||
presentations of a dataset.
|
||||
The dataset config attributes can be overridden depending on the url by which the
|
||||
dataset was accessed. These are stored in dataroot_config.
|
||||
AppConfig has methods to initialize, modify, and access the configuration.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
|
||||
# the default configuration (see default_config.py)
|
||||
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
|
||||
# defaults within the config class?
|
||||
self.default_config = get_default_config()
|
||||
# the server configuration
|
||||
self.server_config = ServerConfig(self, self.default_config["server"])
|
||||
# the dataset config, unless overridden by an entry in dataroot_config
|
||||
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
|
||||
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
|
||||
# attribute of the server_config. The default dataset config will apply to all datasets unless a different set
|
||||
# of config vars was passed for a specific dataset under the multidataset config. For example:
|
||||
"""
|
||||
per_dataset_config:
|
||||
d1:
|
||||
user_annotations:
|
||||
enable: false
|
||||
d2:
|
||||
user_annotations:
|
||||
enable: true
|
||||
"""
|
||||
# dataroot config
|
||||
self.dataroot_config = {}
|
||||
|
||||
# external config
|
||||
self.external_config = ExternalConfig(self, self.default_config["external"])
|
||||
|
||||
# Set to true when config_completed is called
|
||||
self.is_completed = False
|
||||
|
||||
def get_dataset_config(self, dataroot_key):
|
||||
if self.server_config.single_dataset__datapath:
|
||||
return self.default_dataset_config
|
||||
else:
|
||||
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
|
||||
|
||||
def check_config(self):
|
||||
"""Verify all the attributes in the config have been type checked"""
|
||||
if not self.is_completed:
|
||||
raise ConfigurationError("The configuration has not been completed")
|
||||
self.server_config.check_config()
|
||||
self.default_dataset_config.check_config()
|
||||
for dataset_config in self.dataroot_config.values():
|
||||
dataset_config.check_config()
|
||||
self.external_config.check_config()
|
||||
|
||||
def update_server_config(self, **kw):
|
||||
self.server_config.update(**kw)
|
||||
self.is_complete = False
|
||||
|
||||
def update_default_dataset_config(self, **kw):
|
||||
self.default_dataset_config.update(**kw)
|
||||
# update all the other dataset configs, if any
|
||||
for value in self.dataroot_config.values():
|
||||
value.update(**kw)
|
||||
self.is_complete = False
|
||||
|
||||
def update_single_config_from_path_and_value(self, path, value):
|
||||
"""Update a single config parameter with the value.
|
||||
Path is a list of string, that gives a path to the config parameter to be updated.
|
||||
For example, path may be ["server","app","port"].
|
||||
"""
|
||||
self.is_complete = False
|
||||
if not isinstance(path, list):
|
||||
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
|
||||
for part in path:
|
||||
if not isinstance(part, str):
|
||||
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
|
||||
|
||||
if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
|
||||
raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
|
||||
|
||||
if path[0] == "server":
|
||||
attr = "__".join(path[1:])
|
||||
try:
|
||||
self.update_server_config(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
elif path[0] == "dataset":
|
||||
attr = "__".join(path[1:])
|
||||
try:
|
||||
self.update_default_dataset_config(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
|
||||
elif path[0] == "per_dataset_config":
|
||||
if len(path) < 2:
|
||||
raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
|
||||
dataroot = path[1]
|
||||
if dataroot not in self.dataroot_config:
|
||||
dataroots = str(list(self.dataroot_config.keys()))
|
||||
raise ConfigurationError(
|
||||
f"unknown dataroot when using per_dataset_config: got '{path}',"
|
||||
f" dataroots specified in config are {dataroots}"
|
||||
)
|
||||
|
||||
attr = "__".join(path[2:])
|
||||
try:
|
||||
self.dataroot_config[dataroot].update(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
|
||||
def update_from_config_file(self, config_file):
|
||||
try:
|
||||
with open(config_file) as yml_file:
|
||||
config = yaml.safe_load(yml_file)
|
||||
except yaml.YAMLError as e:
|
||||
raise ConfigurationError(f"The specified config file contained an error: {e}")
|
||||
except OSError as e:
|
||||
raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
|
||||
|
||||
if config.get("server"):
|
||||
self.server_config.update_from_config(config["server"], "server")
|
||||
if config.get("dataset"):
|
||||
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
|
||||
|
||||
per_dataset_config = config.get("per_dataset_config", {})
|
||||
for key, dataroot_config in per_dataset_config.items():
|
||||
# first create and initialize the dataroot with the default config
|
||||
self.add_dataroot_config(key, **config["dataset"])
|
||||
# then apply the per dataset configuration
|
||||
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
|
||||
|
||||
if config.get("external"):
|
||||
self.external_config.update_from_config(config["external"], "external")
|
||||
|
||||
self.is_complete = False
|
||||
|
||||
def config_to_dict(self):
|
||||
"""return the configuration as an unflattened dict"""
|
||||
server = self.server_config.create_mapping(self.server_config.default_config)
|
||||
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
|
||||
external = self.external_config.create_mapping(self.external_config.default_config)
|
||||
config = dict(server={}, dataset={})
|
||||
for attrname in server.keys():
|
||||
config["server__" + attrname] = getattr(self.server_config, attrname)
|
||||
for attrname in dataset.keys():
|
||||
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
|
||||
if self.dataroot_config:
|
||||
config["per_dataset_config"] = {}
|
||||
for dataroot_tag, dataroot_config in self.dataroot_config.items():
|
||||
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
|
||||
for attrname in dataset.keys():
|
||||
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
|
||||
for attrname in external.keys():
|
||||
config["external__" + attrname] = getattr(self.external_config, attrname)
|
||||
|
||||
config = unflatten(config, splitter=lambda key: key.split("__"))
|
||||
return config
|
||||
|
||||
def write_config(self, config_file):
|
||||
"""output the config to a yaml file"""
|
||||
config = self.config_to_dict()
|
||||
yaml.dump(config, open(config_file, "w"))
|
||||
|
||||
def changes_from_default(self):
|
||||
"""Return all the attribute that are different from the default"""
|
||||
diff_server = self.server_config.changes_from_default()
|
||||
diff_dataset = self.default_dataset_config.changes_from_default()
|
||||
diff_external = self.external.changes_from_default()
|
||||
diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
|
||||
return diff
|
||||
|
||||
def add_dataroot_config(self, dataroot_tag, **kw):
|
||||
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
|
||||
if dataroot_tag in self.dataroot_config:
|
||||
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
|
||||
if type(self.server_config.multi_dataset__dataroot) != dict:
|
||||
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
|
||||
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
|
||||
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
|
||||
|
||||
self.is_completed = False
|
||||
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
|
||||
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
|
||||
config = {key: value[1] for key, value in flat_config.items()}
|
||||
self.dataroot_config[dataroot_tag].update(**config)
|
||||
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
|
||||
|
||||
def complete_config(self, messagefn=None):
|
||||
"""The configure options are checked, and any additional setup based on the config
|
||||
parameters is done"""
|
||||
|
||||
if messagefn is None:
|
||||
|
||||
def noop(message):
|
||||
pass
|
||||
|
||||
messagefn = noop
|
||||
|
||||
# TODO: to give better error messages we can add a mapping between where each config
|
||||
# attribute originated (e.g. command line argument or config file), then in the error
|
||||
# messages we can give correct context for attributes with bad value.
|
||||
context = dict(messagefn=messagefn)
|
||||
|
||||
# complete config for external_config first, since this may update values in the other sections
|
||||
self.external_config.complete_config(context)
|
||||
self.server_config.complete_config(context)
|
||||
self.default_dataset_config.complete_config(context)
|
||||
for dataroot_config in self.dataroot_config.values():
|
||||
dataroot_config.complete_config(context)
|
||||
|
||||
self.is_completed = True
|
||||
self.check_config()
|
||||
|
||||
def get_matrix_data_cache_manager(self):
|
||||
return self.server_config.matrix_data_cache_manager
|
||||
|
||||
def is_multi_dataset(self):
|
||||
return self.server_config.multi_dataset__dataroot is not None
|
||||
|
||||
def get_title(self, data_adaptor):
|
||||
return (
|
||||
self.server_config.single_dataset__title
|
||||
if self.server_config.single_dataset__title
|
||||
else data_adaptor.get_title()
|
||||
)
|
||||
|
||||
def get_about(self, data_adaptor):
|
||||
return (
|
||||
self.server_config.single_dataset__about
|
||||
if self.server_config.single_dataset__about
|
||||
else data_adaptor.get_about()
|
||||
)
|
||||
@@ -1,132 +0,0 @@
|
||||
import copy
|
||||
|
||||
from flatten_dict import flatten
|
||||
from server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
class BaseConfig(object):
|
||||
"""
|
||||
This class handles the mechanics of updating and checking attributes.
|
||||
Derived classes are expected to store the actual attributes
|
||||
Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
|
||||
"""
|
||||
|
||||
def __init__(self, app_config, default_config, dictval_cases={}):
|
||||
# reference back to the app_config
|
||||
self.app_config = app_config
|
||||
# the complete set of attributes and their default values (unflattened)
|
||||
self.default_config = default_config
|
||||
# attributes where the value may be a dict (and therefore are not flattened)
|
||||
self.dictval_cases = dictval_cases
|
||||
# used to make sure every attribute value is checked
|
||||
self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
|
||||
|
||||
def create_mapping(self, config):
|
||||
"""
|
||||
Create a dictionary where the keys are the name of attributes (using double underscore convention)
|
||||
For example: authentication__type
|
||||
|
||||
The values are a tuple,
|
||||
- the first item of the tuple is a tuple of path elements (location in config 'tree')
|
||||
- the second item is the value of the config parameter
|
||||
|
||||
For example: (('authentication', 'type'), 'session'))
|
||||
"""
|
||||
config_copy = copy.deepcopy(config)
|
||||
mapping = {}
|
||||
|
||||
# special cases where the value could be a dict.
|
||||
# If its value is not None, the entry is added to the mapping, and not included
|
||||
# in the flattening below.
|
||||
for dictval_case in self.dictval_cases:
|
||||
cur = config_copy
|
||||
for part in dictval_case[:-1]:
|
||||
cur = cur.get(part, {})
|
||||
val = cur.get(dictval_case[-1])
|
||||
if val is not None:
|
||||
key = "__".join(dictval_case)
|
||||
mapping[key] = (dictval_case, val)
|
||||
del cur[dictval_case[-1]]
|
||||
|
||||
flat_config = flatten(config_copy)
|
||||
for key, value in flat_config.items():
|
||||
# name of the attribute
|
||||
attr = "__".join(key)
|
||||
mapping[attr] = (key, value)
|
||||
|
||||
return mapping
|
||||
|
||||
def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
|
||||
val = getattr(self, attrname)
|
||||
if type(vtype) in (list, tuple):
|
||||
if type(val) not in vtype:
|
||||
tnames = ",".join([x.__name__ for x in vtype])
|
||||
raise ConfigurationError(
|
||||
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
|
||||
)
|
||||
else:
|
||||
if type(val) != vtype:
|
||||
raise ConfigurationError(
|
||||
f"Invalid type for attribute: {attrname}, "
|
||||
f"expected type {vtype.__name__}, got {type(val).__name__}"
|
||||
)
|
||||
|
||||
self.attr_checked[attrname] = True
|
||||
|
||||
def check_config(self):
|
||||
mapping = self.create_mapping(self.default_config)
|
||||
for key in mapping.keys():
|
||||
if not self.attr_checked[key]:
|
||||
raise ConfigurationError(f"The attr '{key}' has not been checked")
|
||||
|
||||
def update(self, **kw):
|
||||
"""Update the attributes defined in kw with their new values."""
|
||||
for key, value in kw.items():
|
||||
if not hasattr(self, key):
|
||||
|
||||
# check if the key is setting into a dictval entry.
|
||||
found_dictval = False
|
||||
for dictval in self.dictval_cases:
|
||||
dictvalname = "__".join(dictval)
|
||||
if dictvalname + "__" in key:
|
||||
dictkey = key[len(dictvalname) + 2 :]
|
||||
curdictval = getattr(self, dictvalname)
|
||||
if curdictval is None:
|
||||
setattr(self, dictvalname, dict(dictkey=value))
|
||||
else:
|
||||
curdictval[dictkey] = value
|
||||
|
||||
found_dictval = True
|
||||
break
|
||||
|
||||
if found_dictval:
|
||||
continue
|
||||
raise ConfigurationError(f"unknown config parameter {key}.")
|
||||
try:
|
||||
if type(value) == tuple:
|
||||
# convert tuple values to list values
|
||||
value = list(value)
|
||||
setattr(self, key, value)
|
||||
except KeyError:
|
||||
raise ConfigurationError(f"Unable to set config parameter {key}.")
|
||||
|
||||
self.attr_checked[key] = False
|
||||
|
||||
def update_from_config(self, config, prefix):
|
||||
mapping = self.create_mapping(config)
|
||||
for attr, (key, value) in mapping.items():
|
||||
if not hasattr(self, attr):
|
||||
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.attr_checked[attr] = False
|
||||
|
||||
def changes_from_default(self):
|
||||
"""Return all the attribute that are different from the default"""
|
||||
mapping = self.create_mapping(self.default_config)
|
||||
diff = []
|
||||
for attrname, (key, defval) in mapping.items():
|
||||
curval = getattr(self, attrname)
|
||||
if curval != defval:
|
||||
diff.append((attrname, curval, defval))
|
||||
return diff
|
||||
@@ -1,122 +0,0 @@
|
||||
from server import display_version as cellxgene_display_version
|
||||
|
||||
|
||||
def get_client_config(app_config, data_adaptor):
|
||||
"""
|
||||
Return the configuration as required by the /config REST route
|
||||
"""
|
||||
|
||||
server_config = app_config.server_config
|
||||
dataset_config = data_adaptor.dataset_config
|
||||
annotation = dataset_config.user_annotations
|
||||
auth = server_config.auth
|
||||
|
||||
# FIXME The current set of config is not consistently presented:
|
||||
# we have camalCase, hyphen-text, and underscore_text
|
||||
|
||||
# make sure the configuration has been checked.
|
||||
app_config.check_config()
|
||||
|
||||
# display_names
|
||||
title = app_config.get_title(data_adaptor)
|
||||
about = app_config.get_about(data_adaptor)
|
||||
|
||||
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
|
||||
|
||||
# library_versions
|
||||
library_versions = {}
|
||||
library_versions.update(data_adaptor.get_library_versions())
|
||||
library_versions["cellxgene"] = cellxgene_display_version
|
||||
|
||||
# links
|
||||
links = {"about-dataset": about}
|
||||
|
||||
# parameters
|
||||
parameters = {
|
||||
"layout": dataset_config.embeddings__names,
|
||||
"max-category-items": dataset_config.presentation__max_categories,
|
||||
"obs_names": server_config.single_dataset__obs_names,
|
||||
"var_names": server_config.single_dataset__var_names,
|
||||
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
|
||||
"backed": server_config.adaptor__anndata_adaptor__backed,
|
||||
"disable-diffexp": not dataset_config.diffexp__enable,
|
||||
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
|
||||
"annotations": False,
|
||||
"annotations_file": None,
|
||||
"annotations_dir": None,
|
||||
"annotations_cell_ontology_enabled": False,
|
||||
"annotations_cell_ontology_obopath": None,
|
||||
"annotations_cell_ontology_terms": None,
|
||||
"custom_colors": dataset_config.presentation__custom_colors,
|
||||
"diffexp-may-be-slow": False,
|
||||
"about_legal_tos": dataset_config.app__about_legal_tos,
|
||||
"about_legal_privacy": dataset_config.app__about_legal_privacy,
|
||||
}
|
||||
|
||||
# corpora dataset_props
|
||||
# TODO/Note: putting info from the dataset into the /config is not ideal.
|
||||
# However, it is definitely not part of /schema, and we do not have a top-level
|
||||
# route for data properties. Consider creating one at some point.
|
||||
corpora_props = data_adaptor.get_corpora_props()
|
||||
if corpora_props and "default_embedding" in corpora_props:
|
||||
default_embedding = corpora_props["default_embedding"]
|
||||
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
|
||||
default_embedding = default_embedding[2:] # drop X_ prefix
|
||||
if default_embedding in data_adaptor.get_embedding_names():
|
||||
parameters["default_embedding"] = default_embedding
|
||||
|
||||
data_adaptor.update_parameters(parameters)
|
||||
if annotation:
|
||||
annotation.update_parameters(parameters, data_adaptor)
|
||||
|
||||
# gather it all together
|
||||
client_config = {}
|
||||
config = client_config["config"] = {}
|
||||
config["displayNames"] = display_names
|
||||
config["library_versions"] = library_versions
|
||||
config["links"] = links
|
||||
config["parameters"] = parameters
|
||||
config["corpora_props"] = corpora_props
|
||||
config["limits"] = {
|
||||
"column_request_max": server_config.limits__column_request_max,
|
||||
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
|
||||
}
|
||||
|
||||
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
|
||||
config["authentication"] = {
|
||||
"requires_client_login": auth.requires_client_login(),
|
||||
}
|
||||
if auth.requires_client_login():
|
||||
config["authentication"].update(
|
||||
{
|
||||
# Todo why are these stored on the data_adaptor?
|
||||
"login": auth.get_login_url(data_adaptor),
|
||||
"logout": auth.get_logout_url(data_adaptor),
|
||||
}
|
||||
)
|
||||
|
||||
return client_config
|
||||
|
||||
|
||||
def get_client_userinfo(app_config, data_adaptor):
|
||||
"""
|
||||
Return the userinfo as required by the /userinfo REST route
|
||||
"""
|
||||
|
||||
server_config = app_config.server_config
|
||||
dataset_config = data_adaptor.dataset_config
|
||||
auth = server_config.auth
|
||||
|
||||
# make sure the configuration has been checked.
|
||||
app_config.check_config()
|
||||
|
||||
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
|
||||
userinfo = {}
|
||||
userinfo["userinfo"] = {
|
||||
"is_authenticated": auth.is_user_authenticated(),
|
||||
"username": auth.get_user_name(),
|
||||
"user_id": auth.get_user_id(),
|
||||
"email": auth.get_user_email(),
|
||||
"picture": auth.get_user_picture(),
|
||||
}
|
||||
return userinfo
|
||||
@@ -1,230 +0,0 @@
|
||||
import os
|
||||
from os.path import splitext, isdir
|
||||
|
||||
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
|
||||
from server.common.annotations.local_file_csv import AnnotationsLocalFile
|
||||
from server.common.config.base_config import BaseConfig
|
||||
from server.common.errors import ConfigurationError, OntologyLoadFailure
|
||||
from server.compute.scanpy import get_scanpy_module
|
||||
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
|
||||
from server.db.db_utils import DbUtils
|
||||
|
||||
|
||||
class DatasetConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with a dataset."""
|
||||
|
||||
def __init__(self, tag, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
self.tag = tag
|
||||
try:
|
||||
self.app__scripts = default_config["app"]["scripts"]
|
||||
self.app__inline_scripts = default_config["app"]["inline_scripts"]
|
||||
self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
|
||||
self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
|
||||
self.app__authentication_enable = default_config["app"]["authentication_enable"]
|
||||
|
||||
self.presentation__max_categories = default_config["presentation"]["max_categories"]
|
||||
self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
|
||||
|
||||
self.user_annotations__enable = default_config["user_annotations"]["enable"]
|
||||
self.user_annotations__type = default_config["user_annotations"]["type"]
|
||||
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
|
||||
"directory"
|
||||
]
|
||||
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
|
||||
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
|
||||
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
|
||||
"obo_location"
|
||||
]
|
||||
self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
|
||||
"hosted_tiledb_array"
|
||||
]["db_uri"]
|
||||
self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
|
||||
"hosted_tiledb_array"
|
||||
]["hosted_file_directory"]
|
||||
|
||||
self.embeddings__names = default_config["embeddings"]["names"]
|
||||
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
|
||||
|
||||
self.diffexp__enable = default_config["diffexp"]["enable"]
|
||||
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
|
||||
self.diffexp__top_n = default_config["diffexp"]["top_n"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
# The annotation object is created during complete_config and stored here.
|
||||
self.user_annotations = None
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_app()
|
||||
self.handle_presentation()
|
||||
self.handle_user_annotations(context)
|
||||
self.handle_embeddings()
|
||||
self.handle_diffexp(context)
|
||||
|
||||
def handle_app(self):
|
||||
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
|
||||
self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
|
||||
self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
|
||||
|
||||
# scripts can be string (filename) or dict (attributes). Convert string to dict.
|
||||
scripts = []
|
||||
for script in self.app__scripts:
|
||||
try:
|
||||
if isinstance(script, str):
|
||||
scripts.append({"src": script})
|
||||
elif isinstance(script, dict) and isinstance(script["src"], str):
|
||||
scripts.append(script)
|
||||
else:
|
||||
raise Exception
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
|
||||
|
||||
self.app__scripts = scripts
|
||||
|
||||
def handle_presentation(self):
|
||||
self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
|
||||
self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
|
||||
|
||||
def handle_user_annotations(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__local_file_csv__directory", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__local_file_csv__file", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__ontology__obo_location", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
|
||||
)
|
||||
if self.user_annotations__enable:
|
||||
server_config = self.app_config.server_config
|
||||
if not self.app__authentication_enable:
|
||||
raise ConfigurationError("user annotations requires authentication to be enabled")
|
||||
if not server_config.auth.is_valid_authentication_type():
|
||||
auth_type = server_config.authentication__type
|
||||
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
|
||||
|
||||
if self.user_annotations__type == "local_file_csv":
|
||||
self.handle_local_file_csv_annotations()
|
||||
elif self.user_annotations__type == "hosted_tiledb_array":
|
||||
self.handle_hosted_tiledb_annotations()
|
||||
else:
|
||||
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
|
||||
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
|
||||
try:
|
||||
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
|
||||
except OntologyLoadFailure as e:
|
||||
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
|
||||
else:
|
||||
self.check_annotation_config_vars_not_set(context)
|
||||
|
||||
def handle_local_file_csv_annotations(self):
|
||||
dirname = self.user_annotations__local_file_csv__directory
|
||||
filename = self.user_annotations__local_file_csv__file
|
||||
if filename is not None and dirname is not None:
|
||||
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
|
||||
|
||||
if filename is not None:
|
||||
lf_name, lf_ext = splitext(filename)
|
||||
if lf_ext and lf_ext != ".csv":
|
||||
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
|
||||
|
||||
if dirname is not None and not isdir(dirname):
|
||||
try:
|
||||
os.mkdir(dirname)
|
||||
except OSError:
|
||||
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
|
||||
|
||||
self.user_annotations = AnnotationsLocalFile(dirname, filename)
|
||||
|
||||
# if the user has specified a fixed label file, go ahead and validate it
|
||||
# so that we can remove errors early in the process.
|
||||
server_config = self.app_config.server_config
|
||||
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
|
||||
with server_config.matrix_data_cache_manager.data_adaptor(
|
||||
self.tag, server_config.single_dataset__datapath, self.app_config
|
||||
) as data_adaptor:
|
||||
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
|
||||
|
||||
def handle_hosted_tiledb_annotations(self):
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__hosted_tiledb_array__hosted_file_directory", str
|
||||
)
|
||||
self.user_annotations = AnnotationsHostedTileDB(
|
||||
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
|
||||
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
|
||||
)
|
||||
|
||||
def check_annotation_config_vars_not_set(self, context):
|
||||
if self.user_annotations__type is not None:
|
||||
dirname = self.user_annotations__local_file_csv__directory
|
||||
filename = self.user_annotations__local_file_csv__file
|
||||
db_uri = self.user_annotations__hosted_tiledb_array__db_uri
|
||||
hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
|
||||
if filename is not None:
|
||||
context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
|
||||
if dirname is not None:
|
||||
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
|
||||
if db_uri is not None:
|
||||
context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
|
||||
if hosted_file_dirname is not None:
|
||||
context["messagefn"](
|
||||
"Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
|
||||
)
|
||||
|
||||
if self.user_annotations__ontology__enable:
|
||||
context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
|
||||
if self.user_annotations__ontology__obo_location is not None:
|
||||
context["messagefn"](
|
||||
"Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
|
||||
)
|
||||
|
||||
def handle_embeddings(self):
|
||||
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
|
||||
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
|
||||
|
||||
server_config = self.app_config.server_config
|
||||
if self.embeddings__enable_reembedding:
|
||||
if server_config.single_dataset__datapath:
|
||||
matrix_data_loader = MatrixDataLoader(
|
||||
server_config.single_dataset__datapath, app_config=self.app_config
|
||||
)
|
||||
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
|
||||
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
|
||||
if server_config.adaptor__anndata_adaptor__backed:
|
||||
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
|
||||
|
||||
try:
|
||||
get_scanpy_module()
|
||||
except NotImplementedError:
|
||||
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
|
||||
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
|
||||
|
||||
def handle_diffexp(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
|
||||
|
||||
server_config = self.app_config.server_config
|
||||
if server_config.single_dataset__datapath:
|
||||
with server_config.matrix_data_cache_manager.data_adaptor(
|
||||
self.tag, server_config.single_dataset__datapath, self.app_config
|
||||
) as data_adaptor:
|
||||
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
|
||||
context["messagefn"](
|
||||
"CAUTION: due to the size of your dataset, "
|
||||
"running differential expression may take longer or fail."
|
||||
)
|
||||
@@ -1,96 +0,0 @@
|
||||
import os
|
||||
|
||||
from server.common.config.base_config import BaseConfig
|
||||
from server.common.errors import ConfigurationError
|
||||
from server.common.config import get_secret_key
|
||||
from server.common.errors import SecretKeyRetrievalError
|
||||
from server.common.utils.type_conversion_utils import convert_string_to_value
|
||||
|
||||
|
||||
class ExternalConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with external configuration sources, such as
|
||||
environment variables or the AWS Secrets Manager."""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
try:
|
||||
self.environment = default_config["environment"]
|
||||
self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
|
||||
self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_environment(context)
|
||||
self.handle_aws_secrets_manager(context)
|
||||
|
||||
def handle_environment(self, context):
|
||||
"""For each environment variable defined, get the value (if it is set),
|
||||
and set the specified config parameter"""
|
||||
self.validate_correct_type_of_configuration_attribute("environment", list)
|
||||
for envdict in self.environment:
|
||||
name = envdict.get("name")
|
||||
if name is None:
|
||||
raise ConfigurationError("environment: 'name' is missing")
|
||||
required = envdict.get("required", False)
|
||||
if type(required) != bool:
|
||||
raise ConfigurationError("environment: 'required' must be a bool")
|
||||
path = envdict.get("path")
|
||||
if path is None:
|
||||
raise ConfigurationError("environment: 'path' is missing")
|
||||
|
||||
value = os.environ.get(name)
|
||||
if value is None:
|
||||
if required:
|
||||
raise ConfigurationError(f"required environment variable '{name}' not set")
|
||||
else:
|
||||
value = convert_string_to_value(value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, value)
|
||||
|
||||
def handle_aws_secrets_manager(self, context):
|
||||
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
|
||||
|
||||
if not self.aws_secrets_manager__secrets:
|
||||
return
|
||||
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
|
||||
|
||||
for secret in self.aws_secrets_manager__secrets:
|
||||
secret_name = secret.get("name")
|
||||
if secret_name is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' is missing")
|
||||
if not isinstance(secret_name, str):
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
|
||||
|
||||
try:
|
||||
secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
|
||||
except SecretKeyRetrievalError as e:
|
||||
raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
|
||||
|
||||
values = secret.get("values")
|
||||
if values is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' is missing")
|
||||
if not isinstance(values, list):
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
|
||||
|
||||
for value in values:
|
||||
key = value.get("key")
|
||||
if key is None:
|
||||
raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
|
||||
path = value.get("path")
|
||||
if path is None:
|
||||
raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
|
||||
required = value.get("required", False)
|
||||
if type(required) != bool:
|
||||
raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
|
||||
|
||||
secret_value = secret_dict.get(key)
|
||||
if secret_value is None:
|
||||
if required:
|
||||
raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
|
||||
else:
|
||||
secret_value = convert_string_to_value(secret_value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, secret_value)
|
||||
@@ -1,387 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from os.path import basename
|
||||
from urllib.parse import urlparse, quote_plus
|
||||
|
||||
from server.auth.auth import AuthTypeFactory
|
||||
from server.common.config.base_config import BaseConfig
|
||||
from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
|
||||
from server.common.errors import ConfigurationError, DatasetAccessError
|
||||
from server.common.data_locator import discover_s3_region_name
|
||||
from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
|
||||
from server.compute import diffexp_cxg as diffexp_tiledb
|
||||
from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
|
||||
|
||||
|
||||
class ServerConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with the server."""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
dictval_cases = [
|
||||
("app", "csp_directives"),
|
||||
("authentication", "params_oauth", "cookie"),
|
||||
("authentication", "params_oauth", "jwt_decode_options"),
|
||||
("adaptor", "cxg_adaptor", "tiledb_ctx"),
|
||||
("multi_dataset", "dataroot"),
|
||||
]
|
||||
super().__init__(app_config, default_config, dictval_cases)
|
||||
|
||||
try:
|
||||
self.app__verbose = default_config["app"]["verbose"]
|
||||
self.app__debug = default_config["app"]["debug"]
|
||||
self.app__host = default_config["app"]["host"]
|
||||
self.app__port = default_config["app"]["port"]
|
||||
self.app__open_browser = default_config["app"]["open_browser"]
|
||||
self.app__force_https = default_config["app"]["force_https"]
|
||||
self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
|
||||
self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
|
||||
self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
|
||||
self.app__csp_directives = default_config["app"]["csp_directives"]
|
||||
self.app__api_base_url = default_config["app"]["api_base_url"]
|
||||
self.app__web_base_url = default_config["app"]["web_base_url"]
|
||||
|
||||
self.authentication__type = default_config["authentication"]["type"]
|
||||
self.authentication__insecure_test_environment = default_config["authentication"][
|
||||
"insecure_test_environment"
|
||||
]
|
||||
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
|
||||
"oauth_api_base_url"
|
||||
]
|
||||
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
|
||||
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
|
||||
"client_secret"
|
||||
]
|
||||
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
|
||||
"jwt_decode_options"
|
||||
]
|
||||
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
|
||||
"session_cookie"
|
||||
]
|
||||
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
|
||||
|
||||
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
|
||||
self.multi_dataset__index = default_config["multi_dataset"]["index"]
|
||||
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
|
||||
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
|
||||
"max_datasets"
|
||||
]
|
||||
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
|
||||
"timelimit_s"
|
||||
]
|
||||
|
||||
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
|
||||
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
|
||||
self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
|
||||
self.single_dataset__about = default_config["single_dataset"]["about"]
|
||||
self.single_dataset__title = default_config["single_dataset"]["title"]
|
||||
|
||||
self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
|
||||
self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
|
||||
self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
|
||||
|
||||
self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
|
||||
|
||||
self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
|
||||
self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
|
||||
|
||||
self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
|
||||
self.limits__column_request_max = default_config["limits"]["column_request_max"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
# The matrix data cache manager is created during the complete_config and stored here.
|
||||
self.matrix_data_cache_manager = None
|
||||
|
||||
# The authentication object
|
||||
self.auth = None
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_app(context)
|
||||
self.handle_data_source()
|
||||
self.handle_authentication()
|
||||
self.handle_data_locator()
|
||||
self.handle_adaptor() # may depend on data_locator
|
||||
self.handle_single_dataset(context) # may depend on adaptor
|
||||
self.handle_multi_dataset() # may depend on adaptor
|
||||
self.handle_diffexp()
|
||||
self.handle_limits()
|
||||
|
||||
self.check_config()
|
||||
|
||||
def handle_app(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__debug", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__host", str)
|
||||
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
|
||||
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
|
||||
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
|
||||
self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
|
||||
|
||||
if self.app__port:
|
||||
try:
|
||||
if not is_port_available(self.app__host, self.app__port):
|
||||
raise ConfigurationError(
|
||||
f"The port selected {self.app__port} is in use, please configure an open port."
|
||||
)
|
||||
except OverflowError:
|
||||
raise ConfigurationError(f"Invalid port: {self.app__port}")
|
||||
else:
|
||||
try:
|
||||
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
|
||||
except ValueError:
|
||||
raise ConfigurationError(
|
||||
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
|
||||
)
|
||||
try:
|
||||
self.app__port = find_available_port(self.app__host, default_server_port)
|
||||
except OverflowError:
|
||||
raise ConfigurationError(f"Invalid port: {default_server_port}")
|
||||
|
||||
if self.app__debug:
|
||||
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
|
||||
self.app__verbose = True
|
||||
self.app__open_browser = False
|
||||
else:
|
||||
warnings.formatwarning = custom_format_warning
|
||||
|
||||
if not self.app__verbose:
|
||||
sys.tracebacklimit = 0
|
||||
|
||||
# CSP Directives are a dict of string: list(string) or string: string
|
||||
if self.app__csp_directives is not None:
|
||||
for k, v in self.app__csp_directives.items():
|
||||
if not isinstance(k, str):
|
||||
raise ConfigurationError("CSP directive names must be a string.")
|
||||
if isinstance(v, list):
|
||||
for policy in v:
|
||||
if not isinstance(policy, str):
|
||||
raise ConfigurationError("CSP directive value must be a string or list of strings.")
|
||||
elif not isinstance(v, str):
|
||||
raise ConfigurationError("CSP directive value must be a string or list of strings.")
|
||||
|
||||
if self.app__web_base_url is None:
|
||||
self.app__web_base_url = self.app__api_base_url
|
||||
|
||||
def handle_authentication(self):
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
|
||||
|
||||
if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
|
||||
raise ConfigurationError("Test auth can only be used in an insecure test environment")
|
||||
|
||||
# oauth
|
||||
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"authentication__params_oauth__oauth_api_base_url", ptypes
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
|
||||
|
||||
if self.authentication__params_oauth__session_cookie:
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"authentication__params_oauth__cookie", (type(None), dict)
|
||||
)
|
||||
else:
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
|
||||
|
||||
self.auth = AuthTypeFactory.create(self.authentication__type, self)
|
||||
if self.auth is None:
|
||||
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
|
||||
|
||||
def handle_data_locator(self):
|
||||
self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
|
||||
if self.data_locator__s3__region_name is True:
|
||||
path = self.single_dataset__datapath or self.multi_dataset__dataroot
|
||||
|
||||
if type(path) == dict:
|
||||
# if multi_dataset__dataroot is a dict, then use the first key
|
||||
# that is in s3. NOTE: it is not supported to have dataroots
|
||||
# in different regions.
|
||||
paths = [val.get("dataroot") for val in path.values()]
|
||||
for path in paths:
|
||||
if path.startswith("s3://"):
|
||||
break
|
||||
if path.startswith("s3://"):
|
||||
region_name = discover_s3_region_name(path)
|
||||
if region_name is None:
|
||||
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
|
||||
else:
|
||||
region_name = None
|
||||
self.data_locator__s3__region_name = region_name
|
||||
|
||||
def handle_data_source(self):
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
|
||||
|
||||
if self.single_dataset__datapath and self.multi_dataset__dataroot:
|
||||
raise ConfigurationError(
|
||||
"You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
|
||||
)
|
||||
if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
|
||||
raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
|
||||
|
||||
def handle_single_dataset(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
|
||||
|
||||
if self.single_dataset__datapath is None:
|
||||
return
|
||||
|
||||
# create the matrix data cache manager:
|
||||
if self.matrix_data_cache_manager is None:
|
||||
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
|
||||
|
||||
# preload this data set
|
||||
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
|
||||
try:
|
||||
matrix_data_loader.pre_load_validation()
|
||||
except DatasetAccessError as e:
|
||||
raise ConfigurationError(str(e))
|
||||
|
||||
file_size = matrix_data_loader.file_size()
|
||||
file_basename = basename(self.single_dataset__datapath)
|
||||
if file_size > BIG_FILE_SIZE_THRESHOLD:
|
||||
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
|
||||
else:
|
||||
context["messagefn"](f"Loading data from {file_basename}.")
|
||||
|
||||
if self.single_dataset__about:
|
||||
|
||||
def url_check(url):
|
||||
try:
|
||||
result = urlparse(url)
|
||||
if all([result.scheme, result.netloc]):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
if not url_check(self.single_dataset__about):
|
||||
raise ConfigurationError(
|
||||
"Must provide an absolute URL for --about. (Example format: http://example.com)"
|
||||
)
|
||||
|
||||
def handle_multi_dataset(self):
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
|
||||
)
|
||||
|
||||
if self.multi_dataset__dataroot is None:
|
||||
return
|
||||
|
||||
if type(self.multi_dataset__dataroot) == str:
|
||||
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
|
||||
self.multi_dataset__dataroot = dict(d=default_dict)
|
||||
|
||||
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
|
||||
if "base_url" not in dataroot_dict:
|
||||
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
|
||||
if "dataroot" not in dataroot_dict:
|
||||
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
|
||||
|
||||
base_url = dataroot_dict["base_url"]
|
||||
|
||||
# sanity check for well formed base urls
|
||||
bad = False
|
||||
if type(base_url) != str:
|
||||
bad = True
|
||||
elif os.path.normpath(base_url) != base_url:
|
||||
bad = True
|
||||
else:
|
||||
base_url_parts = base_url.split("/")
|
||||
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
|
||||
bad = True
|
||||
if ".." in base_url_parts:
|
||||
bad = True
|
||||
if bad:
|
||||
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
|
||||
|
||||
# verify all the base_urls are unique
|
||||
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
|
||||
if len(base_urls) > len(set(base_urls)):
|
||||
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
|
||||
|
||||
# error checking
|
||||
for mtype in self.multi_dataset__allowed_matrix_types:
|
||||
try:
|
||||
MatrixDataType(mtype)
|
||||
except ValueError:
|
||||
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
|
||||
|
||||
# create the matrix data cache manager:
|
||||
if self.matrix_data_cache_manager is None:
|
||||
self.matrix_data_cache_manager = MatrixDataCacheManager(
|
||||
max_cached=self.multi_dataset__matrix_cache__max_datasets,
|
||||
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
|
||||
)
|
||||
|
||||
def handle_diffexp(self):
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
|
||||
|
||||
max_workers = self.diffexp__alg_cxg__max_workers
|
||||
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
|
||||
cpu_count = os.cpu_count()
|
||||
max_workers = min(max_workers, cpu_multiplier * cpu_count)
|
||||
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
|
||||
|
||||
def handle_adaptor(self):
|
||||
# cxg
|
||||
self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
|
||||
regionkey = "vfs.s3.region"
|
||||
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
|
||||
if type(self.data_locator__s3__region_name) == str:
|
||||
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
|
||||
|
||||
from server.data_cxg.cxg_adaptor import CxgAdaptor
|
||||
|
||||
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
|
||||
|
||||
# anndata
|
||||
self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
|
||||
|
||||
def handle_limits(self):
|
||||
self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
|
||||
self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
|
||||
|
||||
def exceeds_limit(self, limit_name, value):
|
||||
limit_value = getattr(self, "limits__" + limit_name, None)
|
||||
if limit_value is None: # disabled
|
||||
return False
|
||||
return value > limit_value
|
||||
|
||||
def get_api_base_url(self):
|
||||
if self.app__api_base_url == "local":
|
||||
return f"http://{self.app__host}:{self.app__port}"
|
||||
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
|
||||
return self.app__api_base_url[:-1]
|
||||
return self.app__api_base_url
|
||||
|
||||
def get_web_base_url(self):
|
||||
if self.app__web_base_url == "local":
|
||||
return f"http://{self.app__host}:{self.app__port}"
|
||||
if self.app__web_base_url is None:
|
||||
return self.get_api_base_url()
|
||||
if self.app__web_base_url.endswith("/"):
|
||||
return self.app__web_base_url[:-1]
|
||||
return self.app__web_base_url
|
||||
@@ -1,30 +0,0 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class AugmentedEnum(Enum):
|
||||
def __hash__(self):
|
||||
return self.value.__hash__()
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, type(self)) or isinstance(other, str):
|
||||
return self.value == other
|
||||
return False
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.value
|
||||
|
||||
|
||||
class Axis(AugmentedEnum):
|
||||
OBS = "obs"
|
||||
VAR = "var"
|
||||
|
||||
|
||||
class DiffExpMode(AugmentedEnum):
|
||||
TOP_N = "topN"
|
||||
VAR_FILTER = "varFilter"
|
||||
|
||||
|
||||
JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
|
||||
REACTIVE_LIMIT = 1_000_000
|
||||
|
||||
MAX_LAYOUTS = 30
|
||||
@@ -1,78 +0,0 @@
|
||||
"""
|
||||
Corpora schema conventions support. Helper functions for reading.
|
||||
|
||||
https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
|
||||
|
||||
https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
|
||||
"""
|
||||
import collections
|
||||
import json
|
||||
|
||||
from server.cli.upgrade import validate_version_str
|
||||
from server.common.utils.corpora_constants import CorporaConstants
|
||||
|
||||
|
||||
def corpora_get_versions_from_anndata(adata):
|
||||
"""
|
||||
Given an AnnData object, return:
|
||||
* None - if not a Corpora object
|
||||
* [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
|
||||
|
||||
Implements the identification protocol defined in the specification.
|
||||
"""
|
||||
|
||||
# per Corpora AnnData spec, this is a corpora file if the following is true
|
||||
if "version" not in adata.uns_keys():
|
||||
return None
|
||||
version = adata.uns["version"]
|
||||
if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
|
||||
return None
|
||||
|
||||
corpora_schema_version = version.get("corpora_schema_version")
|
||||
corpora_encoding_version = version.get("corpora_encoding_version")
|
||||
|
||||
# TODO: spec says these must be SEMVER values, so check.
|
||||
if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
|
||||
return [corpora_schema_version, corpora_encoding_version]
|
||||
|
||||
|
||||
def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
|
||||
return (
|
||||
corpora_schema_version
|
||||
and corpora_encoding_version
|
||||
and corpora_schema_version.startswith("1.")
|
||||
and corpora_encoding_version.startswith("0.1.")
|
||||
)
|
||||
|
||||
|
||||
def corpora_get_props_from_anndata(adata):
|
||||
"""
|
||||
Get Corpora dataset properties from an AnnData
|
||||
"""
|
||||
versions = corpora_get_versions_from_anndata(adata)
|
||||
if versions is None:
|
||||
return None
|
||||
[corpora_schema_version, corpora_encoding_version] = versions
|
||||
version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
|
||||
if not version_is_supported:
|
||||
raise ValueError("Unsupported Corpora schema version")
|
||||
|
||||
corpora_props = {}
|
||||
for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
|
||||
if key not in adata.uns:
|
||||
raise KeyError(f"missing Corpora schema field {key}")
|
||||
corpora_props[key] = adata.uns[key]
|
||||
|
||||
for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
|
||||
if key not in adata.uns:
|
||||
continue
|
||||
try:
|
||||
corpora_props[key] = json.loads(adata.uns[key])
|
||||
except json.JSONDecodeError:
|
||||
raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
|
||||
|
||||
for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
|
||||
if key in adata.uns:
|
||||
corpora_props[key] = adata.uns[key]
|
||||
|
||||
return corpora_props
|
||||
@@ -1,154 +0,0 @@
|
||||
import os
|
||||
import tempfile
|
||||
import fsspec
|
||||
from datetime import datetime
|
||||
import boto3
|
||||
import botocore
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class DataLocator:
|
||||
"""
|
||||
DataLocator is a simple wrapper around fsspec functionality, and provides a
|
||||
set of functions to encapsulate a data location (URI or path), interogate
|
||||
metadata about the object at that location (size, existance, etc) and
|
||||
access the underlying data.
|
||||
|
||||
https://filesystem-spec.readthedocs.io/en/latest/index.html
|
||||
|
||||
Example:
|
||||
dl = DataLocator("/tmp/foo.h5ad")
|
||||
if dl.exists():
|
||||
print(dl.size())
|
||||
with dl.open() as f:
|
||||
thecontents = f.read()
|
||||
|
||||
DataLocator will accept a URI or native path. Error handling is as defined
|
||||
in fsspec.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, uri_or_path, region_name=None):
|
||||
if isinstance(uri_or_path, DataLocator):
|
||||
locator = uri_or_path
|
||||
self.uri_or_path = locator.uri_or_path
|
||||
self.protocol = locator.protocol
|
||||
self.path = locator.path
|
||||
self.cname = locator.cname
|
||||
else:
|
||||
self.uri_or_path = uri_or_path
|
||||
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
|
||||
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
|
||||
self.cname = self.path if self.protocol == "file" else self.uri_or_path
|
||||
|
||||
# fsspec.filesystem will throw RuntimeError if the protocol is unsupported
|
||||
if self.protocol == "s3":
|
||||
if region_name:
|
||||
config_kwargs = dict(region_name=region_name)
|
||||
self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs)
|
||||
else:
|
||||
self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30)
|
||||
else:
|
||||
self.fs = fsspec.filesystem(self.protocol)
|
||||
|
||||
def __repr__(self):
|
||||
return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
|
||||
f"path={self.path}, uri_or_path={self.uri_or_path})"
|
||||
|
||||
@staticmethod
|
||||
def _get_protocol_and_path(uri_or_path):
|
||||
if "://" in uri_or_path:
|
||||
protocol, path = uri_or_path.split("://", 1)
|
||||
# windows!!! Ignore single letter drive identifiers,
|
||||
# eg, G:\foo.txt
|
||||
if len(protocol) > 1:
|
||||
return protocol, path
|
||||
return None, uri_or_path
|
||||
|
||||
def exists(self):
|
||||
return self.fs.exists(self.cname)
|
||||
|
||||
def size(self):
|
||||
return self.fs.size(self.cname)
|
||||
|
||||
def lastmodtime(self):
|
||||
""" return datetime object representing last modification time, or None if unavailable """
|
||||
info = self.fs.info(self.cname)
|
||||
if self.islocal() and info is not None:
|
||||
return datetime.fromtimestamp(info["mtime"])
|
||||
else:
|
||||
return getattr(info, "LastModified", None)
|
||||
|
||||
def abspath(self):
|
||||
"""
|
||||
return the absolute path for the locator - only really does something
|
||||
for file: protocol, as all others are already absolute
|
||||
"""
|
||||
if self.islocal():
|
||||
return os.path.abspath(self.path)
|
||||
else:
|
||||
return self.uri_or_path
|
||||
|
||||
def isfile(self):
|
||||
return self.fs.isfile(self.cname)
|
||||
|
||||
def open(self, *args):
|
||||
return self.fs.open(self.uri_or_path, *args)
|
||||
|
||||
def islocal(self):
|
||||
return self.protocol is None or self.protocol == "file"
|
||||
|
||||
def local_handle(self):
|
||||
if self.islocal():
|
||||
return LocalFilePath(self.path)
|
||||
|
||||
# if not local, create a tmp file system object to contain the data,
|
||||
# and clean it up when done. If the path has a suffix/extension,
|
||||
# do our best to create a file with the same.
|
||||
ext = os.path.splitext(self.path)
|
||||
suffix = None if ext[1] == "" else ext[1]
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(src.read())
|
||||
tmp.close()
|
||||
src.close()
|
||||
tmp_path = tmp.name
|
||||
return LocalFilePath(tmp_path, delete=True)
|
||||
|
||||
def ls(self):
|
||||
paths = self.fs.ls(self.uri_or_path)
|
||||
return [os.path.basename(p) for p in paths]
|
||||
|
||||
|
||||
class LocalFilePath:
|
||||
def __init__(self, tmp_path, delete=False):
|
||||
self.tmp_path = tmp_path
|
||||
self.delete = delete
|
||||
|
||||
def __enter__(self):
|
||||
return self.tmp_path
|
||||
|
||||
def __exit__(self, *args):
|
||||
if self.delete:
|
||||
os.unlink(self.tmp_path)
|
||||
|
||||
|
||||
def discover_s3_region_name(uri):
|
||||
"""If this is an s3 protocol, discover and return the (aws) region name.
|
||||
If a return name could not be discovered, or if the uri is not an s3 protocol, return None."""
|
||||
|
||||
protocol, _ = DataLocator._get_protocol_and_path(uri)
|
||||
if protocol == "s3":
|
||||
bucket = urlparse(uri).netloc
|
||||
client = boto3.client("s3")
|
||||
try:
|
||||
res = client.head_bucket(Bucket=bucket)
|
||||
except botocore.exceptions.ClientError:
|
||||
return None
|
||||
|
||||
region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
|
||||
if region:
|
||||
return region
|
||||
else:
|
||||
return None
|
||||
|
||||
return None
|
||||
@@ -1,57 +0,0 @@
|
||||
from http import HTTPStatus
|
||||
|
||||
|
||||
class CellxgeneException(Exception):
|
||||
"""Base class for cellxgene exceptions"""
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class RequestException(CellxgeneException):
|
||||
"""Baseclass for exceptions that can be raised from a request."""
|
||||
|
||||
# The default status code is 400 (Bad Request)
|
||||
default_status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
def __init__(self, message, status_code=None):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code or self.default_status_code
|
||||
|
||||
|
||||
def define_exception(name, doc):
|
||||
globals()[name] = type(name, (CellxgeneException,), dict(__doc__=doc))
|
||||
|
||||
|
||||
def define_request_exception(name, doc, default_status_code=HTTPStatus.BAD_REQUEST):
|
||||
globals()[name] = type(name, (RequestException,), dict(__doc__=doc, default_status_code=default_status_code))
|
||||
|
||||
|
||||
define_request_exception("FilterError", "Raised when filter is malformed")
|
||||
define_request_exception("JSONEncodingValueError", "Raised when data cannot be encoded into json")
|
||||
define_request_exception("MimeTypeError", "Raised when incompatible MIME type selected")
|
||||
define_request_exception("DatasetAccessError", "Raised when file loaded into a DataAdaptor is misformatted")
|
||||
define_request_exception("DisabledFeatureError", "Raised when an attempt to use a disabled feature occurs")
|
||||
define_request_exception("AnnotationsError", "Raised when an attempt to use the annotations feature fails")
|
||||
define_request_exception(
|
||||
"ComputeError",
|
||||
"Raised when an error occurs during a compute algorithm (such as diffexp)",
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
|
||||
define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
|
||||
define_request_exception(
|
||||
"AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
|
||||
)
|
||||
|
||||
define_request_exception(
|
||||
"AnnotationCategoryNameError",
|
||||
"Raised when an annotation category name cant be saved",
|
||||
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
)
|
||||
|
||||
define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
|
||||
define_exception("ConfigurationError", "Raised when checking configuration errors")
|
||||
define_exception("PrepareError", "Raised when data is misprepared")
|
||||
define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
|
||||
@@ -1,38 +0,0 @@
|
||||
from http import HTTPStatus
|
||||
from flask import make_response, jsonify
|
||||
|
||||
from server import __version__ as cellxgene_version
|
||||
from server.common.data_locator import DataLocator
|
||||
|
||||
|
||||
def _is_accessible(path, config):
|
||||
if path is None:
|
||||
return True
|
||||
|
||||
try:
|
||||
dl = DataLocator(path, region_name=config.data_locator__s3__region_name)
|
||||
return dl.exists()
|
||||
except RuntimeError:
|
||||
return False
|
||||
|
||||
|
||||
def health_check(config):
|
||||
"""
|
||||
simple health check - return HTTP response.
|
||||
See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html
|
||||
"""
|
||||
health = {"status": None, "version": "1", "releaseID": cellxgene_version}
|
||||
|
||||
checks = False
|
||||
server_config = config.server_config
|
||||
if config.is_multi_dataset():
|
||||
dataroots = [datapath_dict["dataroot"] for datapath_dict in server_config.multi_dataset__dataroot.values()]
|
||||
checks = all([_is_accessible(dataroot, server_config) for dataroot in dataroots])
|
||||
else:
|
||||
checks = _is_accessible(server_config.single_dataset__datapath, server_config)
|
||||
|
||||
health["status"] = "pass" if checks else "fail"
|
||||
code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST
|
||||
response = make_response(jsonify(health), code)
|
||||
response.headers["Content-Type"] = "application/health+json"
|
||||
return response
|
||||
@@ -1,71 +0,0 @@
|
||||
import threading
|
||||
from collections.abc import MutableMapping
|
||||
|
||||
|
||||
class ImmutableKVCache(MutableMapping):
|
||||
"""
|
||||
Guarantees that the factory will be called for each key once, and
|
||||
only once.
|
||||
"""
|
||||
|
||||
def __init__(self, factory):
|
||||
self.factory = factory # user-provided factory function
|
||||
self.lock = threading.Lock() # guards factory_calls
|
||||
self.factory_calls = {} # per-key factory condition variables
|
||||
self.cache = {} # result cache, indexed by key
|
||||
super().__init__()
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
|
||||
# we need to call factory. First grab the main lock and the per-key CV.
|
||||
factory_calls = None
|
||||
creation_thr = False
|
||||
with self.lock:
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
if key not in self.factory_calls:
|
||||
creation_thr = True
|
||||
self.factory_calls[key] = {"cv": threading.Condition(), "is_done": False, "error": None}
|
||||
factory_calls = self.factory_calls[key]
|
||||
|
||||
# with the CV, create the value (or wait for it to be created)
|
||||
cv = factory_calls["cv"]
|
||||
with cv:
|
||||
if creation_thr:
|
||||
try:
|
||||
self.cache[key] = self.factory(key)
|
||||
except Exception as e:
|
||||
factory_calls["error"] = e
|
||||
|
||||
factory_calls["is_done"] = True
|
||||
cv.notify_all()
|
||||
else:
|
||||
""" wait for the value to be available """
|
||||
while not factory_calls["is_done"]:
|
||||
cv.wait()
|
||||
|
||||
with self.lock:
|
||||
if key in self.factory_calls:
|
||||
del self.factory_calls[key]
|
||||
|
||||
return self.cache[key]
|
||||
|
||||
def __iter__(self):
|
||||
""" weak iter, don't call factory """
|
||||
return self.cache.__iter__()
|
||||
|
||||
def __len__(self):
|
||||
return self.cache.__len__()
|
||||
|
||||
def __contains__(self, key):
|
||||
""" weak contain - don't call factory """
|
||||
return self.cache.__contains__(key)
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self.cache[key]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
""" unsupported """
|
||||
raise NotImplementedError
|
||||
@@ -1,330 +0,0 @@
|
||||
import copy
|
||||
import logging
|
||||
import sys
|
||||
from http import HTTPStatus
|
||||
import zlib
|
||||
|
||||
from flask import make_response, jsonify, current_app, abort
|
||||
from werkzeug.urls import url_unquote
|
||||
|
||||
from server.common.config.client_config import get_client_config, get_client_userinfo
|
||||
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
|
||||
from server.common.errors import (
|
||||
FilterError,
|
||||
JSONEncodingValueError,
|
||||
PrepareError,
|
||||
DisabledFeatureError,
|
||||
ExceedsLimitError,
|
||||
DatasetAccessError,
|
||||
ColorFormatException,
|
||||
)
|
||||
|
||||
import json
|
||||
from server.data_common.fbs.matrix import decode_matrix_fbs
|
||||
|
||||
|
||||
def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
|
||||
"""
|
||||
Log the message, then abort with HTTP code. If include_exc_info is true,
|
||||
also include current exception via sys.exc_info().
|
||||
"""
|
||||
if include_exc_info:
|
||||
exc_info = sys.exc_info()
|
||||
else:
|
||||
exc_info = False
|
||||
current_app.logger.log(loglevel, logmsg, exc_info=exc_info)
|
||||
# Do NOT send log message to HTTP response.
|
||||
return abort(code)
|
||||
|
||||
|
||||
def _query_parameter_to_filter(args):
|
||||
"""
|
||||
Convert an annotation value filter, if present in the query args,
|
||||
into the standard dict filter format used by internal code.
|
||||
|
||||
Query param filters look like: <axis>:name=value, where value
|
||||
may be one of:
|
||||
- a range, min,max, where either may be an open range by using an asterisc, eg, 10,*
|
||||
- a value
|
||||
Eg,
|
||||
...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
|
||||
"""
|
||||
filters = {
|
||||
"obs": {},
|
||||
"var": {},
|
||||
}
|
||||
|
||||
# args has already been url-unquoted once. We assume double escaping
|
||||
# on name and value.
|
||||
try:
|
||||
for key, value in args.items(multi=True):
|
||||
axis, name = key.split(":")
|
||||
if axis not in ("obs", "var"):
|
||||
raise FilterError("unknown filter axis")
|
||||
name = url_unquote(name)
|
||||
current = filters[axis].setdefault(name, {"name": name})
|
||||
|
||||
val_split = value.split(",")
|
||||
if len(val_split) == 1:
|
||||
if "min" in current or "max" in current:
|
||||
raise FilterError("do not mix range and value filters")
|
||||
value = url_unquote(value)
|
||||
values = current.setdefault("values", [])
|
||||
values.append(value)
|
||||
|
||||
elif len(val_split) == 2:
|
||||
if len(current) > 1:
|
||||
raise FilterError("duplicate range specification")
|
||||
min = url_unquote(val_split[0])
|
||||
max = url_unquote(val_split[1])
|
||||
if min != "*":
|
||||
current["min"] = float(min)
|
||||
if max != "*":
|
||||
current["max"] = float(max)
|
||||
if len(current) < 2:
|
||||
raise FilterError("must specify at least min or max in range filter")
|
||||
|
||||
else:
|
||||
raise FilterError("badly formated filter value")
|
||||
|
||||
except ValueError as e:
|
||||
raise FilterError(str(e))
|
||||
|
||||
result = {}
|
||||
for axis in ("obs", "var"):
|
||||
axis_filter = filters[axis]
|
||||
if len(axis_filter) > 0:
|
||||
result[axis] = {"annotation_value": [val for val in axis_filter.values()]}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def schema_get_helper(data_adaptor):
|
||||
"""helper function to gather the schema from the data source and annotations"""
|
||||
schema = data_adaptor.get_schema()
|
||||
schema = copy.deepcopy(schema)
|
||||
|
||||
# add label obs annotations as needed
|
||||
annotations = data_adaptor.dataset_config.user_annotations
|
||||
if annotations is not None:
|
||||
label_schema = annotations.get_schema(data_adaptor)
|
||||
schema["annotations"]["obs"]["columns"].extend(label_schema)
|
||||
|
||||
return schema
|
||||
|
||||
|
||||
def schema_get(data_adaptor):
|
||||
schema = schema_get_helper(data_adaptor)
|
||||
return make_response(jsonify({"schema": schema}), HTTPStatus.OK)
|
||||
|
||||
|
||||
def config_get(app_config, data_adaptor):
|
||||
config = get_client_config(app_config, data_adaptor)
|
||||
return make_response(jsonify(config), HTTPStatus.OK)
|
||||
|
||||
|
||||
def userinfo_get(app_config, data_adaptor):
|
||||
config = get_client_userinfo(app_config, data_adaptor)
|
||||
return make_response(jsonify(config), HTTPStatus.OK)
|
||||
|
||||
|
||||
def annotations_obs_get(request, data_adaptor):
|
||||
fields = request.args.getlist("annotation-name", None)
|
||||
num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)
|
||||
if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
|
||||
return abort(HTTPStatus.BAD_REQUEST)
|
||||
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
|
||||
if preferred_mimetype != "application/octet-stream":
|
||||
return abort(HTTPStatus.NOT_ACCEPTABLE)
|
||||
|
||||
try:
|
||||
labels = None
|
||||
annotations = data_adaptor.dataset_config.user_annotations
|
||||
if annotations:
|
||||
labels = annotations.read_labels(data_adaptor)
|
||||
fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
|
||||
return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
|
||||
except KeyError as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
|
||||
|
||||
def annotations_put_fbs_helper(data_adaptor, fbs):
|
||||
"""helper function to write annotations from fbs"""
|
||||
annotations = data_adaptor.dataset_config.user_annotations
|
||||
if annotations is None:
|
||||
raise DisabledFeatureError("Writable annotations are not enabled")
|
||||
|
||||
new_label_df = decode_matrix_fbs(fbs)
|
||||
if not new_label_df.empty:
|
||||
new_label_df = data_adaptor.check_new_labels(new_label_df)
|
||||
annotations.write_labels(new_label_df, data_adaptor)
|
||||
|
||||
|
||||
def inflate(data):
|
||||
return zlib.decompress(data)
|
||||
|
||||
|
||||
def annotations_obs_put(request, data_adaptor):
|
||||
annotations = data_adaptor.dataset_config.user_annotations
|
||||
if annotations is None:
|
||||
return abort(HTTPStatus.NOT_IMPLEMENTED)
|
||||
|
||||
anno_collection = request.args.get("annotation-collection-name", default=None)
|
||||
fbs = inflate(request.get_data())
|
||||
|
||||
if anno_collection is not None:
|
||||
if not annotations.is_safe_collection_name(anno_collection):
|
||||
return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
|
||||
annotations.set_collection(anno_collection)
|
||||
|
||||
try:
|
||||
annotations_put_fbs_helper(data_adaptor, fbs)
|
||||
res = json.dumps({"status": "OK"})
|
||||
return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
|
||||
except (ValueError, DisabledFeatureError, KeyError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
|
||||
|
||||
def annotations_var_get(request, data_adaptor):
|
||||
fields = request.args.getlist("annotation-name", None)
|
||||
num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields)
|
||||
if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
|
||||
return abort(HTTPStatus.BAD_REQUEST)
|
||||
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
|
||||
if preferred_mimetype != "application/octet-stream":
|
||||
return abort(HTTPStatus.NOT_ACCEPTABLE)
|
||||
|
||||
try:
|
||||
labels = None
|
||||
annotations = data_adaptor.dataset_config.user_annotations
|
||||
if annotations is not None:
|
||||
labels = annotations.read_labels(data_adaptor)
|
||||
return make_response(
|
||||
data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
|
||||
HTTPStatus.OK,
|
||||
{"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
except KeyError as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
|
||||
|
||||
def data_var_put(request, data_adaptor):
|
||||
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
|
||||
if preferred_mimetype != "application/octet-stream":
|
||||
return abort(HTTPStatus.NOT_ACCEPTABLE)
|
||||
|
||||
filter_json = request.get_json()
|
||||
filter = filter_json["filter"] if filter_json else None
|
||||
try:
|
||||
return make_response(
|
||||
data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
|
||||
HTTPStatus.OK,
|
||||
{"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
except (FilterError, ValueError, ExceedsLimitError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
|
||||
|
||||
def data_var_get(request, data_adaptor):
|
||||
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
|
||||
if preferred_mimetype != "application/octet-stream":
|
||||
return abort(HTTPStatus.NOT_ACCEPTABLE)
|
||||
|
||||
try:
|
||||
filter = _query_parameter_to_filter(request.args)
|
||||
return make_response(
|
||||
data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
|
||||
HTTPStatus.OK,
|
||||
{"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
except (FilterError, ValueError, ExceedsLimitError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
|
||||
|
||||
def colors_get(data_adaptor):
|
||||
if not data_adaptor.dataset_config.presentation__custom_colors:
|
||||
return make_response(jsonify({}), HTTPStatus.OK)
|
||||
try:
|
||||
return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
|
||||
except ColorFormatException as e:
|
||||
return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
|
||||
|
||||
|
||||
def diffexp_obs_post(request, data_adaptor):
|
||||
if not data_adaptor.dataset_config.diffexp__enable:
|
||||
return abort(HTTPStatus.NOT_IMPLEMENTED)
|
||||
|
||||
args = request.get_json()
|
||||
try:
|
||||
# TODO: implement varfilter mode
|
||||
mode = DiffExpMode(args["mode"])
|
||||
|
||||
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
|
||||
return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
|
||||
|
||||
set1_filter = args.get("set1", {"filter": {}})["filter"]
|
||||
set2_filter = args.get("set2", {"filter": {}})["filter"]
|
||||
count = args.get("count", None)
|
||||
|
||||
if set1_filter is None or set2_filter is None or count is None:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
|
||||
if Axis.VAR in set1_filter or Axis.VAR in set2_filter:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled")
|
||||
|
||||
except (KeyError, TypeError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
|
||||
try:
|
||||
diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count)
|
||||
return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
|
||||
except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
except JSONEncodingValueError:
|
||||
# JSON encoding failure, usually due to bad data. Just let it ripple up
|
||||
# to default exception handler.
|
||||
current_app.logger.warning(JSON_NaN_to_num_warning_msg)
|
||||
raise
|
||||
|
||||
|
||||
def layout_obs_get(request, data_adaptor):
|
||||
fields = request.args.getlist("layout-name", None)
|
||||
num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields)
|
||||
if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
|
||||
return abort(HTTPStatus.BAD_REQUEST)
|
||||
|
||||
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
|
||||
if preferred_mimetype != "application/octet-stream":
|
||||
return abort(HTTPStatus.NOT_ACCEPTABLE)
|
||||
|
||||
try:
|
||||
return make_response(
|
||||
data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
|
||||
)
|
||||
except (KeyError, DatasetAccessError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
except PrepareError:
|
||||
return abort_and_log(
|
||||
HTTPStatus.NOT_IMPLEMENTED,
|
||||
f"No embedding available {request.path}",
|
||||
loglevel=logging.ERROR,
|
||||
include_exc_info=True,
|
||||
)
|
||||
|
||||
|
||||
def layout_obs_put(request, data_adaptor):
|
||||
if not data_adaptor.dataset_config.embeddings__enable_reembedding:
|
||||
return abort(HTTPStatus.NOT_IMPLEMENTED)
|
||||
|
||||
args = request.get_json()
|
||||
filter = args["filter"] if args else None
|
||||
if not filter:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
|
||||
method = args["method"] if args else "umap"
|
||||
|
||||
try:
|
||||
schema = data_adaptor.compute_embedding(method, filter)
|
||||
return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
|
||||
except NotImplementedError as e:
|
||||
return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
|
||||
except (ValueError, DisabledFeatureError, FilterError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
@@ -1,22 +0,0 @@
|
||||
class CorporaConstants(object):
|
||||
REQUIRED_SIMPLE_METADATA_FIELDS = [
|
||||
"version",
|
||||
"title",
|
||||
"layer_descriptions",
|
||||
"organism",
|
||||
"organism_ontology_term_id",
|
||||
]
|
||||
|
||||
# The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
|
||||
# types.
|
||||
OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
|
||||
|
||||
OPTIONAL_SIMPLE_METADATA_FIELDS = [
|
||||
"preprint_doi",
|
||||
"publication_doi",
|
||||
"default_embedding",
|
||||
"default_field",
|
||||
"tags",
|
||||
"project_name",
|
||||
"project_description",
|
||||
]
|
||||
@@ -1,4 +0,0 @@
|
||||
class CxgConstants(object):
|
||||
# The CXG container version number. Must be a semver string (major.minor.patch)
|
||||
# DO NOT UPDATE THIS WITHOUT ALSO UPDATING CXG SPECIFICATION.
|
||||
CXG_VERSION = "0.2.0"
|
||||
@@ -1,178 +0,0 @@
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import tiledb
|
||||
|
||||
from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
|
||||
|
||||
|
||||
def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
|
||||
"""
|
||||
Saves the contents of the dictionary to the CXG output directory specified.
|
||||
|
||||
This function is primarily used to save metadata about a dataset to the CXG directory. At some point, tiledb will
|
||||
have support for metadata on groups at which point the utility of this function should be revisited. Until such
|
||||
feature exists, this function create an empty array and annotate that array.
|
||||
|
||||
For more information, visit https://github.com/TileDB-Inc/TileDB-Py/issues/254.
|
||||
"""
|
||||
|
||||
array_name = f"{cxg_container}/{group_metadata_name}"
|
||||
|
||||
# Because TileDB does not allow one to attach metadata directly to a CXG group, we need to have a workaround
|
||||
# where we create an empty array and attached the metadata onto to this empty array. Below we construct this empty
|
||||
# array.
|
||||
tiledb.from_numpy(array_name, np.zeros((1,)))
|
||||
|
||||
with tiledb.DenseArray(array_name, mode="w") as metadata_array:
|
||||
for key, value in metadata_dict.items():
|
||||
metadata_array.meta[key] = value
|
||||
|
||||
|
||||
def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, index_column_name, ctx):
|
||||
"""
|
||||
Saves the contents of the dataframe to the CXG output directory specified.
|
||||
|
||||
Current access patterns are oriented toward reading very large slices of the dataframe, one attribute at a time.
|
||||
Attribute data also tends to be (often) repetitive (bools, categories, strings). Given this, we use a large tile
|
||||
size (1000) and very aggressive compression levels.
|
||||
"""
|
||||
|
||||
def create_dataframe_array(array_name, dataframe):
|
||||
tiledb_filter = tiledb.FilterList(
|
||||
[
|
||||
# Attempt aggressive compression as many of these dataframes are very repetitive strings, bools and
|
||||
# other non-float data.
|
||||
tiledb.ZstdFilter(level=22),
|
||||
]
|
||||
)
|
||||
attrs = [
|
||||
tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
|
||||
for column in dataframe
|
||||
]
|
||||
domain = tiledb.Domain(
|
||||
tiledb.Dim(domain=(0, dataframe.shape[0] - 1), tile=min(dataframe.shape[0], 1000), dtype=np.uint32)
|
||||
)
|
||||
schema = tiledb.ArraySchema(
|
||||
domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
|
||||
)
|
||||
tiledb.DenseArray.create(array_name, schema)
|
||||
|
||||
array_name = f"{cxg_container}/{dataframe_name}"
|
||||
|
||||
create_dataframe_array(array_name, dataframe)
|
||||
|
||||
with tiledb.DenseArray(array_name, mode="w", ctx=ctx) as array:
|
||||
value = {}
|
||||
schema_hints = {}
|
||||
for column_name, column_values in dataframe.items():
|
||||
dtype, hints = get_dtype_and_schema_of_array(column_values)
|
||||
value[column_name] = column_values.to_numpy(dtype=dtype)
|
||||
if hints:
|
||||
schema_hints.update({column_name: hints})
|
||||
|
||||
schema_hints.update({"index": index_column_name})
|
||||
array[:] = value
|
||||
array.meta["cxg_schema"] = json.dumps(schema_hints)
|
||||
|
||||
tiledb.consolidate(array_name, ctx=ctx)
|
||||
|
||||
|
||||
def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
|
||||
"""
|
||||
Saves contents of ndarray to the CXG output directory specified.
|
||||
|
||||
Generally this function is used to convert dataset embeddings. Because embeddings are typically accessed with
|
||||
very large slices (or all of the embedding), they do not benefit from overly aggressive compression due to their
|
||||
format. Given this, we use a large tile size (1000) but only default compression level.
|
||||
"""
|
||||
|
||||
def create_ndarray_array(ndarray_name, ndarray):
|
||||
filters = tiledb.FilterList([tiledb.ZstdFilter()])
|
||||
attrs = [tiledb.Attr(dtype=ndarray.dtype, filters=filters)]
|
||||
dimensions = [
|
||||
tiledb.Dim(
|
||||
domain=(0, ndarray.shape[dimension] - 1), tile=min(ndarray.shape[dimension], 1000), dtype=np.uint32
|
||||
)
|
||||
for dimension in range(ndarray.ndim)
|
||||
]
|
||||
domain = tiledb.Domain(*dimensions)
|
||||
schema = tiledb.ArraySchema(
|
||||
domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
|
||||
)
|
||||
tiledb.DenseArray.create(ndarray_name, schema)
|
||||
|
||||
create_ndarray_array(ndarray_name, ndarray)
|
||||
|
||||
with tiledb.DenseArray(ndarray_name, mode="w", ctx=ctx) as array:
|
||||
array[:] = ndarray
|
||||
|
||||
tiledb.consolidate(ndarray_name, ctx=ctx)
|
||||
|
||||
|
||||
def convert_matrix_to_cxg_array(
|
||||
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
|
||||
):
|
||||
"""
|
||||
Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
|
||||
is true or not. Note that when the matrix is encoded as a SparseArray, it only writes the values that are
|
||||
nonzero. This means that if you count the number of elements in the SparseArray, it will not equal the total
|
||||
number of elements in the matrix, only the number of nonzero elements.
|
||||
|
||||
Furthermore, if the `column_shift_for_sparse_encoding` matrix is not None, this function will subtract the sparse
|
||||
encoding from the original given matrix and as previously stated, only write the nonzero values to the TileDB
|
||||
SparseArray.
|
||||
"""
|
||||
|
||||
def create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array):
|
||||
filters = tiledb.FilterList([tiledb.ZstdFilter()])
|
||||
attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
|
||||
if encode_as_sparse_array:
|
||||
domain = tiledb.Domain(
|
||||
tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 512), dtype=np.uint32),
|
||||
tiledb.Dim(
|
||||
name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 2048), dtype=np.uint32
|
||||
),
|
||||
)
|
||||
else:
|
||||
domain = tiledb.Domain(
|
||||
tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 50), dtype=np.uint32),
|
||||
tiledb.Dim(
|
||||
name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 100), dtype=np.uint32
|
||||
),
|
||||
)
|
||||
schema = tiledb.ArraySchema(
|
||||
domain=domain, sparse=encode_as_sparse_array, attrs=attrs, cell_order="row-major", tile_order="col-major"
|
||||
)
|
||||
if encode_as_sparse_array:
|
||||
tiledb.SparseArray.create(matrix_name, schema)
|
||||
else:
|
||||
tiledb.DenseArray.create(matrix_name, schema)
|
||||
|
||||
number_of_rows = matrix.shape[0]
|
||||
number_of_columns = matrix.shape[1]
|
||||
stride = min(int(np.power(10, np.around(np.log10(1e9 / number_of_columns)))), 10_000)
|
||||
|
||||
create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array)
|
||||
|
||||
if encode_as_sparse_array:
|
||||
with tiledb.SparseArray(matrix_name, mode="w", ctx=ctx) as array:
|
||||
for start_row_index in range(0, number_of_rows, stride):
|
||||
end_row_index = min(start_row_index + stride, number_of_rows)
|
||||
matrix_subset = matrix[start_row_index:end_row_index, :]
|
||||
if not isinstance(matrix_subset, np.ndarray):
|
||||
matrix_subset = matrix_subset.toarray()
|
||||
if column_shift_for_sparse_encoding is not None:
|
||||
matrix_subset = matrix_subset - column_shift_for_sparse_encoding
|
||||
indices = np.nonzero(matrix_subset)
|
||||
trow = indices[0] + start_row_index
|
||||
array[trow, indices[1]] = matrix_subset[indices[0], indices[1]]
|
||||
|
||||
else:
|
||||
with tiledb.DenseArray(matrix_name, mode="w", ctx=ctx) as array:
|
||||
for start_row_index in range(0, number_of_rows, stride):
|
||||
end_row_index = min(start_row_index + stride, number_of_rows)
|
||||
matrix_subset = matrix[start_row_index:end_row_index, :]
|
||||
if not isinstance(matrix_subset, np.ndarray):
|
||||
matrix_subset = matrix_subset.toarray()
|
||||
array[start_row_index:end_row_index, :] = matrix_subset
|
||||
@@ -1,115 +0,0 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
from scipy.stats import mode
|
||||
|
||||
|
||||
def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
|
||||
"""
|
||||
Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
|
||||
a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
|
||||
elements evaluated. This function may return before evaluating the whole matrix if it can be determined that matrix
|
||||
is not sparse enough.
|
||||
"""
|
||||
|
||||
if sparse_threshold == 100.0:
|
||||
return True
|
||||
if sparse_threshold == 0.0:
|
||||
return False
|
||||
|
||||
total_number_of_rows = matrix.shape[0]
|
||||
total_number_of_columns = matrix.shape[1]
|
||||
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
|
||||
|
||||
# For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
|
||||
# maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
|
||||
# quit early for large dense matrices.
|
||||
row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
|
||||
|
||||
maximum_number_of_non_zero_elements_in_matrix = int(
|
||||
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
|
||||
)
|
||||
number_of_non_zero_elements = 0
|
||||
|
||||
for start_row_index in range(0, total_number_of_rows, row_stride):
|
||||
end_row_index = min(start_row_index + row_stride, total_number_of_rows)
|
||||
|
||||
matrix_subset = matrix[start_row_index:end_row_index, :]
|
||||
if not isinstance(matrix_subset, np.ndarray):
|
||||
matrix_subset = matrix_subset.toarray()
|
||||
|
||||
number_of_non_zero_elements += np.count_nonzero(matrix_subset)
|
||||
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
|
||||
if end_row_index != total_number_of_rows:
|
||||
percentage_of_non_zero_elements = (
|
||||
100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
|
||||
)
|
||||
logging.info(
|
||||
f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
|
||||
f"{percentage_of_non_zero_elements:6.2f}"
|
||||
)
|
||||
else:
|
||||
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
|
||||
logging.info(
|
||||
f"Matrix is not sparse. Percentage of non-zero elements (exact): "
|
||||
f"{percentage_of_non_zero_elements:6.2f}"
|
||||
)
|
||||
return False
|
||||
|
||||
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
|
||||
return is_sparse
|
||||
|
||||
|
||||
def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
|
||||
"""
|
||||
Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
|
||||
shift encoding works by taking the most common value in each column, then subtracting that value from each element
|
||||
of the column. If each column mostly contains its most common value, then the resulting matrix can be very sparse.
|
||||
|
||||
This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
|
||||
sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
|
||||
the number of non-zeros encountered and number of elements evaluated. This function may return before evaluating
|
||||
the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
|
||||
"""
|
||||
|
||||
total_number_of_rows = matrix.shape[0]
|
||||
total_number_of_columns = matrix.shape[1]
|
||||
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
|
||||
|
||||
stride = max(1, 128_000_000 // total_number_of_rows)
|
||||
column_shift = np.zeros(total_number_of_columns)
|
||||
|
||||
maximum_number_of_non_zero_elements_in_matrix = int(
|
||||
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
|
||||
)
|
||||
number_of_non_zero_elements = 0
|
||||
|
||||
for start_column_index in range(0, total_number_of_columns, stride):
|
||||
end_column_index = min(start_column_index + stride, total_number_of_columns)
|
||||
|
||||
matrix_subset = matrix[:, start_column_index:end_column_index]
|
||||
if not isinstance(matrix_subset, np.ndarray):
|
||||
matrix_subset = matrix_subset.toarray()
|
||||
|
||||
matrix_subset_mode = mode(matrix_subset)
|
||||
|
||||
column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
|
||||
number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
|
||||
matrix_subset_mode.count
|
||||
)
|
||||
|
||||
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
|
||||
if end_column_index != total_number_of_columns:
|
||||
logging.info(
|
||||
"Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
|
||||
% (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
|
||||
)
|
||||
else:
|
||||
logging.info(
|
||||
"Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
|
||||
% (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
|
||||
)
|
||||
return None
|
||||
|
||||
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
|
||||
return column_shift if is_sparse else None
|
||||
@@ -1,40 +0,0 @@
|
||||
import re
|
||||
|
||||
|
||||
def sanitize_values_in_list(list_of_keys: list):
|
||||
"""
|
||||
Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
|
||||
safe and unique.
|
||||
"""
|
||||
|
||||
if not all([isinstance(key, str) for key in list_of_keys]):
|
||||
raise Exception("List of keys to sanitize must contain all strings.")
|
||||
|
||||
# Mask out [~/.] and anything outside the ASCII range.
|
||||
mask = re.compile(r"[^ -\-0-\[\]-\}]")
|
||||
clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
|
||||
|
||||
# Dedupe the clean keys list
|
||||
deduped_clean_keys_list = []
|
||||
for index, clean_key in enumerate(clean_keys_list):
|
||||
total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
|
||||
total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
|
||||
deduped_clean_keys_list.append(
|
||||
clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
|
||||
if total_occurrences_of_clean_key > 1
|
||||
else clean_key
|
||||
)
|
||||
|
||||
return dict(zip(list_of_keys, deduped_clean_keys_list))
|
||||
|
||||
|
||||
def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
|
||||
"""
|
||||
Clean and dedupe the keys in the given dictionary.
|
||||
"""
|
||||
|
||||
clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
|
||||
for original_key, sanitized_key in clean_keys.items():
|
||||
if original_key != sanitized_key:
|
||||
dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
|
||||
del dict_to_sanitize[original_key]
|
||||
@@ -1,158 +0,0 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
|
||||
dtypes_by_column_name = {}
|
||||
schema_type_hints_by_column_name = {}
|
||||
|
||||
for column_name, column_values in dataframe.items():
|
||||
(
|
||||
dtypes_by_column_name[column_name],
|
||||
schema_type_hints_by_column_name[column_name],
|
||||
) = get_dtype_and_schema_of_array(column_values)
|
||||
|
||||
return dtypes_by_column_name, schema_type_hints_by_column_name
|
||||
|
||||
|
||||
def get_dtype_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[0]
|
||||
|
||||
|
||||
def get_schema_type_hint_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[1]
|
||||
|
||||
|
||||
def get_dtype_and_schema_of_array(array: pd.Series):
|
||||
return (
|
||||
get_dtype_from_dtype(array.dtype, array_values=array),
|
||||
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
|
||||
)
|
||||
|
||||
|
||||
def get_dtype_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
|
||||
for 64 bit values which will get downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype_name == "bool":
|
||||
return np.uint8
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return str
|
||||
if dtype_name == "category":
|
||||
return get_dtype_from_dtype(dtype.categories.dtype, array_values)
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return np.int32
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return np.float32
|
||||
if not can_cast_to_float32(dtype, array_values):
|
||||
return np.float64
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def get_schema_type_hint_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
|
||||
and will be downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype == np.float32 or dtype == np.int32:
|
||||
return {"type": dtype_name}
|
||||
if dtype_name == "bool":
|
||||
return {"type": "boolean"}
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return {"type": "string"}
|
||||
if dtype_name == "category":
|
||||
return {"type": "categorical", "categories": dtype.categories.tolist()}
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return {"type": "int32"}
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float32"}
|
||||
if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float64"}
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def can_cast_to_float32(dtype, array_values):
|
||||
"""
|
||||
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
|
||||
a float.
|
||||
|
||||
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
|
||||
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
|
||||
"""
|
||||
|
||||
if dtype.kind == "f":
|
||||
if not np.can_cast(dtype, np.float32):
|
||||
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
|
||||
|
||||
return True
|
||||
|
||||
if dtype.kind == "O" and array_values.hasnans:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def can_cast_to_int32(dtype, array_values=None):
|
||||
"""
|
||||
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
|
||||
the higher precision type has values that are entirely within the range of the downcast type.
|
||||
"""
|
||||
|
||||
# Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
|
||||
# return False.
|
||||
if array_values.hasnans:
|
||||
return False
|
||||
|
||||
# If the array is categorical, then we need to order the array values so that functions min and max that occur
|
||||
# later, can function. They do not function on unordered categories.
|
||||
ordered_array_values = array_values
|
||||
if array_values.dtype.name == "category" and not array_values.cat.ordered:
|
||||
ordered_array_values = array_values.cat.as_ordered()
|
||||
|
||||
if dtype.kind in ["i", "u"]:
|
||||
if np.can_cast(dtype, np.int32):
|
||||
return True
|
||||
ii32 = np.iinfo(np.int32)
|
||||
if (
|
||||
not ordered_array_values.empty
|
||||
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
|
||||
or ordered_array_values.empty
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
|
||||
if series_to_convert.hasnans and dtype == np.int32:
|
||||
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
|
||||
|
||||
return series_to_convert.to_numpy(dtype)
|
||||
|
||||
|
||||
def convert_string_to_value(value: str):
|
||||
"""convert a string to value with the most appropriate type"""
|
||||
if value.lower() == "true":
|
||||
return True
|
||||
if value.lower() == "false":
|
||||
return False
|
||||
if value == "null":
|
||||
return None
|
||||
try:
|
||||
return eval(value)
|
||||
except: # noqa E722
|
||||
return value
|
||||
@@ -1,118 +0,0 @@
|
||||
import contextlib
|
||||
import errno
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import pkgutil
|
||||
import socket
|
||||
from urllib.parse import urlsplit, urljoin
|
||||
|
||||
import numpy as np
|
||||
from flask import json
|
||||
|
||||
from server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
def find_available_port(host, port=5005):
|
||||
"""
|
||||
Helper method to find open port on host. Tries 5000 ports incremented from the specified port
|
||||
"""
|
||||
# Takes approx 2 seconds to do a scan of 5000 ports on my laptop
|
||||
num_ports_to_try = 5000
|
||||
for port_to_try in range(port, port + num_ports_to_try):
|
||||
if is_port_available(host, port_to_try):
|
||||
return port_to_try
|
||||
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
|
||||
|
||||
|
||||
def is_port_available(host, port):
|
||||
is_available = False
|
||||
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
|
||||
try:
|
||||
s.bind((host, port))
|
||||
is_available = True
|
||||
except socket.error:
|
||||
pass
|
||||
return is_available
|
||||
|
||||
|
||||
def sort_options(command):
|
||||
"""
|
||||
Helper for the click options - will sort options in a command, and can
|
||||
be used as a decorator.
|
||||
"""
|
||||
command.params.sort(key=lambda p: p.name)
|
||||
return command
|
||||
|
||||
|
||||
def path_join(base, *urls):
|
||||
"""
|
||||
this is like urllib.parse.urljoin, except it works around the scheme-specific
|
||||
cleverness in the aforementioned code, ignores anything in the url except the path,
|
||||
and accepts more than one url.
|
||||
"""
|
||||
if not base.endswith("/"):
|
||||
base += "/"
|
||||
btpl = urlsplit(base)
|
||||
path = btpl.path
|
||||
for url in urls:
|
||||
utpl = urlsplit(url)
|
||||
if btpl.scheme == "":
|
||||
path = os.path.join(path, utpl.path)
|
||||
path = os.path.normpath(path)
|
||||
else:
|
||||
path = urljoin(path, utpl.path)
|
||||
return btpl._replace(path=path).geturl()
|
||||
|
||||
|
||||
class Float32JSONEncoder(json.JSONEncoder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
NaN/Infinities are illegal in standard JSON. Python extends JSON with
|
||||
non-standard symbols that most JavaScript JSON parsers do not understand.
|
||||
The `allow_nan` parameter will force Python simplejson to throw an ValueError
|
||||
if it runs into non-finite floating point values which are unsupported by
|
||||
standard JSON.
|
||||
"""
|
||||
kwargs["allow_nan"] = False
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def default(self, obj):
|
||||
if isinstance(obj, np.float32):
|
||||
return float(obj)
|
||||
elif isinstance(obj, np.integer):
|
||||
return int(obj)
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
def custom_format_warning(msg, *args, **kwargs):
|
||||
return f"[cellxgene] Warning: {msg} \n"
|
||||
|
||||
|
||||
def jsonify_numpy(data):
|
||||
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
|
||||
|
||||
|
||||
def import_plugins(plugin_module):
|
||||
"""
|
||||
Load optional plugin modules from server.common.plugins
|
||||
|
||||
If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
|
||||
This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
|
||||
normal.
|
||||
"""
|
||||
loaded_modules = []
|
||||
try:
|
||||
pkg = importlib.import_module(plugin_module)
|
||||
for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
|
||||
full_name = f"{plugin_module}.{name}"
|
||||
try:
|
||||
module = importlib.import_module(full_name)
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
|
||||
loaded_modules.append(module)
|
||||
except ModuleNotFoundError as e:
|
||||
# This exception occurs when the plugin_module does not exist (not an error).
|
||||
logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
|
||||
|
||||
return loaded_modules
|
||||
Reference in New Issue
Block a user