mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-30 17:28:12 +08:00
Split out the local backend (#2052)
This splits the backend into two parts: the local backend for desktop cellxgene and the AWS backend for hosted cellxgene. The local backend is in local_server while the hosted remains in server. The general idea is to copy everything from server to local_server, pull unneeded stuff out of local_server, and keep server as-is for this PR. Not touching server means all the infra and deployment code will continue working just as it did before so we can make those changes incrementally.
This commit is contained in:
Marcus Kinsella
@@ -0,0 +1,22 @@
|
||||
class CorporaConstants(object):
|
||||
REQUIRED_SIMPLE_METADATA_FIELDS = [
|
||||
"version",
|
||||
"title",
|
||||
"layer_descriptions",
|
||||
"organism",
|
||||
"organism_ontology_term_id",
|
||||
]
|
||||
|
||||
# The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
|
||||
# types.
|
||||
OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
|
||||
|
||||
OPTIONAL_SIMPLE_METADATA_FIELDS = [
|
||||
"preprint_doi",
|
||||
"publication_doi",
|
||||
"default_embedding",
|
||||
"default_field",
|
||||
"tags",
|
||||
"project_name",
|
||||
"project_description",
|
||||
]
|
||||
@@ -0,0 +1,158 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
|
||||
dtypes_by_column_name = {}
|
||||
schema_type_hints_by_column_name = {}
|
||||
|
||||
for column_name, column_values in dataframe.items():
|
||||
(
|
||||
dtypes_by_column_name[column_name],
|
||||
schema_type_hints_by_column_name[column_name],
|
||||
) = get_dtype_and_schema_of_array(column_values)
|
||||
|
||||
return dtypes_by_column_name, schema_type_hints_by_column_name
|
||||
|
||||
|
||||
def get_dtype_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[0]
|
||||
|
||||
|
||||
def get_schema_type_hint_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[1]
|
||||
|
||||
|
||||
def get_dtype_and_schema_of_array(array: pd.Series):
|
||||
return (
|
||||
get_dtype_from_dtype(array.dtype, array_values=array),
|
||||
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
|
||||
)
|
||||
|
||||
|
||||
def get_dtype_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
|
||||
for 64 bit values which will get downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype_name == "bool":
|
||||
return np.uint8
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return str
|
||||
if dtype_name == "category":
|
||||
return get_dtype_from_dtype(dtype.categories.dtype, array_values)
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return np.int32
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return np.float32
|
||||
if not can_cast_to_float32(dtype, array_values):
|
||||
return np.float64
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def get_schema_type_hint_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
|
||||
and will be downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype == np.float32 or dtype == np.int32:
|
||||
return {"type": dtype_name}
|
||||
if dtype_name == "bool":
|
||||
return {"type": "boolean"}
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return {"type": "string"}
|
||||
if dtype_name == "category":
|
||||
return {"type": "categorical", "categories": dtype.categories.tolist()}
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return {"type": "int32"}
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float32"}
|
||||
if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float64"}
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def can_cast_to_float32(dtype, array_values):
|
||||
"""
|
||||
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
|
||||
a float.
|
||||
|
||||
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
|
||||
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
|
||||
"""
|
||||
|
||||
if dtype.kind == "f":
|
||||
if not np.can_cast(dtype, np.float32):
|
||||
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
|
||||
|
||||
return True
|
||||
|
||||
if dtype.kind == "O" and array_values.hasnans:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def can_cast_to_int32(dtype, array_values=None):
|
||||
"""
|
||||
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
|
||||
the higher precision type has values that are entirely within the range of the downcast type.
|
||||
"""
|
||||
|
||||
# Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
|
||||
# return False.
|
||||
if array_values.hasnans:
|
||||
return False
|
||||
|
||||
# If the array is categorical, then we need to order the array values so that functions min and max that occur
|
||||
# later, can function. They do not function on unordered categories.
|
||||
ordered_array_values = array_values
|
||||
if array_values.dtype.name == "category" and not array_values.cat.ordered:
|
||||
ordered_array_values = array_values.cat.as_ordered()
|
||||
|
||||
if dtype.kind in ["i", "u"]:
|
||||
if np.can_cast(dtype, np.int32):
|
||||
return True
|
||||
ii32 = np.iinfo(np.int32)
|
||||
if (
|
||||
not ordered_array_values.empty
|
||||
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
|
||||
or ordered_array_values.empty
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
|
||||
if series_to_convert.hasnans and dtype == np.int32:
|
||||
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
|
||||
|
||||
return series_to_convert.to_numpy(dtype)
|
||||
|
||||
|
||||
def convert_string_to_value(value: str):
|
||||
"""convert a string to value with the most appropriate type"""
|
||||
if value.lower() == "true":
|
||||
return True
|
||||
if value.lower() == "false":
|
||||
return False
|
||||
if value == "null":
|
||||
return None
|
||||
try:
|
||||
return eval(value)
|
||||
except: # noqa E722
|
||||
return value
|
||||
@@ -0,0 +1,118 @@
|
||||
import contextlib
|
||||
import errno
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import pkgutil
|
||||
import socket
|
||||
from urllib.parse import urlsplit, urljoin
|
||||
|
||||
import numpy as np
|
||||
from flask import json
|
||||
|
||||
from local_server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
def find_available_port(host, port=5005):
|
||||
"""
|
||||
Helper method to find open port on host. Tries 5000 ports incremented from the specified port
|
||||
"""
|
||||
# Takes approx 2 seconds to do a scan of 5000 ports on my laptop
|
||||
num_ports_to_try = 5000
|
||||
for port_to_try in range(port, port + num_ports_to_try):
|
||||
if is_port_available(host, port_to_try):
|
||||
return port_to_try
|
||||
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
|
||||
|
||||
|
||||
def is_port_available(host, port):
|
||||
is_available = False
|
||||
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
|
||||
try:
|
||||
s.bind((host, port))
|
||||
is_available = True
|
||||
except socket.error:
|
||||
pass
|
||||
return is_available
|
||||
|
||||
|
||||
def sort_options(command):
|
||||
"""
|
||||
Helper for the click options - will sort options in a command, and can
|
||||
be used as a decorator.
|
||||
"""
|
||||
command.params.sort(key=lambda p: p.name)
|
||||
return command
|
||||
|
||||
|
||||
def path_join(base, *urls):
|
||||
"""
|
||||
this is like urllib.parse.urljoin, except it works around the scheme-specific
|
||||
cleverness in the aforementioned code, ignores anything in the url except the path,
|
||||
and accepts more than one url.
|
||||
"""
|
||||
if not base.endswith("/"):
|
||||
base += "/"
|
||||
btpl = urlsplit(base)
|
||||
path = btpl.path
|
||||
for url in urls:
|
||||
utpl = urlsplit(url)
|
||||
if btpl.scheme == "":
|
||||
path = os.path.join(path, utpl.path)
|
||||
path = os.path.normpath(path)
|
||||
else:
|
||||
path = urljoin(path, utpl.path)
|
||||
return btpl._replace(path=path).geturl()
|
||||
|
||||
|
||||
class Float32JSONEncoder(json.JSONEncoder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
NaN/Infinities are illegal in standard JSON. Python extends JSON with
|
||||
non-standard symbols that most JavaScript JSON parsers do not understand.
|
||||
The `allow_nan` parameter will force Python simplejson to throw an ValueError
|
||||
if it runs into non-finite floating point values which are unsupported by
|
||||
standard JSON.
|
||||
"""
|
||||
kwargs["allow_nan"] = False
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def default(self, obj):
|
||||
if isinstance(obj, np.float32):
|
||||
return float(obj)
|
||||
elif isinstance(obj, np.integer):
|
||||
return int(obj)
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
def custom_format_warning(msg, *args, **kwargs):
|
||||
return f"[cellxgene] Warning: {msg} \n"
|
||||
|
||||
|
||||
def jsonify_numpy(data):
|
||||
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
|
||||
|
||||
|
||||
def import_plugins(plugin_module):
|
||||
"""
|
||||
Load optional plugin modules from local_server.common.plugins
|
||||
|
||||
If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
|
||||
This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
|
||||
normal.
|
||||
"""
|
||||
loaded_modules = []
|
||||
try:
|
||||
pkg = importlib.import_module(plugin_module)
|
||||
for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
|
||||
full_name = f"{plugin_module}.{name}"
|
||||
try:
|
||||
module = importlib.import_module(full_name)
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
|
||||
loaded_modules.append(module)
|
||||
except ModuleNotFoundError as e:
|
||||
# This exception occurs when the plugin_module does not exist (not an error).
|
||||
logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
|
||||
|
||||
return loaded_modules
|
||||
Reference in New Issue
Block a user