mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-30 12:38:11 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -0,0 +1,23 @@
|
||||
import logging
|
||||
|
||||
import boto3
|
||||
from flask import json
|
||||
|
||||
from backend.common.errors import SecretKeyRetrievalError
|
||||
|
||||
|
||||
def get_secret_key(region_name, secret_name):
|
||||
session = boto3.session.Session()
|
||||
client = session.client(service_name="secretsmanager", region_name=region_name)
|
||||
|
||||
try:
|
||||
get_secret_value_response = client.get_secret_value(SecretId=secret_name)
|
||||
if "SecretString" in get_secret_value_response:
|
||||
var = get_secret_value_response["SecretString"]
|
||||
secret = json.loads(var)
|
||||
return secret
|
||||
except Exception as e:
|
||||
logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
|
||||
raise SecretKeyRetrievalError(str(e))
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,154 @@
|
||||
import os
|
||||
import tempfile
|
||||
import fsspec
|
||||
from datetime import datetime
|
||||
import boto3
|
||||
import botocore
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class DataLocator:
|
||||
"""
|
||||
DataLocator is a simple wrapper around fsspec functionality, and provides a
|
||||
set of functions to encapsulate a data location (URI or path), interogate
|
||||
metadata about the object at that location (size, existance, etc) and
|
||||
access the underlying data.
|
||||
|
||||
https://filesystem-spec.readthedocs.io/en/latest/index.html
|
||||
|
||||
Example:
|
||||
dl = DataLocator("/tmp/foo.h5ad")
|
||||
if dl.exists():
|
||||
print(dl.size())
|
||||
with dl.open() as f:
|
||||
thecontents = f.read()
|
||||
|
||||
DataLocator will accept a URI or native path. Error handling is as defined
|
||||
in fsspec.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, uri_or_path, region_name=None):
|
||||
if isinstance(uri_or_path, DataLocator):
|
||||
locator = uri_or_path
|
||||
self.uri_or_path = locator.uri_or_path
|
||||
self.protocol = locator.protocol
|
||||
self.path = locator.path
|
||||
self.cname = locator.cname
|
||||
else:
|
||||
self.uri_or_path = uri_or_path
|
||||
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
|
||||
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
|
||||
self.cname = self.path if self.protocol == "file" else self.uri_or_path
|
||||
|
||||
# fsspec.filesystem will throw RuntimeError if the protocol is unsupported
|
||||
if self.protocol == "s3":
|
||||
if region_name:
|
||||
config_kwargs = dict(region_name=region_name)
|
||||
self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs)
|
||||
else:
|
||||
self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30)
|
||||
else:
|
||||
self.fs = fsspec.filesystem(self.protocol)
|
||||
|
||||
def __repr__(self):
|
||||
return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
|
||||
f"path={self.path}, uri_or_path={self.uri_or_path})"
|
||||
|
||||
@staticmethod
|
||||
def _get_protocol_and_path(uri_or_path):
|
||||
if "://" in uri_or_path:
|
||||
protocol, path = uri_or_path.split("://", 1)
|
||||
# windows!!! Ignore single letter drive identifiers,
|
||||
# eg, G:\foo.txt
|
||||
if len(protocol) > 1:
|
||||
return protocol, path
|
||||
return None, uri_or_path
|
||||
|
||||
def exists(self):
|
||||
return self.fs.exists(self.cname)
|
||||
|
||||
def size(self):
|
||||
return self.fs.size(self.cname)
|
||||
|
||||
def lastmodtime(self):
|
||||
""" return datetime object representing last modification time, or None if unavailable """
|
||||
info = self.fs.info(self.cname)
|
||||
if self.islocal() and info is not None:
|
||||
return datetime.fromtimestamp(info["mtime"])
|
||||
else:
|
||||
return getattr(info, "LastModified", None)
|
||||
|
||||
def abspath(self):
|
||||
"""
|
||||
return the absolute path for the locator - only really does something
|
||||
for file: protocol, as all others are already absolute
|
||||
"""
|
||||
if self.islocal():
|
||||
return os.path.abspath(self.path)
|
||||
else:
|
||||
return self.uri_or_path
|
||||
|
||||
def isfile(self):
|
||||
return self.fs.isfile(self.cname)
|
||||
|
||||
def open(self, *args):
|
||||
return self.fs.open(self.uri_or_path, *args)
|
||||
|
||||
def islocal(self):
|
||||
return self.protocol is None or self.protocol == "file"
|
||||
|
||||
def local_handle(self):
|
||||
if self.islocal():
|
||||
return LocalFilePath(self.path)
|
||||
|
||||
# if not local, create a tmp file system object to contain the data,
|
||||
# and clean it up when done. If the path has a suffix/extension,
|
||||
# do our best to create a file with the same.
|
||||
ext = os.path.splitext(self.path)
|
||||
suffix = None if ext[1] == "" else ext[1]
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(src.read())
|
||||
tmp.close()
|
||||
src.close()
|
||||
tmp_path = tmp.name
|
||||
return LocalFilePath(tmp_path, delete=True)
|
||||
|
||||
def ls(self):
|
||||
paths = self.fs.ls(self.uri_or_path)
|
||||
return [os.path.basename(p) for p in paths]
|
||||
|
||||
|
||||
class LocalFilePath:
|
||||
def __init__(self, tmp_path, delete=False):
|
||||
self.tmp_path = tmp_path
|
||||
self.delete = delete
|
||||
|
||||
def __enter__(self):
|
||||
return self.tmp_path
|
||||
|
||||
def __exit__(self, *args):
|
||||
if self.delete:
|
||||
os.unlink(self.tmp_path)
|
||||
|
||||
|
||||
def discover_s3_region_name(uri):
|
||||
"""If this is an s3 protocol, discover and return the (aws) region name.
|
||||
If a return name could not be discovered, or if the uri is not an s3 protocol, return None."""
|
||||
|
||||
protocol, _ = DataLocator._get_protocol_and_path(uri)
|
||||
if protocol == "s3":
|
||||
bucket = urlparse(uri).netloc
|
||||
client = boto3.client("s3")
|
||||
try:
|
||||
res = client.head_bucket(Bucket=bucket)
|
||||
except botocore.exceptions.ClientError:
|
||||
return None
|
||||
|
||||
region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
|
||||
if region:
|
||||
return region
|
||||
else:
|
||||
return None
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,158 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
|
||||
dtypes_by_column_name = {}
|
||||
schema_type_hints_by_column_name = {}
|
||||
|
||||
for column_name, column_values in dataframe.items():
|
||||
(
|
||||
dtypes_by_column_name[column_name],
|
||||
schema_type_hints_by_column_name[column_name],
|
||||
) = get_dtype_and_schema_of_array(column_values)
|
||||
|
||||
return dtypes_by_column_name, schema_type_hints_by_column_name
|
||||
|
||||
|
||||
def get_dtype_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[0]
|
||||
|
||||
|
||||
def get_schema_type_hint_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[1]
|
||||
|
||||
|
||||
def get_dtype_and_schema_of_array(array: pd.Series):
|
||||
return (
|
||||
get_dtype_from_dtype(array.dtype, array_values=array),
|
||||
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
|
||||
)
|
||||
|
||||
|
||||
def get_dtype_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
|
||||
for 64 bit values which will get downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype_name == "bool":
|
||||
return np.uint8
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return str
|
||||
if dtype_name == "category":
|
||||
return get_dtype_from_dtype(dtype.categories.dtype, array_values)
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return np.int32
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return np.float32
|
||||
if not can_cast_to_float32(dtype, array_values):
|
||||
return np.float64
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def get_schema_type_hint_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
|
||||
and will be downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype == np.float32 or dtype == np.int32:
|
||||
return {"type": dtype_name}
|
||||
if dtype_name == "bool":
|
||||
return {"type": "boolean"}
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return {"type": "string"}
|
||||
if dtype_name == "category":
|
||||
return {"type": "categorical", "categories": dtype.categories.tolist()}
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return {"type": "int32"}
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float32"}
|
||||
if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float64"}
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def can_cast_to_float32(dtype, array_values):
|
||||
"""
|
||||
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
|
||||
a float.
|
||||
|
||||
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
|
||||
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
|
||||
"""
|
||||
|
||||
if dtype.kind == "f":
|
||||
if not np.can_cast(dtype, np.float32):
|
||||
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
|
||||
|
||||
return True
|
||||
|
||||
if dtype.kind == "O" and array_values.hasnans:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def can_cast_to_int32(dtype, array_values=None):
|
||||
"""
|
||||
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
|
||||
the higher precision type has values that are entirely within the range of the downcast type.
|
||||
"""
|
||||
|
||||
# Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
|
||||
# return False.
|
||||
if array_values.hasnans:
|
||||
return False
|
||||
|
||||
# If the array is categorical, then we need to order the array values so that functions min and max that occur
|
||||
# later, can function. They do not function on unordered categories.
|
||||
ordered_array_values = array_values
|
||||
if array_values.dtype.name == "category" and not array_values.cat.ordered:
|
||||
ordered_array_values = array_values.cat.as_ordered()
|
||||
|
||||
if dtype.kind in ["i", "u"]:
|
||||
if np.can_cast(dtype, np.int32):
|
||||
return True
|
||||
ii32 = np.iinfo(np.int32)
|
||||
if (
|
||||
not ordered_array_values.empty
|
||||
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
|
||||
or ordered_array_values.empty
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
|
||||
if series_to_convert.hasnans and dtype == np.int32:
|
||||
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
|
||||
|
||||
return series_to_convert.to_numpy(dtype)
|
||||
|
||||
|
||||
def convert_string_to_value(value: str):
|
||||
"""convert a string to value with the most appropriate type"""
|
||||
if value.lower() == "true":
|
||||
return True
|
||||
if value.lower() == "false":
|
||||
return False
|
||||
if value == "null":
|
||||
return None
|
||||
try:
|
||||
return eval(value)
|
||||
except: # noqa E722
|
||||
return value
|
||||
@@ -0,0 +1,118 @@
|
||||
import contextlib
|
||||
import errno
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import pkgutil
|
||||
import socket
|
||||
from urllib.parse import urlsplit, urljoin
|
||||
|
||||
import numpy as np
|
||||
from flask import json
|
||||
|
||||
from backend.common.errors import ConfigurationError
|
||||
|
||||
|
||||
def find_available_port(host, port=5005):
|
||||
"""
|
||||
Helper method to find open port on host. Tries 5000 ports incremented from the specified port
|
||||
"""
|
||||
# Takes approx 2 seconds to do a scan of 5000 ports on my laptop
|
||||
num_ports_to_try = 5000
|
||||
for port_to_try in range(port, port + num_ports_to_try):
|
||||
if is_port_available(host, port_to_try):
|
||||
return port_to_try
|
||||
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
|
||||
|
||||
|
||||
def is_port_available(host, port):
|
||||
is_available = False
|
||||
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
|
||||
try:
|
||||
s.bind((host, port))
|
||||
is_available = True
|
||||
except socket.error:
|
||||
pass
|
||||
return is_available
|
||||
|
||||
|
||||
def sort_options(command):
|
||||
"""
|
||||
Helper for the click options - will sort options in a command, and can
|
||||
be used as a decorator.
|
||||
"""
|
||||
command.params.sort(key=lambda p: p.name)
|
||||
return command
|
||||
|
||||
|
||||
def path_join(base, *urls):
|
||||
"""
|
||||
this is like urllib.parse.urljoin, except it works around the scheme-specific
|
||||
cleverness in the aforementioned code, ignores anything in the url except the path,
|
||||
and accepts more than one url.
|
||||
"""
|
||||
if not base.endswith("/"):
|
||||
base += "/"
|
||||
btpl = urlsplit(base)
|
||||
path = btpl.path
|
||||
for url in urls:
|
||||
utpl = urlsplit(url)
|
||||
if btpl.scheme == "":
|
||||
path = os.path.join(path, utpl.path)
|
||||
path = os.path.normpath(path)
|
||||
else:
|
||||
path = urljoin(path, utpl.path)
|
||||
return btpl._replace(path=path).geturl()
|
||||
|
||||
|
||||
class Float32JSONEncoder(json.JSONEncoder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
NaN/Infinities are illegal in standard JSON. Python extends JSON with
|
||||
non-standard symbols that most JavaScript JSON parsers do not understand.
|
||||
The `allow_nan` parameter will force Python simplejson to throw an ValueError
|
||||
if it runs into non-finite floating point values which are unsupported by
|
||||
standard JSON.
|
||||
"""
|
||||
kwargs["allow_nan"] = False
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def default(self, obj):
|
||||
if isinstance(obj, np.float32):
|
||||
return float(obj)
|
||||
elif isinstance(obj, np.integer):
|
||||
return int(obj)
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
def custom_format_warning(msg, *args, **kwargs):
|
||||
return f"[cellxgene] Warning: {msg} \n"
|
||||
|
||||
|
||||
def jsonify_numpy(data):
|
||||
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
|
||||
|
||||
|
||||
def import_plugins(plugin_module):
|
||||
"""
|
||||
Load optional plugin modules from server.common.plugins
|
||||
|
||||
If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
|
||||
This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
|
||||
normal.
|
||||
"""
|
||||
loaded_modules = []
|
||||
try:
|
||||
pkg = importlib.import_module(plugin_module)
|
||||
for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
|
||||
full_name = f"{plugin_module}.{name}"
|
||||
try:
|
||||
module = importlib.import_module(full_name)
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
|
||||
loaded_modules.append(module)
|
||||
except ModuleNotFoundError as e:
|
||||
# This exception occurs when the plugin_module does not exist (not an error).
|
||||
logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
|
||||
|
||||
return loaded_modules
|
||||
Reference in New Issue
Block a user