move common code into server, update tests and makefile (#2425)

* move common code into server, update tests and makefile

remove backend directory, refactor

update smoke tests

server/common/utils/corpora_constants.py

@@ -0,0 +1,22 @@
+class CorporaConstants(object):
+    REQUIRED_SIMPLE_METADATA_FIELDS = [
+        "version",
+        "title",
+        "layer_descriptions",
+        "organism",
+        "organism_ontology_term_id",
+    ]
+
+    # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
+    # types.
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

server/common/utils/data_locator.py

@@ -0,0 +1,154 @@
+import os
+import tempfile
+import fsspec
+from datetime import datetime
+import boto3
+import botocore
+from urllib.parse import urlparse
+
+
+class DataLocator:
+    """
+    DataLocator is a simple wrapper around fsspec functionality, and provides a
+    set of functions to encapsulate a data location (URI or path), interogate
+    metadata about the object at that location (size, existance, etc) and
+    access the underlying data.
+
+    https://filesystem-spec.readthedocs.io/en/latest/index.html
+
+    Example:
+        dl = DataLocator("/tmp/foo.h5ad")
+        if dl.exists():
+            print(dl.size())
+            with dl.open() as f:
+                thecontents = f.read()
+
+    DataLocator will accept a URI or native path.  Error handling is as defined
+    in fsspec.
+
+    """
+
+    def __init__(self, uri_or_path, region_name=None):
+        if isinstance(uri_or_path, DataLocator):
+            locator = uri_or_path
+            self.uri_or_path = locator.uri_or_path
+            self.protocol = locator.protocol
+            self.path = locator.path
+            self.cname = locator.cname
+        else:
+            self.uri_or_path = uri_or_path
+            self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
+            # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
+            self.cname = self.path if self.protocol == "file" else self.uri_or_path
+
+        # fsspec.filesystem will throw RuntimeError if the protocol is unsupported
+        if self.protocol == "s3":
+            if region_name:
+                config_kwargs = dict(region_name=region_name)
+                self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs)
+            else:
+                self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30)
+        else:
+            self.fs = fsspec.filesystem(self.protocol)
+
+    def __repr__(self):
+        return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
+        f"path={self.path}, uri_or_path={self.uri_or_path})"
+
+    @staticmethod
+    def _get_protocol_and_path(uri_or_path):
+        if "://" in uri_or_path:
+            protocol, path = uri_or_path.split("://", 1)
+            # windows!!!  Ignore single letter drive identifiers,
+            # eg, G:\foo.txt
+            if len(protocol) > 1:
+                return protocol, path
+        return None, uri_or_path
+
+    def exists(self):
+        return self.fs.exists(self.cname)
+
+    def size(self):
+        return self.fs.size(self.cname)
+
+    def lastmodtime(self):
+        """ return datetime object representing last modification time, or None if unavailable """
+        info = self.fs.info(self.cname)
+        if self.islocal() and info is not None:
+            return datetime.fromtimestamp(info["mtime"])
+        else:
+            return getattr(info, "LastModified", None)
+
+    def abspath(self):
+        """
+        return the absolute path for the locator - only really does something
+        for file: protocol, as all others are already absolute
+        """
+        if self.islocal():
+            return os.path.abspath(self.path)
+        else:
+            return self.uri_or_path
+
+    def isfile(self):
+        return self.fs.isfile(self.cname)
+
+    def open(self, *args):
+        return self.fs.open(self.uri_or_path, *args)
+
+    def islocal(self):
+        return self.protocol is None or self.protocol == "file"
+
+    def local_handle(self):
+        if self.islocal():
+            return LocalFilePath(self.path)
+
+        # if not local, create a tmp file system object to contain the data,
+        # and clean it up when done.  If the path has a suffix/extension,
+        # do our best to create a file with the same.
+        ext = os.path.splitext(self.path)
+        suffix = None if ext[1] == "" else ext[1]
+        with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
+            tmp.write(src.read())
+            tmp.close()
+            src.close()
+            tmp_path = tmp.name
+            return LocalFilePath(tmp_path, delete=True)
+
+    def ls(self):
+        paths = self.fs.ls(self.uri_or_path)
+        return [os.path.basename(p) for p in paths]
+
+
+class LocalFilePath:
+    def __init__(self, tmp_path, delete=False):
+        self.tmp_path = tmp_path
+        self.delete = delete
+
+    def __enter__(self):
+        return self.tmp_path
+
+    def __exit__(self, *args):
+        if self.delete:
+            os.unlink(self.tmp_path)
+
+
+def discover_s3_region_name(uri):
+    """If this is an s3 protocol, discover and return the (aws) region name.
+    If a return name could not be discovered, or if the uri is not an s3 protocol, return None."""
+
+    protocol, _ = DataLocator._get_protocol_and_path(uri)
+    if protocol == "s3":
+        bucket = urlparse(uri).netloc
+        client = boto3.client("s3")
+        try:
+            res = client.head_bucket(Bucket=bucket)
+        except botocore.exceptions.ClientError:
+            return None
+
+        region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
+        if region:
+            return region
+        else:
+            return None
+
+    return None

server/common/utils/type_conversion_utils.py

@@ -0,0 +1,191 @@
+from typing import Union, Tuple
+import logging
+
+import numpy as np
+import pandas as pd
+
+"""
+These routines drive all type inference for the schema generation and the
+FBS (REST OTA) encoding.
+
+
+H5AD Type                       REST              REST
+(ndarray, Series, Index)        FBS encoding      schema type       ERROR/exceptions
+----------------------------    --------------    ---------------   ----------------------
+bool_/bool                      uint8             boolean
+(u)int8, (u)int16, int32        int32             int32
+uint32, (u)int64                int32             int32             CHECKS value bounds
+float16, float32, float64       float32           float32[0]
+
+categorical[T is numeric[4]]:
+    hasna = False               T                 categorical[1]
+    hasna = True                float32           categorical[1]    CHECKS value bounds
+
+categorical[T not numeric]      JSON/str          categorical[1,2]
+
+(other object)                  JSON/str          string
+
+(all other)                                                         Always an ERROR[3]
+
+
+Notes:
+[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
+[1] with NO categories enumerated (client side does it to handle rounding)
+[2] NA (undefined) categories are assigned a JSON null value
+[3] Includes all other numpy types:  datetime, complex, etc.
+[4] means float, int, uint (dtype.kind in ['i','u','f'])
+
+"""
+
+
+def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
+    dtypes_by_column_name = {}
+    schema_type_hints_by_column_name = {}
+
+    for column_name, column_values in dataframe.items():
+        (
+            dtypes_by_column_name[column_name],
+            schema_type_hints_by_column_name[column_name],
+        ) = get_dtype_and_schema_of_array(column_values)
+
+    return dtypes_by_column_name, schema_type_hints_by_column_name
+
+
+def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
+    return _get_type_info(array)[0]
+
+
+def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
+    return _get_type_info(array)[1]
+
+
+def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """Return tuple (encoding_dtype, schema_type_hint)"""
+    return _get_type_info(array)
+
+
+def get_schema_type_hint_from_dtype(dtype) -> dict:
+    res = _get_type_info_from_dtype(dtype)
+    if res is None:
+        raise TypeError(f"Annotations of type {dtype} are unsupported.")
+    else:
+        return res[1]
+
+
+def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
+    """
+    Best-effort to determine encoding type and schema hint from a dtype.
+    If this is not possible, or the type is unsupported, return None.
+
+    This should be a subset of the cases which are supported by
+    _get_type_info().  The latter should be preferred if the array (values)
+    are available for typing.
+    """
+    if dtype.kind == "b":
+        return (np.uint8, {"type": "boolean"})
+
+    if dtype.kind == "U":
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"]:
+        if np.can_cast(dtype, np.int32):
+            return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(dtype)
+        return (np.float32, {"type": "float32"})
+
+    if dtype.kind == "O" and not dtype.name == "category":
+        return (np.dtype(str), {"type": "string"})
+
+    return None
+
+
+def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """
+    Determine encoding type and schema hint from an array.  This allows more
+    flexible casting than may be possible by using just the dtype, as it can
+    account for category types and array values.
+    """
+    if (
+        not isinstance(array, np.ndarray)
+        and not isinstance(array, pd.Series)
+        and not isinstance(array, pd.Index)
+        and not hasattr(array, "dtype")
+    ):
+        raise TypeError("Unsupported data type.")
+
+    dtype = array.dtype
+
+    res = _get_type_info_from_dtype(dtype)
+    if res is not None:
+        return res
+
+    if dtype.kind == "O":
+        if dtype.name == "category":
+            # Sometimes CategoricalDType can be encoded as int or float without further fuss.
+            # Do not specify the categories in the schema - let the client-side figure it out
+            # on its own.  Utilize Series.to_numpy() to do casting that handles categorical
+            # NA/NaN (missing or undefined) categories.
+            if dtype.categories.dtype.kind in ["f", "i", "u"]:
+                return (
+                    _get_type_info(array.to_numpy())[0],
+                    {"type": "categorical"},
+                )
+            else:
+                return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
+
+        # all other extension types are str-encoded
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
+        return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(array.dtype)
+        return (np.float32, {"type": "float32"})
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def _float64_warning(dtype):
+    """
+    Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
+    """
+    if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
+        logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
+
+def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
+    """
+    Return true if the (U)INT array values can be safely cast to int32.  We allow size reducing
+    casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
+    can be represented by the smaller type).
+    """
+    assert array.dtype.kind in ["u", "i"]
+
+    if np.can_cast(array.dtype, np.int32):
+        return True
+
+    if array.size == 0:
+        return True
+
+    int32_machine_limits = np.iinfo(np.int32)
+    if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
+        return True
+
+    return False
+
+
+def convert_string_to_value(value: str):
+    """convert a string to value with the most appropriate type"""
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+    if value == "null":
+        return None
+    try:
+        return eval(value)
+    except:  # noqa E722
+        return value

server/common/utils/utils.py

@@ -0,0 +1,126 @@
+import contextlib
+import errno
+import importlib.util
+import logging
+import os
+import pkgutil
+import socket
+from urllib.parse import urlsplit, urljoin
+
+import numpy as np
+from flask import json
+
+from server.common.errors import ConfigurationError
+
+
+def find_available_port(host, port=5005):
+    """
+    Helper method to find open port on host. Tries 5000 ports incremented from the specified port
+    """
+    # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
+    num_ports_to_try = 5000
+    for port_to_try in range(port, port + num_ports_to_try):
+        if is_port_available(host, port_to_try):
+            return port_to_try
+    raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
+
+
+def is_port_available(host, port):
+    is_available = False
+    with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        try:
+            s.bind((host, port))
+            is_available = True
+        except socket.error:
+            pass
+    return is_available
+
+
+def sort_options(command):
+    """
+    Helper for the click options - will sort options in a command, and can
+    be used as a decorator.
+    """
+    command.params.sort(key=lambda p: p.name)
+    return command
+
+
+def path_join(base, *urls):
+    """
+    this is like urllib.parse.urljoin, except it works around the scheme-specific
+    cleverness in the aforementioned code, ignores anything in the url except the path,
+    and accepts more than one url.
+    """
+    if not base.endswith("/"):
+        base += "/"
+    btpl = urlsplit(base)
+    path = btpl.path
+    for url in urls:
+        utpl = urlsplit(url)
+        if btpl.scheme == "":
+            path = os.path.join(path, utpl.path)
+            path = os.path.normpath(path)
+        else:
+            path = urljoin(path, utpl.path)
+    return btpl._replace(path=path).geturl()
+
+
+class StrictJSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder set-up performing two tasks:
+    1. Strict JSON conformance with non-finite floats (NaN, +/-Inf) via allow_nan=False
+    2. Convert various Numpy types into python types so the encoder will correctly encode.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        NaN/Infinities are illegal in standard JSON.  Python extends JSON with
+        non-standard symbols that most JavaScript JSON parsers do not understand.
+        The `allow_nan` parameter will force Python simplejson to throw an ValueError
+        if it runs into non-finite floating point values which are unsupported by
+        standard JSON.
+        """
+        kwargs["allow_nan"] = False
+        super().__init__(*args, **kwargs)
+
+    def default(self, obj):
+        """This helps us convert types not supported by the native JSON encoder into
+        standard python types, eg, np.int64."""
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.integer):
+            return int(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+def custom_format_warning(msg, *args, **kwargs):
+    return f"[cellxgene] Warning: {msg} \n"
+
+
+def jsonify_strict(data):
+    return json.dumps(data, cls=StrictJSONEncoder, allow_nan=False)
+
+
+def import_plugins(plugin_module):
+    """
+    Load optional plugin modules from server.common.plugins
+
+    If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
+    This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
+    normal.
+    """
+    loaded_modules = []
+    try:
+        pkg = importlib.import_module(plugin_module)
+        for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
+            full_name = f"{plugin_module}.{name}"
+            try:
+                module = importlib.import_module(full_name)
+            except Exception as e:
+                raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
+            loaded_modules.append(module)
+    except ModuleNotFoundError as e:
+        #  This exception occurs when the plugin_module does not exist (not an error).
+        logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
+
+    return loaded_modules