mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-01 13:38:11 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -0,0 +1,11 @@
|
||||
|
||||
.PHONY: unit-test
|
||||
unit-test:
|
||||
PYTHONWARNINGS=ignore:ResourceWarning coverage run \
|
||||
--source=fbs,utils \
|
||||
--omit=.coverage,data_common/fbs/NetEncoding,venv \
|
||||
-m unittest discover \
|
||||
--start-directory ../test/test_common/unit \
|
||||
--top-level-directory ../../ \
|
||||
--verbose; test_result=$$?; \
|
||||
exit $$test_result \
|
||||
@@ -0,0 +1,233 @@
|
||||
import re
|
||||
|
||||
from backend.common.errors import ColorFormatException
|
||||
|
||||
HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
|
||||
|
||||
# https://www.w3.org/TR/css-color-4/#named-colors
|
||||
CSS4_NAMED_COLORS = dict(
|
||||
aliceblue="#f0f8ff",
|
||||
antiquewhite="#faebd7",
|
||||
aqua="#00ffff",
|
||||
aquamarine="#7fffd4",
|
||||
azure="#f0ffff",
|
||||
beige="#f5f5dc",
|
||||
bisque="#ffe4c4",
|
||||
black="#000000",
|
||||
blanchedalmond="#ffebcd",
|
||||
blue="#0000ff",
|
||||
blueviolet="#8a2be2",
|
||||
brown="#a52a2a",
|
||||
burlywood="#deb887",
|
||||
cadetblue="#5f9ea0",
|
||||
chartreuse="#7fff00",
|
||||
chocolate="#d2691e",
|
||||
coral="#ff7f50",
|
||||
cornflowerblue="#6495ed",
|
||||
cornsilk="#fff8dc",
|
||||
crimson="#dc143c",
|
||||
cyan="#00ffff",
|
||||
darkblue="#00008b",
|
||||
darkcyan="#008b8b",
|
||||
darkgoldenrod="#b8860b",
|
||||
darkgray="#a9a9a9",
|
||||
darkgreen="#006400",
|
||||
darkgrey="#a9a9a9",
|
||||
darkkhaki="#bdb76b",
|
||||
darkmagenta="#8b008b",
|
||||
darkolivegreen="#556b2f",
|
||||
darkorange="#ff8c00",
|
||||
darkorchid="#9932cc",
|
||||
darkred="#8b0000",
|
||||
darksalmon="#e9967a",
|
||||
darkseagreen="#8fbc8f",
|
||||
darkslateblue="#483d8b",
|
||||
darkslategray="#2f4f4f",
|
||||
darkslategrey="#2f4f4f",
|
||||
darkturquoise="#00ced1",
|
||||
darkviolet="#9400d3",
|
||||
deeppink="#ff1493",
|
||||
deepskyblue="#00bfff",
|
||||
dimgray="#696969",
|
||||
dimgrey="#696969",
|
||||
dodgerblue="#1e90ff",
|
||||
firebrick="#b22222",
|
||||
floralwhite="#fffaf0",
|
||||
forestgreen="#228b22",
|
||||
fuchsia="#ff00ff",
|
||||
gainsboro="#dcdcdc",
|
||||
ghostwhite="#f8f8ff",
|
||||
gold="#ffd700",
|
||||
goldenrod="#daa520",
|
||||
gray="#808080",
|
||||
green="#008000",
|
||||
greenyellow="#adff2f",
|
||||
grey="#808080",
|
||||
honeydew="#f0fff0",
|
||||
hotpink="#ff69b4",
|
||||
indianred="#cd5c5c",
|
||||
indigo="#4b0082",
|
||||
ivory="#fffff0",
|
||||
khaki="#f0e68c",
|
||||
lavender="#e6e6fa",
|
||||
lavenderblush="#fff0f5",
|
||||
lawngreen="#7cfc00",
|
||||
lemonchiffon="#fffacd",
|
||||
lightblue="#add8e6",
|
||||
lightcoral="#f08080",
|
||||
lightcyan="#e0ffff",
|
||||
lightgoldenrodyellow="#fafad2",
|
||||
lightgray="#d3d3d3",
|
||||
lightgreen="#90ee90",
|
||||
lightgrey="#d3d3d3",
|
||||
lightpink="#ffb6c1",
|
||||
lightsalmon="#ffa07a",
|
||||
lightseagreen="#20b2aa",
|
||||
lightskyblue="#87cefa",
|
||||
lightslategray="#778899",
|
||||
lightslategrey="#778899",
|
||||
lightsteelblue="#b0c4de",
|
||||
lightyellow="#ffffe0",
|
||||
lime="#00ff00",
|
||||
limegreen="#32cd32",
|
||||
linen="#faf0e6",
|
||||
magenta="#ff00ff",
|
||||
maroon="#800000",
|
||||
mediumaquamarine="#66cdaa",
|
||||
mediumblue="#0000cd",
|
||||
mediumorchid="#ba55d3",
|
||||
mediumpurple="#9370db",
|
||||
mediumseagreen="#3cb371",
|
||||
mediumslateblue="#7b68ee",
|
||||
mediumspringgreen="#00fa9a",
|
||||
mediumturquoise="#48d1cc",
|
||||
mediumvioletred="#c71585",
|
||||
midnightblue="#191970",
|
||||
mintcream="#f5fffa",
|
||||
mistyrose="#ffe4e1",
|
||||
moccasin="#ffe4b5",
|
||||
navajowhite="#ffdead",
|
||||
navy="#000080",
|
||||
oldlace="#fdf5e6",
|
||||
olive="#808000",
|
||||
olivedrab="#6b8e23",
|
||||
orange="#ffa500",
|
||||
orangered="#ff4500",
|
||||
orchid="#da70d6",
|
||||
palegoldenrod="#eee8aa",
|
||||
palegreen="#98fb98",
|
||||
paleturquoise="#afeeee",
|
||||
palevioletred="#db7093",
|
||||
papayawhip="#ffefd5",
|
||||
peachpuff="#ffdab9",
|
||||
peru="#cd853f",
|
||||
pink="#ffc0cb",
|
||||
plum="#dda0dd",
|
||||
powderblue="#b0e0e6",
|
||||
purple="#800080",
|
||||
rebeccapurple="#663399",
|
||||
red="#ff0000",
|
||||
rosybrown="#bc8f8f",
|
||||
royalblue="#4169e1",
|
||||
saddlebrown="#8b4513",
|
||||
salmon="#fa8072",
|
||||
sandybrown="#f4a460",
|
||||
seagreen="#2e8b57",
|
||||
seashell="#fff5ee",
|
||||
sienna="#a0522d",
|
||||
silver="#c0c0c0",
|
||||
skyblue="#87ceeb",
|
||||
slateblue="#6a5acd",
|
||||
slategray="#708090",
|
||||
slategrey="#708090",
|
||||
snow="#fffafa",
|
||||
springgreen="#00ff7f",
|
||||
steelblue="#4682b4",
|
||||
tan="#d2b48c",
|
||||
teal="#008080",
|
||||
thistle="#d8bfd8",
|
||||
tomato="#ff6347",
|
||||
turquoise="#40e0d0",
|
||||
violet="#ee82ee",
|
||||
wheat="#f5deb3",
|
||||
white="#ffffff",
|
||||
whitesmoke="#f5f5f5",
|
||||
yellow="#ffff00",
|
||||
yellowgreen="#9acd32",
|
||||
)
|
||||
|
||||
|
||||
def convert_color_to_hex_format(unknown):
|
||||
"""
|
||||
Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet.
|
||||
|
||||
The function accepts for the following formats:
|
||||
- A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html
|
||||
- RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0]
|
||||
- RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255]
|
||||
- Hex triplet string, as in "#08c0ff"
|
||||
|
||||
:param unknown: color info of unknown format
|
||||
:return: a hex triplet representing that color
|
||||
"""
|
||||
try:
|
||||
if type(unknown) in (list, tuple) and len(unknown) == 3:
|
||||
if all(0.0 <= ele <= 1.0 for ele in unknown):
|
||||
tup = tuple(int(ele * 255) for ele in unknown)
|
||||
elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown):
|
||||
tup = tuple(unknown)
|
||||
else:
|
||||
raise ColorFormatException("Unknown color iterable format!")
|
||||
return "#%02x%02x%02x" % tup
|
||||
elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS:
|
||||
return CSS4_NAMED_COLORS[unknown.lower()]
|
||||
elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown):
|
||||
return unknown.lower()
|
||||
else:
|
||||
raise ColorFormatException("Unknown color format type!")
|
||||
except Exception as e:
|
||||
raise ColorFormatException(e)
|
||||
|
||||
|
||||
def convert_anndata_category_colors_to_cxg_category_colors(data):
|
||||
"""
|
||||
Convert color information from anndata files to the cellxgene color data format as described below:
|
||||
{
|
||||
"<category_name>": {
|
||||
"<label_name>": "<color_hex_code>",
|
||||
...
|
||||
},
|
||||
...
|
||||
}
|
||||
|
||||
For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307.
|
||||
|
||||
For more on the anndata color data structure, see
|
||||
https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178.
|
||||
|
||||
Handling of malformed data:
|
||||
- For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot
|
||||
convert to a hex triplet string, a ColorFormatException is raised
|
||||
- No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no
|
||||
adata.obs[f"{category}"] key
|
||||
|
||||
:param data: the anndata file
|
||||
:return: cellxgene color data structure as described above
|
||||
"""
|
||||
cxg_colors = dict()
|
||||
color_key_suffix = "_colors"
|
||||
for uns_key in data.uns.keys():
|
||||
# find uns array that describes colors for a category
|
||||
if not uns_key.endswith(color_key_suffix):
|
||||
continue
|
||||
|
||||
# check to see if we actually have observations for that category
|
||||
category_name = uns_key[: -len(color_key_suffix)]
|
||||
if category_name not in data.obs.keys():
|
||||
continue
|
||||
|
||||
# create the cellxgene color entry for this category
|
||||
cxg_colors[category_name] = dict(
|
||||
zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
|
||||
)
|
||||
return cxg_colors
|
||||
@@ -0,0 +1,30 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class AugmentedEnum(Enum):
|
||||
def __hash__(self):
|
||||
return self.value.__hash__()
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, type(self)) or isinstance(other, str):
|
||||
return self.value == other
|
||||
return False
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.value
|
||||
|
||||
|
||||
class Axis(AugmentedEnum):
|
||||
OBS = "obs"
|
||||
VAR = "var"
|
||||
|
||||
|
||||
class DiffExpMode(AugmentedEnum):
|
||||
TOP_N = "topN"
|
||||
VAR_FILTER = "varFilter"
|
||||
|
||||
|
||||
JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
|
||||
REACTIVE_LIMIT = 1_000_000
|
||||
|
||||
MAX_LAYOUTS = 30
|
||||
@@ -0,0 +1,59 @@
|
||||
from http import HTTPStatus
|
||||
|
||||
|
||||
class CellxgeneException(Exception):
|
||||
"""Base class for cellxgene exceptions"""
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class RequestException(CellxgeneException):
|
||||
"""Baseclass for exceptions that can be raised from a request."""
|
||||
|
||||
# The default status code is 400 (Bad Request)
|
||||
default_status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
def __init__(self, message, status_code=None):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code or self.default_status_code
|
||||
|
||||
|
||||
def define_exception(name, doc):
|
||||
globals()[name] = type(name, (CellxgeneException,), dict(__doc__=doc))
|
||||
|
||||
|
||||
def define_request_exception(name, doc, default_status_code=HTTPStatus.BAD_REQUEST):
|
||||
globals()[name] = type(name, (RequestException,), dict(__doc__=doc, default_status_code=default_status_code))
|
||||
|
||||
|
||||
define_request_exception("FilterError", "Raised when filter is malformed")
|
||||
define_request_exception("JSONEncodingValueError", "Raised when data cannot be encoded into json")
|
||||
define_request_exception("MimeTypeError", "Raised when incompatible MIME type selected")
|
||||
define_request_exception("DatasetAccessError", "Raised when file loaded into a DataAdaptor is misformatted")
|
||||
define_request_exception("DisabledFeatureError", "Raised when an attempt to use a disabled feature occurs")
|
||||
define_request_exception("AnnotationsError", "Raised when an attempt to use the annotations feature fails")
|
||||
define_request_exception(
|
||||
"ComputeError",
|
||||
"Raised when an error occurs during a compute algorithm (such as diffexp)",
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
|
||||
define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
|
||||
define_request_exception(
|
||||
"AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
|
||||
)
|
||||
|
||||
define_request_exception(
|
||||
"AnnotationCategoryNameError",
|
||||
"Raised when an annotation category name cant be saved",
|
||||
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
)
|
||||
|
||||
define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
|
||||
define_exception("ConfigurationError", "Raised when checking configuration errors")
|
||||
define_exception("PrepareError", "Raised when data is misprepared")
|
||||
define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
|
||||
define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
|
||||
define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")
|
||||
@@ -0,0 +1,41 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
import flatbuffers
|
||||
|
||||
class Column(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@classmethod
|
||||
def GetRootAsColumn(cls, buf, offset):
|
||||
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
|
||||
x = Column()
|
||||
x.Init(buf, n + offset)
|
||||
return x
|
||||
|
||||
# Column
|
||||
def Init(self, buf, pos):
|
||||
self._tab = flatbuffers.table.Table(buf, pos)
|
||||
|
||||
# Column
|
||||
def UType(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Column
|
||||
def U(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
|
||||
if o != 0:
|
||||
from flatbuffers.table import Table
|
||||
obj = Table(bytearray(), 0)
|
||||
self._tab.Union(obj, o)
|
||||
return obj
|
||||
return None
|
||||
|
||||
def ColumnStart(builder): builder.StartObject(2)
|
||||
def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
|
||||
def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
|
||||
def ColumnEnd(builder): return builder.EndObject()
|
||||
@@ -0,0 +1,46 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
import flatbuffers
|
||||
|
||||
class Float32Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@classmethod
|
||||
def GetRootAsFloat32Array(cls, buf, offset):
|
||||
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
|
||||
x = Float32Array()
|
||||
x.Init(buf, n + offset)
|
||||
return x
|
||||
|
||||
# Float32Array
|
||||
def Init(self, buf, pos):
|
||||
self._tab = flatbuffers.table.Table(buf, pos)
|
||||
|
||||
# Float32Array
|
||||
def Data(self, j):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return 0
|
||||
|
||||
# Float32Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
|
||||
return 0
|
||||
|
||||
# Float32Array
|
||||
def DataLength(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Float32ArrayStart(builder): builder.StartObject(1)
|
||||
def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def Float32ArrayEnd(builder): return builder.EndObject()
|
||||
@@ -0,0 +1,46 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
import flatbuffers
|
||||
|
||||
class Float64Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@classmethod
|
||||
def GetRootAsFloat64Array(cls, buf, offset):
|
||||
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
|
||||
x = Float64Array()
|
||||
x.Init(buf, n + offset)
|
||||
return x
|
||||
|
||||
# Float64Array
|
||||
def Init(self, buf, pos):
|
||||
self._tab = flatbuffers.table.Table(buf, pos)
|
||||
|
||||
# Float64Array
|
||||
def Data(self, j):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
|
||||
return 0
|
||||
|
||||
# Float64Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
|
||||
return 0
|
||||
|
||||
# Float64Array
|
||||
def DataLength(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Float64ArrayStart(builder): builder.StartObject(1)
|
||||
def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
|
||||
def Float64ArrayEnd(builder): return builder.EndObject()
|
||||
@@ -0,0 +1,46 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
import flatbuffers
|
||||
|
||||
class Int32Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@classmethod
|
||||
def GetRootAsInt32Array(cls, buf, offset):
|
||||
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
|
||||
x = Int32Array()
|
||||
x.Init(buf, n + offset)
|
||||
return x
|
||||
|
||||
# Int32Array
|
||||
def Init(self, buf, pos):
|
||||
self._tab = flatbuffers.table.Table(buf, pos)
|
||||
|
||||
# Int32Array
|
||||
def Data(self, j):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return 0
|
||||
|
||||
# Int32Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
|
||||
return 0
|
||||
|
||||
# Int32Array
|
||||
def DataLength(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Int32ArrayStart(builder): builder.StartObject(1)
|
||||
def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def Int32ArrayEnd(builder): return builder.EndObject()
|
||||
@@ -0,0 +1,46 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
import flatbuffers
|
||||
|
||||
class JSONEncodedArray(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@classmethod
|
||||
def GetRootAsJSONEncodedArray(cls, buf, offset):
|
||||
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
|
||||
x = JSONEncodedArray()
|
||||
x.Init(buf, n + offset)
|
||||
return x
|
||||
|
||||
# JSONEncodedArray
|
||||
def Init(self, buf, pos):
|
||||
self._tab = flatbuffers.table.Table(buf, pos)
|
||||
|
||||
# JSONEncodedArray
|
||||
def Data(self, j):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
|
||||
return 0
|
||||
|
||||
# JSONEncodedArray
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
|
||||
return 0
|
||||
|
||||
# JSONEncodedArray
|
||||
def DataLength(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def JSONEncodedArrayStart(builder): builder.StartObject(1)
|
||||
def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
|
||||
def JSONEncodedArrayEnd(builder): return builder.EndObject()
|
||||
@@ -0,0 +1,98 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
import flatbuffers
|
||||
|
||||
class Matrix(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@classmethod
|
||||
def GetRootAsMatrix(cls, buf, offset):
|
||||
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
|
||||
x = Matrix()
|
||||
x.Init(buf, n + offset)
|
||||
return x
|
||||
|
||||
# Matrix
|
||||
def Init(self, buf, pos):
|
||||
self._tab = flatbuffers.table.Table(buf, pos)
|
||||
|
||||
# Matrix
|
||||
def NRows(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
def NCols(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
def Columns(self, j):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
|
||||
if o != 0:
|
||||
x = self._tab.Vector(o)
|
||||
x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
|
||||
x = self._tab.Indirect(x)
|
||||
from .Column import Column
|
||||
obj = Column()
|
||||
obj.Init(self._tab.Bytes, x)
|
||||
return obj
|
||||
return None
|
||||
|
||||
# Matrix
|
||||
def ColumnsLength(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
|
||||
if o != 0:
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
def ColIndexType(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
def ColIndex(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
|
||||
if o != 0:
|
||||
from flatbuffers.table import Table
|
||||
obj = Table(bytearray(), 0)
|
||||
self._tab.Union(obj, o)
|
||||
return obj
|
||||
return None
|
||||
|
||||
# Matrix
|
||||
def RowIndexType(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
|
||||
if o != 0:
|
||||
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
|
||||
return 0
|
||||
|
||||
# Matrix
|
||||
def RowIndex(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16))
|
||||
if o != 0:
|
||||
from flatbuffers.table import Table
|
||||
obj = Table(bytearray(), 0)
|
||||
self._tab.Union(obj, o)
|
||||
return obj
|
||||
return None
|
||||
|
||||
def MatrixStart(builder): builder.StartObject(7)
|
||||
def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
|
||||
def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
|
||||
def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
|
||||
def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
|
||||
def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
|
||||
def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
|
||||
def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
|
||||
def MatrixEnd(builder): return builder.EndObject()
|
||||
@@ -0,0 +1,12 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
class TypedArray(object):
|
||||
NONE = 0
|
||||
Float32Array = 1
|
||||
Int32Array = 2
|
||||
Uint32Array = 3
|
||||
Float64Array = 4
|
||||
JSONEncodedArray = 5
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
# namespace: NetEncoding
|
||||
|
||||
import flatbuffers
|
||||
|
||||
class Uint32Array(object):
|
||||
__slots__ = ['_tab']
|
||||
|
||||
@classmethod
|
||||
def GetRootAsUint32Array(cls, buf, offset):
|
||||
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
|
||||
x = Uint32Array()
|
||||
x.Init(buf, n + offset)
|
||||
return x
|
||||
|
||||
# Uint32Array
|
||||
def Init(self, buf, pos):
|
||||
self._tab = flatbuffers.table.Table(buf, pos)
|
||||
|
||||
# Uint32Array
|
||||
def Data(self, j):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
a = self._tab.Vector(o)
|
||||
return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
|
||||
return 0
|
||||
|
||||
# Uint32Array
|
||||
def DataAsNumpy(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
|
||||
return 0
|
||||
|
||||
# Uint32Array
|
||||
def DataLength(self):
|
||||
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
|
||||
if o != 0:
|
||||
return self._tab.VectorLen(o)
|
||||
return 0
|
||||
|
||||
def Uint32ArrayStart(builder): builder.StartObject(1)
|
||||
def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
|
||||
def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
|
||||
def Uint32ArrayEnd(builder): return builder.EndObject()
|
||||
@@ -0,0 +1,247 @@
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from flatbuffers import Builder
|
||||
from scipy import sparse
|
||||
|
||||
import backend.common.fbs.NetEncoding.Column as Column
|
||||
import backend.common.fbs.NetEncoding.Float32Array as Float32Array
|
||||
import backend.common.fbs.NetEncoding.Float64Array as Float64Array
|
||||
import backend.common.fbs.NetEncoding.Int32Array as Int32Array
|
||||
import backend.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
|
||||
import backend.common.fbs.NetEncoding.Matrix as Matrix
|
||||
import backend.common.fbs.NetEncoding.TypedArray as TypedArray
|
||||
import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
|
||||
|
||||
# Serialization helper
|
||||
def serialize_column(builder, typed_arr):
|
||||
""" Serialize NetEncoding.Column """
|
||||
|
||||
(u_type, u_value) = typed_arr
|
||||
Column.ColumnStart(builder)
|
||||
Column.ColumnAddUType(builder, u_type)
|
||||
Column.ColumnAddU(builder, u_value)
|
||||
return Column.ColumnEnd(builder)
|
||||
|
||||
|
||||
# Serialization helper
|
||||
def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
|
||||
""" Serialize NetEncoding.Matrix """
|
||||
|
||||
Matrix.MatrixStart(builder)
|
||||
Matrix.MatrixAddNRows(builder, n_rows)
|
||||
Matrix.MatrixAddNCols(builder, n_cols)
|
||||
Matrix.MatrixAddColumns(builder, columns)
|
||||
if col_idx is not None:
|
||||
(u_type, u_val) = col_idx
|
||||
Matrix.MatrixAddColIndexType(builder, u_type)
|
||||
Matrix.MatrixAddColIndex(builder, u_val)
|
||||
return Matrix.MatrixEnd(builder)
|
||||
|
||||
|
||||
# Serialization helper
|
||||
def serialize_typed_array(builder, source_array, encoding_info):
|
||||
"""
|
||||
Serialize any of the various typed arrays, eg, Float32Array. Specific means of serialization and type conversion
|
||||
are provided by type_info.
|
||||
"""
|
||||
|
||||
arr = source_array
|
||||
(array_type, as_type) = encoding_info(source_array)
|
||||
|
||||
if isinstance(arr, pd.Index):
|
||||
arr = arr.to_series()
|
||||
|
||||
# convert to a simple ndarray
|
||||
if as_type == "json":
|
||||
as_json = arr.to_json(orient="records")
|
||||
arr = np.array(bytearray(as_json, "utf-8"))
|
||||
else:
|
||||
if sparse.issparse(arr):
|
||||
arr = arr.toarray()
|
||||
elif isinstance(arr, pd.Series):
|
||||
arr = arr.to_numpy()
|
||||
if arr.dtype != as_type:
|
||||
arr = arr.astype(as_type)
|
||||
|
||||
# serialize the ndarray into a vector
|
||||
if arr.ndim == 2:
|
||||
if arr.shape[0] == 1:
|
||||
arr = arr[0]
|
||||
elif arr.shape[1] == 1:
|
||||
arr = arr.T[0]
|
||||
|
||||
vec = builder.CreateNumpyVector(arr)
|
||||
|
||||
# serialize the typed array table
|
||||
builder.StartObject(1)
|
||||
builder.PrependUOffsetTRelativeSlot(0, vec, 0)
|
||||
array_value = builder.EndObject()
|
||||
return (array_type, array_value)
|
||||
|
||||
|
||||
def column_encoding(arr):
|
||||
column_encoding_type_map = {
|
||||
# array protocol string: ( array_type, as_type )
|
||||
np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64),
|
||||
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
|
||||
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
|
||||
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
}
|
||||
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
|
||||
|
||||
return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
|
||||
|
||||
|
||||
def index_encoding(arr):
|
||||
index_encoding_type_map = {
|
||||
# array protocol string: ( array_type, as_type )
|
||||
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
|
||||
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
|
||||
}
|
||||
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
|
||||
|
||||
return index_encoding_type_map.get(arr.dtype.str, index_encoding_default)
|
||||
|
||||
|
||||
def guess_at_mem_needed(matrix):
|
||||
(n_rows, n_cols) = matrix.shape
|
||||
if isinstance(matrix, np.ndarray) or sparse.issparse(matrix):
|
||||
guess = (n_rows * n_cols * matrix.dtype.itemsize) + 1024
|
||||
elif isinstance(matrix, pd.DataFrame):
|
||||
# XXX TODO - DataFrame type estimate
|
||||
guess = 1
|
||||
else:
|
||||
guess = 1
|
||||
|
||||
# round up to nearest 1024 bytes
|
||||
guess = (guess + 0x400) & (~0x3FF)
|
||||
return guess
|
||||
|
||||
|
||||
def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
|
||||
"""
|
||||
Given a 2D DataFrame, ndarray or sparse equivalent, create and return a Matrix flatbuffer.
|
||||
|
||||
:param matrix: 2D DataFrame, ndarray or sparse equivalent
|
||||
:param row_idx: index for row dimension, Index or ndarray
|
||||
:param col_idx: index for col dimension, Index or ndarray
|
||||
|
||||
NOTE: row indices are (currently) unsupported and must be None
|
||||
"""
|
||||
|
||||
if row_idx is not None:
|
||||
raise ValueError("row indexing not supported for FBS Matrix")
|
||||
if matrix.ndim != 2:
|
||||
raise ValueError("FBS Matrix must be 2D")
|
||||
|
||||
(n_rows, n_cols) = matrix.shape
|
||||
|
||||
# estimate size needed, so we don't unnecessarily realloc.
|
||||
builder = Builder(guess_at_mem_needed(matrix))
|
||||
|
||||
columns = []
|
||||
for cidx in range(n_cols - 1, -1, -1):
|
||||
# serialize the typed array
|
||||
col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx]
|
||||
typed_arr = serialize_typed_array(builder, col, column_encoding)
|
||||
|
||||
# serialize the Column union
|
||||
columns.append(serialize_column(builder, typed_arr))
|
||||
|
||||
# Serialize Matrix.columns[]
|
||||
Matrix.MatrixStartColumnsVector(builder, n_cols)
|
||||
for c in columns:
|
||||
builder.PrependUOffsetTRelative(c)
|
||||
matrix_column_vec = builder.EndVector(n_cols)
|
||||
|
||||
# serialize the colIndex if provided
|
||||
cidx = None
|
||||
if col_idx is not None:
|
||||
cidx = serialize_typed_array(builder, col_idx, index_encoding)
|
||||
|
||||
# Serialize Matrix
|
||||
matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx)
|
||||
|
||||
builder.Finish(matrix)
|
||||
return builder.Output()
|
||||
|
||||
|
||||
def deserialize_typed_array(tarr):
|
||||
type_map = {
|
||||
TypedArray.TypedArray.NONE: None,
|
||||
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
|
||||
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
|
||||
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
|
||||
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
|
||||
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
|
||||
}
|
||||
(u_type, u) = tarr
|
||||
if u_type is TypedArray.TypedArray.NONE:
|
||||
return None
|
||||
|
||||
TarType = type_map.get(u_type, None)
|
||||
if TarType is None:
|
||||
raise TypeError(f"FBS contains unknown data type: {u_type}")
|
||||
|
||||
arr = TarType()
|
||||
arr.Init(u.Bytes, u.Pos)
|
||||
narr = arr.DataAsNumpy()
|
||||
if u_type == TypedArray.TypedArray.JSONEncodedArray:
|
||||
narr = json.loads(narr.tostring().decode("utf-8"))
|
||||
return narr
|
||||
|
||||
|
||||
def decode_matrix_fbs(fbs):
|
||||
"""
|
||||
Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data and indices.
|
||||
"""
|
||||
|
||||
matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0)
|
||||
n_rows = matrix.NRows()
|
||||
n_cols = matrix.NCols()
|
||||
if n_rows == 0 or n_cols == 0:
|
||||
return pd.DataFrame()
|
||||
|
||||
if matrix.RowIndexType() is not TypedArray.TypedArray.NONE:
|
||||
raise ValueError("row indexing not supported for FBS Matrix")
|
||||
|
||||
columns_length = matrix.ColumnsLength()
|
||||
|
||||
columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
|
||||
if columns_index is None:
|
||||
columns_index = range(0, n_cols)
|
||||
|
||||
# sanity checks
|
||||
if len(columns_index) != n_cols or columns_length != n_cols:
|
||||
raise ValueError("FBS column count does not match number of columns in underlying matrix")
|
||||
|
||||
columns_data = {}
|
||||
columns_type = {}
|
||||
for col_idx in range(0, columns_length):
|
||||
col = matrix.Columns(col_idx)
|
||||
tarr = (col.UType(), col.U())
|
||||
data = deserialize_typed_array(tarr)
|
||||
columns_data[columns_index[col_idx]] = data
|
||||
if len(data) != n_rows:
|
||||
raise ValueError("FBS column length does not match number of rows")
|
||||
if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
|
||||
columns_type[columns_index[col_idx]] = "category"
|
||||
|
||||
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
|
||||
|
||||
# more sanity checks
|
||||
if not df.columns.is_unique or len(df.columns) != n_cols:
|
||||
raise KeyError("FBS column indices are not unique")
|
||||
|
||||
return df
|
||||
@@ -0,0 +1,23 @@
|
||||
import logging
|
||||
|
||||
import boto3
|
||||
from flask import json
|
||||
|
||||
from backend.common.errors import SecretKeyRetrievalError
|
||||
|
||||
|
||||
def get_secret_key(region_name, secret_name):
|
||||
session = boto3.session.Session()
|
||||
client = session.client(service_name="secretsmanager", region_name=region_name)
|
||||
|
||||
try:
|
||||
get_secret_value_response = client.get_secret_value(SecretId=secret_name)
|
||||
if "SecretString" in get_secret_value_response:
|
||||
var = get_secret_value_response["SecretString"]
|
||||
secret = json.loads(var)
|
||||
return secret
|
||||
except Exception as e:
|
||||
logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
|
||||
raise SecretKeyRetrievalError(str(e))
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,154 @@
|
||||
import os
|
||||
import tempfile
|
||||
import fsspec
|
||||
from datetime import datetime
|
||||
import boto3
|
||||
import botocore
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class DataLocator:
|
||||
"""
|
||||
DataLocator is a simple wrapper around fsspec functionality, and provides a
|
||||
set of functions to encapsulate a data location (URI or path), interogate
|
||||
metadata about the object at that location (size, existance, etc) and
|
||||
access the underlying data.
|
||||
|
||||
https://filesystem-spec.readthedocs.io/en/latest/index.html
|
||||
|
||||
Example:
|
||||
dl = DataLocator("/tmp/foo.h5ad")
|
||||
if dl.exists():
|
||||
print(dl.size())
|
||||
with dl.open() as f:
|
||||
thecontents = f.read()
|
||||
|
||||
DataLocator will accept a URI or native path. Error handling is as defined
|
||||
in fsspec.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, uri_or_path, region_name=None):
|
||||
if isinstance(uri_or_path, DataLocator):
|
||||
locator = uri_or_path
|
||||
self.uri_or_path = locator.uri_or_path
|
||||
self.protocol = locator.protocol
|
||||
self.path = locator.path
|
||||
self.cname = locator.cname
|
||||
else:
|
||||
self.uri_or_path = uri_or_path
|
||||
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
|
||||
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
|
||||
self.cname = self.path if self.protocol == "file" else self.uri_or_path
|
||||
|
||||
# fsspec.filesystem will throw RuntimeError if the protocol is unsupported
|
||||
if self.protocol == "s3":
|
||||
if region_name:
|
||||
config_kwargs = dict(region_name=region_name)
|
||||
self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs)
|
||||
else:
|
||||
self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30)
|
||||
else:
|
||||
self.fs = fsspec.filesystem(self.protocol)
|
||||
|
||||
def __repr__(self):
|
||||
return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
|
||||
f"path={self.path}, uri_or_path={self.uri_or_path})"
|
||||
|
||||
@staticmethod
|
||||
def _get_protocol_and_path(uri_or_path):
|
||||
if "://" in uri_or_path:
|
||||
protocol, path = uri_or_path.split("://", 1)
|
||||
# windows!!! Ignore single letter drive identifiers,
|
||||
# eg, G:\foo.txt
|
||||
if len(protocol) > 1:
|
||||
return protocol, path
|
||||
return None, uri_or_path
|
||||
|
||||
def exists(self):
|
||||
return self.fs.exists(self.cname)
|
||||
|
||||
def size(self):
|
||||
return self.fs.size(self.cname)
|
||||
|
||||
def lastmodtime(self):
|
||||
""" return datetime object representing last modification time, or None if unavailable """
|
||||
info = self.fs.info(self.cname)
|
||||
if self.islocal() and info is not None:
|
||||
return datetime.fromtimestamp(info["mtime"])
|
||||
else:
|
||||
return getattr(info, "LastModified", None)
|
||||
|
||||
def abspath(self):
|
||||
"""
|
||||
return the absolute path for the locator - only really does something
|
||||
for file: protocol, as all others are already absolute
|
||||
"""
|
||||
if self.islocal():
|
||||
return os.path.abspath(self.path)
|
||||
else:
|
||||
return self.uri_or_path
|
||||
|
||||
def isfile(self):
|
||||
return self.fs.isfile(self.cname)
|
||||
|
||||
def open(self, *args):
|
||||
return self.fs.open(self.uri_or_path, *args)
|
||||
|
||||
def islocal(self):
|
||||
return self.protocol is None or self.protocol == "file"
|
||||
|
||||
def local_handle(self):
|
||||
if self.islocal():
|
||||
return LocalFilePath(self.path)
|
||||
|
||||
# if not local, create a tmp file system object to contain the data,
|
||||
# and clean it up when done. If the path has a suffix/extension,
|
||||
# do our best to create a file with the same.
|
||||
ext = os.path.splitext(self.path)
|
||||
suffix = None if ext[1] == "" else ext[1]
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(src.read())
|
||||
tmp.close()
|
||||
src.close()
|
||||
tmp_path = tmp.name
|
||||
return LocalFilePath(tmp_path, delete=True)
|
||||
|
||||
def ls(self):
|
||||
paths = self.fs.ls(self.uri_or_path)
|
||||
return [os.path.basename(p) for p in paths]
|
||||
|
||||
|
||||
class LocalFilePath:
|
||||
def __init__(self, tmp_path, delete=False):
|
||||
self.tmp_path = tmp_path
|
||||
self.delete = delete
|
||||
|
||||
def __enter__(self):
|
||||
return self.tmp_path
|
||||
|
||||
def __exit__(self, *args):
|
||||
if self.delete:
|
||||
os.unlink(self.tmp_path)
|
||||
|
||||
|
||||
def discover_s3_region_name(uri):
|
||||
"""If this is an s3 protocol, discover and return the (aws) region name.
|
||||
If a return name could not be discovered, or if the uri is not an s3 protocol, return None."""
|
||||
|
||||
protocol, _ = DataLocator._get_protocol_and_path(uri)
|
||||
if protocol == "s3":
|
||||
bucket = urlparse(uri).netloc
|
||||
client = boto3.client("s3")
|
||||
try:
|
||||
res = client.head_bucket(Bucket=bucket)
|
||||
except botocore.exceptions.ClientError:
|
||||
return None
|
||||
|
||||
region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
|
||||
if region:
|
||||
return region
|
||||
else:
|
||||
return None
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,158 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
|
||||
dtypes_by_column_name = {}
|
||||
schema_type_hints_by_column_name = {}
|
||||
|
||||
for column_name, column_values in dataframe.items():
|
||||
(
|
||||
dtypes_by_column_name[column_name],
|
||||
schema_type_hints_by_column_name[column_name],
|
||||
) = get_dtype_and_schema_of_array(column_values)
|
||||
|
||||
return dtypes_by_column_name, schema_type_hints_by_column_name
|
||||
|
||||
|
||||
def get_dtype_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[0]
|
||||
|
||||
|
||||
def get_schema_type_hint_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[1]
|
||||
|
||||
|
||||
def get_dtype_and_schema_of_array(array: pd.Series):
|
||||
return (
|
||||
get_dtype_from_dtype(array.dtype, array_values=array),
|
||||
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
|
||||
)
|
||||
|
||||
|
||||
def get_dtype_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
|
||||
for 64 bit values which will get downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype_name == "bool":
|
||||
return np.uint8
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return str
|
||||
if dtype_name == "category":
|
||||
return get_dtype_from_dtype(dtype.categories.dtype, array_values)
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return np.int32
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return np.float32
|
||||
if not can_cast_to_float32(dtype, array_values):
|
||||
return np.float64
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def get_schema_type_hint_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
|
||||
and will be downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype == np.float32 or dtype == np.int32:
|
||||
return {"type": dtype_name}
|
||||
if dtype_name == "bool":
|
||||
return {"type": "boolean"}
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return {"type": "string"}
|
||||
if dtype_name == "category":
|
||||
return {"type": "categorical", "categories": dtype.categories.tolist()}
|
||||
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return {"type": "int32"}
|
||||
if can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float32"}
|
||||
if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
|
||||
return {"type": "float64"}
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def can_cast_to_float32(dtype, array_values):
|
||||
"""
|
||||
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
|
||||
a float.
|
||||
|
||||
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
|
||||
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
|
||||
"""
|
||||
|
||||
if dtype.kind == "f":
|
||||
if not np.can_cast(dtype, np.float32):
|
||||
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
|
||||
|
||||
return True
|
||||
|
||||
if dtype.kind == "O" and array_values.hasnans:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def can_cast_to_int32(dtype, array_values=None):
|
||||
"""
|
||||
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
|
||||
the higher precision type has values that are entirely within the range of the downcast type.
|
||||
"""
|
||||
|
||||
# Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
|
||||
# return False.
|
||||
if array_values.hasnans:
|
||||
return False
|
||||
|
||||
# If the array is categorical, then we need to order the array values so that functions min and max that occur
|
||||
# later, can function. They do not function on unordered categories.
|
||||
ordered_array_values = array_values
|
||||
if array_values.dtype.name == "category" and not array_values.cat.ordered:
|
||||
ordered_array_values = array_values.cat.as_ordered()
|
||||
|
||||
if dtype.kind in ["i", "u"]:
|
||||
if np.can_cast(dtype, np.int32):
|
||||
return True
|
||||
ii32 = np.iinfo(np.int32)
|
||||
if (
|
||||
not ordered_array_values.empty
|
||||
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
|
||||
or ordered_array_values.empty
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
|
||||
if series_to_convert.hasnans and dtype == np.int32:
|
||||
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
|
||||
|
||||
return series_to_convert.to_numpy(dtype)
|
||||
|
||||
|
||||
def convert_string_to_value(value: str):
|
||||
"""convert a string to value with the most appropriate type"""
|
||||
if value.lower() == "true":
|
||||
return True
|
||||
if value.lower() == "false":
|
||||
return False
|
||||
if value == "null":
|
||||
return None
|
||||
try:
|
||||
return eval(value)
|
||||
except: # noqa E722
|
||||
return value
|
||||
@@ -0,0 +1,118 @@
|
||||
import contextlib
|
||||
import errno
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import pkgutil
|
||||
import socket
|
||||
from urllib.parse import urlsplit, urljoin
|
||||
|
||||
import numpy as np
|
||||
from flask import json
|
||||
|
||||
from backend.common.errors import ConfigurationError
|
||||
|
||||
|
||||
def find_available_port(host, port=5005):
|
||||
"""
|
||||
Helper method to find open port on host. Tries 5000 ports incremented from the specified port
|
||||
"""
|
||||
# Takes approx 2 seconds to do a scan of 5000 ports on my laptop
|
||||
num_ports_to_try = 5000
|
||||
for port_to_try in range(port, port + num_ports_to_try):
|
||||
if is_port_available(host, port_to_try):
|
||||
return port_to_try
|
||||
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
|
||||
|
||||
|
||||
def is_port_available(host, port):
|
||||
is_available = False
|
||||
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
|
||||
try:
|
||||
s.bind((host, port))
|
||||
is_available = True
|
||||
except socket.error:
|
||||
pass
|
||||
return is_available
|
||||
|
||||
|
||||
def sort_options(command):
|
||||
"""
|
||||
Helper for the click options - will sort options in a command, and can
|
||||
be used as a decorator.
|
||||
"""
|
||||
command.params.sort(key=lambda p: p.name)
|
||||
return command
|
||||
|
||||
|
||||
def path_join(base, *urls):
|
||||
"""
|
||||
this is like urllib.parse.urljoin, except it works around the scheme-specific
|
||||
cleverness in the aforementioned code, ignores anything in the url except the path,
|
||||
and accepts more than one url.
|
||||
"""
|
||||
if not base.endswith("/"):
|
||||
base += "/"
|
||||
btpl = urlsplit(base)
|
||||
path = btpl.path
|
||||
for url in urls:
|
||||
utpl = urlsplit(url)
|
||||
if btpl.scheme == "":
|
||||
path = os.path.join(path, utpl.path)
|
||||
path = os.path.normpath(path)
|
||||
else:
|
||||
path = urljoin(path, utpl.path)
|
||||
return btpl._replace(path=path).geturl()
|
||||
|
||||
|
||||
class Float32JSONEncoder(json.JSONEncoder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
NaN/Infinities are illegal in standard JSON. Python extends JSON with
|
||||
non-standard symbols that most JavaScript JSON parsers do not understand.
|
||||
The `allow_nan` parameter will force Python simplejson to throw an ValueError
|
||||
if it runs into non-finite floating point values which are unsupported by
|
||||
standard JSON.
|
||||
"""
|
||||
kwargs["allow_nan"] = False
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def default(self, obj):
|
||||
if isinstance(obj, np.float32):
|
||||
return float(obj)
|
||||
elif isinstance(obj, np.integer):
|
||||
return int(obj)
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
def custom_format_warning(msg, *args, **kwargs):
|
||||
return f"[cellxgene] Warning: {msg} \n"
|
||||
|
||||
|
||||
def jsonify_numpy(data):
|
||||
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
|
||||
|
||||
|
||||
def import_plugins(plugin_module):
|
||||
"""
|
||||
Load optional plugin modules from server.common.plugins
|
||||
|
||||
If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
|
||||
This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
|
||||
normal.
|
||||
"""
|
||||
loaded_modules = []
|
||||
try:
|
||||
pkg = importlib.import_module(plugin_module)
|
||||
for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
|
||||
full_name = f"{plugin_module}.{name}"
|
||||
try:
|
||||
module = importlib.import_module(full_name)
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
|
||||
loaded_modules.append(module)
|
||||
except ModuleNotFoundError as e:
|
||||
# This exception occurs when the plugin_module does not exist (not an error).
|
||||
logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
|
||||
|
||||
return loaded_modules
|
||||
Reference in New Issue
Block a user