fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-30 17:08:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve handling of non-finite floating point values (#490)

Browse Source 
* improve HTTP error reporting

* generate standards-compatible JSON

* add --nan-to-num work-around for non-finite floating point values

* lint

* update tests

* correctly set Infinities to min/max

* REAMDE update for --nan-to-num

* define constant for repetitive warning message

* clarify where NaN errors will occure
This commit is contained in:
Bruce Martin
2018-12-04 14:56:16 -08:00
committed by GitHub
parent 296ed752fa
commit 3bfeadc2b9
8 changed files with 138 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/rest_api/rest.py
+54 -7
View File
@@ -1,5 +1,6 @@
from http import HTTPStatus
import pkg_resources
import warnings
from flask import (
Blueprint, current_app, jsonify, make_response, request
@@ -7,7 +8,7 @@ from flask import (
from flask_restful_swagger_2 import Api, swagger, Resource
from werkzeug.datastructures import ImmutableMultiDict
from server.app.util.constants import Axis, DiffExpMode
from server.app.util.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.app.util.filter import parse_filter, QueryStringError
from server.app.util.models import FilterModel
from server.app.util.utils import get_mime_type
@@ -160,7 +161,12 @@ class AnnotationsObsAPI(Resource):
annotation_response = current_app.data.annotation({}, "obs", fields)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
return make_response(jsonify(annotation_response), HTTPStatus.OK)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Fetch annotations (metadata) for filtered subset of observations.",
@@ -211,7 +217,12 @@ class AnnotationsObsAPI(Resource):
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
return make_response(jsonify(annotation_response), HTTPStatus.OK)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class AnnotationsVarAPI(Resource):
@@ -253,7 +264,12 @@ class AnnotationsVarAPI(Resource):
annotation_response = current_app.data.annotation({}, "var", fields)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
return make_response(jsonify(annotation_response), HTTPStatus.OK)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Fetch annotations (metadata) for filtered subset of variables.",
@@ -302,7 +318,12 @@ class AnnotationsVarAPI(Resource):
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except FilterError:
return make_response("Malformed filter", HTTPStatus.BAD_REQUEST)
return make_response(jsonify(annotation_response), HTTPStatus.OK)
try:
return make_response(jsonify(annotation_response), HTTPStatus.OK)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataObsAPI(Resource):
@@ -363,6 +384,10 @@ class DataObsAPI(Resource):
return make_response((jsonify(current_app.data.data_frame(filter_, axis=Axis.OBS))), HTTPStatus.OK)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
@@ -407,6 +432,10 @@ class DataObsAPI(Resource):
HTTPStatus.OK)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataVarAPI(Resource):
@@ -465,6 +494,10 @@ class DataVarAPI(Resource):
return make_response((jsonify(current_app.data.data_frame(filter_, axis=Axis.VAR))), HTTPStatus.OK)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
@swagger.doc({
"summary": "Get data (expression values) from the dataframe.",
@@ -510,6 +543,10 @@ class DataVarAPI(Resource):
HTTPStatus.OK)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DiffExpObsAPI(Resource):
@@ -614,7 +651,12 @@ class DiffExpObsAPI(Resource):
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError:
return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
return make_response(jsonify(diffexp), HTTPStatus.OK)
try:
return make_response(jsonify(diffexp), HTTPStatus.OK)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class LayoutObsAPI(Resource):
@@ -647,7 +689,12 @@ class LayoutObsAPI(Resource):
layout = current_app.data.layout({})
except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
return make_response((jsonify({"layout": layout})), HTTPStatus.OK)
try:
return make_response((jsonify({"layout": layout})), HTTPStatus.OK)
except ValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
# @swagger.doc({
# "summary": "Observation layout for filtered subset.",
server/app/scanpy_engine/scanpy_engine.py
+41 -27
View File
@@ -36,8 +36,8 @@ class ScanpyEngine(CXGDriver):
self._create_schema()
# TODO: temporary work-arounds
self._IEEE754_X_warning_issued = False
self._IEEE754_special_values_workaround_annotations()
if args['nan_to_num']:
self._IEEE754_special_values_workaround()
def _alias_annotation_names(self, axis, name):
"""
@@ -176,7 +176,7 @@ class ScanpyEngine(CXGDriver):
f"`cellxgene prepare --layout {self.layout_method} <datafile>` "
f"to solve this problem. ")
def _IEEE754_special_values_workaround_annotations(self):
def _IEEE754_special_values_workaround(self):
"""
TODO: temporary workaround
@@ -187,40 +187,55 @@ class ScanpyEngine(CXGDriver):
This will likely be removed in the future, contingent upon improved marshalling.
Where non-finite floating point is present in obs, var or X:
* issue a warning to the user that these values will be treated as zeros.
* set the value to zero within the in-memory data (self.data)
* issue a warning to the user that these values will be convert to finite numbers.
* set NaN to zero, and Infinities to min/max of the element.
"""
# annotations
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
dtype = curr_axis[ann].dtype
if dtype.kind == 'f':
not_finite = np.isfinite(curr_axis[ann]) == False # noqa: E712
if np.count_nonzero(not_finite) > 0:
finite_idx = np.isfinite(curr_axis[ann])
if not finite_idx.all():
curr_axis.loc[np.isnan(curr_axis[ann]), ann] = 0
curr_axis.loc[np.isneginf(curr_axis[ann]), ann] = curr_axis[ann][finite_idx].min()
curr_axis.loc[np.isposinf(curr_axis[ann]), ann] = curr_axis[ann][finite_idx].max()
warnings.warn(
f"{str(ax).title()} annotation '{ann}' contains floating point NaN or Infinities. "
f"These values will be treated as zero."
f"These will be converted to finite values."
)
curr_axis[ann][not_finite] = 0
def _IEEE754_special_values_workaround_X(self, _X):
"""
TODO: temporary workaround
# X
non_finite_X_found = False
if sparse.issparse(self.data._X):
coo = self.data._X.tocoo()
finite_idx = np.isfinite(coo.data)
if not finite_idx.all():
non_finite_X_found = True
coo.data[np.isnan(coo.data)] = 0
coo.data[np.isneginf(coo.data)] = np.min(coo.data[finite_idx])
coo.data[np.isposinf(coo.data)] = np.max(coo.data[finite_idx])
coo.eliminate_zeros()
_X = coo.asformat(self.data._X.getformat())
self.data._X = _X
else:
_X = self.data._X
finite_idx = np.isfinite(_X.flat)
if not finite_idx.all():
non_finite_X_found = True
min_X = _X.flat[finite_idx].min()
max_X = _X.flat[finite_idx].max()
_X[np.isnan(_X)] = 0
_X[np.isneginf(_X)] = min_X
_X[np.isposinf(_X)] = max_X
See comments in _IEEE754_special_values_workaround_annotations
"""
not_finite = np.isfinite(_X) == False # noqa: E712
if np.count_nonzero(not_finite) > 0:
_X[not_finite] = 0
if not self._IEEE754_X_warning_issued:
# only want to issue this warning once.
warnings.warn(
"Dataframe X contains floating point NaN or Infinities. "
"These values will be treated as zero."
)
self._IEEE754_X_warning_issued = True
return _X
if non_finite_X_found:
warnings.warn(
"Dataframe X contains floating point NaN or Infinities. "
"These will be converted to finite values."
)
def filter_dataframe(self, filter):
"""
@@ -373,7 +388,6 @@ class ScanpyEngine(CXGDriver):
_X = _X.toarray()
var_index_sliced = self.data.var.index[var_selector]
obs_index_sliced = self.data.obs.index[obs_selector]
_X = self._IEEE754_special_values_workaround_X(_X)
if axis == Axis.OBS:
result = {
"var": var_index_sliced.tolist(),
server/app/util/constants.py
+3
View File
@@ -25,3 +25,6 @@ class Axis(AugmentedEnum):
class DiffExpMode(AugmentedEnum):
TOP_N = "topN"
VAR_FILTER = "varFilter"
JSON_NaN_to_num_warning_msg = "JSON encoding failure - suggest trying --nan-to-num command line option"
server/app/util/utils.py
+11
View File
@@ -7,6 +7,17 @@ from server.app.util.errors import MimeTypeError
class Float32JSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs):
"""
NaN/Infinities are illegal in standard JSON. Python extends JSON with
non-standard symbols that most JavaScript JSON parsers do not understand.
The `allow_nan` parameter will force Python simplejson to throw an ValueError
if it runs into non-finite floating point values which are unsupported by
standard JSON.
"""
kwargs['allow_nan'] = False
super().__init__(*args, **kwargs)
def default(self, obj):
if isinstance(obj, float32):
return float(obj)
server/cli/launch.py
+6 -2
View File
@@ -29,8 +29,11 @@ from server.app.util.errors import ScanpyFileError
help="Limits the number of categorical annotation items displayed.")
@click.option("--diffexp-lfc-cutoff", default=0.01, show_default=True,
help="Relative expression cutoff used when selecting top N differentially expressed genes")
@click.option("--nan-to-num", is_flag=True, default=False, show_default=True,
help="Replace all floating point NaN with zero, and infinities with finite numbers")
def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
open_browser, port, host, max_category_items, diffexp_lfc_cutoff):
open_browser, port, host, max_category_items, diffexp_lfc_cutoff,
nan_to_num):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects, read the
@@ -91,7 +94,8 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
"max_category_items": max_category_items,
"diffexp_lfc_cutoff": diffexp_lfc_cutoff,
"obs_names": obs_names,
"var_names": var_names
"var_names": var_names,
"nan_to_num": nan_to_num
}
try:
server/test/test_scanpy_engine.py
+2 -1
View File
@@ -14,7 +14,8 @@ from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
class UtilTest(unittest.TestCase):
def setUp(self):
args = {'layout': 'umap', 'diffexp': 'ttest', 'max_category_items': 100,
'obs_names': None, 'var_names': None, 'diffexp_lfc_cutoff': 0.01}
'obs_names': None, 'var_names': None, 'diffexp_lfc_cutoff': 0.01,
'nan_to_num': True}
self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args)
self.data._create_schema()