improve handling of non-finite floating point values (#490)

* improve HTTP error reporting

* generate standards-compatible JSON

* add --nan-to-num work-around for non-finite floating point values

* lint

* update tests

* correctly set Infinities to min/max

* REAMDE update for --nan-to-num

* define constant for repetitive warning message

* clarify where NaN errors will occure

server/app/scanpy_engine/scanpy_engine.py

@@ -36,8 +36,8 @@ class ScanpyEngine(CXGDriver):
         self._create_schema()
 
         # TODO: temporary work-arounds
-        self._IEEE754_X_warning_issued = False
-        self._IEEE754_special_values_workaround_annotations()
+        if args['nan_to_num']:
+            self._IEEE754_special_values_workaround()
 
     def _alias_annotation_names(self, axis, name):
         """
@@ -176,7 +176,7 @@ class ScanpyEngine(CXGDriver):
                 f"`cellxgene prepare --layout {self.layout_method} <datafile>` "
                 f"to solve this problem. ")
 
-    def _IEEE754_special_values_workaround_annotations(self):
+    def _IEEE754_special_values_workaround(self):
         """
         TODO: temporary workaround
 
@@ -187,40 +187,55 @@ class ScanpyEngine(CXGDriver):
         This will likely be removed in the future, contingent upon improved marshalling.
 
         Where non-finite floating point is present in obs, var or X:
-        * issue a warning to the user that these values will be treated as zeros.
-        * set the value to zero within the in-memory data (self.data)
+        * issue a warning to the user that these values will be convert to finite numbers.
+        * set NaN to zero, and Infinities to min/max of the element.
         """
+
+        # annotations
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
             for ann in curr_axis:
                 dtype = curr_axis[ann].dtype
                 if dtype.kind == 'f':
-                    not_finite = np.isfinite(curr_axis[ann]) == False   # noqa: E712
-                    if np.count_nonzero(not_finite) > 0:
+                    finite_idx = np.isfinite(curr_axis[ann])
+                    if not finite_idx.all():
+                        curr_axis.loc[np.isnan(curr_axis[ann]), ann] = 0
+                        curr_axis.loc[np.isneginf(curr_axis[ann]), ann] = curr_axis[ann][finite_idx].min()
+                        curr_axis.loc[np.isposinf(curr_axis[ann]), ann] = curr_axis[ann][finite_idx].max()
                         warnings.warn(
                             f"{str(ax).title()} annotation '{ann}' contains floating point NaN or Infinities. "
-                            f"These values will be treated as zero."
+                            f"These will be converted to finite values."
                         )
-                        curr_axis[ann][not_finite] = 0
 
-    def _IEEE754_special_values_workaround_X(self, _X):
-        """
-        TODO: temporary workaround
+        # X
+        non_finite_X_found = False
+        if sparse.issparse(self.data._X):
+            coo = self.data._X.tocoo()
+            finite_idx = np.isfinite(coo.data)
+            if not finite_idx.all():
+                non_finite_X_found = True
+                coo.data[np.isnan(coo.data)] = 0
+                coo.data[np.isneginf(coo.data)] = np.min(coo.data[finite_idx])
+                coo.data[np.isposinf(coo.data)] = np.max(coo.data[finite_idx])
+                coo.eliminate_zeros()
+                _X = coo.asformat(self.data._X.getformat())
+                self.data._X = _X
+        else:
+            _X = self.data._X
+            finite_idx = np.isfinite(_X.flat)
+            if not finite_idx.all():
+                non_finite_X_found = True
+                min_X = _X.flat[finite_idx].min()
+                max_X = _X.flat[finite_idx].max()
+                _X[np.isnan(_X)] = 0
+                _X[np.isneginf(_X)] = min_X
+                _X[np.isposinf(_X)] = max_X
 
-        See comments in _IEEE754_special_values_workaround_annotations
-        """
-        not_finite = np.isfinite(_X) == False   # noqa: E712
-        if np.count_nonzero(not_finite) > 0:
-            _X[not_finite] = 0
-            if not self._IEEE754_X_warning_issued:
-                # only want to issue this warning once.
-                warnings.warn(
-                    "Dataframe X contains floating point NaN or Infinities. "
-                    "These values will be treated as zero."
-                )
-                self._IEEE754_X_warning_issued = True
-
-        return _X
+        if non_finite_X_found:
+            warnings.warn(
+                "Dataframe X contains floating point NaN or Infinities. "
+                "These will be converted to finite values."
+            )
 
     def filter_dataframe(self, filter):
         """
@@ -373,7 +388,6 @@ class ScanpyEngine(CXGDriver):
             _X = _X.toarray()
         var_index_sliced = self.data.var.index[var_selector]
         obs_index_sliced = self.data.obs.index[obs_selector]
-        _X = self._IEEE754_special_values_workaround_X(_X)
         if axis == Axis.OBS:
             result = {
                 "var": var_index_sliced.tolist(),