fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-04 00:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Correctly handle non-finite numbers in heuristic determination of X distribution (#2342)

Browse Source 
* handle non-finites explicitly

* improve and test edge case handling for distribution estimation

* revert debugging changes

* code readability
This commit is contained in:
Bruce Martin
2021-07-28 14:34:29 -07:00
committed by GitHub
parent 0b1ab02a60
commit 1140676106
2 changed files with 116 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/test_server/unit/compute/test_est_dist.py
+79
View File
@@ -25,6 +25,9 @@ class EstDistTest(unittest.TestCase):
def test_estimate_approximate_distribution(self):
raw = np.random.exponential(scale=1000, size=(100, 40))
# empty
self.assertEqual(estimate_approximate_distribution(np.zeros((0,))), XApproximateDistribution.NORMAL)
# ndarray
self.assertEqual(estimate_approximate_distribution(raw), XApproximateDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproximateDistribution.NORMAL)
@@ -35,7 +38,83 @@ class EstDistTest(unittest.TestCase):
estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
)
# csc_matrix
self.assertEqual(estimate_approximate_distribution(sparse.csc_matrix(raw)), XApproximateDistribution.COUNT)
self.assertEqual(
estimate_approximate_distribution(sparse.csc_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
)
# BIG (ie, trigger MT)
big = np.random.exponential(scale=100, size=(1_000_000, 100))
self.assertEqual(estimate_approximate_distribution(big), XApproximateDistribution.COUNT)
self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproximateDistribution.NORMAL)
def test_unsupported_throws(self):
# dtypes and matrix formats we do not support
with self.assertRaises(TypeError):
estimate_approximate_distribution(np.array(["a", "b"]))
with self.assertRaises(TypeError):
estimate_approximate_distribution(sparse.coo_matrix(np.array([[0, 1, 2], [3, 0, 2]])))
def test_nonfinites(self):
def put(arr, ind, vals):
# like np.put, but creates and returns a modified copy of original array
a = arr.copy()
np.put(a, ind, vals)
return a
# non-finites
self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
self.assertEqual(
estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
)
self.assertEqual(
estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
)
raw = np.random.exponential(scale=1000, size=(50, 3))
logged = np.log1p(raw)
self.assertEqual(
estimate_approximate_distribution(put(raw, [1], [np.nan])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(raw, [1], [np.PINF])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(raw, [1], [np.NINF])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(raw, [0, 1], [np.nan, np.nan])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [1], [np.nan])),
XApproximateDistribution.NORMAL,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [1], [np.PINF])),
XApproximateDistribution.NORMAL,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [1], [np.NINF])),
XApproximateDistribution.NORMAL,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
XApproximateDistribution.NORMAL,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [0, 1], [np.nan, np.nan])),
XApproximateDistribution.NORMAL,
)