fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: Fix compatibility tests (#2685)

Browse Source 
* chore: Fix compatibility tests

* DEBUGGGG

* fix: update deps, fix unit tests

* fix: FE deps

* chore: update compatibility matrix

---------

Co-authored-by: kaloster <rkalo@contractor.chanzuckerberg.com>
This commit is contained in:
Timmy Huang
2024-09-05 09:26:40 -07:00
committed by GitHub
parent 4281a8f816
commit 7bf5add6ef
10 changed files with 21475 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.github/workflows/compatibility_tests.yml vendored
View File

@@ -14,7 +14,7 @@ jobs:
docker-build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
@@ -29,21 +29,21 @@ jobs:
fail-fast: false
matrix:
# note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
os: [ubuntu-latest, macos-latest, macos-11]
python-version: [3.8, 3.9, 3.10, 3.11]
os: [ubuntu-latest, macos-latest, macos-13]
python-version: ["3.10", "3.11"]
cellxgene_build: [main, latest]
# add anndata pinned version test for subset of matrix configurations,
# in order to reduce matrix cross-product explosion
include:
- python-version: 3.9
- python-version: 3.10
cellxgene_build: latest
# TODO: dynamically use the literal version in requirements.txt,
# to avoid having to update this in manually in the future
# TODO: Do not bother running this if anndata latest version
# matches this pinned version, to avoid a redundant test
anndata_version: "==0.10.3"
anndata_version: "==0.10.9"
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
@@ -55,7 +55,7 @@ jobs:
run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
# FIXME: Only working for Linux
- name: Python cache
uses: actions/cache@v1
uses: actions/cache@v2
with:
path: ${{ env.PIP_CACHE }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
@@ -96,6 +96,7 @@ jobs:
# keep same pip pkg versions as in the cxg release
sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
pip install -r server/requirements-dev.txt
pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas
- name: Install anndata version per matrix variable
run: pip install anndata${{ matrix.anndata_version }}
- name: Install node

1
client/.nvmrc Normal file
View File

@@ -0,0 +1 @@
16.20.0

2
client/__tests__/e2e/puppeteer.setup.js
View File

@@ -13,7 +13,7 @@ import * as ENV_DEFAULT from "../../../environment.default.json";
// a test can take more time to finish, so we don't want
// jest to shut off the test too soon
jest.setTimeout(2 * 60 * 1000);
setDefaultOptions({ timeout: 20 * 1000 });
setDefaultOptions({ timeout: 60 * 1000 });
jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);

21756
client/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

7
client/package.json
View File

@@ -18,7 +18,8 @@
},
"engineStrict": true,
"engines": {
"npm": ">=3.0.0"
"npm": ">=3.0.0",
"node": "^16.0.0"
},
"eslintConfig": {
"extends": "./configuration/eslint/eslint.js"
@@ -123,7 +124,7 @@
"jest-circus": "^27.0.6",
"jest-environment-puppeteer": "^5.0.1",
"jest-fetch-mock": "^3.0.3",
"jest-puppeteer": "^5.0.1",
"jest-puppeteer": "^6.2.0",
"json-loader": "^0.5.7",
"lint-staged": "^10.2.11",
"lodash": "^4.17.21",
@@ -134,7 +135,7 @@
"lodash.zip": "^4.2.0",
"mini-css-extract-plugin": "^1.5.0",
"prettier": "^2.0.5",
"puppeteer": "^8.0.0",
"puppeteer": "^10.4.0",
"rimraf": "^3.0.2",
"serve-favicon": "^2.5.0",
"terser-webpack-plugin": "^5.1.1",

3
server/common/utils/type_conversion_utils.py
View File

@@ -116,7 +116,7 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
raise TypeError("Unsupported data type.")
dtype = array.dtype
res = _get_type_info_from_dtype(dtype)
if res is not None:
return res
@@ -140,7 +140,6 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
return (np.int32, {"type": "int32"})
if dtype.kind == "f":
_float64_warning(array.dtype)
return (np.float32, {"type": "float32"})

6
server/requirements.txt
View File

@@ -12,10 +12,10 @@ flatten-dict>=0.2.0
fsspec>0.8.0
gunicorn>=20.0.4
h5py>=3.0.0
numba>=0.51.2
numpy>1.22
numba>=0.60.0
numpy==2.0.1
packaging>=20.0
pandas<2.0.0
pandas>=2.2.2
PyYAML>=5.4 # CVE-2020-14343
requests>=2.22.0
s3fs==0.4.2

20
test/unit/compute/test_est_dist.py
View File

@@ -65,13 +65,13 @@ class EstDistTest(unittest.TestCase):
# non-finites
self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
self.assertEqual(
estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
estimate_approximate_distribution(np.array([np.inf, np.inf, 0])), XApproximateDistribution.NORMAL
)
self.assertEqual(
estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
estimate_approximate_distribution(np.array([np.nan, np.inf, np.inf])), XApproximateDistribution.NORMAL
)
raw = np.random.exponential(scale=1000, size=(50, 3))
@@ -82,15 +82,15 @@ class EstDistTest(unittest.TestCase):
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(raw, [1], [np.PINF])),
estimate_approximate_distribution(put(raw, [1], [np.inf])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(raw, [1], [np.NINF])),
estimate_approximate_distribution(put(raw, [1], [np.inf])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.inf, np.inf])),
XApproximateDistribution.COUNT,
)
self.assertEqual(
@@ -103,15 +103,15 @@ class EstDistTest(unittest.TestCase):
XApproximateDistribution.NORMAL,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [1], [np.PINF])),
estimate_approximate_distribution(put(logged, [1], [np.inf])),
XApproximateDistribution.NORMAL,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [1], [np.NINF])),
estimate_approximate_distribution(put(logged, [1], [np.inf])),
XApproximateDistribution.NORMAL,
)
self.assertEqual(
estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.inf, np.inf])),
XApproximateDistribution.NORMAL,
)
self.assertEqual(

4
test/unit/utils/test_jsonify_strict.py
View File

@@ -16,10 +16,10 @@ class TestJsonifyStrict(unittest.TestCase):
jsonify_strict({"nan": [np.nan]})
with self.assertRaises(ValueError):
jsonify_strict({"pinf": [np.PINF]})
jsonify_strict({"pinf": [np.inf]})
with self.assertRaises(ValueError):
jsonify_strict({"ninf": [np.NINF]})
jsonify_strict({"ninf": [np.inf]})
def test_jsonify_numpy_ndarray(self):
values = {

19
test/unit/utils/test_type_conversion_utils.py
View File

@@ -42,7 +42,7 @@ class TestTypeConversionUtils(unittest.TestCase):
with self.assertRaises(TypeError):
get_schema_type_hint_from_dtype(np.dtype(dtype))
for dtype in [np.float16, np.float32, np.float64]:
for dtype in [np.float32, np.float64]:
self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
for dtype in [np.dtype(object), np.dtype(str)]:
@@ -123,17 +123,18 @@ int_OK_cases = [
float_OK_cases = [
{
"test_case": "float_OK_cases",
"data": data,
"expected_encoding_dtype": np.float32,
"expected_schema_hint": {"type": "float32"},
"logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
"logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
}
for dtype in [np.float16, np.float32, np.float64]
for dtype in [np.float32, np.float64]
for data in [
np.arange(-128, 1000, dtype=dtype),
pd.Series(np.arange(-128, 1000, dtype=dtype)),
pd.Index(np.arange(-129, 1000, dtype=dtype)),
np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
np.array([-np.nan, np.inf, -1, 0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype),
np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
sparse.csr_matrix((10, 100), dtype=dtype),
]
@@ -198,12 +199,13 @@ category_numeric_OK_cases = [
# numeric, no NA/NaN, float
*[
{
"test_case": "numeric, no NA/NaN, float",
"data": data,
"expected_encoding_dtype": np.float32,
"expected_schema_hint": {"type": "categorical"},
"logs": {"level": logging.WARNING, "output": "may lose precision"},
"logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
}
for dtype in [np.float16, np.float32, np.float64]
for dtype in [np.float32, np.float64]
for data in [
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
@@ -213,10 +215,11 @@ category_numeric_OK_cases = [
# numeric, has NA-induced cast to float32
*[
{
"test_case": "numeric, has NA-induced cast to float32",
"data": data,
"expected_encoding_dtype": np.float32,
"expected_schema_hint": {"type": "categorical"},
"logs": {"level": logging.WARNING, "output": "may lose precision"},
"logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
}
for dtype in [
np.int8,
@@ -227,7 +230,6 @@ category_numeric_OK_cases = [
np.uint32,
np.int64,
np.uint64,
np.float16,
np.float32,
np.float64,
]
@@ -312,7 +314,6 @@ class TestTypeInference(unittest.TestCase, AssertNoLog):
self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
self.assertEqual(schema_hint, self.expected_schema_hint)
self.assertIn(logs["output"], logger.output[0])
else:
with self.assertNoLogs(logging.getLogger(), logging.WARNING):
encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)