diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3ec24118..27c8ce92 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,17 +1,17 @@ [bumpversion] -current_version = 0.16.0 +current_version = 0.16.7 commit = True # The below regex details an acceptable version number by naming the groups (major, minor, patch, prerel, and # prerelversion) and also specifying the valid values for each group (integers, `\d+`, for major, minor, patch, and # prerelversion and only `rc` as the acceptable value for prerel). parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?:-(?Prc)\.(?P\d+))? -serialize = +serialize = {major}.{minor}.{patch}-{prerel}.{prerelversion} {major}.{minor}.{patch} [bumpversion:part:prerel] optional_value = release -values = +values = rc release diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 70fcf947..0e21c56b 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -36,7 +36,7 @@ jobs: npm install - name: Format with black and lint with flake8 run: | - make lint-server + make lint-servers - name: Lint src with eslint working-directory: ./client run: | @@ -72,6 +72,36 @@ jobs: bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest + unit-test-local: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Python cache + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Node cache + uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: Install dependencies + run: make pydist-local install-dist dev-env-local-server + - name: Unit tests + run: | + make unit-test-local + bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k local_server -cF backend,python,unitTest + cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest + smoke-tests: runs-on: macos-latest timeout-minutes: 20 @@ -126,7 +156,7 @@ jobs: restore-keys: | ${{ runner.os }}-node- - name: Install dependencies - run: make pydist install-dist + run: make pydist-local install-dist - name: Smoke tests (with annotations feature) run: | cd client && make smoke-test-annotations diff --git a/MANIFEST.in b/MANIFEST.in index 3a3b5b2b..4dbe4057 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ -recursive-include server/common/web/templates * -recursive-include server/common/web/static * +recursive-include local_server/common/web/templates * +recursive-include local_server/common/web/static * -include server/requirements.txt -include server/requirements-prepare.txt -include server/converters/schema/hgnc_complete_set.txt.gz -include server/converters/schema/schema_definitions/* +include local_server/requirements.txt +include local_server/requirements-prepare.txt +include local_server/converters/schema/hgnc_complete_set.txt.gz +include local_server/converters/schema/schema_definitions/* diff --git a/MANIFEST_hosted.in b/MANIFEST_hosted.in new file mode 100644 index 00000000..3a3b5b2b --- /dev/null +++ b/MANIFEST_hosted.in @@ -0,0 +1,7 @@ +recursive-include server/common/web/templates * +recursive-include server/common/web/static * + +include server/requirements.txt +include server/requirements-prepare.txt +include server/converters/schema/hgnc_complete_set.txt.gz +include server/converters/schema/schema_definitions/* diff --git a/Makefile b/Makefile index 8f9a4083..1344ea4a 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,14 @@ include common.mk BUILDDIR := build CLIENTBUILD := $(BUILDDIR)/client SERVERBUILD := $(BUILDDIR)/server +LOCALSERVERBUILD := $(BUILDDIR)/local_server CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info PART ?= patch # CLEANING .PHONY: clean -clean: clean-lite clean-server clean-client +clean: clean-lite clean-local-server clean-server clean-client # cleaning the client's node_modules is the longest one, so we avoid that if possible .PHONY: clean-lite @@ -17,7 +18,7 @@ clean-lite: rm -rf $(CLEANFILES) clean-%: - cd $(*) && $(MAKE) clean + cd $(subst -,_,$*) && $(MAKE) clean # BUILDING PACKAGE @@ -26,18 +27,35 @@ clean-%: build-client: cd client && $(MAKE) ci build +.PHONY: build-local +build-local: clean build-client + git ls-files local_server/ | grep -v 'local_server/test/' | cpio -pdm $(BUILDDIR) + cp -r client/build/ $(CLIENTBUILD) + $(call copy_client_assets,$(CLIENTBUILD),$(LOCALSERVERBUILD)) + cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) + .PHONY: build build: clean build-client git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR) cp -r client/build/ $(CLIENTBUILD) $(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD)) - cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) + cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR) + mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py + mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in # If you are actively developing in the server folder use this, dirties the source tree +.PHONY: build-for-server-dev-local +build-for-server-dev-local: clean-local-server build-client + $(call copy_client_assets,client/build,local_server) + .PHONY: build-for-server-dev build-for-server-dev: clean-server build-client $(call copy_client_assets,client/build,server) +.PHONY: copy-client-assets-local +copy-client-assets-local: + $(call copy_client_assets,client/build,local_server) + .PHONY: copy-client-assets copy-client-assets: $(call copy_client_assets,client/build,server) @@ -46,11 +64,17 @@ copy-client-assets: .PHONY: test test: unit-test smoke-test +.PHONY: test-local +test-local: unit-test-local smoke-test + +.PHONY: unit-test-local +unit-test-local: unit-test-local-server unit-test-client + .PHONY: unit-test unit-test: unit-test-server unit-test-client unit-test-%: - cd $(*) && $(MAKE) unit-test + cd $(subst -,_,$*) && $(MAKE) unit-test .PHONY: smoke-test smoke-test: @@ -64,10 +88,9 @@ smoke-test-annotations: test-db: cd server && $(MAKE) test-db - # FORMATTING CODE -.PHOHY: fmt +.PHONY: fmt fmt: fmt-client fmt-py .PHONY: fmt-client @@ -79,19 +102,30 @@ fmt-py: black . .PHONY: lint -lint: lint-server lint-client +lint: lint-servers lint-client + +.PHONY: lint-servers +lint-servers: lint-local-server lint-server + +.PHONY: lint-local-server +lint-local-server: fmt-py + flake8 local_server --per-file-ignores='local_server/test/fixtures/dataset_config_outline.py:F821 local_server/test/fixtures/server_config_outline.py:F821 local_server/test/performance/scale_test_annotations.py:E501' .PHONY: lint-server lint-server: fmt-py flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501' - .PHONY: lint-client lint-client: cd client && $(MAKE) lint # CREATING DISTRIBUTION RELEASE +.PHONY: pydist-local +pydist-local: build-local + cd $(BUILDDIR); python setup.py sdist -d ../dist + @echo "done" + .PHONY: pydist pydist: build cd $(BUILDDIR); python setup.py sdist -d ../dist @@ -137,16 +171,19 @@ release-directly-to-prod: dev-env pydist twine-prod @echo " make install-release" .PHONY: dev-env -dev-env: dev-env-client dev-env-server +dev-env: dev-env-client dev-env-local-server .PHONY: dev-env-client dev-env-client: cd client && $(MAKE) ci +.PHONY: dev-env-local-server +dev-env-local-server: + pip install -r local_server/requirements-dev.txt + .PHONY: dev-env-server dev-env-server: pip install -r server/requirements-dev.txt - # Set PART=[major, minor, patch] as param to make bump. # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump) .PHONY: bump-version diff --git a/README.md b/README.md index 669d2675..cec1258d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,16 @@ This project adheres to the Contributor Covenant [code of conduct](https://githu ### Reuse -This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT). +This project was started with the sole goal of empowering the scientific community to explore and understand their data. +As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from +this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT). + + +Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an +extension could be directly contributed, which would make it available for a wider audience, or that it's on our +[roadmap](./docs/posts/roadmap.md) and under active development. + +See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. ### Security diff --git a/client/Makefile b/client/Makefile index 1a8d3a81..ad1ed798 100644 --- a/client/Makefile +++ b/client/Makefile @@ -1,6 +1,6 @@ include ../common.mk -ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv) +ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../local_server/test/fixtures/pbmc3k-annotations.csv) ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS)) CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml) diff --git a/client/__tests__/e2e/test_config.yaml b/client/__tests__/e2e/test_config.yaml index 11576878..0c37dafb 100644 --- a/client/__tests__/e2e/test_config.yaml +++ b/client/__tests__/e2e/test_config.yaml @@ -1,20 +1,7 @@ server: app: force_https: true - - # By default, cellxgene will serve api requests from the same base url as the webpage. - # In general api_base_url and web_base_url will not need to be set. - # There are two reasons to set these parameters: - # 1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login, - # which then redirects back to the web_base_url. If the web_base_url is not set, it will default to - # the api_base_url. If oauth authentication is used, the api_base_url must be set. - # For a local test (where the server runs on "http://localhost:"), then the api_base_url may be - # set to the string "local". - # 2. The cellxgene deploymnent is in an environment where the webpage and api have - # different base urls. In this case both api_base_url and web_base_url must be set. - # It is up to the server admin to ensure that the networking is setup correctly for this environment. - api_base_url: http://localhost:5005 - web_base_url: http://localhost:3000 + port: 5005 authentication: # The authentication types may be "none", "session", "oauth" @@ -22,12 +9,9 @@ server: # session: A session based userid is automatically generated. (no params needed) # oauth: oauth2 is used for authentication; parameters are defined in params_oauth. type: test + insecure_test_environment: true dataset: - app: - about_legal_tos: null - about_legal_privacy: null - presentation: max_categories: 1000 custom_colors: true diff --git a/client/__tests__/reducers/genesets.test.js b/client/__tests__/reducers/genesets.test.js new file mode 100644 index 00000000..7df76c6c --- /dev/null +++ b/client/__tests__/reducers/genesets.test.js @@ -0,0 +1,524 @@ +import genesetsReducer from "../../src/reducers/genesets"; + +describe("initial reducer state", () => { + test("some other action", () => { + expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({ + initialized: false, + lastTid: undefined, + genesets: new Map(), + }); + }); +}); + +describe("geneset: initial load", () => { + test("missing JSON response", () => { + expect(() => + genesetsReducer(undefined, { + type: "geneset: initial load", + }) + ).toThrow("missing or malformed JSON response"); + }); + + test("empty geneset", () => { + expect( + genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 0, + genesets: [], + }, + }) + ).toMatchObject({ + initialized: true, + lastTid: 0, + genesets: new Map(), + }); + }); + + test("non-empty geneset", () => { + expect( + genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 99, + genesets: [ + { + geneset_name: "G1", + genes: [{ gene_symbol: "F5" }], + }, + { + geneset_name: "G2", + geneset_description: "G2 desc", + genes: [{ gene_symbol: "F6" }], + }, + { + geneset_name: "G3", + geneset_description: "G3 desc", + genes: [{ gene_symbol: "F7", gene_description: "gene desc" }], + }, + ], + }, + }) + ).toMatchObject({ + initialized: true, + lastTid: 99, + genesets: new Map([ + [ + "G1", + { + genesetName: "G1", + genesetDescription: "", + genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]), + }, + ], + [ + "G2", + { + genesetName: "G2", + genesetDescription: "G2 desc", + genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]), + }, + ], + [ + "G3", + { + genesetName: "G3", + genesetDescription: "G3 desc", + genes: new Map([ + ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }], + ]), + }, + ], + ]), + }); + }); +}); + +describe("geneset: create", () => { + const initialState = genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 0, + genesets: [], + }, + }); + + test("simple create", () => { + expect( + genesetsReducer(initialState, { + type: "geneset: create", + genesetName: "a geneset", + genesetDescription: "", + }) + ).toMatchObject({ + ...initialState, + genesets: new Map([ + [ + "a geneset", + { + genesetName: "a geneset", + genesetDescription: "", + genes: new Map(), + }, + ], + ]), + }); + }); + + test("error - duplicate name", () => { + expect(() => { + genesetsReducer( + genesetsReducer(initialState, { + type: "geneset: create", + genesetName: "foo", + genesetDescription: "foo", + }), + { + type: "geneset: create", + genesetName: "foo", + genesetDescription: "bar", + } + ); + }).toThrow("name already defined"); + }); + + test("error - missing required action values", () => { + expect(() => { + genesetsReducer(initialState, { + type: "geneset: create", + genesetDescription: "foo", + }); + }).toThrow(); + expect(() => { + genesetsReducer(initialState, { + type: "geneset: create", + genesetName: "foo", + }); + }).toThrow("name or description unspecified"); + }); +}); + +describe("geneset: delete", () => { + const initialState = genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 0, + genesets: [], + }, + }); + + test("simple delete", () => { + expect( + genesetsReducer( + genesetsReducer(initialState, { + type: "geneset: create", + genesetName: "foo", + genesetDescription: "foo", + }), + { + type: "geneset: delete", + genesetName: "foo", + } + ) + ).toMatchObject({ + initialized: true, + lastTid: 0, + genesets: new Map(), + }); + }); + + test("error - missing name", () => { + expect(() => { + genesetsReducer(initialState, { + type: "geneset: delete", + genesetName: "foo", + }); + }).toThrow("name does not exist"); + }); +}); + +describe("geneset: update", () => { + const initialState = genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 0, + genesets: [], + }, + }); + + test("simple update", () => { + expect( + genesetsReducer( + genesetsReducer( + genesetsReducer(initialState, { + type: "geneset: create", + genesetName: "foo1", + genesetDescription: "foo1", + }), + { + type: "geneset: create", + genesetName: "foo2", + genesetDescription: "foo2", + } + ), + { + type: "geneset: update", + genesetName: "foo1", + update: { + genesetName: "bar", + genesetDescription: "bar", + }, + } + ) + ).toMatchObject({ + initialized: true, + lastTid: 0, + genesets: new Map([ + [ + "bar", + { genesetName: "bar", genesetDescription: "bar", genes: new Map() }, + ], + [ + "foo2", + { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() }, + ], + ]), + }); + }); + + test("error - unknown name", () => { + expect(() => { + genesetsReducer(initialState, { + type: "geneset: update", + genesetName: "foo", + update: { + genesetName: "foo", + genesetDescription: "bar", + }, + }); + }).toThrow("name unspecified or does not exist"); + }); + + test("error - duplicate name", () => { + expect(() => { + genesetsReducer( + genesetsReducer(initialState, { + type: "geneset: create", + genesetName: "foo", + genesetDescription: "foo", + }), + { + type: "geneset: update", + genesetName: "foo", + update: { + genesetName: "foo", + genesetDescription: "foo", + }, + } + ); + }).toThrow("update specified existing name"); + }); +}); + +describe("geneset: add genes", () => { + const initialState = genesetsReducer( + genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 0, + genesets: [], + }, + }), + { + type: "geneset: create", + genesetName: "test", + genesetDescription: "", + } + ); + + test("add a gene", () => { + expect( + genesetsReducer(initialState, { + type: "geneset: add genes", + genesetName: "test", + genes: [{ geneSymbol: "F5" }], + }) + ).toMatchObject({ + ...initialState, + genesets: new Map([ + [ + "test", + { + genesetName: "test", + genesetDescription: "", + genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]), + }, + ], + ]), + }); + + expect( + genesetsReducer(initialState, { + type: "geneset: add genes", + genesetName: "test", + genes: [ + { geneSymbol: "F5", geneDescription: "desc" }, + { geneSymbol: "SET1", geneDescription: "" }, + ], + }) + ).toMatchObject({ + ...initialState, + genesets: new Map([ + [ + "test", + { + genesetName: "test", + genesetDescription: "", + genes: new Map([ + ["F5", { geneSymbol: "F5", geneDescription: "desc" }], + ["SET1", { geneSymbol: "SET1", geneDescription: "" }], + ]), + }, + ], + ]), + }); + }); + + test("no such geneset error", () => { + expect(() => { + genesetsReducer(initialState, { + type: "geneset: add genes", + genesetName: "mumble", + genes: [], + }); + }).toThrow("geneset name does not exist"); + }); +}); + +describe("geneset: delete genes", () => { + const initialState = genesetsReducer( + genesetsReducer( + genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 0, + genesets: [], + }, + }), + { + type: "geneset: create", + genesetName: "test", + genesetDescription: "", + } + ), + { + type: "geneset: add genes", + genesetName: "test", + genes: [{ geneSymbol: "F5" }], + } + ); + + test("simple", () => { + expect( + genesetsReducer(initialState, { + type: "geneset: delete genes", + genesetName: "test", + geneSymbols: ["F5"], + }) + ).toMatchObject({ + ...initialState, + genesets: new Map([ + [ + "test", + { + genesetName: "test", + genesetDescription: "", + genes: new Map(), + }, + ], + ]), + }); + }); + + test("no such geneset error", () => { + expect(() => { + genesetsReducer(initialState, { + type: "geneset: delete genes", + genesetName: "mumble", + geneSymbols: [], + }); + }).toThrow("name does not exist"); + }); +}); + +describe("geneset: set gene description", () => { + const initialState = genesetsReducer( + genesetsReducer( + genesetsReducer(undefined, { + type: "geneset: initial load", + data: { + tid: 0, + genesets: [], + }, + }), + { + type: "geneset: create", + genesetName: "test", + genesetDescription: "", + } + ), + { + type: "geneset: add genes", + genesetName: "test", + genes: [{ geneSymbol: "F5" }], + } + ); + + test("simple set", () => { + expect( + genesetsReducer(initialState, { + type: "geneset: set gene description", + genesetName: "test", + update: { + geneSymbol: "F5", + geneDescription: "mumble", + }, + }) + ).toMatchObject({ + ...initialState, + genesets: new Map([ + [ + "test", + { + genesetName: "test", + genesetDescription: "", + genes: new Map([ + ["F5", { geneSymbol: "F5", geneDescription: "mumble" }], + ]), + }, + ], + ]), + }); + }); + + test("no such geneset error", () => { + expect(() => { + genesetsReducer(initialState, { + type: "geneset: set gene description", + genesetName: "does not exist", + update: { + geneSymbol: "F5", + geneDescription: "mumble", + }, + }); + }).toThrow("geneset name does not exist"); + }); + + test("no such gene error", () => { + expect(() => { + genesetsReducer(initialState, { + type: "geneset: set gene description", + genesetName: "test", + update: { + geneSymbol: "NO SUCH GENE", + geneDescription: "mumble", + }, + }); + }).toThrow("no such gene"); + }); +}); + +describe("geneset: set tid", () => { + test("simple set", () => { + expect( + genesetsReducer(undefined, { + type: "geneset: set tid", + tid: 1, + }) + ).toMatchObject({ lastTid: 1 }); + }); + + test("not a number error", () => { + expect(() => { + genesetsReducer( + { lastTid: 1 }, + { + type: "geneset: set tid", + tid: "0", + } + ); + }).toThrow("must be a positive integer"); + }); + + test("decrement error", () => { + expect(() => { + genesetsReducer( + { lastTid: 1 }, + { + type: "geneset: set tid", + tid: 0, + } + ); + }).toThrow("may not be decremented"); + }); +}); diff --git a/client/__tests__/util/stateManager/colorHelpers.test.js b/client/__tests__/util/stateManager/colorHelpers.test.js new file mode 100644 index 00000000..5ac9973c --- /dev/null +++ b/client/__tests__/util/stateManager/colorHelpers.test.js @@ -0,0 +1,198 @@ +/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */ + +/* +test color helpers +*/ +import { + createColorTable, + loadUserColorConfig, +} from "../../../src/util/stateManager/colorHelpers"; +import * as Dataframe from "../../../src/util/dataframe"; + +describe("categorical color helpers", () => { + /* + Primary test constraint for categorical colors is that they are ordered/identified + by schema order, NOT by value. Ie, + + scale(schemaIndex) should match rgb[obsOffset] + */ + + const schema = indexSchema({ + annotations: { + obs: { + columns: [ + { + name: "name_0", + type: "string", + writable: false, + }, + { + name: "continuousColumn", + type: "float32", + writable: false, + }, + { + categories: [ + "CD4 T cells", + "CD14+ Monocytes", + "B cells", + "CD8 T cells", + "NK cells", + "FCGR3A+ Monocytes", + "Dendritic cells", + "Megakaryocytes", + ], + name: "categoricalColumn", + type: "categorical", + writable: false, + }, + ], + index: "name_0", + }, + var: { + columns: [ + { + name: "name_0", + type: "string", + writable: false, + }, + ], + index: "name_0", + }, + }, + dataframe: { + nObs: 2638, + nVar: 1838, + type: "float32", + }, + layout: {}, + }); + + const catColCategories = schema.annotations.obs.columns[2].categories; + const obsDataframe = new Dataframe.Dataframe( + [schema.dataframe.nObs, 2], + [ + new Float32Array(schema.dataframe.nObs).map(() => Math.random()), + new Array(schema.dataframe.nObs) + .fill("") + .map( + () => + catColCategories[(Math.random() * catColCategories.length) >>> 0] + ), + ], + null, + new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"]) + ); + + test("default category order", () => { + const ct = createColorTable( + "color by categorical metadata", + "categoricalColumn", + obsDataframe, + schema + ); + expect(ct).toBeDefined(); + const data = obsDataframe.col("categoricalColumn").asArray(); + const cats = schema.annotations.obsByName.categoricalColumn.categories; + for (let i = 0; i < schema.dataframe.nObs; i += 1) { + expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i]))); + } + }); + + test("shuffle category order", () => { + const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema))); + shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories); + const ct = createColorTable( + "color by categorical metadata", + "categoricalColumn", + obsDataframe, + schemaClone + ); + expect(ct).toBeDefined(); + const data = obsDataframe.col("categoricalColumn").asArray(); + const cats = schemaClone.annotations.obsByName.categoricalColumn.categories; + for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) { + expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i]))); + } + }); + + test("user defined color order", () => { + const cats = schema.annotations.obsByName.categoricalColumn.categories; + const shuffleCats = shuffle( + Array.from(schema.annotations.obsByName.categoricalColumn.categories) + ); + const userDefinedColorTable = { + categoricalColumn: shuffleCats.reduce((acc, label) => { + acc[label] = randRGBColor(); + return acc; + }, {}), + }; + + const userColors = loadUserColorConfig(userDefinedColorTable); + expect(userColors).toBeDefined(); + + const ct = createColorTable( + "color by categorical metadata", + "categoricalColumn", + obsDataframe, + schema, + userColors + ); + expect(ct).toBeDefined(); + const data = obsDataframe.col("categoricalColumn").asArray(); + for (let i = 0; i < schema.dataframe.nObs; i += 1) { + expect(makeScale(ct.rgb[i])).toEqual( + ct.scale(cats.indexOf(data[i])).toString() + ); + } + }); +}); + +/* +TODO: +1. mix up category order in schema to make sure it works with varied order +2. user defined colors +*/ + +function indexSchema(schema) { + schema.annotations.obsByName = Object.fromEntries( + schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? [] + ); + schema.annotations.varByName = Object.fromEntries( + schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? [] + ); + schema.layout.obsByName = Object.fromEntries( + schema.layout?.obs?.map((v) => [v.name, v]) ?? [] + ); + schema.layout.varByName = Object.fromEntries( + schema.layout?.var?.map((v) => [v.name, v]) ?? [] + ); + + return schema; +} + +function makeScale(rgb) { + // make a scale string from a rgb float triple + return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${ + (rgb[2] * 256) >>> 0 + })`; +} + +function shuffle(array) { + for (let i = array.length - 1; i > 0; i -= 1) { + const j = (Math.random() * (i + 1)) >>> 0; + [array[i], array[j]] = [array[j], array[i]]; + } + return array; +} + +function randHexColor() { + const hex = ((Math.random() * 255) >>> 0).toString(16); + return `0${hex}`.slice(-2); +} + +function randRGBColor() { + return `#${randHexColor()}${randHexColor()}${randHexColor()}`; +} + +/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */ diff --git a/client/package-lock.json b/client/package-lock.json index 45343a05..3f792d31 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1,6 +1,6 @@ { "name": "cellxgene", - "version": "0.16.0", + "version": "0.16.7", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/client/package.json b/client/package.json index dbeb9327..ea3a1988 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "cellxgene", - "version": "0.16.0", + "version": "0.16.7", "license": "MIT", "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.", "repository": "https://github.com/chanzuckerberg/cellxgene", diff --git a/client/src/actions/annotation.js b/client/src/actions/annotation.js index 1836c983..3d40fb2d 100644 --- a/client/src/actions/annotation.js +++ b/client/src/actions/annotation.js @@ -390,41 +390,51 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => { export const saveGenesetsAction = () => async (dispatch, getState) => { const state = getState(); - const { config, genesets, annotations } = state; // bail if gene sets not available, or in readonly mode. + const { config } = state; const genesetsAreAvailable = config?.parameters?.["annotations_genesets"] ?? false; const genesetsReadonly = config?.parameters?.["annotations_genesets_readonly"] ?? true; if (!genesetsAreAvailable || genesetsReadonly) { // our non-save was completed! - dispatch({ + return dispatch({ type: "autosave: genesets complete", - lastSavedGenesets: genesets, + lastSavedGenesets: state.genesets, }); } - /* - JSON data structure is an array of arrays, where the first - element is gene set name, remainder are the genes. Eg, + const { lastTid, genesets: lastGenesets } = state.genesets; - { - "genesets": [ - [ "gs1", ["TNFRSF4","SUMO3","BRWD1"]], - [ "gs2", ["DSCR3", "BRWD1", "BACE2", "SIK1", "C21orf33", "ICOSLG", "SUMO3"]] - ] + /* Create the JSON OTA data structure */ + const tid = (lastTid ?? 0) + 1; + const genesets = []; + for (const [name, gs] of lastGenesets) { + const genes = []; + for (const g of gs.genes.values()) { + genes.push({ + gene_symbol: g.geneSymbol, + gene_description: g.geneDescription, + }); } - - Order of gene sets and genes is significant - */ - const gsArr = []; - for (const [gsName, gsGenes] of genesets.genesets) { - gsArr.push([gsName, Array.from(gsGenes)]); + genesets.push({ + geneset_name: name, + geneset_description: gs.genesetDescription, + genes, + }); } + const ota = { + tid, + genesets, + }; + /* Save to server */ try { - const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations; + const { + dataCollectionNameIsReadOnly, + dataCollectionName, + } = state.annotations; const queryString = !dataCollectionNameIsReadOnly && !!dataCollectionName ? `?annotation-collection-name=${encodeURIComponent( @@ -440,26 +450,29 @@ export const saveGenesetsAction = () => async (dispatch, getState) => { Accept: "application/json", "Content-Type": "application/json", }), - body: JSON.stringify({ - genesets: gsArr, - }), + body: JSON.stringify(ota), credentials: "include", } ); - if (res.ok) { - dispatch({ - type: "autosave: genesets complete", - lastSavedGenesets: genesets, - }); - } else { - dispatch({ + if (!res.ok) { + return dispatch({ type: "autosave: genesets error", message: `HTTP error ${res.status} - ${res.statusText}`, res, }); } + return Promise.all([ + dispatch({ + type: "autosave: genesets complete", + lastSavedGenesets: genesets, + }), + dispatch({ + type: "geneset: set tid", + tid, + }), + ]); } catch (error) { - dispatch({ + return dispatch({ type: "autosave: genesets error", message: error.toString(), error, diff --git a/client/src/actions/index.js b/client/src/actions/index.js index e0858de5..17cb7358 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -53,18 +53,22 @@ async function userInfoFetch(dispatch) { } async function genesetsFetch(dispatch, config) { + /* request genesets ONLY if the backend supports the feature */ + const defaultResponse = { + genesets: [], + tid: 0, + }; if (config?.parameters?.["annotations_genesets"] ?? false) { fetchJson("genesets").then((response) => { - const genesets = response?.genesets ?? {}; dispatch({ type: "geneset: initial load", - init: genesets, + data: response ?? defaultResponse, }); }); } else { dispatch({ type: "geneset: initial load", - init: [], + data: defaultResponse, }); } } diff --git a/client/src/components/autosave/index.js b/client/src/components/autosave/index.js index dcf93831..18b5f152 100644 --- a/client/src/components/autosave/index.js +++ b/client/src/components/autosave/index.js @@ -11,7 +11,7 @@ import FilenameDialog from "./filenameDialog"; error: state.autosave?.error, writableCategoriesEnabled: state.config?.parameters?.annotations ?? false, writableGenesetsEnabled: !( - state.config?.parameters?.annotations_genesets_readonly ?? true + state.config?.parameters?.["annotations_genesets_readonly"] ?? true ), annoMatrix: state.annoMatrix, genesets: state.genesets, @@ -27,11 +27,11 @@ class Autosave extends React.Component { } componentDidMount() { - const { writableCategoriesEnabled } = this.props; + const { writableCategoriesEnabled, writableGenesetsEnabled } = this.props; let { timer } = this.state; if (timer) clearInterval(timer); - if (writableCategoriesEnabled) { + if (writableCategoriesEnabled || writableGenesetsEnabled) { timer = setInterval(this.tick, 2500); } else { timer = null; @@ -58,11 +58,6 @@ class Autosave extends React.Component { } }; - saveInProgress() { - const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props; - return obsAnnotationSaveInProgress || genesetSaveInProgress; - } - needToSaveObsAnnotations = () => { /* return true if we need to save obs cell labels, false if we don't */ const { annoMatrix, lastSavedAnnoMatrix } = this.props; @@ -79,6 +74,11 @@ class Autosave extends React.Component { return this.needToSaveGenesets() || this.needToSaveObsAnnotations(); } + saveInProgress() { + const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props; + return obsAnnotationSaveInProgress || genesetSaveInProgress; + } + statusMessage() { const { error } = this.props; if (error) { @@ -88,10 +88,14 @@ class Autosave extends React.Component { } render() { - const { writableCategoriesEnabled, lastSavedAnnoMatrix } = this.props; + const { + writableCategoriesEnabled, + writableGenesetsEnabled, + lastSavedAnnoMatrix, + } = this.props; const initialDataLoadComplete = lastSavedAnnoMatrix; - if (!writableCategoriesEnabled) return null; + if (!writableCategoriesEnabled && !writableGenesetsEnabled) return null; return (
0.5) { discard; @@ -67,5 +67,15 @@ export default function drawPointsRegl(regl) { count: regl.prop("count"), primitive: "points", + + blend: { + enable: true, + func: { + srcRGB: "src alpha", + srcAlpha: 1, + dstRGB: 0, + dstAlpha: "zero", + }, + }, }); } diff --git a/client/src/components/graph/graph.js b/client/src/components/graph/graph.js index 473f8c27..ceaf31c7 100644 --- a/client/src/components/graph/graph.js +++ b/client/src/components/graph/graph.js @@ -22,6 +22,12 @@ import CentroidLabels from "./overlays/centroidLabels"; import actions from "../../actions"; import renderThrottle from "../../util/renderThrottle"; +import { + flagBackground, + flagSelected, + flagHighlight, +} from "../../util/glHelpers"; + /* Simple 2D transforms control all point painting. There are three: * model - convert from underlying per-point coordinate to a layout. @@ -62,10 +68,6 @@ function createModelTF() { return m; } -const flagSelected = 1; -const flagNaN = 2; -const flagHighlight = 4; - @connect((state) => ({ annoMatrix: state.annoMatrix, crossfilter: state.obsCrossfilter, @@ -159,8 +161,9 @@ class Graph extends React.Component { const flags = new Float32Array(nObs); if (colorByData) { for (let i = 0, len = flags.length; i < len; i += 1) { - if (!Number.isFinite(colorByData[i])) { - flags[i] = flagNaN; + const val = colorByData[i]; + if (typeof val === "number" && !Number.isFinite(val)) { + flags[i] = flagBackground; } } } diff --git a/client/src/components/scatterplot/drawPointsRegl.js b/client/src/components/scatterplot/drawPointsRegl.js index 4f70e9ed..38dfdbd2 100644 --- a/client/src/components/scatterplot/drawPointsRegl.js +++ b/client/src/components/scatterplot/drawPointsRegl.js @@ -13,7 +13,7 @@ export default function drawPointsRegl(regl) { uniform float nPoints; uniform float minViewportDimension; - varying vec4 fragColor; + varying lowp vec4 fragColor; const float zBottom = 0.99; const float zMiddle = 0.; @@ -26,22 +26,22 @@ export default function drawPointsRegl(regl) { ${glPointSize} void main() { - bool isNaN, isSelected, isHighlight; - getFlags(flag, isNaN, isSelected, isHighlight); + bool isBackground, isSelected, isHighlight; + getFlags(flag, isBackground, isSelected, isHighlight); gl_PointSize = pointSize(nPoints, minViewportDimension, isSelected, isHighlight); - float z = isNaN ? zBottom : (isHighlight ? zTop : zMiddle); + float z = isBackground ? zBottom : (isHighlight ? zTop : zMiddle); vec3 xy = projection * vec3(position, 1.); gl_Position = vec4(xy.xy, z, 1.); - float alpha = isNaN ? 0.9 : 1.0; + float alpha = isBackground ? 0.9 : 1.0; fragColor = vec4(color, alpha); }`, frag: ` precision mediump float; - varying vec4 fragColor; + varying lowp vec4 fragColor; void main() { if (length(gl_PointCoord.xy - 0.5) > 0.5) { discard; @@ -64,5 +64,15 @@ export default function drawPointsRegl(regl) { count: regl.prop("count"), primitive: "points", + + blend: { + enable: true, + func: { + srcRGB: "src alpha", + srcAlpha: 1, + dstRGB: 0, + dstAlpha: "zero", + }, + }, }); } diff --git a/client/src/components/scatterplot/scatterplot.js b/client/src/components/scatterplot/scatterplot.js index fa9aa6cb..d68de428 100644 --- a/client/src/components/scatterplot/scatterplot.js +++ b/client/src/components/scatterplot/scatterplot.js @@ -16,10 +16,11 @@ import { createColorQuery, } from "../../util/stateManager/colorHelpers"; import renderThrottle from "../../util/renderThrottle"; - -const flagSelected = 1; -const flagNaN = 2; -const flagHighlight = 4; +import { + flagBackground, + flagSelected, + flagHighlight, +} from "../../util/glHelpers"; function createProjectionTF(viewportWidth, viewportHeight) { /* @@ -135,8 +136,9 @@ class Scatterplot extends React.PureComponent { const flags = new Float32Array(nObs); if (colorByData) { for (let i = 0, len = flags.length; i < len; i += 1) { - if (!Number.isFinite(colorByData[i])) { - flags[i] = flagNaN; + const val = colorByData[i]; + if (typeof val === "number" && !Number.isFinite(val)) { + flags[i] = flagBackground; } } } diff --git a/client/src/reducers/genesets.js b/client/src/reducers/genesets.js index bd651962..56458330 100644 --- a/client/src/reducers/genesets.js +++ b/client/src/reducers/genesets.js @@ -2,14 +2,30 @@ * Gene set state. Geneset UI state is in a different reducer. * * geneset reducer state is a Map object, where: - * key: the gene set name, a string. - * val: the genes in the gene set, which is a Set of string + * key: the geneset name, a string. + * val: the geneset defined as an object ("geneset object") * - * Ie, Map> + * A geneset object is: + * { + * genesetName: # same as the map key + * genesetDescription: + * genes: Map<, { + * geneSymbol: , # same as the map key + * geneDescription: + * }> + * } + * + * Geneset and genes Map order is significant, and will be preserved across + * CRUD operations on either. + * + * This reducer does light error checking, but not as much as the backend + * routes. Do not rely on it to enforce geneset integrity - eg, no duplicate + * genes in a geneset. */ const GeneSets = ( state = { initialized: false, + lastTid: undefined, genesets: new Map(), }, action @@ -19,39 +35,73 @@ const GeneSets = ( * Initial, load-time bootstrap. * { * type: "geneset: initial load" - * init: Array>> + * data: JSON response * } */ case "geneset: initial load": { - const { init } = action; + const { data } = action; + + if ( + !data || + typeof data.tid !== "number" || + !Array.isArray(data.genesets) + ) + throw new Error("missing or malformed JSON response"); + + const lastTid = data.tid; + const genesetsData = data.genesets; const genesets = new Map(); - for (const gs of init) { - const genes = new Set(); - for (const gene of gs[1]) { - genes.add(gene); + + for (const gsData of genesetsData) { + const genes = new Map(); + for (const gene of gsData.genes) { + genes.set(gene.gene_symbol, { + geneSymbol: gene.gene_symbol, + geneDescription: gene?.["gene_description"] ?? "", + }); } - genesets.set(gs[0], genes); + const gs = { + genesetName: gsData.geneset_name, + genesetDescription: gsData?.["geneset_description"] ?? "", + genes, + }; + genesets.set(gsData.geneset_name, gs); } + return { - genesets, initialized: true, + lastTid, + genesets, }; } /** + * Creates a new & empty geneset with the given name and description. * { * type: "geneset: create", - * name: string, // gene set name - * genes: Set || Array || undefined + * genesetName: string, // gene set name + * genesetDescription: string, // geneset description * } + * */ case "geneset: create": { - const { name, genes } = action; - if (state.genesets.has(name)) + const { genesetName, genesetDescription } = action; + if ( + typeof genesetName !== "string" || + !genesetName || + genesetDescription === undefined + ) + throw new Error("geneset: create -- name or description unspecified."); + if (state.genesets.has(genesetName)) throw new Error("geneset: create -- name already defined."); const genesets = new Map(state.genesets); // clone - genesets.set(name, new Set(genes || [])); + genesets.set(genesetName, { + genesetName, + genesetDescription, + genes: new Map(), + }); + return { ...state, genesets, @@ -59,18 +109,19 @@ const GeneSets = ( } /** + * Deletes the named geneset, if it exists. Throws if it does not. * { * type: "geneset: delete", - * name: string + * genesetName: string * } */ case "geneset: delete": { - const { name } = action; - if (!state.genesets.has(name)) - throw new Error("geneset: delete -- name does not exist."); + const { genesetName } = action; + if (!state.genesets.has(genesetName)) + throw new Error("geneset: delete -- geneset name does not exist."); const genesets = new Map(state.genesets); // clone - genesets.delete(name); + genesets.delete(genesetName); return { ...state, genesets, @@ -78,81 +129,142 @@ const GeneSets = ( } /** + * Update the named geneset with a new name and description. Preserves the existing + * order of the geneset, even when the genesetName changes. * { - * type: "geneset: add genes" - * name: string, // gene set name - * genes: Set || Array // genes to add + * type: "geneset: update", + * genesetName: string, current name of geneset to be updated + * update: { + * genesetName: string, new name + * genesetDescription: string, new description + * } * } - */ - case "geneset: add genes": { - const { name, genes } = action; - if (!state.genesets.has(name)) - throw new Error("geneset: add genes -- name does not exist."); - - const genesets = new Map(state.genesets); // clone - const newGenes = new Set(state.genesets.get(name)); // clone - for (const gene of genes) { - newGenes.add(gene); - } - genesets.set(name, newGenes); - return { - ...state, - genesets, - }; - } - - /** - * { - * type: "geneset: del genes" - * name: string, // gene set name - * genes: Set || Array // genes to delete - * } - */ - case "geneset: del genes": { - const { name, genes } = action; - - if (!state.genesets.has(name)) - throw new Error("geneset: add genes -- name does not exist."); - - const genesets = new Map(state.genesets); // clone - const newGenes = new Set(state.genesets.get(name)); // clone - for (const gene of genes) { - newGenes.delete(gene); - } - genesets.set(name, newGenes); - return { - ...state, - genesets, - }; - } - - /** - * Rename the gene set, preserving its order in the geneset collection. * - * { - * type: "geneset: rename", - * name: string, // gene set current (previous) name - * newName: string, // the new gene set name - * } + * For example, if you want to update JUST the description: + * dispatch({ + * action: "geneset: update", + * genesetName: "foo", + * update: { genesetName: "foo", genesetDescription: "a new description"} + * }) */ - case "geneset: rename": { - const { name, newName } = action; - if (!state.genesets.has(name)) - throw new Error("geneset: rename -- name does not exist."); + case "geneset: update": { + const { genesetName, update } = action; if ( - typeof newName !== "string" || - !newName.length || - state.genesets.has(newName) + typeof genesetName !== "string" || + !genesetName || + !state.genesets.has(genesetName) ) throw new Error( - "geneset: rename -- new name must be unique, non-null string." + "geneset: update -- geneset name unspecified or does not exist." ); + if (state.genesets.has(update.genesetName)) + throw new Error("geneset: update -- update specified existing name."); + + const prevGs = state.genesets.get(genesetName); + const newGs = { + ...update, + genes: prevGs.genes, + }; // clone // clone the map, preserving current insert order, but mapping name->newName. const genesets = new Map(); - for (const [gsName, gsGenes] of state.genesets) { - if (gsName === name) genesets.set(newName, gsGenes); - else genesets.set(gsName, gsGenes); + for (const [name, gs] of state.genesets) { + if (name === genesetName) genesets.set(newGs.genesetName, newGs); + else genesets.set(name, gs); + } + + return { + ...state, + genesets, + }; + } + + /** + * Adds genes to the geneset. They are appended to the END of the geneset, in the + * order provided. Duplicates or genes already in the geneset, will be ignored. + * { + * type: "geneset: add genes" + * genesetName: , // gene set name + * genes: Array<{ + * geneSymbol: , + * geneDescription: + * }> + * } + * + * Example: + * dispatch({ + * type: "add genes", + * genesetName: "foo", + * genes: [ { geneSymbol: "FOXP", geneDescription: "test" }] + * }); + */ + case "geneset: add genes": { + const { genesetName, genes } = action; + if (!state.genesets.has(genesetName)) + throw new Error("geneset: add genes -- geneset name does not exist."); + + // clone + const genesets = new Map(state.genesets); + const gs = { + ...genesets.get(genesetName), + genes: new Map(genesets.get(genesetName).genes), + }; + genesets.set(genesetName, gs); + + // add + const newGenes = gs.genes; + for (const gene of genes) { + const { geneSymbol } = gene; + const geneDescription = gene?.geneDescription ?? ""; + // ignore genes already present + if (!newGenes.has(geneSymbol)) + newGenes.set(geneSymbol, { + geneSymbol, + geneDescription, + }); + } + + return { + ...state, + genesets, + }; + } + + /** + * Delete genes from the named geneset. Will throw if the genesetName does + * not exist. Will ignore geneSymbols that do not exist. + * { + * type: "geneset: delete genes", + * genesetName: , // the geneset from which to delete genes + * geneSymbols: [, ...], // the gene symbols to delete. + * } + * + * Example: + * dispatch({ + * type: "geneset: delete genes", + * genesetName: "a geneset name", + * geneSymbols: ["F5"] + * }) + */ + case "geneset: delete genes": { + const { genesetName, geneSymbols } = action; + if (!state.genesets.has(genesetName)) + throw new Error( + "geneset: delete genes -- geneset name does not exist." + ); + + // clone + const genesets = new Map(state.genesets); + const gs = { + ...genesets.get(genesetName), + genes: new Map(genesets.get(genesetName).genes), + }; + genesets.set(genesetName, gs); + + // delete + const { genes } = gs; + for (const geneSymbol of geneSymbols) { + genes.delete(geneSymbol); } return { ...state, @@ -160,6 +272,75 @@ const GeneSets = ( }; } + /** + * Set/update the description of the gene. NOTE that this does not allow the name + * of the gene to change - only "geneset: add" and "geneset: delete" can change + * the genes in a geneset. Use this to update a gene description AFTER you add it + * to the geneset. + * { + * type: "geneset: set gene description", + * genesetName: , // the geneset to update + * update: { + * geneSymbol: , // the gene to update, MUST exist already in the geneset + * geneDescription: + * } + * } + * + * Example: + * dispatch({ + * type: "geneset: set gene description", + * genesetName: "my fav geneset", + * update: { + * geneSymbol: "F5", + * geneDescription: "tada, moar description" + * } + * }) + */ + case "geneset: set gene description": { + const { genesetName, update } = action; + if (!state.genesets.has(genesetName)) + throw new Error( + "geneset: set gene description -- geneset name does not exist." + ); + + // clone + const genesets = new Map(state.genesets); + const gs = { + ...genesets.get(genesetName), + genes: new Map(genesets.get(genesetName).genes), + }; + genesets.set(genesetName, gs); + + const { geneSymbol, geneDescription } = update; + const gene = gs.genes.get(geneSymbol); + if (!gene) + throw new Error("geneset: set gene description -- no such gene"); + gs.genes.set(geneSymbol, { + geneSymbol, + geneDescription, + }); + + return { + ...state, + genesets, + }; + } + + /** + * Used by autosave to update the server synchronization TID + */ + case "geneset: set tid": { + const { tid } = action; + if (!Number.isInteger(tid) || tid < 0) + throw new Error("TID must be a positive integer number"); + if (state.lastTid !== undefined && tid < state.lastTid) + throw new Error("TID may not be decremented."); + return { + ...state, + lastTid: tid, + }; + } + default: return state; } diff --git a/client/src/reducers/index.js b/client/src/reducers/index.js index a5a5a59d..7863a117 100644 --- a/client/src/reducers/index.js +++ b/client/src/reducers/index.js @@ -4,7 +4,7 @@ import thunk from "redux-thunk"; import cascadeReducers from "./cascade"; import undoable from "./undoable"; import config from "./config"; -import userInfo from "./userInfo"; +import userInfo from "./userinfo"; import annoMatrix from "./annoMatrix"; import obsCrossfilter from "./obsCrossfilter"; import categoricalSelection from "./categoricalSelection"; @@ -59,6 +59,7 @@ const Reducer = undoable( "differential", "layoutChoice", "centroidLabels", + "genesets", "annotations", ], undoableConfig diff --git a/client/src/util/glHelpers.js b/client/src/util/glHelpers.js index 21cecc44..96631434 100644 --- a/client/src/util/glHelpers.js +++ b/client/src/util/glHelpers.js @@ -8,18 +8,24 @@ PointFlags: We want a bitmask-like flag structure, but due to webgl limitations must emulate it with floats. + + Supported flags are: + + selected: the point is currently selected + highlight: the point is currently highlighted + background: the point is background information */ // for JS export const flagSelected = 1; -export const flagNaN = 2; +export const flagBackground = 2; export const flagHighlight = 4; // for GLSL export const glPointFlags = ` const float flagSelected = 1.; - const float flagNaN = 2.; + const float flagBackground = 2.; const float flagHighlight = 4.; bool isLowBitSet(float f) { @@ -32,12 +38,12 @@ export const glPointFlags = ` } void getFlags(in float flag, - out bool isNaN, + out bool isBackground, out bool isSelected, out bool isHighlight) { isSelected = isLowBitSet(flag); flag = shiftRightOne(flag); - isNaN = isLowBitSet(flag); + isBackground = isLowBitSet(flag); flag = shiftRightOne(flag); isHighlight = isLowBitSet(flag); } diff --git a/client/src/util/stateManager/colorHelpers.js b/client/src/util/stateManager/colorHelpers.js index d324dd37..1f914c46 100644 --- a/client/src/util/stateManager/colorHelpers.js +++ b/client/src/util/stateManager/colorHelpers.js @@ -55,8 +55,8 @@ create colors scale and RGB array and return as object. Parameters: * userColors - optional user color table Returns: { - scale: color scale - rgb: cell to color mapping + scale: function, mapping label index to color scale + rgb: cell label to color mapping } */ function _createColorTable( @@ -70,7 +70,7 @@ function _createColorTable( case "color by categorical metadata": { const data = colorByData.col(colorByAccessor).asArray(); if (userColors && colorByAccessor in userColors) { - return createUserColors(data, colorByAccessor, userColors); + return createUserColors(data, colorByAccessor, schema, userColors); } return createColorsByCategoricalMetadata(data, colorByAccessor, schema); } @@ -91,42 +91,41 @@ function _createColorTable( } export const createColorTable = memoize(_createColorTable); +/** + * Create two category label-indexed objects: + * - colors: maps label to RGB triplet for that label (used by graph, etc) + * - scale: function which given label returns d3 color scale for label + * Order doesn't matter - everything is keyed by label value. + */ export function loadUserColorConfig(userColors) { const convertedUserColors = {}; Object.keys(userColors).forEach((category) => { - // We cannot iterate over keys without sorting - // because we handle categorical values in alphabetical order __ignoring case__ - // while Object.keys() _usually_ is ordered alphabetically where all upper characters are less than lowercase (A, B, C, a, b, c) - const [colors, scaleMap] = Object.keys(userColors[category]) - .sort((a, b) => { - a = a.toLowerCase(); - b = b.toLowerCase(); - if (a === b) return 0; - if (a > b) return 1; - return -1; - }) - .reduce( - (acc, label) => { - const color = parseRGB(userColors[category][label]); - acc[0][label] = color; - acc[1][label] = d3.rgb( - 255 * color[0], - 255 * color[1], - 255 * color[2] - ); - return acc; - }, - [{}, {}] - ); + const [colors, scaleMap] = Object.keys(userColors[category]).reduce( + (acc, label) => { + const color = parseRGB(userColors[category][label]); + acc[0][label] = color; + acc[1][label] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]); + return acc; + }, + [{}, {}] + ); const scale = (label) => scaleMap[label]; convertedUserColors[category] = { colors, scale }; }); return convertedUserColors; } -function _createUserColors(data, colorAccessor, userColors) { - const { colors, scale } = userColors[colorAccessor]; +function _createUserColors(data, colorAccessor, schema, userColors) { + const { colors, scale: scaleByLabel } = userColors[colorAccessor]; const rgb = createRgbArray(data, colors); + + // color scale function param is INDEX (offset) into schema categories. It is NOT label value. + // See createColorsByCategoricalMetadata() for another example. + const { categories } = schema.annotations.obsByName[colorAccessor]; + const categoryMap = new Map(); + categories.forEach((label, idx) => categoryMap.set(idx, label)); + const scale = (idx) => scaleByLabel(categoryMap.get(idx)); + return { rgb, scale }; } const createUserColors = memoize(_createUserColors); diff --git a/docs/_config.yml b/docs/_config.yml index 6e29ea64..4290ac5f 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -33,5 +33,7 @@ nav: url: posts/roadmap - title: Contributing (ideas or code) url: posts/contribute + - title: Extensions + url: posts/extensions - title: Contact & finding help url: posts/contact diff --git a/docs/_site/deprecated/cellxgene_cziscience_com.html b/docs/_site/deprecated/cellxgene_cziscience_com.html new file mode 100644 index 00000000..f23a54b0 --- /dev/null +++ b/docs/_site/deprecated/cellxgene_cziscience_com.html @@ -0,0 +1,446 @@ + + + + + + + + +cellxgene.cziscience.com | cellxgene + + + + + + + + + + + + + + + +
+
+ cellxgene + +

An interactive explorer for single-cell transcriptomics data

+

+ + + Quick start
+ + + + + Installation
+ + + + Gallery
+ + + + Demo datasets
+ + + + All other datasets
+ + + + Preparing your data
+ + + + Launching cellxgene
+ + + + Hosting cellxgene
+ + + + Annotating data
+ + + + Methods
+ + + + Troubleshooting
+ + + + Roadmap
+ + + + Contributing (ideas or code)
+ + + + Extensions
+ + + + Contact & finding help
+ + + + Code +

+
+ +
+

cellxgene.cziscience.com

+ +

Chan Zuckerberg has an online repository of public single-cell datasets for exploration with cellxgene.

+ +

If you have a public dataset which you would like hosted for visualization on this site, +with a link to embed on your own site, please drop us a note at cellxgene@chanzuckerberg.com.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
cellxgene linkMore Information
Krasnow Lab Human Lung Cell Atlas, 10X + Krasnow Lab, + HLCA website +
Krasnow Lab Human Lung Cell Atlas, Smart-seq2 + Krasnow Lab, + HLCA website +
Human Cell Landscape + Guo Lab, + HCL website +
Human fetal liver single cell transcriptome data + E-MTAB-7407, + covid19cellatlas.org +
A cell atlas of human thymic development defines T cell repertoire formation + E-MTAB-8581, + covid19cellatlas.org +
A cellular census of human lungs identifies novel cell states in health and in asthma - parenchyma + asthma.cellgeni.sanger.ac.uk, + covid19cellatlas.org +
A cellular census of human lungs identifies novel cell states in health and in asthma - nasal + asthma.cellgeni.sanger.ac.uk, + covid19cellatlas.org +
A cellular census of human lungs identifies novel cell states in health and in asthma - bronchi + asthma.cellgeni.sanger.ac.uk, + covid19cellatlas.org +
Ischaemic sensitivity of human tissue by single cell RNA seq - lung + HCA, + covid19cellatlas.org +
Ischaemic sensitivity of human tissue by single cell RNA seq - spleen + HCA, + covid19cellatlas.org +
Ischaemic sensitivity of human tissue by single cell RNA seq - oesophagus + HCA, + covid19cellatlas.org +
Spatio-temporal immune zonation of the human kidney + www.kidneycellatlas.org, + covid19cellatlas.org +
Reconstructing the human first trimester fetal-maternal interface using single cell transcriptomics - 10x + E-MTAB-6701, + covid19cellatlas.org +
Reconstructing the human first trimester fetal-maternal interface using single cell transcriptomics - SmartSeq2 + E-MTAB-6701, + covid19cellatlas.org +
Gut Cell Atlas + www.gutcellatlas.org, + covid19cellatlas.org +
A single-cell atlas of the peripheral immune response to severe COVID-19 + Blish Lab, + medRxiv preprint +
Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Ileum + Single Cell Portal +
Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Lung + Single Cell Portal +
Allergic inflammatory memory in human respiratory epithelial progenitor cells - epithelial cells + Single Cell Portal +
Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal scrapings + Single Cell Portal +
Allergic inflammatory memory in human respiratory epithelial progenitor cells - surgical + Single Cell Portal +
Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal SSS + Single Cell Portal +
ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum - epithelial cells + Single Cell Portal +
ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum + Single Cell Portal +
Human Lung HIV-TB Co-infection ACE2+ Cells + Single Cell Portal +
Epithelial Cells in NHP mTB Granuloma and Uninvolved Lung + Single Cell Portal +
Selective Neuronal Vulnerability in Alzheimer's Disease + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Superior Frontal Gyrus + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Astrocytes in EC + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Excitatory Neurons in EC + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Inhibitory Neurons in EC + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Microglia in EC + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Astrocytes in SFG + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Excitatory Neurons in SFG + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Inhibitory Neurons in SFG + Kampmann Lab, + BioRxiv preprint +
Selective Neuronal Vulnerability in Alzheimer's Disease: Microglia in SFG + Kampmann Lab, + BioRxiv preprint +
Single-cell gene expression profiling of SARS-CoV-2 infected human cell lines - H1299 + Landthaler Lab, + BioRxiv preprint +
Single-cell gene expression profiling of SARS-CoV-2 infected human cell lines - Calu-3 + Landthaler Lab, + BioRxiv preprint +
Single-cell drug screening - A549 + Trapnell Lab Github, + Science +
Single-cell drug screening - K562 + Trapnell Lab Github, + Science +
Single-cell drug screening - MCF7 + Trapnell Lab Github, + Science +
A molecular atlas of cell types and zonation in the brain vasculature + Betsholtz Lab, + Nature +
Single Soma Transcriptomics - AT8 + bioRxiv preprint +
Single Soma Transcriptomics - MAP2 + bioRxiv preprint +
Single Soma Transcriptomics - MAP2AT8 + bioRxiv preprint +
Single-cell longitudinal analysis of SARS-CoV-2 infection in human bronchial epithelial cells + bioRxiv preprint +
+ +
+ +
+ + + + diff --git a/docs/_site/index.html b/docs/_site/index.html index 49e763a0..4d484f9b 100644 --- a/docs/_site/index.html +++ b/docs/_site/index.html @@ -7,7 +7,7 @@ Index | cellxgene - + @@ -16,10 +16,10 @@ +{"@type":"WebSite","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","description":"An interactive explorer for single-cell transcriptomics data","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"} - + @@ -86,6 +86,10 @@ + Extensions
+ + + Contact & finding help
diff --git a/docs/_site/posts/extensions.html b/docs/_site/posts/extensions.html new file mode 100644 index 00000000..7c4a6562 --- /dev/null +++ b/docs/_site/posts/extensions.html @@ -0,0 +1,156 @@ + + + + + + + + +Extensions | cellxgene + + + + + + + + + + + + + + + +
+
+ cellxgene + +

An interactive explorer for single-cell transcriptomics data

+

+ + + Quick start
+ + + + + Installation
+ + + + Gallery
+ + + + Demo datasets
+ + + + All other datasets
+ + + + Preparing your data
+ + + + Launching cellxgene
+ + + + Hosting cellxgene
+ + + + Annotating data
+ + + + Methods
+ + + + Troubleshooting
+ + + + Roadmap
+ + + + Contributing (ideas or code)
+ + + + Extensions
+ + + + Contact & finding help
+ + + + Code +

+
+ +
+

Extensions

+ +

This project was started with the sole goal of empowering the scientific community to explore and understand their data. +As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from +this project. All code is freely available for reuse under the MIT license.

+ +

Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an +extension could be directly contributed, which would make it available for a wider audience, or that it’s on our +roadmap and under active development.

+ +

Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against.

+ +

Example Reuse & extensions

+ +

cellxgene Gateway

+ +

cellxgene Gateway allows you to use with multiple datasets. It +displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server +instance that loads that particular data file and once it is available proxies requests to that server.

+ +

cellxgene-VIP (Visualization in Plugin)

+ +

cellxgene-VIP enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R.

+ +

Galaxy

+ +

Galaxy is an open source, collaborative, web-based platform for data intensive biomedical research. +Galaxy provides various tools for single-cell data analysis and also infrastructure to the +Galaxy Human Cell Atlas project. cellxgene can be + accessed within Galaxy to view analyzed datasets. See also the relevant publication + +

+ +

Single Cell Portal

+ +

The Single Cell Portal is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. +Example.

+ +

FASTGenomics

+ +

FASTGenomics is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. +See also this example for data from Schulte-Schrepping et al. (Cell, 2020). +Note that it is not necessary to create an account, anonymous login is permitted.

+ +
+ +
+ + + + diff --git a/docs/_site/posts/extensions.md b/docs/_site/posts/extensions.md new file mode 100644 index 00000000..5b25c958 --- /dev/null +++ b/docs/_site/posts/extensions.md @@ -0,0 +1,39 @@ +# Extensions + +This project was started with the sole goal of empowering the scientific community to explore and understand their data. +As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from +this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT). + +Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an +extension could be directly contributed, which would make it available for a wider audience, or that it's on our +[roadmap](./roadmap.md) and under active development. + +Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against. + +## Example Reuse & extensions + +#### cellxgene Gateway + +[cellxgene Gateway](https://github.com/Novartis/cellxgene-gateway) allows you to use with multiple datasets. It +displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server +instance that loads that particular data file and once it is available proxies requests to that server. + +#### cellxgene-VIP (Visualization in Plugin) + +[cellxgene-VIP](https://github.com/interactivereport/cellxgene_VIP) enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R. + +#### Galaxy + +[Galaxy](https://singlecell.usegalaxy.eu/) is an open source, web-based platform for data intensive biomedical research. cellxgene can be accessed within Galaxy to view analyzed datasets. +See also the relevant [publication](https://www.biorxiv.org/content/10.1101/2020.06.06.137570v1.full.pdf) + +#### Single Cell Portal + +The [Single Cell Portal](https://singlecell.broadinstitute.org/single_cell) is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. +[Example](https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells). + +#### FASTGenomics + +[FASTGenomics](https://beta.fastgenomics.org/) is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. +See also this [example](https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene) for data from [Schulte-Schrepping et al. (Cell, 2020)](https://beta.fastgenomics.org/p/schulte-schrepping_covid19). +Note that it is not necessary to create an account, anonymous login is permitted. diff --git a/docs/posts/extensions.md b/docs/posts/extensions.md new file mode 100644 index 00000000..5b25c958 --- /dev/null +++ b/docs/posts/extensions.md @@ -0,0 +1,39 @@ +# Extensions + +This project was started with the sole goal of empowering the scientific community to explore and understand their data. +As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from +this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT). + +Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an +extension could be directly contributed, which would make it available for a wider audience, or that it's on our +[roadmap](./roadmap.md) and under active development. + +Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against. + +## Example Reuse & extensions + +#### cellxgene Gateway + +[cellxgene Gateway](https://github.com/Novartis/cellxgene-gateway) allows you to use with multiple datasets. It +displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server +instance that loads that particular data file and once it is available proxies requests to that server. + +#### cellxgene-VIP (Visualization in Plugin) + +[cellxgene-VIP](https://github.com/interactivereport/cellxgene_VIP) enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R. + +#### Galaxy + +[Galaxy](https://singlecell.usegalaxy.eu/) is an open source, web-based platform for data intensive biomedical research. cellxgene can be accessed within Galaxy to view analyzed datasets. +See also the relevant [publication](https://www.biorxiv.org/content/10.1101/2020.06.06.137570v1.full.pdf) + +#### Single Cell Portal + +The [Single Cell Portal](https://singlecell.broadinstitute.org/single_cell) is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. +[Example](https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells). + +#### FASTGenomics + +[FASTGenomics](https://beta.fastgenomics.org/) is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. +See also this [example](https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene) for data from [Schulte-Schrepping et al. (Cell, 2020)](https://beta.fastgenomics.org/p/schulte-schrepping_covid19). +Note that it is not necessary to create an account, anonymous login is permitted. diff --git a/local_server/Makefile b/local_server/Makefile new file mode 100644 index 00000000..5f7acf13 --- /dev/null +++ b/local_server/Makefile @@ -0,0 +1,26 @@ +include ../common.mk + +.PHONY: clean +clean: + rm -f common/web/templates/index.html + rm -rf common/web/static + rm -f common/web/csp-hashes.json + +.PHONY: unit-test +unit-test: + PYTHONWARNINGS=ignore:ResourceWarning coverage run \ + --source=app,cli,common,compute,converters,data_anndata,data_common \ + --omit=.coverage,data_common/fbs/NetEncoding,venv \ + -m unittest discover \ + --start-directory test/ \ + --top-level-directory ../ \ + --verbose; test_result=$$?; \ + exit $$test_result \ + +.PHONY: test-annotations-performance +test-annotations-performance: + python test/performance/performance_test_annotations_backend.py + +.PHONY: test-annotations-scale +test-annotations-scale: + locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt diff --git a/local_server/__init__.py b/local_server/__init__.py new file mode 100644 index 00000000..7c3e1b4e --- /dev/null +++ b/local_server/__init__.py @@ -0,0 +1,13 @@ +import logging +import sys +from local_server.common.utils.utils import import_plugins + +__version__ = "0.16.0" +display_version = "cellxgene v" + __version__ + +try: + import_plugins("server.plugins") +except Exception as e: + # Make sure to exit in this case, as the server may not be configured as expected. + logging.critical(f"Error in import_plugins: {str(e)}") + sys.exit(1) diff --git a/local_server/__main__.py b/local_server/__main__.py new file mode 100644 index 00000000..b582e892 --- /dev/null +++ b/local_server/__main__.py @@ -0,0 +1,15 @@ +# Work around bug https://github.com/pallets/werkzeug/issues/461 +if __package__ is None: + import sys + from pathlib import Path + + PKG_PATH = Path(__file__).parent + sys.path.insert(0, str(PKG_PATH.parent)) + import server # noqa F401 + + __package__ = PKG_PATH.name + +# Main thing +from .cli.cli import cli # noqa F402 + +cli() diff --git a/local_server/app/__init__.py b/local_server/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/app/app.py b/local_server/app/app.py new file mode 100644 index 00000000..6bf53cd3 --- /dev/null +++ b/local_server/app/app.py @@ -0,0 +1,237 @@ +import datetime +import logging +from functools import wraps +from http import HTTPStatus + +from flask import ( + Flask, + current_app, + make_response, + render_template, + Blueprint, + request, + send_from_directory, +) +from flask_restful import Api, Resource + +import local_server.common.rest as common_rest +from local_server.common.errors import DatasetAccessError, RequestException +from local_server.common.health import health_check +from local_server.common.utils.utils import Float32JSONEncoder + +webbp = Blueprint("webapp", "local_server.common.web", template_folder="templates") + + +@webbp.route("/", methods=["GET"]) +def dataset_index(): + app_config = current_app.app_config + + dataset_config = app_config.get_dataset_config() + scripts = dataset_config.app__scripts + inline_scripts = dataset_config.app__inline_scripts + + try: + args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts} + return render_template("index.html", **args) + + except DatasetAccessError as e: + return common_rest.abort_and_log( + e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True + ) + + +@webbp.errorhandler(RequestException) +def handle_request_exception(error): + return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True) + + +def requires_authentication(func): + @wraps(func) + def wrapped_function(self, *args, **kwargs): + auth = current_app.auth + if auth.is_user_authenticated(): + return func(self, *args, **kwargs) + else: + return make_response("not authenticated", HTTPStatus.UNAUTHORIZED) + + return wrapped_function + + +def rest_get_data_adaptor(func): + @wraps(func) + def wrapped_function(self): + try: + return func(self, current_app.data_adaptor) + except DatasetAccessError as e: + return common_rest.abort_and_log( + e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True + ) + + return wrapped_function + + +class HealthAPI(Resource): + def get(self): + config = current_app.app_config + return health_check(config) + + +class SchemaAPI(Resource): + # TODO @mdunitz separate dataset schema and user schema + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.schema_get(data_adaptor) + + +class ConfigAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.config_get(current_app.app_config, data_adaptor) + + +class UserInfoAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.userinfo_get(current_app.app_config, data_adaptor) + + +class AnnotationsObsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.annotations_obs_get(request, data_adaptor) + + @requires_authentication + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.annotations_obs_put(request, data_adaptor) + + +class AnnotationsVarAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.annotations_var_get(request, data_adaptor) + + +class DataVarAPI(Resource): + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.data_var_put(request, data_adaptor) + + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.data_var_get(request, data_adaptor) + + +class ColorsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.colors_get(data_adaptor) + + +class DiffExpObsAPI(Resource): + @rest_get_data_adaptor + def post(self, data_adaptor): + return common_rest.diffexp_obs_post(request, data_adaptor) + + +class LayoutObsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.layout_obs_get(request, data_adaptor) + + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.layout_obs_put(request, data_adaptor) + + +class GenesetsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.genesets_get(request, data_adaptor) + + @requires_authentication + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.genesets_put(request, data_adaptor) + + +def get_api_base_resources(bp_base): + """Add resources that are accessed from the api url""" + api = Api(bp_base) + + # Diagnostics routes + api.add_resource(HealthAPI, "/health") + return api + + +def get_api_dataroot_resources(bp_dataroot): + """Add resources that refer to a dataset""" + api = Api(bp_dataroot) + + def add_resource(resource, url): + """convenience function to make the outer function less verbose""" + api.add_resource(resource, url) + + # Initialization routes + add_resource(SchemaAPI, "/schema") + add_resource(ConfigAPI, "/config") + add_resource(UserInfoAPI, "/userinfo") + # Data routes + add_resource(AnnotationsObsAPI, "/annotations/obs") + add_resource(AnnotationsVarAPI, "/annotations/var") + add_resource(DataVarAPI, "/data/var") + add_resource(GenesetsAPI, "/genesets") + # Display routes + add_resource(ColorsAPI, "/colors") + # Computation routes + add_resource(DiffExpObsAPI, "/diffexp/obs") + add_resource(LayoutObsAPI, "/layout/obs") + return api + + +class Server: + @staticmethod + def _before_adding_routes(app, app_config): + """ will be called before routes are added, during __init__. Subclass protocol """ + pass + + def __init__(self, app_config): + self.app = Flask(__name__, static_folder=None) + self._before_adding_routes(self.app, app_config) + self.app.json_encoder = Float32JSONEncoder + server_config = app_config.server_config + + # enable session data + self.app.permanent_session_lifetime = datetime.timedelta(days=50 * 365) + + # Config + secret_key = server_config.app__flask_secret_key + self.app.config.update(SECRET_KEY=secret_key) + + self.app.register_blueprint(webbp) + + api_version = "/api/v0.2" + api_path = "/" + + bp_base = Blueprint("bp_base", __name__, url_prefix=api_path) + base_resources = get_api_base_resources(bp_base) + self.app.register_blueprint(base_resources.blueprint) + + bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}") + resources = get_api_dataroot_resources(bp_api) + self.app.register_blueprint(resources.blueprint) + self.app.add_url_rule( + "/static/", + "static_assets", + view_func=lambda filename: send_from_directory("../common/web/static", filename), + methods=["GET"], + ) + + self.app.data_adaptor = server_config.data_adaptor + self.app.app_config = app_config + + auth = server_config.auth + self.app.auth = auth + if auth.requires_client_login(): + auth.add_url_rules(self.app) + auth.complete_setup(self.app) diff --git a/local_server/auth/__init__.py b/local_server/auth/__init__.py new file mode 100644 index 00000000..c0c238b4 --- /dev/null +++ b/local_server/auth/__init__.py @@ -0,0 +1,5 @@ +# import the built in auth types so they can be registered + +import local_server.auth.auth_none # noqa: F401 +import local_server.auth.auth_test # noqa: F401 +import local_server.auth.auth_session # noqa: F401 diff --git a/local_server/auth/auth.py b/local_server/auth/auth.py new file mode 100644 index 00000000..03184e8d --- /dev/null +++ b/local_server/auth/auth.py @@ -0,0 +1,91 @@ +from abc import ABC, abstractmethod + + +class AuthTypeBase(ABC): + """Base type for all authentication types.""" + + def __init__(self): + super().__init__() + + @abstractmethod + def is_valid_authentication_type(self): + """Return True if the auth type is valid, e.g. it can return userinfo and username. + (AuthTypeNone is the only one type that returns False)""" + pass + + def requires_client_login(self): + """Return True if the user needs to login from the client (e.g. Login button is shown)""" + return False + + @abstractmethod + def complete_setup(self, app): + """complete any setup that may be needed by this auth type. The Flask app is passed in. + This is the last auth function called before the server starts to run.""" + pass + + @abstractmethod + def is_user_authenticated(self): + """Return True if the user is authenticated""" + pass + + @abstractmethod + def get_user_id(self): + """Return the id for this user (string)""" + pass + + @abstractmethod + def get_user_name(self): + """Return the name of the user (string)""" + pass + + @abstractmethod + def get_user_email(self): + """Return the name of the user (string)""" + pass + + def get_user_picture(self): + """Return the location to the user's picture""" + return None + + +class AuthTypeClientBase(AuthTypeBase): + """Base type for all authentication types that require the client to login""" + + def __init__(self): + super().__init__() + + def requires_client_login(self): + return True + + @abstractmethod + def add_url_rules(self, selfapp): + """Add url rules to the app (like /login, /logout, etc)""" + pass + + @abstractmethod + def get_login_url(self, data_adaptor): + """Return the url for the login route""" + pass + + @abstractmethod + def get_logout_url(self, data_adaptor): + """Return the url for the logout route""" + pass + + +class AuthTypeFactory: + """Factory class to create an authentication type""" + + auth_types = {} + + @staticmethod + def register(name, auth_type): + assert issubclass(auth_type, AuthTypeBase) + AuthTypeFactory.auth_types[name] = auth_type + + @staticmethod + def create(name, app_config): + auth_type = AuthTypeFactory.auth_types.get(name) + if auth_type is None: + return None + return auth_type(app_config) diff --git a/local_server/auth/auth_none.py b/local_server/auth/auth_none.py new file mode 100644 index 00000000..2d6f1f75 --- /dev/null +++ b/local_server/auth/auth_none.py @@ -0,0 +1,27 @@ +from local_server.auth.auth import AuthTypeBase, AuthTypeFactory + + +class AuthTypeNone(AuthTypeBase): + def __init__(self, app_config): + super().__init__() + + def is_valid_authentication_type(self): + return False + + def complete_setup(self, app): + pass + + def is_user_authenticated(self): + return True + + def get_user_id(self): + return None + + def get_user_name(self): + return None + + def get_user_email(self): + return None + + +AuthTypeFactory.register(None, AuthTypeNone) diff --git a/local_server/auth/auth_session.py b/local_server/auth/auth_session.py new file mode 100644 index 00000000..de460fb0 --- /dev/null +++ b/local_server/auth/auth_session.py @@ -0,0 +1,39 @@ +from local_server.auth.auth import AuthTypeBase, AuthTypeFactory +from flask import session +from uuid import uuid4 + + +class AuthTypeSession(AuthTypeBase): + """Session based authentication. The user is always logged. The user id is a random number + associated with the session. This is a good choice for desktop servers.""" + + # key in the session token for userid + CXGUID = "cxguid" + + def __init__(self, app_config): + super().__init__() + + def is_valid_authentication_type(self): + return True + + def complete_setup(self, app): + pass + + def is_user_authenticated(self): + # always authenticated + return True + + def get_user_id(self): + if self.CXGUID not in session: + session[self.CXGUID] = uuid4().hex + session.permanent = True + return session[self.CXGUID] + + def get_user_name(self): + return "anonymous" + + def get_user_email(self): + return None + + +AuthTypeFactory.register("session", AuthTypeSession) diff --git a/local_server/auth/auth_test.py b/local_server/auth/auth_test.py new file mode 100644 index 00000000..f5eaf1e3 --- /dev/null +++ b/local_server/auth/auth_test.py @@ -0,0 +1,73 @@ +from local_server.auth.auth import AuthTypeClientBase, AuthTypeFactory +from flask import session, request, redirect + + +class AuthTypeTest(AuthTypeClientBase): + """An authentication type for testing client based logins. When the login route is accessed + the user is automatically logged in with a default or configured username""" + + # key in session token with userid and username + CXGUID = "cxguid_test" + CXGUNAME = "cxguname_test" + CXGUEMAIL = "cxguemail_test" + CXGUPICTURE = "cxgupicture_test" + + def __init__(self, app_config): + super().__init__() + self.user_name = "test_account" + self.user_id = "id0001" + self.user_email = "test_account@test.com" + self.user_picture = None + + def is_valid_authentication_type(self): + return True + + def requires_client_login(self): + return True + + def add_url_rules(self, app): + app.add_url_rule("/login", "login", self.login, methods=["GET"]) + app.add_url_rule("/logout", "logout", self.logout, methods=["GET"]) + + def complete_setup(self, app): + pass + + def is_user_authenticated(self): + return self.CXGUID in session + + def get_user_id(self): + return session.get(self.CXGUID) + + def get_user_name(self): + return session.get(self.CXGUNAME) + + def get_user_email(self): + return session.get(self.CXGUEMAIL) + + def get_user_picture(self): + return session.get(self.CXGUPICTURE) + + def login(self): + args = request.args + return_to = args.get("dataset", "/") + session[self.CXGUID] = args.get("userid", self.user_id) + session[self.CXGUNAME] = args.get("username", self.user_name) + session[self.CXGUEMAIL] = args.get("email", self.user_email) + session[self.CXGUPICTURE] = args.get("picture", self.user_picture) + return redirect(return_to) + + def logout(self): + session.clear() + return_to = request.args.get("dataset", "/") + return redirect(return_to) + + def get_login_url(self, data_adaptor): + """Return the url for the login route""" + return "/login" + + def get_logout_url(self, data_adaptor): + """Return the url for the logout route""" + return "/logout" + + +AuthTypeFactory.register("test", AuthTypeTest) diff --git a/local_server/cli/__init__.py b/local_server/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/cli/cli.py b/local_server/cli/cli.py new file mode 100644 index 00000000..a093df13 --- /dev/null +++ b/local_server/cli/cli.py @@ -0,0 +1,33 @@ +import click + +from .launch import launch +from .prepare import prepare +from .upgrade import log_upgrade_check +from .schema import schema_cli +from .. import __version__ + + +@click.group( + name="cellxgene", + subcommand_metavar="COMMAND ", + options_metavar="", + context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]), +) +@click.help_option("--help", "-h", help="Show this message and exit.") +@click.version_option( + version=__version__, + prog_name="cellxgene", + message="[%(prog)s] Version %(version)s", + help="Show the software version and exit.", +) +@click.option( + "--upgrade-check/--no-upgrade-check", default=True, show_default=True, help="Check for release upgrades on start.", +) +def cli(upgrade_check): + if upgrade_check: + log_upgrade_check() + + +cli.add_command(launch) +cli.add_command(prepare) +cli.add_command(schema_cli) diff --git a/local_server/cli/launch.py b/local_server/cli/launch.py new file mode 100644 index 00000000..af880b59 --- /dev/null +++ b/local_server/cli/launch.py @@ -0,0 +1,467 @@ +import errno +import functools +import logging +import sys +import webbrowser +import os +import click +from flask_compress import Compress +from flask_cors import CORS + +from local_server.default_config import default_config +from local_server.app.app import Server +from local_server.common.config.app_config import AppConfig +from local_server.common.errors import DatasetAccessError, ConfigurationError +from local_server.common.utils.utils import sort_options + +DEFAULT_CONFIG = AppConfig() + + +def annotation_args(func): + @click.option( + "--disable-annotations", + is_flag=True, + default=not DEFAULT_CONFIG.dataset_config.user_annotations__enable, + show_default=True, + help="Disable user annotation of data.", + ) + @click.option( + "--annotations-file", + default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__file, + show_default=True, + multiple=False, + metavar="", + help="CSV file to initialize editing of existing annotations; will be altered in-place. " + "Incompatible with --user-generated-data-dir.", + ) + @click.option( + "--user-generated-data-dir", + "--annotations-dir", + default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory, + show_default=False, + multiple=False, + metavar="", + help="Directory of where to save output annotations; filename will be specified in the application. " + "Incompatible with --annotations-file and --gene-sets-file.", + ) + @click.option( + "--experimental-annotations-ontology", + is_flag=True, + default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__enable, + show_default=True, + help="When creating annotations, optionally autocomplete names from ontology terms.", + ) + @click.option( + "--experimental-annotations-ontology-obo", + default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__obo_location, + show_default=True, + metavar="", + help="Location of OBO file defining cell annotation autosuggest terms.", + ) + @click.option( + "--disable-gene-sets-save", + is_flag=True, + default=DEFAULT_CONFIG.dataset_config.user_annotations__gene_sets__readonly, + show_default=False, + help="Disable saving gene sets. If disabled, users will be able to make changes to gene sets but all " + "changes will be lost on browser refresh.", + ) + @click.option( + "--gene-sets-file", + default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__gene_sets_file, + show_default=True, + multiple=False, + metavar="", + help="CSV file to initialize editing of gene sets; will be altered in-place. Incompatible with " + "--user-generated-data-dir.", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def config_args(func): + @click.option( + "--max-category-items", + default=DEFAULT_CONFIG.dataset_config.presentation__max_categories, + metavar="", + show_default=True, + help="Will not display categories with more distinct values than specified.", + ) + @click.option( + "--disable-custom-colors", + is_flag=True, + default=False, + show_default=False, + help="Disable user-defined category-label colors drawn from source data file.", + ) + @click.option( + "--diffexp-lfc-cutoff", + "-de", + default=DEFAULT_CONFIG.dataset_config.diffexp__lfc_cutoff, + show_default=True, + metavar="", + help="Minimum log fold change threshold for differential expression.", + ) + @click.option( + "--disable-diffexp", + is_flag=True, + default=not DEFAULT_CONFIG.dataset_config.diffexp__enable, + show_default=False, + help="Disable on-demand differential expression.", + ) + @click.option( + "--embedding", + "-e", + default=DEFAULT_CONFIG.dataset_config.embeddings__names, + multiple=True, + show_default=False, + metavar="", + help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.", + ) + @click.option( + "--experimental-enable-reembedding", + is_flag=True, + default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding, + show_default=False, + hidden=True, + help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def dataset_args(func): + @click.option( + "--obs-names", + "-obs", + default=DEFAULT_CONFIG.server_config.single_dataset__obs_names, + metavar="", + help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.", + ) + @click.option( + "--var-names", + "-var", + default=DEFAULT_CONFIG.server_config.single_dataset__var_names, + metavar="", + help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.", + ) + @click.option( + "--backed", + "-b", + is_flag=True, + default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed, + show_default=False, + help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.", + ) + @click.option( + "--title", + "-t", + default=DEFAULT_CONFIG.server_config.single_dataset__title, + metavar="", + help="Title to display. If omitted will use file name.", + ) + @click.option( + "--about", + default=DEFAULT_CONFIG.server_config.single_dataset__about, + metavar="", + help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def server_args(func): + @click.option( + "--debug", + "-d", + is_flag=True, + default=DEFAULT_CONFIG.server_config.app__debug, + show_default=True, + help="Run in debug mode. This is helpful for cellxgene developers, " + "or when you want more information about an error condition.", + ) + @click.option( + "--verbose", + "-v", + is_flag=True, + default=DEFAULT_CONFIG.server_config.app__verbose, + show_default=True, + help="Provide verbose output, including warnings and all server requests.", + ) + @click.option( + "--port", + "-p", + metavar="", + default=DEFAULT_CONFIG.server_config.app__port, + type=int, + show_default=True, + help="Port to run server on. If not specified cellxgene will find an available port.", + ) + @click.option( + "--host", + metavar="", + default=DEFAULT_CONFIG.server_config.app__host, + show_default=False, + help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).", + ) + @click.option( + "--scripts", + "-s", + default=DEFAULT_CONFIG.dataset_config.app__scripts, + multiple=True, + metavar="", + help="Additional script files to include in HTML page. If not specified, " + "no additional script files will be included.", + show_default=False, + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def launch_args(func): + @annotation_args + @config_args + @dataset_args + @server_args + @click.argument("datapath", required=False, metavar="") + @click.option( + "--open", + "-o", + "open_browser", + is_flag=True, + default=DEFAULT_CONFIG.server_config.app__open_browser, + show_default=True, + help="Open web browser after launch.", + ) + @click.option( + "--config-file", + "-c", + "config_file", + default=None, + show_default=True, + help="Location to yaml file with configuration settings", + ) + @click.option( + "--dump-default-config", + "dump_default_config", + is_flag=True, + default=False, + show_default=True, + help="Print default configuration settings and exit", + ) + @click.help_option("--help", "-h", help="Show this message and exit.") + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def handle_scripts(scripts): + if scripts: + click.echo( + r""" + / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _ + \ \/ \/ / _` | '__| '_ \| | '_ \ / _` | + \ /\ / (_| | | | | | | | | | | (_| | + \/ \/ \__,_|_| |_| |_|_|_| |_|\__, | + |___/ + The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a + security risk by including the --scripts flag. Make sure you trust the scripts that you are including. + """ + ) + scripts_pretty = ", ".join(scripts) + click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) + + +class CliLaunchServer(Server): + """ + the CLI runs a local web server, and needs to enable a few more features. + """ + + def __init__(self, app_config): + super().__init__(app_config) + + @staticmethod + def _before_adding_routes(app, app_config): + app.config["COMPRESS_MIMETYPES"] = [ + "text/html", + "text/css", + "text/xml", + "application/json", + "application/javascript", + "application/octet-stream", + ] + Compress(app) + if app_config.server_config.app__debug: + CORS(app, supports_credentials=True) + + +@sort_options +@click.command( + short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.", + options_metavar="", +) +@launch_args +def launch( + datapath, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + disable_custom_colors, + diffexp_lfc_cutoff, + title, + scripts, + about, + disable_annotations, + annotations_file, + user_generated_data_dir, + gene_sets_file, + disable_gene_sets_save, + backed, + disable_diffexp, + experimental_annotations_ontology, + experimental_annotations_ontology_obo, + experimental_enable_reembedding, + config_file, + dump_default_config, +): + """Launch the cellxgene data viewer. + This web app lets you explore single-cell expression data. + Data must be in a format that cellxgene expects. + Read the "getting started" guide to learn more: + https://chanzuckerberg.github.io/cellxgene/getting-started.html + + Examples: + + > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k + + > cellxgene launch --title + + > cellxgene launch """ + + if dump_default_config: + print(default_config) + sys.exit(0) + + # Startup message + click.echo("[cellxgene] Starting the CLI...") + + # app config + app_config = AppConfig() + server_config = app_config.server_config + + try: + if config_file: + app_config.update_from_config_file(config_file) + + # Determine which config options were give on the command line. + # Those will override the ones provided in the config file (if provided). + cli_config = AppConfig() + cli_config.update_server_config( + app__verbose=verbose, + app__debug=debug, + app__host=host, + app__port=port, + app__open_browser=open_browser, + single_dataset__datapath=datapath, + single_dataset__title=title, + single_dataset__about=about, + single_dataset__obs_names=obs_names, + single_dataset__var_names=var_names, + adaptor__anndata_adaptor__backed=backed, + ) + cli_config.update_dataset_config( + app__scripts=scripts, + user_annotations__enable=not disable_annotations, + user_annotations__local_file_csv__file=annotations_file, + user_annotations__local_file_csv__directory=user_generated_data_dir, + user_annotations__local_file_csv__gene_sets_file=gene_sets_file, + user_annotations__gene_sets__readonly=disable_gene_sets_save, + user_annotations__ontology__enable=experimental_annotations_ontology, + user_annotations__ontology__obo_location=experimental_annotations_ontology_obo, + presentation__max_categories=max_category_items, + presentation__custom_colors=not disable_custom_colors, + embeddings__names=embedding, + embeddings__enable_reembedding=experimental_enable_reembedding, + diffexp__enable=not disable_diffexp, + diffexp__lfc_cutoff=diffexp_lfc_cutoff, + ) + + diff = cli_config.server_config.changes_from_default() + changes = {key: val for key, val, _ in diff} + app_config.update_server_config(**changes) + + diff = cli_config.dataset_config.changes_from_default() + changes = {key: val for key, val, _ in diff} + app_config.update_dataset_config(**changes) + + # process the configuration + # any errors will be thrown as an exception. + # any info messages will be passed to the messagefn function. + + def messagefn(message): + click.echo("[cellxgene] " + message) + + # Use a default secret if one is not provided + if not server_config.app__flask_secret_key: + app_config.update_server_config(app__flask_secret_key="SparkleAndShine") + + app_config.complete_config(messagefn) + + except (ConfigurationError, DatasetAccessError) as e: + raise click.ClickException(e) + + handle_scripts(scripts) + + # create the server + server = CliLaunchServer(app_config) + + if not server_config.app__verbose: + log = logging.getLogger("werkzeug") + log.setLevel(logging.ERROR) + + cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}" + if server_config.app__open_browser: + click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") + webbrowser.open(cellxgene_url) + else: + click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.") + + click.echo("[cellxgene] Type CTRL-C at any time to exit.") + + if not server_config.app__verbose: + f = open(os.devnull, "w") + sys.stdout = f + + try: + server.app.run( + host=server_config.app__host, + debug=server_config.app__debug, + port=server_config.app__port, + threaded=not server_config.app__debug, + use_debugger=False, + use_reloader=False, + ) + except OSError as e: + if e.errno == errno.EADDRINUSE: + raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e + raise diff --git a/local_server/cli/prepare.py b/local_server/cli/prepare.py new file mode 100644 index 00000000..df5c8dd7 --- /dev/null +++ b/local_server/cli/prepare.py @@ -0,0 +1,274 @@ +from os.path import expanduser, isdir, isfile, sep, splitext + +import click +import pandas as pd +from numpy import ndarray, unique +from scipy.sparse.csc import csc_matrix + +from local_server.common.utils.utils import sort_options + + +@sort_options +@click.command( + short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.", + options_metavar="", +) +@click.argument("data", nargs=1, metavar="", required=True) +@click.option( + "--embedding", + "-e", + default=["umap", "tsne"], + multiple=True, + type=click.Choice(["umap", "tsne"]), + help="Embedding algorithm(s). Repeat option for multiple embeddings.", + show_default=True, +) +@click.option( + "--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]), show_default=True, +) +@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="") +@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True) +@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True) +@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) +@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="") +@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="") +@click.option( + "--skip-qc", + default=False, + is_flag=True, + help="Do not run quality control metrics. By default cellxgene runs them " + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", +) +@click.option( + "--make-obs-names-unique/--no-make-obs-names-unique", + default=True, + help="Ensure obs index is unique.", + show_default=True, +) +@click.option( + "--make-var-names-unique/--no-make-var-names-unique", + default=True, + help="Ensure var index is unique.", + show_default=True, +) +@click.help_option("--help", "-h", help="Show this message and exit.") +def prepare( + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, +): + """ + Preprocess data for use with cellxgene. + This tool runs a series of scanpy routines for preparing a dataset for use + with cellxgene. It loads data from different formats + (h5ad, loom, or a 10x directory), runs dimensionality reduction, + computes nearest neighbors, computes an embedding, performs clustering, + and saves the results. Includes additional options for naming annotations, + ensuring sparsity, and plotting results. + """ + + # collect slow imports here to make CLI startup more responsive + click.echo("[cellxgene] Starting CLI...") + try: + import matplotlib + + matplotlib.use("Agg") + import scanpy as sc + except ImportError: + raise click.ClickException( + "[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` " + "to install the necessary requirements." + ) + + # scanpy settings + sc.settings.verbosity = 0 + sc.settings.autosave = True + + # check args + if sparse and not recipe == "none": + raise click.UsageError("Cannot use a recipe when forcing sparsity") + + output = expanduser(output) + + if not output: + click.echo( + "Warning: No file will be saved, to save the results of cellxgene prepare include " + "--output to save output to a new file" + ) + if isfile(output) and not overwrite: + raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite") + + def load_data(data): + if isfile(data): + name, extension = splitext(data) + if extension == ".h5ad": + adata = sc.read_h5ad(data) + elif extension == ".loom": + adata = sc.read_loom(data) + else: + raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]") + elif isdir(data): + if not data.endswith(sep): + data += sep + adata = sc.read_10x_mtx(data) + else: + raise click.FileError(data, hint="not a valid file or path") + + if not set_obs_names == "": + if set_obs_names not in adata.obs_keys(): + raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}") + adata.obs_names = adata.obs[set_obs_names] + if not set_var_names == "": + if set_var_names not in adata.var_keys(): + raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}") + adata.var_names = adata.var[set_var_names] + if make_obs_names_unique: + adata.obs.index = make_index_unique(adata.obs.index) + if make_var_names_unique: + adata.var.index = make_index_unique(adata.var.index) + if not adata._obs.index.is_unique: + click.echo("Warning: obs index is not unique") + if not adata._var.index.is_unique: + click.echo("Warning: var index is not unique") + return adata + + def calculate_qc_metrics(adata): + if not skip_qc: + sc.pp.calculate_qc_metrics(adata, inplace=True) + return adata + + def make_sparse(adata): + if (type(adata.X) is ndarray) and sparse: + adata.X = csc_matrix(adata.X) + + def run_recipe(adata): + if recipe == "seurat": + sc.pp.recipe_seurat(adata) + elif recipe == "zheng17": + sc.pp.recipe_zheng17(adata) + else: + sc.pp.filter_cells(adata, min_genes=5) + sc.pp.filter_genes(adata, min_cells=25) + if sparse: + sc.pp.scale(adata, zero_center=False) + else: + sc.pp.scale(adata) + + def run_pca(adata): + if sparse: + sc.pp.pca(adata, svd_solver="arpack", zero_center=False) + else: + sc.pp.pca(adata, svd_solver="arpack") + + def run_neighbors(adata): + sc.pp.neighbors(adata) + + def run_louvain(adata): + sc.tl.louvain(adata) + + def run_embedding(adata): + if len(unique(adata.obs["louvain"].values)) < 10: + palette = "tab10" + else: + palette = "tab20" + + if "umap" in embedding: + sc.tl.umap(adata) + if plotting: + sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain") + + if "tsne" in embedding: + sc.tl.tsne(adata) + if plotting: + sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain") + + def show_step(item): + if not skip_qc: + qc_name = "Calculating QC metrics" + else: + qc_name = "Skipping QC" + names = { + "calculate_qc_metrics": qc_name, + "make_sparse": "Ensuring sparsity", + "run_recipe": f'Running preprocessing recipe "{recipe}"', + "run_pca": "Running PCA", + "run_neighbors": "Calculating neighbors", + "run_louvain": "Calculating clusters", + "run_embedding": "Computing embedding", + } + if item is not None: + return names[item.__name__] + + steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding] + + click.echo(f"[cellxgene] Loading data from {data}, please wait...") + adata = load_data(data) + + click.echo("[cellxgene] Beginning preprocessing...") + with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar: + for step in bar: + step(adata) + + # saving + if not output == "": + click.echo(f"[cellxgene] Saving results to {output}...") + adata.write(output) + + click.echo("[cellxgene] Success!") + + +# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344 +# Note: tentative solution here https://github.com/theislab/anndata/pull/345 +def make_index_unique(index: pd.Index, join: str = "-"): + """ + Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc. + + If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence. + + The first occurrence of a non-unique value is ignored. + Parameters + ---------- + join + The connecting string between name and integer. + Examples + -------- + >>> from anndata import AnnData + >>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c'])) + >>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b'])) + >>> adata = adata1.concatenate(adata2) + >>> adata.obs_names + Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object') + >>> adata.obs_names_make_unique() + >>> adata.obs_names + Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object') + """ + if index.is_unique: + return index + from collections import defaultdict + + values = index.values + values_set = set(values) + indices_dup = index.duplicated(keep="first") + values_dup = values[indices_dup] + counter = defaultdict(lambda: 0) + for i, v in enumerate(values_dup): + while True: + counter[v] += 1 + tentative_new_name = v + join + str(counter[v]) + if tentative_new_name not in values_set: + values_set.add(tentative_new_name) + values_dup[i] = tentative_new_name + break + + values[indices_dup] = values_dup + index = pd.Index(values) + return index diff --git a/local_server/cli/schema.py b/local_server/cli/schema.py new file mode 100644 index 00000000..83c9c07f --- /dev/null +++ b/local_server/cli/schema.py @@ -0,0 +1,72 @@ +import click + +from local_server.converters.schema import remix, validate + + +@click.group( + name="schema", + subcommand_metavar="COMMAND ", + short_help="Apply and validate the cellxgene data integration schema to an h5ad file.", + context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]), +) +def schema_cli(): + try: + import scanpy # noqa: F401 + except ImportError: + raise click.ClickException( + "[cellxgene] cellxgene schema requires scanpy" + ) + + +@click.command( + name="apply", + short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.", + help="(experimental) Using a yaml file that describes schema values to insert or convert and in input " + "h5ad file, apply the schema changes and create a new, conforming h5ad.", +) +@click.option( + "--source-h5ad", + help="Input h5ad file.", + nargs=1, + required=True, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--remix-config", + help="Config yaml with information on how to apply the schema.", + nargs=1, + required=True, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--output-filename", + help="Filename for the new, schema-conforming h5ad file.", + required=True, + nargs=1 +) +def schema_apply(source_h5ad, remix_config, output_filename): + remix.apply_schema(source_h5ad, remix_config, output_filename) + + +@click.command( + name="validate", + short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.", +) +@click.argument( + "h5ad", + nargs=1, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--shallow", + help="When true, just check that the correct version information is present.", + default=False, + show_default=True, + is_flag=True, +) +def schema_validate(h5ad, shallow): + validate.validate(h5ad, shallow) + + +schema_cli.add_command(schema_apply) +schema_cli.add_command(schema_validate) diff --git a/local_server/cli/upgrade.py b/local_server/cli/upgrade.py new file mode 100644 index 00000000..222d7e81 --- /dev/null +++ b/local_server/cli/upgrade.py @@ -0,0 +1,85 @@ +import re + +import click +import requests +from requests.exceptions import ConnectionError + +from .. import __version__ + +# Official SemVer regex: https://semver.org/ +SEMVER_FORMAT = re.compile( + r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[" + r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(" + r"?:\.[0-9a-zA-Z-]+)*))?$" +) + + +def log_upgrade_check(): + # Sanity-check that the CLI version is a properly-formatted SemVer string + assert validate_version_str(__version__, release_only=False) + + # Get the current latest release + try: + release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases()) + latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name)) + if version_gt(latest_release, __version__): + click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True) + click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True) + except (ConnectionError, RateLimitException): + click.echo("Upgrade check failed.\n") + + +class RateLimitException(Exception): + """ + Github API Rate Limit Exception + """ + + +def _request_cellxgene_releases(): + def raise_on_rate_limit(response): + if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0": + raise RateLimitException + + url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases" + res = requests.get(url) + raise_on_rate_limit(res) + for release in res.json(): + yield release + while "next" in res.links.keys(): + res = requests.get(res.links["next"]["url"]) + raise_on_rate_limit(res) + for release in res.json(): + yield release + + +def validate_version_str(version_str, release_only=True): + """ + Test if a string conforms to SemVer format (https://semver.org/) + :param version_str: a string to be validated + :param release_only: only declare releases (not prereleases) valid + :return: True if the version string is of a valid SemVer format else False + """ + match = SEMVER_FORMAT.match(version_str) + has_match = match is not None + if has_match and release_only: + return not match.group("prerelease") + return has_match + + +def split_version(version_string): + """ + Split a SemVer-formatted string into its component integers + :param version_string: a SemVer string to be split + :return: an array of three integers + """ + match = SEMVER_FORMAT.match(version_string) + return [int(match.group(group)) for group in ["major", "minor", "patch"]] + + +def version_gt(left_version, right_version): + for left, right in zip(split_version(left_version), split_version(right_version)): + if left > right: + return True + elif right > left: + return False + return False diff --git a/local_server/common/__init__.py b/local_server/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/common/annotations/__init__.py b/local_server/common/annotations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/common/annotations/annotations.py b/local_server/common/annotations/annotations.py new file mode 100644 index 00000000..3ed2763b --- /dev/null +++ b/local_server/common/annotations/annotations.py @@ -0,0 +1,144 @@ +from abc import ABCMeta, abstractmethod + +import fastobo +import fsspec + +from local_server.common.errors import OntologyLoadFailure, DisabledFeatureError +from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array + + +class Annotations(metaclass=ABCMeta): + """ baseclass for annotations, including ontologies and genesets""" + + """ our default ontology is the PURL for the Cell Ontology. + See http://www.obofoundry.org/ontology/cl.html """ + DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo" + + def __init__(self, config={}): + self.ontology_data = None + self.config = config + + def user_annotations_enabled(self): + return self.config.get("user-annotations", False) + + def gene_sets_save_enabled(self): + return self.config.get("genesets-save", False) + + def check_user_annotations_enabled(self): + if not self.user_annotations_enabled(): + raise DisabledFeatureError("User annotations are disabled.") + + def check_gene_sets_save_enabled(self): + if not self.gene_sets_save_enabled(): + raise DisabledFeatureError("User genesets save is disabled.") + + def load_ontology(self, path): + """Load and parse ontologies - currently support OBO files only.""" + if path is None: + path = self.DefaultOnotology + + try: + with fsspec.open(path) as f: + obo = fastobo.iter(f) + terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo) + names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause] + self.ontology_data = names + + except FileNotFoundError as e: + raise OntologyLoadFailure("Unable to find OBO ontology path") from e + + except SyntaxError as e: + raise OntologyLoadFailure("Syntax error loading OBO ontology") from e + + except Exception as e: + raise OntologyLoadFailure("Error loading OBO file") from e + + def get_schema(self, data_adaptor): + schema = [] + labels = self.read_labels(data_adaptor) + if labels is not None and not labels.empty: + for col in labels.columns: + col_schema = dict(name=col, writable=True) + col_schema.update(get_schema_type_hint_of_array(labels[col])) + schema.append(col_schema) + + return schema + + @abstractmethod + def set_collection(self, name): + """set or create a new annotation collection""" + pass + + @abstractmethod + def read_labels(self, data_adaptor): + """Return the labels as a pandas.DataFrame""" + pass + + @abstractmethod + def write_labels(self, df, data_adaptor): + """Write the labels (df) to a persistent storage such that it can later be read""" + pass + + @abstractmethod + def read_gene_sets(self, data_adaptor): + """Return the genesets from persistent storage """ + pass + + @abstractmethod + def write_gene_sets(self, gs, data_adaptor): + """Write the genesets (gs) to a persistent storage such that it can later be read""" + pass + + @abstractmethod + def update_parameters(self, parameters, data_adaptor): + """Update configuration parameters that describe information about the annotations feature""" + pass + + Genesets_Header = [ + "gene_set_name", + "gene_set_description", + "gene_symbol", + "gene_description", + ] + + @staticmethod + def gene_sets_to_csv(genesets): + """ + Convert the internal genesets format (returned by read_gene_set) into + the simple Tidy CSV. + """ + from io import StringIO + import csv + + if type(genesets) == dict: + genesets = genesets.values() + + with StringIO() as sio: + writer = csv.writer(sio, dialect='excel') + writer.writerow(Annotations.Genesets_Header) + for geneset in genesets: + # genes may be empty, in which case we skip the geneset entirely + genes = geneset["genes"] + if not genes: + writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""]) + else: + writer.writerows( + [ + [ + geneset["geneset_name"], + geneset.get("geneset_description", ""), + gene["gene_symbol"], + gene.get("gene_description", ""), + ] + for gene in genes + ] + ) + return sio.getvalue() + + @staticmethod + def gene_sets_to_response(genesets): + """ + Convert the internal genesets format (returned by read_gene_set) into + the dict expected by the JSON REST API + """ + return list(genesets.values()) diff --git a/local_server/common/annotations/local_file_csv.py b/local_server/common/annotations/local_file_csv.py new file mode 100644 index 00000000..25acdc34 --- /dev/null +++ b/local_server/common/annotations/local_file_csv.py @@ -0,0 +1,352 @@ +import base64 +import os +import re +import threading +from datetime import datetime +from hashlib import blake2b +import csv + +import pandas as pd +from flask import session, has_request_context, current_app + +from local_server import __version__ as cellxgene_version +from local_server.common.annotations.annotations import Annotations +from local_server.common.errors import AnnotationsError, ObsoleteRequest + + +class AnnotationsLocalFile(Annotations): + CXG_ANNO_COLLECTION = "cxg_anno_collection" + + def __init__(self, config, output_dir, label_output_file, gene_sets_output_file): + super().__init__(config) + self.output_dir = output_dir + self.label_output_file = label_output_file + self.gene_sets_output_file = gene_sets_output_file + # lock used to protect label file write ops + self.label_lock = threading.RLock() + self.gene_sets_lock = threading.RLock() + + # cache the most recent annotations. + self.last_fname = None + self.last_labels = None + + # txn ID - used to de-dup geneset writes + self.last_geneset_tid = 0 + + def is_safe_collection_name(self, name): + """ + return true if this is a safe collection name + this is ultra conservative. If we want to allow full legal file name syntax, + we could look at modules like `pathvalidate` + """ + if name is None: + return False + return re.match(r"^[\w\-]+$", name) is not None + + def set_collection(self, name): + session[self.CXG_ANNO_COLLECTION] = name + session.permanent = True + + def get_collection(self): + if session is None: + return None + return session.get(self.CXG_ANNO_COLLECTION) + + def read_labels(self, data_adaptor): + self.check_user_annotations_enabled() # raises + + if has_request_context(): + if not current_app.auth.is_user_authenticated(): + return pd.DataFrame() + + fname = self._get_celllabels_filename(data_adaptor) + with self.label_lock: + if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: + # returned the cached labels if possible, otherwise read them from the file + if fname == self.last_fname: + return self.last_labels + else: + labels = pd.read_csv( + fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False + ) + # update the cache + self.last_fname = fname + self.last_labels = labels + return labels + else: + return pd.DataFrame() + + def write_labels(self, df, data_adaptor): + self.check_user_annotations_enabled() # raises + + # update our internal state and save it. Multi-threading often enabled, + # so treat this as a critical section. + with self.label_lock: + lastmod = data_adaptor.get_last_mod_time() + lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") + header = ( + f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " + f"using cellxgene version {cellxgene_version}\n" + f"# Input data file was {data_adaptor.get_location()}, " + f"which was last modified on {lastmodstr}\n" + ) + + fname = self._get_celllabels_filename(data_adaptor) + self._backup(fname) + if not df.empty: + with open(fname, "w", newline="") as f: + if header is not None: + f.write(header) + df.to_csv(f) + else: + open(fname, "w").close() + + # update the cache + self.last_fname = fname + self.last_labels = df + + def read_gene_sets(self, data_adaptor, context=None): + if has_request_context(): + if not current_app.auth.is_user_authenticated(): + return ({}, self.last_geneset_tid) + + fname = self._get_genesets_filename(data_adaptor) + gene_sets = {} + tid = None + with self.gene_sets_lock: + tid = self.last_geneset_tid # inside the critical section + if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: + with open(fname, newline="") as f: + gene_sets = read_gene_set_tidycsv(f, context) + + return (gene_sets, tid) + + def write_gene_sets(self, gene_sets, tid, data_adaptor): + self.check_gene_sets_save_enabled() # raises + + if type(tid) != int or tid < 0: + raise ValueError("tid must be a positive integer") + + with self.gene_sets_lock: + # skip if the request is stale + if tid is not None: + if tid <= self.last_geneset_tid: + raise ObsoleteRequest("TID is stale.") + self.last_geneset_tid = tid + + lastmod = data_adaptor.get_last_mod_time() + lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") + header = ( + f"# Gene set generated on {datetime.now().isoformat(timespec='seconds')} " + f"using cellxgene version {cellxgene_version}\n" + f"# Input data file was {data_adaptor.get_location()}, " + f"which was last modified on {lastmodstr}\n" + ) + + fname = self._get_genesets_filename(data_adaptor) + self._backup(fname) + with open(fname, "w", newline="") as f: + f.write(header) + f.write(self.gene_sets_to_csv(gene_sets)) + + def _get_userdata_idhash(self, data_adaptor): + """ + Return a short hash that weakly identifies the user and dataset. + Used to create safe annotations output file names. + """ + uid = current_app.auth.get_user_id() or "" + id = (uid + data_adaptor.get_location()).encode() + idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8") + return idhash + + def _get_output_dir(self): + if self.output_dir: + return self.output_dir + + output_file = self.label_output_file or self.gene_sets_output_file + if output_file: + return os.path.dirname(os.path.abspath(output_file)) + + return os.getcwd() + + def _get_celllabels_filename(self, data_adaptor): + """ return the current annotation file name """ + if self.label_output_file: + return self.label_output_file + + return self._get_filename(data_adaptor, "celllabels") + + def _get_genesets_filename(self, data_adaptor): + """ return the current gene sets file name """ + if self.gene_sets_output_file: + return self.gene_sets_output_file + + return self._get_filename(data_adaptor, "genesets") + + def _get_filename(self, data_adaptor, anno_name): + # we need to generate a file name, which we can only do if we have a UID and collection name + if session is None: + raise AnnotationsError("unable to determine file name for annotations") + + collection = self.get_collection() + if collection is None: + return None + + if data_adaptor is None: + raise AnnotationsError("unable to determine file name for annotations") + + idhash = self._get_userdata_idhash(data_adaptor) + return os.path.join(self._get_output_dir(), f"{collection}-{anno_name}-{idhash}.csv") + + def _backup(self, fname, max_backups=9): + """ + save N backups of file to backup_dir. + 1. fname -> backup_dir/fname-TIME + 2. delete excess files in backup_dir + """ + root, ext = os.path.splitext(fname) + backup_dir = f"{root}-backups" + + # Make sure there is work to do + if not os.path.exists(fname): + return + + # Ensure backup_dir exists + if not os.path.exists(backup_dir): + os.mkdir(backup_dir) + + # Save current file to backup_dir + fname_base = os.path.basename(fname) + fname_base_root, fname_base_ext = os.path.splitext(fname_base) + # don't use ISO standard time format, as it contains characters illegal on some filesytems. + nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}") + if os.path.exists(backup_fname): + os.remove(backup_fname) + os.rename(fname, backup_fname) + + # prune the backup_dir to max number of backup files, keeping the most recent backups + backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir))) + excess_count = len(backups) - max_backups + if excess_count > 0: + backups.sort() + for bu in backups[0:excess_count]: + os.remove(os.path.join(backup_dir, bu)) + + def update_parameters(self, parameters, data_adaptor): + params = {} + params["annotations"] = self.user_annotations_enabled() + params["annotations_genesets_readonly"] = not self.gene_sets_save_enabled() + params["user_annotation_collection_name_enabled"] = True + + if self.ontology_data: + params["annotations_cell_ontology_enabled"] = True + params["annotations_cell_ontology_terms"] = self.ontology_data + else: + params["annotations_cell_ontology_enabled"] = False + + if self.label_output_file is not None: + # user has hard-wired the name of the annotation cell label data collection + fname = os.path.basename(self.label_output_file) + collection_fname = os.path.splitext(fname)[0] + params["annotations-data-collection-is-read-only"] = True + params["annotations-data-collection-name"] = collection_fname + + elif session is not None: + collection = self.get_collection() + params["annotations-data-collection-is-read-only"] = False + params["annotations-data-collection-name"] = collection + + if current_app.auth.is_user_authenticated(): + params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) + + parameters.update(params) + + +def read_gene_set_tidycsv(f, context=None): + """ + Read & parse the Tidy CSV format, applying validation checks for mandatory + values, and de-duping rules. + + Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed + comments. Format: + + gene_set_name, gene_set_description, gene_symbol, gene_description + + gene_set_name must be non-null; others are optional. + + Returns: a dictionary of the shape (values in angle-brackets vary): + + { + : { + "geneset_name": , + "geneset_description": , + "genes": [ + { + "gene_symbol": , + "gene_description": + }, + ... + ] + }, + ... + } + """ + + class myDialect(csv.excel): + skipinitialspace = True + + def just(n, seq): + it = iter(seq) + for _ in range(n - 1): + yield next(it, "") + yield tuple(it) + + messagefn = context["messagefn"] if context else (lambda x: None) + + reader = csv.reader(f, dialect=myDialect()) + gene_sets = {} + haveReadHeader = False + lineno = 0 + for row in reader: + lineno += 1 + # ignore empty rows + if len(row) == 0: + continue + # if row starts with '#' it is a comment + if row[0].startswith("#"): + continue + # if this is the first non-comment row, assume it is a header + if not haveReadHeader: + if row != Annotations.Genesets_Header: + raise AnnotationsError("Geneset CSV file missing the required column header.") + haveReadHeader = True + continue + + geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row) + if not geneset_name: + raise AnnotationsError(f"Geneset CSV missing required geneset or gene name on line {lineno}") + if (not gene_symbol) and gene_description: + messagefn(f"Warning: Missing gene name in geneset name {geneset_name} on line {lineno}.") + + if geneset_name in gene_sets: + gs = gene_sets[geneset_name] + else: + gs = gene_sets[geneset_name] = { + "geneset_name": geneset_name, + "geneset_description": geneset_description, + "genes": [], + } + # Use first geneset_description with a value + if not gs["geneset_description"] and geneset_description: + gs["geneset_description"] = geneset_description + # add the gene if the gene_symbol is defined + if gene_symbol: + gs["genes"].append( + { + "gene_symbol": gene_symbol, + "gene_description": gene_description, + } + ) + + return gene_sets diff --git a/local_server/common/aws_secret_utils.py b/local_server/common/aws_secret_utils.py new file mode 100644 index 00000000..4cfb1159 --- /dev/null +++ b/local_server/common/aws_secret_utils.py @@ -0,0 +1,23 @@ +import logging + +import boto3 +from flask import json + +from local_server.common.errors import SecretKeyRetrievalError + + +def get_secret_key(region_name, secret_name): + session = boto3.session.Session() + client = session.client(service_name="secretsmanager", region_name=region_name) + + try: + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + if "SecretString" in get_secret_value_response: + var = get_secret_value_response["SecretString"] + secret = json.loads(var) + return secret + except Exception as e: + logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True) + raise SecretKeyRetrievalError(str(e)) + + return None diff --git a/local_server/common/colors.py b/local_server/common/colors.py new file mode 100644 index 00000000..b1e8d0bb --- /dev/null +++ b/local_server/common/colors.py @@ -0,0 +1,233 @@ +import re + +from local_server.common.errors import ColorFormatException + +HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$") + +# https://www.w3.org/TR/css-color-4/#named-colors +CSS4_NAMED_COLORS = dict( + aliceblue="#f0f8ff", + antiquewhite="#faebd7", + aqua="#00ffff", + aquamarine="#7fffd4", + azure="#f0ffff", + beige="#f5f5dc", + bisque="#ffe4c4", + black="#000000", + blanchedalmond="#ffebcd", + blue="#0000ff", + blueviolet="#8a2be2", + brown="#a52a2a", + burlywood="#deb887", + cadetblue="#5f9ea0", + chartreuse="#7fff00", + chocolate="#d2691e", + coral="#ff7f50", + cornflowerblue="#6495ed", + cornsilk="#fff8dc", + crimson="#dc143c", + cyan="#00ffff", + darkblue="#00008b", + darkcyan="#008b8b", + darkgoldenrod="#b8860b", + darkgray="#a9a9a9", + darkgreen="#006400", + darkgrey="#a9a9a9", + darkkhaki="#bdb76b", + darkmagenta="#8b008b", + darkolivegreen="#556b2f", + darkorange="#ff8c00", + darkorchid="#9932cc", + darkred="#8b0000", + darksalmon="#e9967a", + darkseagreen="#8fbc8f", + darkslateblue="#483d8b", + darkslategray="#2f4f4f", + darkslategrey="#2f4f4f", + darkturquoise="#00ced1", + darkviolet="#9400d3", + deeppink="#ff1493", + deepskyblue="#00bfff", + dimgray="#696969", + dimgrey="#696969", + dodgerblue="#1e90ff", + firebrick="#b22222", + floralwhite="#fffaf0", + forestgreen="#228b22", + fuchsia="#ff00ff", + gainsboro="#dcdcdc", + ghostwhite="#f8f8ff", + gold="#ffd700", + goldenrod="#daa520", + gray="#808080", + green="#008000", + greenyellow="#adff2f", + grey="#808080", + honeydew="#f0fff0", + hotpink="#ff69b4", + indianred="#cd5c5c", + indigo="#4b0082", + ivory="#fffff0", + khaki="#f0e68c", + lavender="#e6e6fa", + lavenderblush="#fff0f5", + lawngreen="#7cfc00", + lemonchiffon="#fffacd", + lightblue="#add8e6", + lightcoral="#f08080", + lightcyan="#e0ffff", + lightgoldenrodyellow="#fafad2", + lightgray="#d3d3d3", + lightgreen="#90ee90", + lightgrey="#d3d3d3", + lightpink="#ffb6c1", + lightsalmon="#ffa07a", + lightseagreen="#20b2aa", + lightskyblue="#87cefa", + lightslategray="#778899", + lightslategrey="#778899", + lightsteelblue="#b0c4de", + lightyellow="#ffffe0", + lime="#00ff00", + limegreen="#32cd32", + linen="#faf0e6", + magenta="#ff00ff", + maroon="#800000", + mediumaquamarine="#66cdaa", + mediumblue="#0000cd", + mediumorchid="#ba55d3", + mediumpurple="#9370db", + mediumseagreen="#3cb371", + mediumslateblue="#7b68ee", + mediumspringgreen="#00fa9a", + mediumturquoise="#48d1cc", + mediumvioletred="#c71585", + midnightblue="#191970", + mintcream="#f5fffa", + mistyrose="#ffe4e1", + moccasin="#ffe4b5", + navajowhite="#ffdead", + navy="#000080", + oldlace="#fdf5e6", + olive="#808000", + olivedrab="#6b8e23", + orange="#ffa500", + orangered="#ff4500", + orchid="#da70d6", + palegoldenrod="#eee8aa", + palegreen="#98fb98", + paleturquoise="#afeeee", + palevioletred="#db7093", + papayawhip="#ffefd5", + peachpuff="#ffdab9", + peru="#cd853f", + pink="#ffc0cb", + plum="#dda0dd", + powderblue="#b0e0e6", + purple="#800080", + rebeccapurple="#663399", + red="#ff0000", + rosybrown="#bc8f8f", + royalblue="#4169e1", + saddlebrown="#8b4513", + salmon="#fa8072", + sandybrown="#f4a460", + seagreen="#2e8b57", + seashell="#fff5ee", + sienna="#a0522d", + silver="#c0c0c0", + skyblue="#87ceeb", + slateblue="#6a5acd", + slategray="#708090", + slategrey="#708090", + snow="#fffafa", + springgreen="#00ff7f", + steelblue="#4682b4", + tan="#d2b48c", + teal="#008080", + thistle="#d8bfd8", + tomato="#ff6347", + turquoise="#40e0d0", + violet="#ee82ee", + wheat="#f5deb3", + white="#ffffff", + whitesmoke="#f5f5f5", + yellow="#ffff00", + yellowgreen="#9acd32", +) + + +def convert_color_to_hex_format(unknown): + """ + Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet. + + The function accepts for the following formats: + - A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html + - RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0] + - RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255] + - Hex triplet string, as in "#08c0ff" + + :param unknown: color info of unknown format + :return: a hex triplet representing that color + """ + try: + if type(unknown) in (list, tuple) and len(unknown) == 3: + if all(0.0 <= ele <= 1.0 for ele in unknown): + tup = tuple(int(ele * 255) for ele in unknown) + elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown): + tup = tuple(unknown) + else: + raise ColorFormatException("Unknown color iterable format!") + return "#%02x%02x%02x" % tup + elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS: + return CSS4_NAMED_COLORS[unknown.lower()] + elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown): + return unknown.lower() + else: + raise ColorFormatException("Unknown color format type!") + except Exception as e: + raise ColorFormatException(e) + + +def convert_anndata_category_colors_to_cxg_category_colors(data): + """ + Convert color information from anndata files to the cellxgene color data format as described below: + { + "": { + "": "", + ... + }, + ... + } + + For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307. + + For more on the anndata color data structure, see + https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178. + + Handling of malformed data: + - For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot + convert to a hex triplet string, a ColorFormatException is raised + - No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no + adata.obs[f"{category}"] key + + :param data: the anndata file + :return: cellxgene color data structure as described above + """ + cxg_colors = dict() + color_key_suffix = "_colors" + for uns_key in data.uns.keys(): + # find uns array that describes colors for a category + if not uns_key.endswith(color_key_suffix): + continue + + # check to see if we actually have observations for that category + category_name = uns_key[: -len(color_key_suffix)] + if category_name not in data.obs.keys(): + continue + + # create the cellxgene color entry for this category + cxg_colors[category_name] = dict( + zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]]) + ) + return cxg_colors diff --git a/local_server/common/config/__init__.py b/local_server/common/config/__init__.py new file mode 100644 index 00000000..fa1a0dcc --- /dev/null +++ b/local_server/common/config/__init__.py @@ -0,0 +1,4 @@ +from local_server.common.aws_secret_utils import get_secret_key # noqa F504 + +DEFAULT_SERVER_PORT = 5005 +BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB diff --git a/local_server/common/config/app_config.py b/local_server/common/config/app_config.py new file mode 100644 index 00000000..8ecd67f7 --- /dev/null +++ b/local_server/common/config/app_config.py @@ -0,0 +1,171 @@ +import yaml +from flatten_dict import unflatten + +from local_server.default_config import get_default_config +from local_server.common.config.dataset_config import DatasetConfig +from local_server.common.config.server_config import ServerConfig +from local_server.common.config.external_config import ExternalConfig +from local_server.common.errors import ConfigurationError + + +class AppConfig(object): + """ + AppConfig stores all the configuration for cellxgene. + AppConfig contains one or more DatasetConfig(s) and one ServerConfig. + The server_config contains attributes that refer to the server process as a whole. + The dataset_config refers to attributes that are associated with the features and + presentations of a dataset. + AppConfig has methods to initialize, modify, and access the configuration. + """ + + def __init__(self): + + # the default configuration (see default_config.py) + # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as + # defaults within the config class? + self.default_config = get_default_config() + # the server configuration + self.server_config = ServerConfig(self, self.default_config["server"]) + # the dataset config + self.dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) + # external config + self.external_config = ExternalConfig(self, self.default_config["external"]) + + # Set to true when config_completed is called + self.is_completed = False + + def get_dataset_config(self): + return self.dataset_config + + def check_config(self): + """Verify all the attributes in the config have been type checked""" + if not self.is_completed: + raise ConfigurationError("The configuration has not been completed") + self.server_config.check_config() + self.dataset_config.check_config() + self.external_config.check_config() + + def update_server_config(self, **kw): + self.server_config.update(**kw) + self.is_complete = False + + def update_dataset_config(self, **kw): + self.dataset_config.update(**kw) + self.is_complete = False + + def update_single_config_from_path_and_value(self, path, value): + """Update a single config parameter with the value. + Path is a list of string, that gives a path to the config parameter to be updated. + For example, path may be ["server","app","port"]. + """ + self.is_complete = False + if not isinstance(path, list): + raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'") + for part in path: + if not isinstance(part, str): + raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'") + + if len(path) < 1 or path[0] not in ("server", "dataset"): + raise ConfigurationError("path must start with 'server', or 'dataset'") + + if path[0] == "server": + attr = "__".join(path[1:]) + try: + self.update_server_config(**{attr: value}) + except ConfigurationError: + raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'") + elif path[0] == "dataset": + attr = "__".join(path[1:]) + try: + self.update_dataset_config(**{attr: value}) + except ConfigurationError: + raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'") + + def update_from_config_file(self, config_file): + try: + with open(config_file) as yml_file: + config = yaml.safe_load(yml_file) + except yaml.YAMLError as e: + raise ConfigurationError(f"The specified config file contained an error: {e}") + except OSError as e: + raise ConfigurationError(f"Issue retrieving the specified config file: {e}") + + if config.get("server"): + self.server_config.update_from_config(config["server"], "server") + if config.get("dataset"): + self.dataset_config.update_from_config(config["dataset"], "dataset") + + if config.get("external"): + self.external_config.update_from_config(config["external"], "external") + + self.is_complete = False + + def config_to_dict(self): + """return the configuration as an unflattened dict""" + server = self.server_config.create_mapping(self.server_config.default_config) + dataset = self.dataset_config.create_mapping(self.dataset_config.default_config) + external = self.external_config.create_mapping(self.external_config.default_config) + config = dict(server={}, dataset={}) + for attrname in server.keys(): + config["server__" + attrname] = getattr(self.server_config, attrname) + for attrname in dataset.keys(): + config["dataset__" + attrname] = getattr(self.dataset_config, attrname) + for attrname in external.keys(): + config["external__" + attrname] = getattr(self.external_config, attrname) + + config = unflatten(config, splitter=lambda key: key.split("__")) + return config + + def write_config(self, config_file): + """output the config to a yaml file""" + config = self.config_to_dict() + yaml.dump(config, open(config_file, "w")) + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + diff_server = self.server_config.changes_from_default() + diff_dataset = self.dataset_config.changes_from_default() + diff_external = self.external.changes_from_default() + diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external) + return diff + + def complete_config(self, messagefn=None): + """The configure options are checked, and any additional setup based on the config + parameters is done""" + + if messagefn is None: + + def noop(message): + pass + + messagefn = noop + + # TODO: to give better error messages we can add a mapping between where each config + # attribute originated (e.g. command line argument or config file), then in the error + # messages we can give correct context for attributes with bad value. + context = dict(messagefn=messagefn) + + # complete config for external_config first, since this may update values in the other sections + self.external_config.complete_config(context) + self.server_config.complete_config(context) + self.dataset_config.complete_config(context) + + self.is_completed = True + self.check_config() + + def get_matrix_data_cache_manager(self): + return self.server_config.matrix_data_cache_manager + + def get_title(self, data_adaptor): + return ( + self.server_config.single_dataset__title + if self.server_config.single_dataset__title + else data_adaptor.get_title() + ) + + def get_about(self, data_adaptor): + return ( + self.server_config.single_dataset__about + if self.server_config.single_dataset__about + else data_adaptor.get_about() + ) diff --git a/local_server/common/config/base_config.py b/local_server/common/config/base_config.py new file mode 100644 index 00000000..8238f2ed --- /dev/null +++ b/local_server/common/config/base_config.py @@ -0,0 +1,99 @@ +import copy + +from flatten_dict import flatten +from local_server.common.errors import ConfigurationError + + +class BaseConfig(object): + """ + This class handles the mechanics of updating and checking attributes. + Derived classes are expected to store the actual attributes + Currently DatasetConfig and ServerConfig both inherit from BaseConfig. + """ + + def __init__(self, app_config, default_config): + # reference back to the app_config + self.app_config = app_config + # the complete set of attributes and their default values (unflattened) + self.default_config = default_config + # used to make sure every attribute value is checked + self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()} + + def create_mapping(self, config): + """ + Create a dictionary where the keys are the name of attributes (using double underscore convention) + For example: authentication__type + + The values are a tuple, + - the first item of the tuple is a tuple of path elements (location in config 'tree') + - the second item is the value of the config parameter + + For example: (('authentication', 'type'), 'session')) + """ + config_copy = copy.deepcopy(config) + mapping = {} + + flat_config = flatten(config_copy) + for key, value in flat_config.items(): + # name of the attribute + attr = "__".join(key) + mapping[attr] = (key, value) + + return mapping + + def validate_correct_type_of_configuration_attribute(self, attrname, vtype): + val = getattr(self, attrname) + if type(vtype) in (list, tuple): + if type(val) not in vtype: + tnames = ",".join([x.__name__ for x in vtype]) + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" + ) + else: + if type(val) != vtype: + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, " + f"expected type {vtype.__name__}, got {type(val).__name__}" + ) + + self.attr_checked[attrname] = True + + def check_config(self): + mapping = self.create_mapping(self.default_config) + for key in mapping.keys(): + if not self.attr_checked[key]: + raise ConfigurationError(f"The attr '{key}' has not been checked") + + def update(self, **kw): + """Update the attributes defined in kw with their new values.""" + for key, value in kw.items(): + if not hasattr(self, key): + raise ConfigurationError(f"unknown config parameter {key}.") + try: + if type(value) == tuple: + # convert tuple values to list values + value = list(value) + setattr(self, key, value) + except KeyError: + raise ConfigurationError(f"Unable to set config parameter {key}.") + + self.attr_checked[key] = False + + def update_from_config(self, config, prefix): + mapping = self.create_mapping(config) + for attr, (key, value) in mapping.items(): + if not hasattr(self, attr): + raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") + setattr(self, attr, value) + + self.attr_checked[attr] = False + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + mapping = self.create_mapping(self.default_config) + diff = [] + for attrname, (key, defval) in mapping.items(): + curval = getattr(self, attrname) + if curval != defval: + diff.append((attrname, curval, defval)) + return diff diff --git a/local_server/common/config/client_config.py b/local_server/common/config/client_config.py new file mode 100644 index 00000000..102a5aae --- /dev/null +++ b/local_server/common/config/client_config.py @@ -0,0 +1,123 @@ +from local_server import display_version as cellxgene_display_version + + +def get_client_config(app_config, data_adaptor): + """ + Return the configuration as required by the /config REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + annotation = dataset_config.user_annotations + auth = server_config.auth + + # FIXME The current set of config is not consistently presented: + # we have camalCase, hyphen-text, and underscore_text + + # make sure the configuration has been checked. + app_config.check_config() + + # display_names + title = app_config.get_title(data_adaptor) + about = app_config.get_about(data_adaptor) + + display_names = dict(engine=data_adaptor.get_name(), dataset=title) + + # library_versions + library_versions = {} + library_versions.update(data_adaptor.get_library_versions()) + library_versions["cellxgene"] = cellxgene_display_version + + # links + links = {"about-dataset": about} + + # parameters + parameters = { + "layout": dataset_config.embeddings__names, + "max-category-items": dataset_config.presentation__max_categories, + "obs_names": server_config.single_dataset__obs_names, + "var_names": server_config.single_dataset__var_names, + "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, + "backed": server_config.adaptor__anndata_adaptor__backed, + "disable-diffexp": not dataset_config.diffexp__enable, + "enable-reembedding": dataset_config.embeddings__enable_reembedding, + "annotations": False, + "annotations_file": None, + "annotations_dir": None, + "annotations_genesets": True, # feature flag + "annotations_genesets_readonly": dataset_config.user_annotations__gene_sets__readonly, + "annotations_genesets_summary_methods": ["mean"], + "annotations_cell_ontology_enabled": False, + "annotations_cell_ontology_obopath": None, + "annotations_cell_ontology_terms": None, + "custom_colors": dataset_config.presentation__custom_colors, + "diffexp-may-be-slow": False, + } + + # corpora dataset_props + # TODO/Note: putting info from the dataset into the /config is not ideal. + # However, it is definitely not part of /schema, and we do not have a top-level + # route for data properties. Consider creating one at some point. + corpora_props = data_adaptor.get_corpora_props() + if corpora_props and "default_embedding" in corpora_props: + default_embedding = corpora_props["default_embedding"] + if isinstance(default_embedding, str) and default_embedding.startswith("X_"): + default_embedding = default_embedding[2:] # drop X_ prefix + if default_embedding in data_adaptor.get_embedding_names(): + parameters["default_embedding"] = default_embedding + + data_adaptor.update_parameters(parameters) + if annotation: + annotation.update_parameters(parameters, data_adaptor) + + # gather it all together + client_config = {} + config = client_config["config"] = {} + config["displayNames"] = display_names + config["library_versions"] = library_versions + config["links"] = links + config["parameters"] = parameters + config["corpora_props"] = corpora_props + config["limits"] = { + "column_request_max": server_config.limits__column_request_max, + "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, + } + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + config["authentication"] = { + "requires_client_login": auth.requires_client_login(), + } + if auth.requires_client_login(): + config["authentication"].update( + { + # Todo why are these stored on the data_adaptor? + "login": auth.get_login_url(data_adaptor), + "logout": auth.get_logout_url(data_adaptor), + } + ) + + return client_config + + +def get_client_userinfo(app_config, data_adaptor): + """ + Return the userinfo as required by the /userinfo REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + auth = server_config.auth + + # make sure the configuration has been checked. + app_config.check_config() + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + userinfo = {} + userinfo["userinfo"] = { + "is_authenticated": auth.is_user_authenticated(), + "username": auth.get_user_name(), + "user_id": auth.get_user_id(), + "email": auth.get_user_email(), + "picture": auth.get_user_picture(), + } + return userinfo diff --git a/local_server/common/config/dataset_config.py b/local_server/common/config/dataset_config.py new file mode 100644 index 00000000..b8f2db55 --- /dev/null +++ b/local_server/common/config/dataset_config.py @@ -0,0 +1,222 @@ +import os +from os.path import splitext, isdir + +from local_server.common.annotations.local_file_csv import AnnotationsLocalFile +from local_server.common.config.base_config import BaseConfig +from local_server.common.errors import ConfigurationError, OntologyLoadFailure, AnnotationsError +from local_server.compute.scanpy import get_scanpy_module +from local_server.data_common.matrix_loader import MatrixDataLoader + + +class DatasetConfig(BaseConfig): + """Manages the config attribute associated with a dataset.""" + + def __init__(self, tag, app_config, default_config): + super().__init__(app_config, default_config) + self.tag = tag + try: + self.app__scripts = default_config["app"]["scripts"] + self.app__inline_scripts = default_config["app"]["inline_scripts"] + self.app__authentication_enable = default_config["app"]["authentication_enable"] + + self.presentation__max_categories = default_config["presentation"]["max_categories"] + self.presentation__custom_colors = default_config["presentation"]["custom_colors"] + + self.user_annotations__enable = default_config["user_annotations"]["enable"] + self.user_annotations__type = default_config["user_annotations"]["type"] + self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][ + "directory" + ] + self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"] + self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"] + self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][ + "obo_location" + ] + self.user_annotations__gene_sets__readonly = default_config["user_annotations"]["gene_sets"]["readonly"] + self.user_annotations__local_file_csv__gene_sets_file = default_config["user_annotations"]["local_file_csv"][ + "gene_sets_file" + ] + + self.embeddings__names = default_config["embeddings"]["names"] + self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"] + + self.diffexp__enable = default_config["diffexp"]["enable"] + self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] + self.diffexp__top_n = default_config["diffexp"]["top_n"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The annotation object is created during complete_config and stored here. + self.user_annotations = None + + def complete_config(self, context): + self.handle_app() + self.handle_presentation() + self.handle_user_annotations(context) + self.handle_embeddings() + self.handle_diffexp(context) + + def get_data_adaptor(self): + server_config = self.app_config.server_config + if not server_config.data_adaptor: + matrix_data_loader = MatrixDataLoader(server_config.single_dataset__datapath, app_config=self.app_config) + server_config.data_adaptor = matrix_data_loader.open(self.app_config) + + return server_config.data_adaptor + + def handle_app(self): + self.validate_correct_type_of_configuration_attribute("app__scripts", list) + self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list) + self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool) + + # scripts can be string (filename) or dict (attributes). Convert string to dict. + scripts = [] + for script in self.app__scripts: + try: + if isinstance(script, str): + scripts.append({"src": script}) + elif isinstance(script, dict) and isinstance(script["src"], str): + scripts.append(script) + else: + raise Exception + except Exception as e: + raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}") + + self.app__scripts = scripts + + def handle_presentation(self): + self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int) + self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool) + + def handle_user_annotations(self, context): + self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool) + self.validate_correct_type_of_configuration_attribute("user_annotations__type", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__directory", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__file", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__gene_sets_file", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__ontology__obo_location", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute("user_annotations__gene_sets__readonly", bool) + + if self.user_annotations__enable or not self.user_annotations__gene_sets__readonly: + server_config = self.app_config.server_config + if not self.app__authentication_enable: + raise ConfigurationError("user annotations requires authentication to be enabled") + if not server_config.auth.is_valid_authentication_type(): + auth_type = server_config.authentication__type + raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") + + # Must always have an annotations instance to support genesets. User annotation (cell labels) are optional + # as are writable gene sets + if self.user_annotations__type == "local_file_csv": + self.handle_local_file_csv_annotations(context) + else: + raise ConfigurationError('The only annotation type support is "local_file_csv"') + + if self.user_annotations__enable: + if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: + try: + self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) + except OntologyLoadFailure as e: + raise ConfigurationError("Unable to load ontology terms\n" + str(e)) + + self.check_annotation_config_vars_not_set(context) + + def handle_local_file_csv_annotations(self, context): + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + genesets_filename = self.user_annotations__local_file_csv__gene_sets_file + + if dirname is not None and (filename is not None or genesets_filename is not None): + raise ConfigurationError( + "'user-generated-data-dir' may not be used with annotations-file' or 'genesets-file'." + ) + + if filename is not None: + lf_name, lf_ext = splitext(filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"annotation file type must be .csv: {filename}") + + if genesets_filename is not None: + lf_name, lf_ext = splitext(genesets_filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}") + + if dirname is not None and not isdir(dirname): + try: + os.mkdir(dirname) + except OSError: + raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir") + + anno_config = { + "user-annotations": self.user_annotations__enable, + "genesets-save": not self.user_annotations__gene_sets__readonly, + } + self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename, genesets_filename) + + # if the user has specified a fixed label file, go ahead and validate it + # so that we can remove errors early in the process. + server_config = self.app_config.server_config + if server_config.single_dataset__datapath: + data_adaptor = self.get_data_adaptor() + if self.user_annotations__local_file_csv__file: + data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + if self.user_annotations__local_file_csv__gene_sets_file: + try: + data_adaptor.check_new_gene_sets(self.user_annotations.read_gene_sets(data_adaptor, context), context) + except (ValueError, AnnotationsError, KeyError) as e: + raise ConfigurationError(f"Unable to read genesets CSV file: {str(e)}") from e + + def check_annotation_config_vars_not_set(self, context): + if self.user_annotations__type is not None: + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + if not self.user_annotations__enable: + if filename is not None: + context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.") + if self.user_annotations__ontology__enable: + context["messagefn"]( + "Warning: --experimental-annotations-ontology ignored as annotations are disabled." + ) + if self.user_annotations__ontology__obo_location is not None: + context["messagefn"]( + "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled." + ) + if dirname is not None: + context["messagefn"]("Warning: --user-generated-data-dir ignored as annotations are disabled.") + + def handle_embeddings(self): + self.validate_correct_type_of_configuration_attribute("embeddings__names", list) + self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool) + + server_config = self.app_config.server_config + if self.embeddings__enable_reembedding: + if server_config.single_dataset__datapath: + if server_config.adaptor__anndata_adaptor__backed: + raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") + + try: + get_scanpy_module() + except NotImplementedError: + # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported + raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") + + def handle_diffexp(self, context): + self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool) + self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float) + self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int) + + data_adaptor = self.get_data_adaptor() + if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): + context["messagefn"]( + "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail." + ) diff --git a/local_server/common/config/external_config.py b/local_server/common/config/external_config.py new file mode 100644 index 00000000..19d29a8e --- /dev/null +++ b/local_server/common/config/external_config.py @@ -0,0 +1,96 @@ +import os + +from local_server.common.config.base_config import BaseConfig +from local_server.common.errors import ConfigurationError +from local_server.common.config import get_secret_key +from local_server.common.errors import SecretKeyRetrievalError +from local_server.common.utils.type_conversion_utils import convert_string_to_value + + +class ExternalConfig(BaseConfig): + """Manages the config attribute associated with external configuration sources, such as + environment variables or the AWS Secrets Manager.""" + + def __init__(self, app_config, default_config): + super().__init__(app_config, default_config) + try: + self.environment = default_config["environment"] + self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"] + self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + def complete_config(self, context): + self.handle_environment(context) + self.handle_aws_secrets_manager(context) + + def handle_environment(self, context): + """For each environment variable defined, get the value (if it is set), + and set the specified config parameter""" + self.validate_correct_type_of_configuration_attribute("environment", list) + for envdict in self.environment: + name = envdict.get("name") + if name is None: + raise ConfigurationError("environment: 'name' is missing") + required = envdict.get("required", False) + if type(required) != bool: + raise ConfigurationError("environment: 'required' must be a bool") + path = envdict.get("path") + if path is None: + raise ConfigurationError("environment: 'path' is missing") + + value = os.environ.get(name) + if value is None: + if required: + raise ConfigurationError(f"required environment variable '{name}' not set") + else: + value = convert_string_to_value(value) + self.app_config.update_single_config_from_path_and_value(path, value) + + def handle_aws_secrets_manager(self, context): + """For each aws secret defined, get the key/values, and set the specified config parameter""" + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list) + + if not self.aws_secrets_manager__secrets: + return + + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str) + + for secret in self.aws_secrets_manager__secrets: + secret_name = secret.get("name") + if secret_name is None: + raise ConfigurationError("aws_secrets_manager: 'name' is missing") + if not isinstance(secret_name, str): + raise ConfigurationError("aws_secrets_manager: 'name' must be a string") + + try: + secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name) + except SecretKeyRetrievalError as e: + raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}") + + values = secret.get("values") + if values is None: + raise ConfigurationError("aws_secrets_manager: 'values' is missing") + if not isinstance(values, list): + raise ConfigurationError("aws_secrets_manager: 'values' must be a list") + + for value in values: + key = value.get("key") + if key is None: + raise ConfigurationError(f"missing 'key' in secret values: {secret_name}") + path = value.get("path") + if path is None: + raise ConfigurationError(f"missing 'path' in secret values: {secret_name}") + required = value.get("required", False) + if type(required) != bool: + raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}") + + secret_value = secret_dict.get(key) + if secret_value is None: + if required: + raise ConfigurationError(f"required secret '{secret_name}:{key}' not set") + else: + secret_value = convert_string_to_value(secret_value) + self.app_config.update_single_config_from_path_and_value(path, secret_value) diff --git a/local_server/common/config/server_config.py b/local_server/common/config/server_config.py new file mode 100644 index 00000000..210c9667 --- /dev/null +++ b/local_server/common/config/server_config.py @@ -0,0 +1,183 @@ +import os +import sys +import warnings +from os.path import basename +from urllib.parse import urlparse + +from local_server.auth.auth import AuthTypeFactory +from local_server.common.config.base_config import BaseConfig +from local_server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD +from local_server.common.errors import ConfigurationError, DatasetAccessError +from local_server.common.data_locator import discover_s3_region_name +from local_server.common.utils.utils import is_port_available, find_available_port, custom_format_warning +from local_server.data_common.matrix_loader import MatrixDataLoader + + +class ServerConfig(BaseConfig): + """Manages the config attribute associated with the server.""" + + def __init__(self, app_config, default_config): + super().__init__(app_config, default_config) + + try: + self.app__verbose = default_config["app"]["verbose"] + self.app__debug = default_config["app"]["debug"] + self.app__host = default_config["app"]["host"] + self.app__port = default_config["app"]["port"] + self.app__open_browser = default_config["app"]["open_browser"] + self.app__force_https = default_config["app"]["force_https"] + self.app__flask_secret_key = default_config["app"]["flask_secret_key"] + + self.authentication__type = default_config["authentication"]["type"] + self.authentication__insecure_test_environment = default_config["authentication"][ + "insecure_test_environment" + ] + + self.single_dataset__datapath = default_config["single_dataset"]["datapath"] + self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"] + self.single_dataset__var_names = default_config["single_dataset"]["var_names"] + self.single_dataset__about = default_config["single_dataset"]["about"] + self.single_dataset__title = default_config["single_dataset"]["title"] + + self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"] + + self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"] + + self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"] + self.limits__column_request_max = default_config["limits"]["column_request_max"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + self.data_adaptor = None + + # The authentication object + self.auth = None + + def complete_config(self, context): + self.handle_app(context) + self.handle_data_source() + self.handle_authentication() + self.handle_data_locator() + self.handle_adaptor() # may depend on data_locator + self.handle_single_dataset(context) # may depend on adaptor + self.handle_limits() + + self.check_config() + + def handle_app(self, context): + self.validate_correct_type_of_configuration_attribute("app__verbose", bool) + self.validate_correct_type_of_configuration_attribute("app__debug", bool) + self.validate_correct_type_of_configuration_attribute("app__host", str) + self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("app__open_browser", bool) + self.validate_correct_type_of_configuration_attribute("app__force_https", bool) + self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str) + + if self.app__port: + try: + if not is_port_available(self.app__host, self.app__port): + raise ConfigurationError( + f"The port selected {self.app__port} is in use, please configure an open port." + ) + except OverflowError: + raise ConfigurationError(f"Invalid port: {self.app__port}") + else: + try: + default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) + except ValueError: + raise ConfigurationError( + "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") + ) + try: + self.app__port = find_available_port(self.app__host, default_server_port) + except OverflowError: + raise ConfigurationError(f"Invalid port: {default_server_port}") + + if self.app__debug: + context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") + self.app__verbose = True + self.app__open_browser = False + else: + warnings.formatwarning = custom_format_warning + + if not self.app__verbose: + sys.tracebacklimit = 0 + + def handle_authentication(self): + self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool) + + if self.authentication__type == "test" and not self.authentication__insecure_test_environment: + raise ConfigurationError("Test auth can only be used in an insecure test environment") + + self.auth = AuthTypeFactory.create(self.authentication__type, self) + if self.auth is None: + raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") + + def handle_data_locator(self): + self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str)) + if self.data_locator__s3__region_name is True: + path = self.single_dataset__datapath + + if path.startswith("s3://"): + region_name = discover_s3_region_name(path) + if region_name is None: + raise ConfigurationError(f"Unable to discover s3 region name from {path}") + else: + region_name = None + self.data_locator__s3__region_name = region_name + + def handle_data_source(self): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", str) + + def handle_single_dataset(self, context): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None))) + + # preload this data set + matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) + try: + matrix_data_loader.pre_load_validation() + except DatasetAccessError as e: + raise ConfigurationError(str(e)) + + file_size = matrix_data_loader.file_size() + file_basename = basename(self.single_dataset__datapath) + if file_size > BIG_FILE_SIZE_THRESHOLD: + context["messagefn"](f"Loading data from {file_basename}, this may take a while...") + else: + context["messagefn"](f"Loading data from {file_basename}.") + + if self.single_dataset__about: + + def url_check(url): + try: + result = urlparse(url) + if all([result.scheme, result.netloc]): + return True + else: + return False + except ValueError: + return False + + if not url_check(self.single_dataset__about): + raise ConfigurationError( + "Must provide an absolute URL for --about. (Example format: http://example.com)" + ) + + def handle_adaptor(self): + self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool) + + def handle_limits(self): + self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int)) + + def exceeds_limit(self, limit_name, value): + limit_value = getattr(self, "limits__" + limit_name, None) + if limit_value is None: # disabled + return False + return value > limit_value diff --git a/local_server/common/constants.py b/local_server/common/constants.py new file mode 100644 index 00000000..a883e23a --- /dev/null +++ b/local_server/common/constants.py @@ -0,0 +1,30 @@ +from enum import Enum + + +class AugmentedEnum(Enum): + def __hash__(self): + return self.value.__hash__() + + def __eq__(self, other): + if isinstance(other, type(self)) or isinstance(other, str): + return self.value == other + return False + + def __str__(self) -> str: + return self.value + + +class Axis(AugmentedEnum): + OBS = "obs" + VAR = "var" + + +class DiffExpMode(AugmentedEnum): + TOP_N = "topN" + VAR_FILTER = "varFilter" + + +JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)" +REACTIVE_LIMIT = 1_000_000 + +MAX_LAYOUTS = 30 diff --git a/local_server/common/corpora.py b/local_server/common/corpora.py new file mode 100644 index 00000000..e9ed5240 --- /dev/null +++ b/local_server/common/corpora.py @@ -0,0 +1,78 @@ +""" +Corpora schema conventions support. Helper functions for reading. + +https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md + +https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md +""" +import collections +import json + +from local_server.cli.upgrade import validate_version_str +from local_server.common.utils.corpora_constants import CorporaConstants + + +def corpora_get_versions_from_anndata(adata): + """ + Given an AnnData object, return: + * None - if not a Corpora object + * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object + + Implements the identification protocol defined in the specification. + """ + + # per Corpora AnnData spec, this is a corpora file if the following is true + if "version" not in adata.uns_keys(): + return None + version = adata.uns["version"] + if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version: + return None + + corpora_schema_version = version.get("corpora_schema_version") + corpora_encoding_version = version.get("corpora_encoding_version") + + # TODO: spec says these must be SEMVER values, so check. + if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version): + return [corpora_schema_version, corpora_encoding_version] + + +def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version): + return ( + corpora_schema_version + and corpora_encoding_version + and corpora_schema_version.startswith("1.") + and corpora_encoding_version.startswith("0.1.") + ) + + +def corpora_get_props_from_anndata(adata): + """ + Get Corpora dataset properties from an AnnData + """ + versions = corpora_get_versions_from_anndata(adata) + if versions is None: + return None + [corpora_schema_version, corpora_encoding_version] = versions + version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version) + if not version_is_supported: + raise ValueError("Unsupported Corpora schema version") + + corpora_props = {} + for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS: + if key not in adata.uns: + raise KeyError(f"missing Corpora schema field {key}") + corpora_props[key] = adata.uns[key] + + for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD: + if key not in adata.uns: + continue + try: + corpora_props[key] = json.loads(adata.uns[key]) + except json.JSONDecodeError: + raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string") + + for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS: + if key in adata.uns: + corpora_props[key] = adata.uns[key] + + return corpora_props diff --git a/local_server/common/data_locator.py b/local_server/common/data_locator.py new file mode 100644 index 00000000..bf47d724 --- /dev/null +++ b/local_server/common/data_locator.py @@ -0,0 +1,154 @@ +import os +import tempfile +import fsspec +from datetime import datetime +import boto3 +import botocore +from urllib.parse import urlparse + + +class DataLocator: + """ + DataLocator is a simple wrapper around fsspec functionality, and provides a + set of functions to encapsulate a data location (URI or path), interogate + metadata about the object at that location (size, existance, etc) and + access the underlying data. + + https://filesystem-spec.readthedocs.io/en/latest/index.html + + Example: + dl = DataLocator("/tmp/foo.h5ad") + if dl.exists(): + print(dl.size()) + with dl.open() as f: + thecontents = f.read() + + DataLocator will accept a URI or native path. Error handling is as defined + in fsspec. + + """ + + def __init__(self, uri_or_path, region_name=None): + if isinstance(uri_or_path, DataLocator): + locator = uri_or_path + self.uri_or_path = locator.uri_or_path + self.protocol = locator.protocol + self.path = locator.path + self.cname = locator.cname + else: + self.uri_or_path = uri_or_path + self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) + # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol + self.cname = self.path if self.protocol == "file" else self.uri_or_path + + # fsspec.filesystem will throw RuntimeError if the protocol is unsupported + if self.protocol == "s3": + if region_name: + config_kwargs = dict(region_name=region_name) + self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs) + else: + self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30) + else: + self.fs = fsspec.filesystem(self.protocol) + + def __repr__(self): + return f"DataLocator(protocol={self.protocol}, cname={self.cname}, " + f"path={self.path}, uri_or_path={self.uri_or_path})" + + @staticmethod + def _get_protocol_and_path(uri_or_path): + if "://" in uri_or_path: + protocol, path = uri_or_path.split("://", 1) + # windows!!! Ignore single letter drive identifiers, + # eg, G:\foo.txt + if len(protocol) > 1: + return protocol, path + return None, uri_or_path + + def exists(self): + return self.fs.exists(self.cname) + + def size(self): + return self.fs.size(self.cname) + + def lastmodtime(self): + """ return datetime object representing last modification time, or None if unavailable """ + info = self.fs.info(self.cname) + if self.islocal() and info is not None: + return datetime.fromtimestamp(info["mtime"]) + else: + return getattr(info, "LastModified", None) + + def abspath(self): + """ + return the absolute path for the locator - only really does something + for file: protocol, as all others are already absolute + """ + if self.islocal(): + return os.path.abspath(self.path) + else: + return self.uri_or_path + + def isfile(self): + return self.fs.isfile(self.cname) + + def open(self, *args): + return self.fs.open(self.uri_or_path, *args) + + def islocal(self): + return self.protocol is None or self.protocol == "file" + + def local_handle(self): + if self.islocal(): + return LocalFilePath(self.path) + + # if not local, create a tmp file system object to contain the data, + # and clean it up when done. If the path has a suffix/extension, + # do our best to create a file with the same. + ext = os.path.splitext(self.path) + suffix = None if ext[1] == "" else ext[1] + with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp: + tmp.write(src.read()) + tmp.close() + src.close() + tmp_path = tmp.name + return LocalFilePath(tmp_path, delete=True) + + def ls(self): + paths = self.fs.ls(self.uri_or_path) + return [os.path.basename(p) for p in paths] + + +class LocalFilePath: + def __init__(self, tmp_path, delete=False): + self.tmp_path = tmp_path + self.delete = delete + + def __enter__(self): + return self.tmp_path + + def __exit__(self, *args): + if self.delete: + os.unlink(self.tmp_path) + + +def discover_s3_region_name(uri): + """If this is an s3 protocol, discover and return the (aws) region name. + If a return name could not be discovered, or if the uri is not an s3 protocol, return None.""" + + protocol, _ = DataLocator._get_protocol_and_path(uri) + if protocol == "s3": + bucket = urlparse(uri).netloc + client = boto3.client("s3") + try: + res = client.head_bucket(Bucket=bucket) + except botocore.exceptions.ClientError: + return None + + region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region") + if region: + return region + else: + return None + + return None diff --git a/local_server/common/errors.py b/local_server/common/errors.py new file mode 100644 index 00000000..b582a43f --- /dev/null +++ b/local_server/common/errors.py @@ -0,0 +1,58 @@ +from http import HTTPStatus + + +class CellxgeneException(Exception): + """Base class for cellxgene exceptions""" + + def __init__(self, message): + self.message = message + super().__init__(message) + + +class RequestException(CellxgeneException): + """Baseclass for exceptions that can be raised from a request.""" + + # The default status code is 400 (Bad Request) + default_status_code = HTTPStatus.BAD_REQUEST + + def __init__(self, message, status_code=None): + super().__init__(message) + self.status_code = status_code or self.default_status_code + + +def define_exception(name, doc): + globals()[name] = type(name, (CellxgeneException,), dict(__doc__=doc)) + + +def define_request_exception(name, doc, default_status_code=HTTPStatus.BAD_REQUEST): + globals()[name] = type(name, (RequestException,), dict(__doc__=doc, default_status_code=default_status_code)) + + +define_request_exception("FilterError", "Raised when filter is malformed") +define_request_exception("JSONEncodingValueError", "Raised when data cannot be encoded into json") +define_request_exception("MimeTypeError", "Raised when incompatible MIME type selected") +define_request_exception("DatasetAccessError", "Raised when file loaded into a DataAdaptor is misformatted") +define_request_exception("DisabledFeatureError", "Raised when an attempt to use a disabled feature occurs") +define_request_exception("AnnotationsError", "Raised when an attempt to use the annotations feature fails") +define_request_exception( + "ComputeError", + "Raised when an error occurs during a compute algorithm (such as diffexp)", + HTTPStatus.INTERNAL_SERVER_ERROR, +) +define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota") +define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format") +define_request_exception( + "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED +) + +define_request_exception( + "AnnotationCategoryNameError", + "Raised when an annotation category name cant be saved", + default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY, +) + +define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails") +define_exception("ConfigurationError", "Raised when checking configuration errors") +define_exception("PrepareError", "Raised when data is misprepared") +define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails") +define_exception("ObsoleteRequest", "Raised when the request is no longer valid.") \ No newline at end of file diff --git a/local_server/common/health.py b/local_server/common/health.py new file mode 100644 index 00000000..8916d95b --- /dev/null +++ b/local_server/common/health.py @@ -0,0 +1,33 @@ +from http import HTTPStatus +from flask import make_response, jsonify + +from local_server import __version__ as cellxgene_version +from local_server.common.data_locator import DataLocator + + +def _is_accessible(path, config): + if path is None: + return True + + try: + dl = DataLocator(path, region_name=config.data_locator__s3__region_name) + return dl.exists() + except RuntimeError: + return False + + +def health_check(config): + """ + simple health check - return HTTP response. + See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html + """ + health = {"status": None, "version": "1", "releaseID": cellxgene_version} + + server_config = config.server_config + check = _is_accessible(server_config.single_dataset__datapath, server_config) + + health["status"] = "pass" if check else "fail" + code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST + response = make_response(jsonify(health), code) + response.headers["Content-Type"] = "application/health+json" + return response diff --git a/local_server/common/rest.py b/local_server/common/rest.py new file mode 100644 index 00000000..63e18f1f --- /dev/null +++ b/local_server/common/rest.py @@ -0,0 +1,382 @@ +import copy +import logging +import sys +from http import HTTPStatus +import zlib + +from flask import make_response, jsonify, current_app, abort +from werkzeug.urls import url_unquote + +from local_server.common.config.client_config import get_client_config, get_client_userinfo +from local_server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg +from local_server.common.errors import ( + FilterError, + JSONEncodingValueError, + PrepareError, + DisabledFeatureError, + ExceedsLimitError, + DatasetAccessError, + ColorFormatException, + AnnotationsError, + ObsoleteRequest, +) + +import json +from local_server.data_common.fbs.matrix import decode_matrix_fbs + + +def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False): + """ + Log the message, then abort with HTTP code. If include_exc_info is true, + also include current exception via sys.exc_info(). + """ + if include_exc_info: + exc_info = sys.exc_info() + else: + exc_info = False + current_app.logger.log(loglevel, logmsg, exc_info=exc_info) + # Do NOT send log message to HTTP response. + return abort(code) + + +def _query_parameter_to_filter(args): + """ + Convert an annotation value filter, if present in the query args, + into the standard dict filter format used by internal code. + + Query param filters look like: :name=value, where value + may be one of: + - a range, min,max, where either may be an open range by using an asterisk, eg, 10,* + - a value + Eg, + ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,* + """ + filters = { + "obs": {}, + "var": {}, + } + + # args has already been url-unquoted once. We assume double escaping + # on name and value. + try: + for key, value in args.items(multi=True): + axis, name = key.split(":") + if axis not in ("obs", "var"): + raise FilterError("unknown filter axis") + name = url_unquote(name) + current = filters[axis].setdefault(name, {"name": name}) + + val_split = value.split(",") + if len(val_split) == 1: + if "min" in current or "max" in current: + raise FilterError("do not mix range and value filters") + value = url_unquote(value) + values = current.setdefault("values", []) + values.append(value) + + elif len(val_split) == 2: + if len(current) > 1: + raise FilterError("duplicate range specification") + min = url_unquote(val_split[0]) + max = url_unquote(val_split[1]) + if min != "*": + current["min"] = float(min) + if max != "*": + current["max"] = float(max) + if len(current) < 2: + raise FilterError("must specify at least min or max in range filter") + + else: + raise FilterError("badly formated filter value") + + except ValueError as e: + raise FilterError(str(e)) + + result = {} + for axis in ("obs", "var"): + axis_filter = filters[axis] + if len(axis_filter) > 0: + result[axis] = {"annotation_value": [val for val in axis_filter.values()]} + + return result + + +def schema_get_helper(data_adaptor): + """helper function to gather the schema from the data source and annotations""" + schema = data_adaptor.get_schema() + schema = copy.deepcopy(schema) + + # add label obs annotations as needed + annotations = data_adaptor.dataset_config.user_annotations + if annotations.user_annotations_enabled(): + label_schema = annotations.get_schema(data_adaptor) + schema["annotations"]["obs"]["columns"].extend(label_schema) + + return schema + + +def schema_get(data_adaptor): + schema = schema_get_helper(data_adaptor) + return make_response(jsonify({"schema": schema}), HTTPStatus.OK) + + +def config_get(app_config, data_adaptor): + config = get_client_config(app_config, data_adaptor) + return make_response(jsonify(config), HTTPStatus.OK) + + +def userinfo_get(app_config, data_adaptor): + config = get_client_userinfo(app_config, data_adaptor) + return make_response(jsonify(config), HTTPStatus.OK) + + +def annotations_obs_get(request, data_adaptor): + fields = request.args.getlist("annotation-name", None) + num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields) + if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested): + return abort(HTTPStatus.BAD_REQUEST) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + labels = None + annotations = data_adaptor.dataset_config.user_annotations + if annotations.user_annotations_enabled(): + labels = annotations.read_labels(data_adaptor) + fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels) + return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) + except KeyError as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def annotations_put_fbs_helper(data_adaptor, fbs): + """helper function to write annotations from fbs""" + annotations = data_adaptor.dataset_config.user_annotations + if not annotations.user_annotations_enabled(): + raise DisabledFeatureError("Writable annotations are not enabled") + + new_label_df = decode_matrix_fbs(fbs) + if not new_label_df.empty: + new_label_df = data_adaptor.check_new_labels(new_label_df) + annotations.write_labels(new_label_df, data_adaptor) + + +def inflate(data): + return zlib.decompress(data) + + +def annotations_obs_put(request, data_adaptor): + annotations = data_adaptor.dataset_config.user_annotations + if not annotations.user_annotations_enabled(): + return abort(HTTPStatus.NOT_IMPLEMENTED) + + anno_collection = request.args.get("annotation-collection-name", default=None) + fbs = inflate(request.get_data()) + + if anno_collection is not None: + if not annotations.is_safe_collection_name(anno_collection): + return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name") + annotations.set_collection(anno_collection) + + try: + annotations_put_fbs_helper(data_adaptor, fbs) + res = json.dumps({"status": "OK"}) + return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"}) + except (ValueError, DisabledFeatureError, KeyError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def annotations_var_get(request, data_adaptor): + fields = request.args.getlist("annotation-name", None) + num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields) + if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested): + return abort(HTTPStatus.BAD_REQUEST) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + labels = None + return make_response( + data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) + except KeyError as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def data_var_put(request, data_adaptor): + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + filter_json = request.get_json() + filter = filter_json["filter"] if filter_json else None + try: + return make_response( + data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) + except (FilterError, ValueError, ExceedsLimitError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def data_var_get(request, data_adaptor): + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + filter = _query_parameter_to_filter(request.args) + return make_response( + data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) + except (FilterError, ValueError, ExceedsLimitError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def colors_get(data_adaptor): + if not data_adaptor.dataset_config.presentation__custom_colors: + return make_response(jsonify({}), HTTPStatus.OK) + try: + return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK) + except ColorFormatException as e: + return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True) + + +def diffexp_obs_post(request, data_adaptor): + if not data_adaptor.dataset_config.diffexp__enable: + return abort(HTTPStatus.NOT_IMPLEMENTED) + + args = request.get_json() + try: + # TODO: implement varfilter mode + mode = DiffExpMode(args["mode"]) + + if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: + return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled") + + set1_filter = args.get("set1", {"filter": {}})["filter"] + set2_filter = args.get("set2", {"filter": {}})["filter"] + count = args.get("count", None) + + if set1_filter is None or set2_filter is None or count is None: + return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter") + if Axis.VAR in set1_filter or Axis.VAR in set2_filter: + return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled") + + except (KeyError, TypeError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + try: + diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count) + return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}) + except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + except JSONEncodingValueError: + # JSON encoding failure, usually due to bad data. Just let it ripple up + # to default exception handler. + current_app.logger.warning(JSON_NaN_to_num_warning_msg) + raise + + +def layout_obs_get(request, data_adaptor): + fields = request.args.getlist("layout-name", None) + num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields) + if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested): + return abort(HTTPStatus.BAD_REQUEST) + + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + return make_response( + data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"} + ) + except (KeyError, DatasetAccessError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + except PrepareError: + return abort_and_log( + HTTPStatus.NOT_IMPLEMENTED, + f"No embedding available {request.path}", + loglevel=logging.ERROR, + include_exc_info=True, + ) + + +def layout_obs_put(request, data_adaptor): + if not data_adaptor.dataset_config.embeddings__enable_reembedding: + return abort(HTTPStatus.NOT_IMPLEMENTED) + + args = request.get_json() + filter = args["filter"] if args else None + if not filter: + return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required") + method = args["method"] if args else "umap" + + try: + schema = data_adaptor.compute_embedding(method, filter) + return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"}) + except NotImplementedError as e: + return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e)) + except (ValueError, DisabledFeatureError, FilterError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def genesets_get(request, data_adaptor): + preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"]) + if preferred_mimetype not in ("application/json", "text/csv"): + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + annotations = data_adaptor.dataset_config.user_annotations + (genesets, tid) = data_adaptor.check_new_gene_sets(annotations.read_gene_sets(data_adaptor)) + + if preferred_mimetype == "text/csv": + return make_response( + annotations.gene_sets_to_csv(genesets), + HTTPStatus.OK, + { + "Content-Type": "text/csv", + "Content-Disposition": "attachment; filename=genesets.csv", + }, + ) + else: + return make_response( + jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK + ) + except (ValueError, KeyError, AnnotationsError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e)) + + +def genesets_put(request, data_adaptor): + annotations = data_adaptor.dataset_config.user_annotations + if not annotations.gene_sets_save_enabled(): + return abort(HTTPStatus.NOT_IMPLEMENTED) + + anno_collection = request.args.get("annotation-collection-name", default=None) + if anno_collection is not None: + if not annotations.is_safe_collection_name(anno_collection): + return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name") + annotations.set_collection(anno_collection) + + args = request.get_json() + try: + genesets = args.get("genesets", None) + tid = args.get("tid", None) + if genesets is None: + abort(HTTPStatus.BAD_REQUEST) + + (gs, _) = data_adaptor.check_new_gene_sets((genesets, tid)) + annotations.write_gene_sets(gs, tid, data_adaptor) + return make_response(jsonify({"status": "OK"}), HTTPStatus.OK) + except (ValueError, DisabledFeatureError, KeyError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + except (ObsoleteRequest, TypeError) as e: + return abort(HTTPStatus.NOT_FOUND, description=str(e)) diff --git a/local_server/common/utils/__init__.py b/local_server/common/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/common/utils/corpora_constants.py b/local_server/common/utils/corpora_constants.py new file mode 100644 index 00000000..fa1641d5 --- /dev/null +++ b/local_server/common/utils/corpora_constants.py @@ -0,0 +1,22 @@ +class CorporaConstants(object): + REQUIRED_SIMPLE_METADATA_FIELDS = [ + "version", + "title", + "layer_descriptions", + "organism", + "organism_ontology_term_id", + ] + + # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex + # types. + OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"] + + OPTIONAL_SIMPLE_METADATA_FIELDS = [ + "preprint_doi", + "publication_doi", + "default_embedding", + "default_field", + "tags", + "project_name", + "project_description", + ] diff --git a/local_server/common/utils/type_conversion_utils.py b/local_server/common/utils/type_conversion_utils.py new file mode 100644 index 00000000..971bf641 --- /dev/null +++ b/local_server/common/utils/type_conversion_utils.py @@ -0,0 +1,158 @@ +import logging + +import numpy as np +import pandas as pd + + +def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame): + dtypes_by_column_name = {} + schema_type_hints_by_column_name = {} + + for column_name, column_values in dataframe.items(): + ( + dtypes_by_column_name[column_name], + schema_type_hints_by_column_name[column_name], + ) = get_dtype_and_schema_of_array(column_values) + + return dtypes_by_column_name, schema_type_hints_by_column_name + + +def get_dtype_of_array(array: pd.Series): + return get_dtype_and_schema_of_array(array)[0] + + +def get_schema_type_hint_of_array(array: pd.Series): + return get_dtype_and_schema_of_array(array)[1] + + +def get_dtype_and_schema_of_array(array: pd.Series): + return ( + get_dtype_from_dtype(array.dtype, array_values=array), + get_schema_type_hint_from_dtype(array.dtype, array_values=array), + ) + + +def get_dtype_from_dtype(dtype, array_values=None): + """ + Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant + for 64 bit values which will get downcast to 32 bit. + """ + + dtype_name = dtype.name + dtype_kind = dtype.kind + + if dtype_name == "bool": + return np.uint8 + if dtype_name == "object" and dtype_kind == "O": + return str + if dtype_name == "category": + return get_dtype_from_dtype(dtype.categories.dtype, array_values) + + if can_cast_to_int32(dtype, array_values): + return np.int32 + if can_cast_to_float32(dtype, array_values): + return np.float32 + if not can_cast_to_float32(dtype, array_values): + return np.float64 + + raise TypeError(f"Annotations of type {dtype} are unsupported.") + + +def get_schema_type_hint_from_dtype(dtype, array_values=None): + """ + Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit + and will be downcast to 32 bit. + """ + + dtype_name = dtype.name + dtype_kind = dtype.kind + + if dtype == np.float32 or dtype == np.int32: + return {"type": dtype_name} + if dtype_name == "bool": + return {"type": "boolean"} + if dtype_name == "object" and dtype_kind == "O": + return {"type": "string"} + if dtype_name == "category": + return {"type": "categorical", "categories": dtype.categories.tolist()} + + if can_cast_to_int32(dtype, array_values): + return {"type": "int32"} + if can_cast_to_float32(dtype, array_values): + return {"type": "float32"} + if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values): + return {"type": "float64"} + + raise TypeError(f"Annotations of type {dtype} are unsupported.") + + +def can_cast_to_float32(dtype, array_values): + """ + Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is + a float. + + We also handle a special case here where the array is a Series object with integer categorical values AND NaNs. + Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True. + """ + + if dtype.kind == "f": + if not np.can_cast(dtype, np.float32): + logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.") + + return True + + if dtype.kind == "O" and array_values.hasnans: + return True + + return False + + +def can_cast_to_int32(dtype, array_values=None): + """ + A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of + the higher precision type has values that are entirely within the range of the downcast type. + """ + + # Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately + # return False. + if array_values.hasnans: + return False + + # If the array is categorical, then we need to order the array values so that functions min and max that occur + # later, can function. They do not function on unordered categories. + ordered_array_values = array_values + if array_values.dtype.name == "category" and not array_values.cat.ordered: + ordered_array_values = array_values.cat.as_ordered() + + if dtype.kind in ["i", "u"]: + if np.can_cast(dtype, np.int32): + return True + ii32 = np.iinfo(np.int32) + if ( + not ordered_array_values.empty + and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) + or ordered_array_values.empty + ): + return True + return False + + +def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype): + if series_to_convert.hasnans and dtype == np.int32: + logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.") + + return series_to_convert.to_numpy(dtype) + + +def convert_string_to_value(value: str): + """convert a string to value with the most appropriate type""" + if value.lower() == "true": + return True + if value.lower() == "false": + return False + if value == "null": + return None + try: + return eval(value) + except: # noqa E722 + return value diff --git a/local_server/common/utils/utils.py b/local_server/common/utils/utils.py new file mode 100644 index 00000000..909999c8 --- /dev/null +++ b/local_server/common/utils/utils.py @@ -0,0 +1,118 @@ +import contextlib +import errno +import importlib.util +import logging +import os +import pkgutil +import socket +from urllib.parse import urlsplit, urljoin + +import numpy as np +from flask import json + +from local_server.common.errors import ConfigurationError + + +def find_available_port(host, port=5005): + """ + Helper method to find open port on host. Tries 5000 ports incremented from the specified port + """ + # Takes approx 2 seconds to do a scan of 5000 ports on my laptop + num_ports_to_try = 5000 + for port_to_try in range(port, port + num_ports_to_try): + if is_port_available(host, port_to_try): + return port_to_try + raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.") + + +def is_port_available(host, port): + is_available = False + with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + try: + s.bind((host, port)) + is_available = True + except socket.error: + pass + return is_available + + +def sort_options(command): + """ + Helper for the click options - will sort options in a command, and can + be used as a decorator. + """ + command.params.sort(key=lambda p: p.name) + return command + + +def path_join(base, *urls): + """ + this is like urllib.parse.urljoin, except it works around the scheme-specific + cleverness in the aforementioned code, ignores anything in the url except the path, + and accepts more than one url. + """ + if not base.endswith("/"): + base += "/" + btpl = urlsplit(base) + path = btpl.path + for url in urls: + utpl = urlsplit(url) + if btpl.scheme == "": + path = os.path.join(path, utpl.path) + path = os.path.normpath(path) + else: + path = urljoin(path, utpl.path) + return btpl._replace(path=path).geturl() + + +class Float32JSONEncoder(json.JSONEncoder): + def __init__(self, *args, **kwargs): + """ + NaN/Infinities are illegal in standard JSON. Python extends JSON with + non-standard symbols that most JavaScript JSON parsers do not understand. + The `allow_nan` parameter will force Python simplejson to throw an ValueError + if it runs into non-finite floating point values which are unsupported by + standard JSON. + """ + kwargs["allow_nan"] = False + super().__init__(*args, **kwargs) + + def default(self, obj): + if isinstance(obj, np.float32): + return float(obj) + elif isinstance(obj, np.integer): + return int(obj) + return json.JSONEncoder.default(self, obj) + + +def custom_format_warning(msg, *args, **kwargs): + return f"[cellxgene] Warning: {msg} \n" + + +def jsonify_numpy(data): + return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False) + + +def import_plugins(plugin_module): + """ + Load optional plugin modules from local_server.common.plugins + + If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app. + This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as + normal. + """ + loaded_modules = [] + try: + pkg = importlib.import_module(plugin_module) + for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__): + full_name = f"{plugin_module}.{name}" + try: + module = importlib.import_module(full_name) + except Exception as e: + raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}") + loaded_modules.append(module) + except ModuleNotFoundError as e: + # This exception occurs when the plugin_module does not exist (not an error). + logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}") + + return loaded_modules diff --git a/local_server/common/web/__init__.py b/local_server/common/web/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/compute/__init__.py b/local_server/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/compute/diffexp_generic.py b/local_server/compute/diffexp_generic.py new file mode 100644 index 00000000..71fe9a59 --- /dev/null +++ b/local_server/compute/diffexp_generic.py @@ -0,0 +1,134 @@ +import numpy as np +from scipy import sparse, stats + + +def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): + """ + Return differential expression statistics for top N variables. + + Algorithm: + - compute log fold change (log2(meanA/meanB)) + - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction) + - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff + + If there are not N which meet criteria, augment by removing the logfoldchange + threshold requirement. + + Notes on alogrithm: + - Welch's ttest provides basic statistics test. + https://en.wikipedia.org/wiki/Welch%27s_t-test + - p-values adjusted with Bonferroni correction. + https://en.wikipedia.org/wiki/Bonferroni_correction + + :param adaptor: DataAdaptor instance + :param maskA: observation selection mask for set 1 + :param maskB: observation selection mask for set 2 + :param top_n: number of variables to return stats for + :param diffexp_lfc_cutoff: minimum + :return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ] + """ + + dataA = adaptor.get_X_array(maskA, None) + dataB = adaptor.get_X_array(maskB, None) + + # mean, variance, N - calculate for both selections + meanA, vA, nA = mean_var_n(dataA) + meanB, vB, nB = mean_var_n(dataB) + res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff) + + return res + + +def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff): + n_var = meanA.shape[0] + top_n = min(top_n, n_var) + + # variance / N + vnA = varA / min(nA, nB) # overestimate variance, would normally be nA + vnB = varB / min(nA, nB) # overestimate variance, would normally be nB + sum_vn = vnA + vnB + + # degrees of freedom for Welch's t-test + with np.errstate(divide="ignore", invalid="ignore"): + dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1)) + dof[np.isnan(dof)] = 1 + + # Welch's t-test score calculation + with np.errstate(divide="ignore", invalid="ignore"): + tscores = (meanA - meanB) / np.sqrt(sum_vn) + tscores[np.isnan(tscores)] = 0 + + # p-value + pvals = stats.t.sf(np.abs(tscores), dof) * 2 + pvals_adj = pvals * n_var + pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1 + + # logfoldchanges: log2(meanA / meanB) + logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) + + # find all with lfc > cutoff + lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] + stats_to_sort = np.abs(tscores) + + # derive sort order + if lfc_above_cutoff_idx.shape[0] > top_n: + # partition top N + rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:] + t_partition = lfc_above_cutoff_idx[rel_t_partition] + # sort the top N partition + rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1] + sort_order = t_partition[rel_sort_order] + else: + # partition and sort top N, ignoring lfc cutoff + partition = np.argpartition(stats_to_sort, -top_n)[-top_n:] + rel_sort_order = np.argsort(stats_to_sort[partition])[::-1] + indices = np.indices(stats_to_sort.shape)[0] + sort_order = indices[partition][rel_sort_order] + + # top n slice based upon sort order + logfoldchanges_top_n = logfoldchanges[sort_order] + pvals_top_n = pvals[sort_order] + pvals_adj_top_n = pvals_adj[sort_order] + + # varIndex, logfoldchange, pval, pval_adj + result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)] + return result + + +# Convenience function which handles sparse data +def mean_var_n(X): + """ + Two-pass variance calculation. Numerically (more) stable + than naive methods (and same method used by numpy.var()) + https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass + """ + # fp_err_occurred is a flag indicating that a floating point error + # occured somewhere in our compute. Used to trigger non-finite + # number handling. + fp_err_occurred = False + + def fp_err_set(err, flag): + nonlocal fp_err_occurred + fp_err_occurred = True + + with np.errstate(divide="call", invalid="call", call=fp_err_set): + n = X.shape[0] + if sparse.issparse(X): + mean = X.mean(axis=0).A1 + dfm = X - mean + sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1 + v = sumsq / (n - 1) + else: + mean = X.mean(axis=0) + dfm = X - mean + sumsq = np.sum(np.multiply(dfm, dfm), axis=0) + v = sumsq / (n - 1) + + if fp_err_occurred: + mean[np.isfinite(mean) == False] = 0 # noqa: E712 + v[np.isfinite(v) == False] = 0 # noqa: E712 + else: + mean[np.isnan(mean)] = 0 + v[np.isnan(v)] = 0 + + return mean, v, n diff --git a/local_server/compute/scanpy.py b/local_server/compute/scanpy.py new file mode 100644 index 00000000..309eb270 --- /dev/null +++ b/local_server/compute/scanpy.py @@ -0,0 +1,53 @@ +import importlib +import numpy as np + +""" +Wrapper for various scanpy modules. Will raise NotImplementedError if the scanpy +module is not installed/available +""" + + +def get_scanpy_module(): + try: + sc = importlib.import_module("scanpy") + # Future: we could enforce versions here, eg, lookat sc.__version__ + return sc + except ModuleNotFoundError as e: + raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e + except Exception as e: + # will capture other ImportError corner cases + raise NotImplementedError() from e + + +def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}): + """ + Given adata and an obs mask, return a new embedding for adata[obs_mask, :] + as an ndarray of shape (len(obs_mask), N), where N>=2. + + Do NOT mutate adata. + """ + + # backed mode is incompatible with the current implementation + if adata.isbacked: + raise NotImplementedError("Backed mode is incompatible with re-embedding") + + # safely get scanpy module, which may not be present. + sc = get_scanpy_module() + + # https://github.com/theislab/anndata/issues/311 + obs_mask = slice(None) if obs_mask is None else obs_mask + adata = adata[obs_mask, :].copy() + + for k in list(adata.obsm.keys()): + del adata.obsm[k] + for k in list(adata.uns.keys()): + del adata.uns[k] + + sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options) + sc.pp.neighbors(adata, **neighbors_options) + sc.tl.umap(adata, **umap_options) + + umap = adata.obsm["X_umap"] + result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN) + result[obs_mask] = umap + return result diff --git a/local_server/converters/__init__.py b/local_server/converters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/converters/schema/__init__.py b/local_server/converters/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/converters/schema/gene_symbol.py b/local_server/converters/schema/gene_symbol.py new file mode 100644 index 00000000..2d0de5a7 --- /dev/null +++ b/local_server/converters/schema/gene_symbol.py @@ -0,0 +1,211 @@ +"""Helpers for converting and checking HGNC gene symbols.""" + +import argparse +import enum +import logging +import os +import re +import numpy as np +import pandas as pd + + +def get_upgraded_var_index(var, hgnc_path=None): + """Given an anndata var dataframe, return a new index for the dataframe + where human gene symbols have been upgraded to the current HGNC set. + """ + + if not hgnc_path: + hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz") + + hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path) + + return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index]) + + +class SymbolStatus(enum.Enum): + """The status of a symbol in the HGNC database. + + APPROVED: Currently a valid symbol + WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown + not to exist _unless_ that symbol is also approved + AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for + multiple approved symbols + UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved + symbol + UNKNOWN: A symbol that does not appear in HGNC + """ + + APPROVED = 1 + WITHDRAWN = 2 + AMBIGUOUS = 3 + UPGRADABLE = 4 + UNKNOWN = 5 + + +class HGNCSymbolChecker: + """Handle checking and correcting HGNC symbols.""" + + def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map): + self.approved_symbols = approved_symbols + self.withdrawn_symbols = withdrawn_symbols + self.ambiguous_symbols = ambiguous_symbols + self.symbol_map = symbol_map + + def print_symbol_map(self): + """Print out a map from old symbol to new symbol.""" + + for symbol_pair in self.symbol_map.items(): + print("\t".join(symbol_pair)) + + def check_symbol(self, symbol): + """See if a symbol if approved or something else.""" + if symbol in self.approved_symbols: + return SymbolStatus.APPROVED + + if symbol in self.withdrawn_symbols: + return SymbolStatus.WITHDRAWN + + if symbol in self.ambiguous_symbols: + return SymbolStatus.AMBIGUOUS + + if symbol in self.symbol_map: + return SymbolStatus.UPGRADABLE + + return SymbolStatus.UNKNOWN + + def upgrade_symbol(self, symbol): + """Return the approved symbol for the given symbol. + + If the symbol cannot be upgraded, just return the original symbol. + """ + + fixed_symbol, stripped_symbol = format_symbol(symbol) + + if fixed_symbol in self.approved_symbols: + return fixed_symbol + elif fixed_symbol in self.symbol_map: + return self.symbol_map[fixed_symbol] + elif stripped_symbol in self.approved_symbols: + return stripped_symbol + elif stripped_symbol in self.symbol_map: + return self.symbol_map[stripped_symbol] + + return symbol + + @classmethod + def from_hgnc_records(cls, hgnc_dataset_path): + """Parse a hgnc database download into a HGNCSymbolChecker object.""" + + def all_symbols(record): + """Get all the symbols associated with an HGNC record including previous, alias, + and approved.""" + yield format_symbol(record["symbol"])[0] + for symbol in alias_and_previous_symbols(record): + yield symbol + + def alias_and_previous_symbols(record): + """Get alias and previous symbols from an HGNC record.""" + for field in ("alias_symbol", "prev_symbol"): + if record[field] is not np.nan: + for symbol in record[field].split("|"): + yield format_symbol(symbol)[0] + # Sometimes something like HGNC:1234 appears in datasets, which we + # want to fix as well. + yield record["hgnc_id"] + + hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records") + + # Get all symbols that are currently approved. + approved_symbols = set() + for record in hgnc_records: + if record["status"] == "Approved": + approved_symbols.add(format_symbol(record["symbol"])[0]) + + # Get all symbols that have been withdrawn + withdrawn_symbols = set() + for record in hgnc_records: + if record["status"] == "Entry Withdrawn": + for symbol in all_symbols(record): + withdrawn_symbols.add(symbol) + + # If a symbol is both approved and withdrawn, be optimistic and call it approved + logging.warning( + f"Some symbols are simulaneously withdrawn and approved\n" + f"We will treat them at approved:\n" + f"{withdrawn_symbols.intersection(approved_symbols)}" + ) + withdrawn_symbols = withdrawn_symbols.difference(approved_symbols) + + # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol + alias_previous_to_approved = {} + ambiguous_symbols = set() + + for record in hgnc_records: + if record["status"] == "Approved": + + # The approved symbol is what we'll map to + approved_symbol = format_symbol(record["symbol"])[0] + + for symbol in alias_and_previous_symbols(record): + + # If the alias or previous symbol is also an approved symbol, + # we'll just leave it alone + if symbol in approved_symbols: + continue + + # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous + if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol: + ambiguous_symbols.add(symbol) + else: + alias_previous_to_approved[symbol] = approved_symbol + + # Remove all the ambiguous symbols from the map + for ambiguous_symbol in ambiguous_symbols: + alias_previous_to_approved.pop(ambiguous_symbol) + + return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved) + + +def format_symbol(symbol): + """HGNC rules say symbols should all be upper case except for C#orf#. However, case is + variable in both alias and previous symbols as well as in the symbols we get in + submissions. So, upper case everything except for the one situation where mixed-case + is allowed, which are the genes like C2orf157. + + Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered + names persist throughout the life of the object. They won't match against the HGNC database + and we want to merge them, so we need to strip off the suffix and try matching again. + + This function takes a symbol and returns the symbol with the fixed case and also with the + seurat/scanpy suffix stripped off. + """ + + match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE) + + if match: + fixed_case = f"C{match.group(2)}orf{match.group(4)}" + else: + fixed_case = symbol.upper() + + suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case) + + return fixed_case, suffix_stripped + + +def main(): + """When called as main, parse a given hgnc download and print out a map from old to new + symbol. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/" + ) + args = parser.parse_args() + + hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset) + + hgnc_symbol_checker.print_symbol_map() + + +if __name__ == "__main__": + main() diff --git a/local_server/converters/schema/hgnc_complete_set.txt.gz b/local_server/converters/schema/hgnc_complete_set.txt.gz new file mode 100644 index 00000000..29c3c7a9 Binary files /dev/null and b/local_server/converters/schema/hgnc_complete_set.txt.gz differ diff --git a/local_server/converters/schema/ontology.py b/local_server/converters/schema/ontology.py new file mode 100644 index 00000000..8a524402 --- /dev/null +++ b/local_server/converters/schema/ontology.py @@ -0,0 +1,86 @@ +"""Methods for working with ontologies and the OLS.""" +from urllib.parse import quote_plus + +import requests + +OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api" + +# Curie means something like CL:0000001 + + +def _ontology_name(curie): + """Get the name of the ontology from the curie, CL or UBERON for example.""" + return curie.split(":")[0] + + +def _ontology_value(curie): + """Get the id component of the curie, 0000001 from CL:0000001 for example.""" + return curie.split(":")[1] + + +def _double_encode(url): + """Double url encode a url. This is required by the OLS API.""" + return quote_plus(quote_plus(url)) + + +def _iri(curie): + """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org""" + if _ontology_name(curie) == "EFO": + return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}" + return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}" + + +class OntologyLookupError(Exception): + """Exception for some problem with looking up ontology information.""" + + +def _ontology_info_url(curie): + """Get the to make a GET to to get information about an ontology term.""" + + # If the curie is empty, just return an empty string. This happens when there is no + # valid ontology value. + if not curie: + return "" + else: + return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}" + + +def get_ontology_label(curie): + """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'""" + + url = _ontology_info_url(curie) + + if not url: + return "" + + response = requests.get(url) + + if not response.ok: + raise OntologyLookupError( + f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}" + ) + return response.json()["label"] + + +def lookup_candidate_term(label, ontology="cl", method="select"): + """Lookup candidate terms for a label. This is useful when there is an existing label in a + submitted dataset, and you want to find an appropriate ontology term. + + Args: + label: the label to find ontology terms for + ontology: the ontology to search in, cl or uberon or efo for example + method: select or search. search provides much broader results + + Returns: + list of (curie, label) tuples returned by OLS + """ + # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api] + url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}" + response = requests.get(url) + + if not response.ok: + raise OntologyLookupError( + f"Label {label} lookup failed, got status code {response.status_code}: {response.text}" + ) + + return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]] diff --git a/local_server/converters/schema/remix.py b/local_server/converters/schema/remix.py new file mode 100644 index 00000000..5cc60746 --- /dev/null +++ b/local_server/converters/schema/remix.py @@ -0,0 +1,264 @@ +import argparse +import collections +import json +import logging +import math +import string + +import anndata +import numpy as np +import pandas as pd +import yaml + +from . import gene_symbol +from . import ontology +from . import validate + +REPLACE_SUFFIX = "_original" +ONTOLOGY_SUFFIX = "_ontology_term_id" + + +def is_curie(value): + """Return True iff the value is an OBO-id CURIE like EFO:000001""" + return (value.count(":") + and all(len(part) > 0 for part in value.split(":")) + and all(c in string.digits for c in value.split(":")[1])) + + +def is_ontology_field(field_name): + """Return True iff the field_name is an ontology field like tissue_ontology_term_id""" + return field_name.endswith(ONTOLOGY_SUFFIX) + + +def get_label_field_name(field_name): + """Get the associated label field from an ontology field, assay_ontology_term_id --> assay""" + return field_name[: -len(ONTOLOGY_SUFFIX)] + + +def split_suffix(maybe_curie): + """Split off the (cell culture) or (organoid) suffix.""" + + suffixes = [" (cell culture)", " (organoid)"] + for suffix in suffixes: + if maybe_curie.endswith(suffix): + return maybe_curie[:-len(suffix)], suffix + return maybe_curie, "" + + +def get_curie_and_label(maybe_curie): + """Given a string that might be a curie, return a (curie, label) pair""" + + maybe_curie, suffix = split_suffix(maybe_curie) + if not is_curie(maybe_curie): + return ("", maybe_curie + suffix) + return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix) + + +def safe_add_field(adata_attr, field_name, field_value): + """Add a field and value to an AnnData, but don't clobber an exising value.""" + + if ( + isinstance(field_value, list) + and field_value + and isinstance(field_value[0], dict) + ): + field_value = json.dumps(field_value) + if field_name in adata_attr: + adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name] + adata_attr[field_name] = field_value + + +def remix_uns(adata, uns_config): + """Add fields from the config to adata.uns""" + for field_name, field_value in uns_config.items(): + + if is_ontology_field(field_name): + # If it's an ontology field, look it up + label_field_name = get_label_field_name(field_name) + ontology_term, ontology_label = get_curie_and_label(field_value) + safe_add_field(adata.uns, field_name, ontology_term) + safe_add_field(adata.uns, label_field_name, ontology_label) + else: + safe_add_field(adata.uns, field_name, field_value) + + +def remix_obs(adata, obs_config): + """Add fields from the config to adata.obs""" + + for field_name, field_value in obs_config.items(): + + if isinstance(field_value, dict): + # If the value is a dict, that means we are supposed to map from an + # existing column to the new one + source_column, column_map = next(iter(field_value.items())) + nan_value = None + for key in column_map: + if isinstance(key, float) and math.isnan(key): + nan_value = column_map[key] + if nan_value is not None: + column_map["nan"] = nan_value + + for key in column_map: + if key not in adata.obs[source_column].unique(): + logging.warning(f'Key {key} not in adata.obs["{source_column}"]') + + for value in adata.obs[source_column].unique(): + if value not in column_map: + logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict') + + if is_ontology_field(field_name): + ontology_term_map, ontology_label_map = {}, {} + logging.info(f"Looking up labels for {field_name}") + for original_value, maybe_curie in column_map.items(): + curie, label = get_curie_and_label(maybe_curie) + ontology_term_map[original_value] = curie + ontology_label_map[original_value] = label + logging.info(f"Mapping {original_value} -> {curie} -> {label}") + + ontology_column = adata.obs[source_column].replace( + ontology_term_map, inplace=False + ) + label_column = adata.obs[source_column].replace( + ontology_label_map, inplace=False + ) + + safe_add_field(adata.obs, field_name, ontology_column) + safe_add_field( + adata.obs, get_label_field_name(field_name), label_column + ) + else: + label_column = adata.obs[source_column].replace( + column_map, inplace=False + ) + safe_add_field(adata.obs, field_name, label_column) + + else: + if is_ontology_field(field_name): + # If it's an ontology field, look it up + label_field_name = get_label_field_name(field_name) + ontology_term, ontology_label = get_curie_and_label(field_value) + safe_add_field(adata.obs, field_name, ontology_term) + safe_add_field(adata.obs, label_field_name, ontology_label) + else: + safe_add_field(adata.obs, field_name, field_value) + + +def merge_df(df, domain, index, columns): + """ + Given a dataframe with duplicate column labels, merge and return a dataframe where + the duplicates have been merged together, resulting in a dataframe with unique column + labels. + + "merge" depends on the value of domain. If the domain is "raw", then duplicate columns + can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then + summed, and then logged or sqrt'd again. + """ + + if not isinstance(df, np.ndarray): + to_merge = df.toarray() + else: + to_merge = df + if domain == "raw": + merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum( + axis=1, level=0, skipna=False + ) + elif domain == "log1p": + merged_df = ( + pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns) + .sum(axis=1, level=0, skipna=False) + ) + merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns) + elif domain == "sqrt": + merged_df = ( + pd.DataFrame(np.square(to_merge), index=index, columns=columns) + .sum(axis=1, level=0, skipna=False) + ) + merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns) + + return merged_df + + +def fixup_gene_symbols(adata, fixup_config): + """Update the var index to hold a consistent set of HGNC gene symbols.""" + + upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var) + + merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index) + fixup_adata = anndata.AnnData( + X=merged_X, + obs=adata.obs, + var=merged_X.columns.to_frame(name="hgnc_gene_symbol"), + uns=adata.uns, + obsm=adata.obsm, + ) + + for layer, domain in fixup_config.items(): + if layer == "X": + continue + if layer == "raw.X": + df = adata.raw.X + else: + df = adata.layers[layer] + + merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index) + assert merged_df.index.equals(merged_X.index) + assert merged_df.columns.equals(merged_X.columns) + + if domain == "raw": + fixup_raw = anndata.AnnData( + X=merged_df, + obs=adata.obs, + var=merged_X.columns.to_frame(name="hgnc_gene_symbol"), + ) + fixup_adata.raw = fixup_raw + else: + fixup_adata.layers[layer] = merged_df + + return fixup_adata + +def _strip_version(adata): + """Remove version information from the AnnData object.""" + + if "version" in adata.uns_keys(): + del adata.uns["version"] + +def apply_schema(source_h5ad, remix_config, output_filename): + + try: + import scanpy + except ImportError: + raise ImportError("scanpy must be installed for cellxgene schema") + adata = scanpy.read_h5ad(source_h5ad) + config = yaml.load(open(remix_config), Loader=yaml.FullLoader) + remix_uns(adata, config["uns"]) + remix_obs(adata, config["obs"]) + + if config.get("fixup_gene_symbols"): + adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"]) + + if ("version" in adata.uns_keys() + and isinstance(adata.uns["version"], collections.Mapping) + and "corpora_schema_version" in adata.uns["version"]): + schema_version = adata.uns["version"]["corpora_schema_version"] + try: + validate.get_schema_definition(schema_version) + except ValueError: + logging.warning(f"Stripping version information out of AnnData because schema " + f"version {schema_version} is unknown.") + _strip_version(adata) + + if not validate.validate_adata(adata, shallow=False): + logging.warning(f"Stripping version information out of AnnData because it does not " + f"follow schema version {schema_version} .") + _strip_version(adata) + + adata.write_h5ad(output_filename, compression="gzip") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--source-h5ad", required=True) + parser.add_argument("--remix-config", required=True) + parser.add_argument("--output-filename", required=True) + args = parser.parse_args() + apply_schema(args.source_h5ad, args.remix_config, args.output_filename) diff --git a/local_server/converters/schema/schema_definitions/1_0_0.yaml b/local_server/converters/schema/schema_definitions/1_0_0.yaml new file mode 100644 index 00000000..acff2238 --- /dev/null +++ b/local_server/converters/schema/schema_definitions/1_0_0.yaml @@ -0,0 +1,95 @@ +title: Corpora schema version 1.0.0 +type: anndata +components: + uns: + type: dict + keys: + version: + type: dict + keys: + corpora_schema_version: null + corpora_encoding_version: null + title: + type: string + contributors: + type: stringified list of dicts + layer_descriptions: + type: dict + keys: + X: null + organism: + type: string + nullable: false + organism_ontology_term_id: + type: curie + prefixes: + - NCBITaxon + var: + type: dataframe + index: + type: human-readable string + unique: true + obs: + type: dataframe + index: + unique: true + columns: + tissue: + type: human-readable string + nullable: false + tissue_ontology_term_id: + type: suffixed curie + nullable: true + prefixes: + - UBERON + assay: + type: human-readable string + nullable: false + assay_ontology_term_id: + type: curie + nullable: true + prefixes: + - EFO + disease: + type: human-readable string + nullable: false + disease_ontology_term_id: + type: curie + nullable: true + prefixes: + - MONDO + - PATO + cell_type: + type: human-readable string + nullable: false + cell_type_ontology_term_id: + type: curie + nullable: true + prefixes: + - CL + - UBERON + sex: + type: string + enum: + - male + - female + - mixed + - unknown + - other + ethnicity: + type: human-readable string + nullable: false + ethnicity_ontology_term_id: + type: curie + nullable: true + prefixes: + - HANCESTRO + development_stage: + type: human-readable string + nullable: false + development_stage_ontology_term_id: + type: curie + nullable: true + prefixes: + - HsapDv + - EFO diff --git a/local_server/converters/schema/schema_definitions/1_1_0.yaml b/local_server/converters/schema/schema_definitions/1_1_0.yaml new file mode 100644 index 00000000..7531c72e --- /dev/null +++ b/local_server/converters/schema/schema_definitions/1_1_0.yaml @@ -0,0 +1,93 @@ +title: Corpora schema version 1.1.0 +type: anndata +components: + uns: + type: dict + keys: + version: + type: dict + keys: + corpora_schema_version: null + corpora_encoding_version: null + title: + type: string + layer_descriptions: + type: dict + keys: + X: null + organism: + type: string + nullable: false + organism_ontology_term_id: + type: curie + prefixes: + - NCBITaxon + var: + type: dataframe + index: + type: human-readable string + unique: true + obs: + type: dataframe + index: + unique: true + columns: + tissue: + type: human-readable string + nullable: false + tissue_ontology_term_id: + type: suffixed curie + nullable: true + prefixes: + - UBERON + assay: + type: human-readable string + nullable: false + assay_ontology_term_id: + type: curie + nullable: true + prefixes: + - EFO + disease: + type: human-readable string + nullable: false + disease_ontology_term_id: + type: curie + nullable: true + prefixes: + - MONDO + - PATO + cell_type: + type: human-readable string + nullable: false + cell_type_ontology_term_id: + type: curie + nullable: true + prefixes: + - CL + - UBERON + sex: + type: string + enum: + - male + - female + - mixed + - unknown + - other + ethnicity: + type: human-readable string + nullable: false + ethnicity_ontology_term_id: + type: curie + nullable: true + prefixes: + - HANCESTRO + development_stage: + type: human-readable string + nullable: false + development_stage_ontology_term_id: + type: curie + nullable: true + prefixes: + - HsapDv + - EFO diff --git a/local_server/converters/schema/validate.py b/local_server/converters/schema/validate.py new file mode 100644 index 00000000..ec463dd5 --- /dev/null +++ b/local_server/converters/schema/validate.py @@ -0,0 +1,236 @@ +import json +import re +import os +import sys + +import pandas as pd +import yaml + + +def _is_null(v): + """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad.""" + return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0) + + +def _validate_stringified_list_of_dicts(s): + """Verify that a string can be parsed into a list. + + We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to + json.dumps them. This verifies that we can load them back. + """ + + try: + list_ = json.loads(s) + if not isinstance(list_, list): + return False + for el in list_: + if not isinstance(el, dict): + return False + return True + except (json.JSONDecodeError, TypeError): + pass + return False + + +def _validate_human_readable_string(s): + """Verify that a string is human-readable. + + There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind + of vague and subjective. I feel like I can read many strings. So here we just check for the main ways + that fails: someone puts in an ontology term id or and ensembl gene/transcript id. + + Returns False if s is not a string or is one of those bad string types. + """ + + return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s)) + + +def _validate_curie(c, prefixes): + """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the + prefix of the curies is in prefixes. + """ + + if not c: + return True + + match = re.match(r"([A-Z]\w+):\d+$", c) + + if prefixes: + return match and match.group(1) in prefixes + else: + return match + + +def _validate_suffixed_curie(c, prefixes): + """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'""" + + # Pull off the suffix + suffix = re.findall(r"\ \(.*\)$", c) + if suffix: + c = c[: -len(suffix[0])] + return _validate_curie(c, prefixes) + + +def _validate_column(column, column_name, df_name, schema_def): + """Given a schema definition and the column of a dataframe, verify that the column satifies + the schema. + """ + + errors = [] + + if schema_def.get("unique"): + if column.nunique() != len(column): + errors.append(f"Column {column_name} in dataframe {df_name} is not unique.") + + if "nullable" in schema_def and not schema_def["nullable"]: + if any(_is_null(v) for v in column): + errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.") + + if schema_def.get("type") == "human-readable string": + non_readables = [v for v in column if not _validate_human_readable_string(v)] + if non_readables: + errors.append( + f"Column {column_name} in dataframe {df_name} contains non-human-readable " + f"values like {non_readables[0]}" + ) + + if schema_def.get("type") in ("curie", "suffixed curie"): + validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie + non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))] + if non_valid_curies: + errors.append( + f"Column {column_name} in dataframe {df_name} contains invalid ontology values like " + f"{non_valid_curies[0]}." + ) + if "prefixes" in schema_def: + errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}." + + if "enum" in schema_def: + bad_enums = [v for v in column if v not in schema_def["enum"]] + if bad_enums: + errors.append( + f"Column {column_name} in dataframe {df_name} contains unpermitted values like " + f"{bad_enums[0]}. Values must be one of {schema_def['enum']}." + ) + + return errors + + +def _validate_dict(dict_, dict_name, schema_def): + """Given a schema definition and dict, verify that the dict satifies the schema.""" + + errors = [] + + for key in schema_def.get("keys", []): + if key not in dict_: + errors.append(f"{dict_name} is missing key {key}.") + elif schema_def["keys"][key]: + if schema_def["keys"][key]["type"] == "stringified list of dicts": + if not _validate_stringified_list_of_dicts(dict_[key]): + errors.append( + f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}" + ) + elif schema_def["keys"][key]["type"] == "dict": + errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key])) + elif schema_def["keys"][key]["type"] == "curie": + if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]): + errors.append(f"Key {key} in {dict_name} contains invalid ontology value.") + if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]: + if _is_null(dict_[key]): + errors.append(f"Key {key} in dict {dict_name} is an empty value.") + + return errors + + +def _validate_dataframe(df, df_name, schema_def): + """Given a dataframe and schema definition, verify that the dataframe follows the schema.""" + + errors = [] + + if "index" in schema_def: + errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"])) + + for column in schema_def.get("columns", []): + if column not in df.columns: + errors.append(f"Dataframe {df_name} is missing column {column}.") + else: + errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column])) + + return errors + + +def get_schema_definition(version): + """Look up and read a schema definition based on a version number like "1.0.0".""" + + path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml" + ) + + if not os.path.isfile(path): + raise ValueError(f"No definition for version {version} found.") + + return yaml.load(open(path), Loader=yaml.FullLoader) + + +def deep_check(adata, schema_def): + """Perform a "deep" check of the AnnData object using the schema definition. + + This checks all the columns and unstructured metadata rather than just the version. + + Returns a list of error messages. If that list is empty, the object passed validation. + """ + + errors = [] + + for component, component_def in schema_def["components"].items(): + if component_def["type"] == "dataframe": + errors.extend(_validate_dataframe(getattr(adata, component), component, component_def)) + elif component_def["type"] == "dict": + errors.extend(_validate_dict(getattr(adata, component), component, component_def)) + else: + raise ValueError(f"Unexpected component type {component['type']}") + + return errors + + +def validate_adata(adata, shallow): + """Validate an AnnData object. If shallow, just check that the required version information is + present. + """ + + # Does it have the version information written into uns? + if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]: + print("AnnData file is missing corpora version information") + return False + + # We can stop here if it's a "shallow" check, that is, if we're just + # checking that version is present. + if shallow: + return True + + schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"]) + + errors = deep_check(adata, schema_def) + + for error in errors: + print(error) + + return not errors + + +def validate(h5ad_path, shallow=False): + """Entry point for validation.""" + + try: + import scanpy + except ImportError: + raise ImportError("scanpy must be installed for cellxgene schema") + + try: + adata = scanpy.read_h5ad(h5ad_path, backed="r") + except (OSError, TypeError): + print(f"Unable to open {h5ad_path} with scanpy.") + sys.exit(1) + + if not validate_adata(adata, shallow): + sys.exit(1) diff --git a/local_server/data_anndata/__init__.py b/local_server/data_anndata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_anndata/anndata_adaptor.py b/local_server/data_anndata/anndata_adaptor.py new file mode 100644 index 00000000..12f468c7 --- /dev/null +++ b/local_server/data_anndata/anndata_adaptor.py @@ -0,0 +1,369 @@ +import warnings +from datetime import datetime + +import anndata +import numpy as np +from packaging import version +from pandas.core.dtypes.dtypes import CategoricalDtype +from scipy import sparse +from server_timing import Timing as ServerTiming + +import local_server.compute.diffexp_generic as diffexp_generic +from local_server.common.colors import convert_anndata_category_colors_to_cxg_category_colors +from local_server.common.constants import Axis, MAX_LAYOUTS +from local_server.common.corpora import corpora_get_props_from_anndata +from local_server.common.errors import PrepareError, DatasetAccessError, FilterError +from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array +from local_server.compute.scanpy import scanpy_umap +from local_server.data_common.data_adaptor import DataAdaptor +from local_server.data_common.fbs.matrix import encode_matrix_fbs + +anndata_version = version.parse(str(anndata.__version__)).release + + +def anndata_version_is_pre_070(): + major = anndata_version[0] + minor = anndata_version[1] if len(anndata_version) > 1 else 0 + return major == 0 and minor < 7 + + +class AnndataAdaptor(DataAdaptor): + def __init__(self, data_locator, app_config=None, dataset_config=None): + super().__init__(data_locator, app_config, dataset_config) + self.data = None + self._load_data(data_locator) + self._validate_and_initialize() + + def cleanup(self): + pass + + @staticmethod + def pre_load_validation(data_locator): + if data_locator.islocal(): + # if data locator is local, apply file system conventions and other "cheap" + # validation checks. If a URI, defer until we actually fetch the data and + # try to read it. Many of these tests don't make sense for URIs (eg, extension- + # based typing). + if not data_locator.exists(): + raise DatasetAccessError("does not exist") + if not data_locator.isfile(): + raise DatasetAccessError("is not a file") + + @staticmethod + def file_size(data_locator): + return data_locator.size() if data_locator.islocal() else 0 + + @staticmethod + def open(data_locator, app_config, dataset_config=None): + return AnndataAdaptor(data_locator, app_config, dataset_config) + + def get_corpora_props(self): + return corpora_get_props_from_anndata(self.data) + + def get_name(self): + return "cellxgene anndata adaptor version" + + def get_library_versions(self): + return dict(anndata=str(anndata.__version__)) + + @staticmethod + def _create_unique_column_name(df, col_name_prefix): + """ given the columns of a dataframe, and a name prefix, return a column name which + does not exist in the dataframe, AND which is prefixed by `prefix` + + The approach is to append a numeric suffix, starting at zero and increasing by + one, until an unused name is found (eg, prefix_0, prefix_1, ...). + """ + suffix = 0 + while f"{col_name_prefix}{suffix}" in df: + suffix += 1 + return f"{col_name_prefix}{suffix}" + + def _alias_annotation_names(self): + """ + The front-end relies on the existance of a unique, human-readable + index for obs & var (eg, var is typically gene name, obs the cell name). + The user can specify these via the --obs-names and --var-names config. + If they are not specified, use the existing index to create them, giving + the resulting column a unique name (eg, "name"). + + In both cases, enforce that the result is unique, and communicate the + index column name to the front-end via the obs_names and var_names config + (which is incorporated into the schema). + """ + self.original_obs_index = self.data.obs.index + + for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")): + config_name = f"single_dataset__{var_name}_names" + parameter_name = f"{var_name}_names" + name = getattr(self.server_config, config_name) + df_axis = getattr(self.data, str(ax_name)) + if name is None: + # Default: create unique names from index + if not df_axis.index.is_unique: + raise KeyError( + f"Values in {ax_name}.index must be unique. " + "Please prepare data to contain unique index values, or specify an " + "alternative with --{ax_name}-name." + ) + name = self._create_unique_column_name(df_axis.columns, "name_") + self.parameters[parameter_name] = name + # reset index to simple range; alias name to point at the + # previously specified index. + df_axis.rename_axis(name, inplace=True) + df_axis.reset_index(inplace=True) + elif name in df_axis.columns: + # User has specified alternative column for unique names, and it exists + if not df_axis[name].is_unique: + raise KeyError( + f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values." + ) + df_axis.reset_index(drop=True, inplace=True) + self.parameters[parameter_name] = name + else: + # user specified a non-existent column name + raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.") + + def _create_schema(self): + self.schema = { + "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)}, + "annotations": { + "obs": {"index": self.parameters.get("obs_names"), "columns": []}, + "var": {"index": self.parameters.get("var_names"), "columns": []}, + }, + "layout": {"obs": []}, + } + for ax in Axis: + curr_axis = getattr(self.data, str(ax)) + for ann in curr_axis: + ann_schema = {"name": ann, "writable": False} + ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann])) + self.schema["annotations"][ax]["columns"].append(ann_schema) + + for layout in self.get_embedding_names(): + layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]} + self.schema["layout"]["obs"].append(layout_schema) + + def get_schema(self): + return self.schema + + def _load_data(self, data_locator): + # as of AnnData 0.6.19, backed mode performs initial load fast, but at the + # cost of significantly slower access to X data. + try: + # there is no guarantee data_locator indicates a local file. The AnnData + # API will only consume local file objects. If we get a non-local object, + # make a copy in tmp, and delete it after we load into memory. + with data_locator.local_handle() as lh: + # as of AnnData 0.6.19, backed mode performs initial load fast, but at the + # cost of significantly slower access to X data. + backed = "r" if self.server_config.adaptor__anndata_adaptor__backed else None + self.data = anndata.read_h5ad(lh, backed=backed) + + except ValueError: + raise DatasetAccessError( + "File must be in the .h5ad format. Please read " + "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to " + "learn more about this format. You may be able to convert your file into this format " + "using `cellxgene prepare`, please run `cellxgene prepare --help` for more " + "information." + ) + except MemoryError: + raise DatasetAccessError("Out of memory - file is too large for available memory.") + except Exception: + raise DatasetAccessError( + "File not found or is inaccessible. File must be an .h5ad object. " + "Please check your input and try again." + ) + + def _validate_and_initialize(self): + if anndata_version_is_pre_070(): + warnings.warn( + "Use of anndata versions older than 0.7 will have serious issues. Please update to at " + "least anndata 0.7 or later." + ) + + # var and obs column names must be unique + if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique: + raise KeyError("All annotation column names must be unique.") + + self._alias_annotation_names() + self._validate_data_types() + self.cell_count = self.data.shape[0] + self.gene_count = self.data.shape[1] + self._create_schema() + + # heuristic + n_values = self.data.shape[0] * self.data.shape[1] + if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8): + self.parameters.update({"diffexp_may_be_slow": True}) + + def _is_valid_layout(self, arr): + """ return True if this layout data is a valid array for front-end presentation: + * ndarray, dtype float/int/uint + * with shape (n_obs, >= 2) + * with all values finite or NaN (no +Inf or -Inf) + """ + is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu" + is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2 + is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr)) + return is_valid + + def _validate_data_types(self): + # The backed API does not support interrogation of the underlying sparsity or sparse matrix type + # Fake it by asking for a small subarray and testing it. NOTE: if the user has ignored our + # anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset, + # this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7 + # which will load the entire X matrix to fullfill any slicing request if X is sparse. See + # user warning in _load_data(). + X0 = self.data.X[0, 0:1] + if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0): + warnings.warn( + "Anndata data matrix is sparse, but not a CSC (columnar) matrix. " + "Performance may be improved by using CSC." + ) + if self.data.X.dtype != "float32": + warnings.warn( + f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated." + ) + for ax in Axis: + curr_axis = getattr(self.data, str(ax)) + for ann in curr_axis: + datatype = curr_axis[ann].dtype + downcast_map = { + "int64": "int32", + "uint32": "int32", + "uint64": "int32", + "float64": "float32", + } + if datatype in downcast_map: + warnings.warn( + f"Anndata annotation {ax}:{ann} is in unsupported format: {datatype}. " + f"Data will be downcast to {downcast_map[datatype]}." + ) + if isinstance(datatype, CategoricalDtype): + category_num = len(curr_axis[ann].dtype.categories) + if category_num > 500 and category_num > self.dataset_config.presentation__max_categories: + warnings.warn( + f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " + f"cumbersome or slow to display. We recommend setting the " + f"--max-category-items option to 500, this will hide categorical " + f"annotations with more than 500 categories in the UI" + ) + + def annotation_to_fbs_matrix(self, axis, fields=None, labels=None): + if axis == Axis.OBS: + if labels is not None and not labels.empty: + df = self.data.obs.join(labels, self.parameters.get("obs_names")) + else: + df = self.data.obs + else: + df = self.data.var + + if fields is not None and len(fields) > 0: + df = df[fields] + return encode_matrix_fbs(df, col_idx=df.columns) + + def get_embedding_names(self): + """ + Return pre-computed embeddings. + + function: + a) generate list of default layouts + b) validate layouts are legal. remove/warn on any that are not + c) cap total list of layouts at global const MAX_LAYOUTS + """ + # load default layouts from the data. + layouts = self.dataset_config.embeddings__names + + if layouts is None or len(layouts) == 0: + layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")] + + # remove invalid layouts + valid_layouts = [] + obsm_keys = self.data.obsm_keys() + for layout in layouts: + layout_name = f"X_{layout}" + if layout_name not in obsm_keys: + warnings.warn(f"Ignoring unknown layout name: {layout}.") + elif not self._is_valid_layout(self.data.obsm[layout_name]): + warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}") + else: + valid_layouts.append(layout) + + if len(valid_layouts) == 0: + raise PrepareError("No valid layout data.") + + # cap layouts to MAX_LAYOUTS + return valid_layouts[0:MAX_LAYOUTS] + + def get_embedding_array(self, ename, dims=2): + full_embedding = self.data.obsm[f"X_{ename}"] + return full_embedding[:, 0:dims] + + def compute_embedding(self, method, obsFilter): + if Axis.VAR in obsFilter: + raise FilterError("Observation filters may not contain variable conditions") + if method != "umap": + raise NotImplementedError(f"re-embedding method {method} is not available.") + try: + shape = self.get_shape() + obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0]) + except (KeyError, IndexError): + raise FilterError("Error parsing filter") + with ServerTiming.time("layout.compute"): + X_umap = scanpy_umap(self.data, obs_mask) + + # Server picks reemedding name, which must not collide with any other + # embedding name generated by this backend. + name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}" + dims = [f"{name}_0", f"{name}_1"] + layout_schema = {"name": name, "type": "float32", "dims": dims} + self.schema["layout"]["obs"].append(layout_schema) + self.data.obsm[f"X_{name}"] = X_umap + return layout_schema + + def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None): + if top_n is None: + top_n = self.dataset_config.diffexp__top_n + if lfc_cutoff is None: + lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff + return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff) + + def get_colors(self): + return convert_anndata_category_colors_to_cxg_category_colors(self.data) + + def get_X_array(self, obs_mask=None, var_mask=None): + if obs_mask is None: + obs_mask = slice(None) + if var_mask is None: + var_mask = slice(None) + X = self.data.X[obs_mask, var_mask] + return X + + def get_shape(self): + return self.data.shape + + def query_var_array(self, term_name): + return getattr(self.data.var, term_name) + + def query_obs_array(self, term_name): + return getattr(self.data.obs, term_name) + + def get_obs_index(self): + name = self.server_config.single_dataset__obs_names + if name is None: + return self.original_obs_index + else: + return self.data.obs[name] + + def get_obs_columns(self): + return self.data.obs.columns + + def get_obs_keys(self): + # return list of keys + return self.data.obs.keys().to_list() + + def get_var_keys(self): + # return list of keys + return self.data.var.keys().to_list() diff --git a/local_server/data_common/__init__.py b/local_server/data_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_common/data_adaptor.py b/local_server/data_common/data_adaptor.py new file mode 100644 index 00000000..b832bd37 --- /dev/null +++ b/local_server/data_common/data_adaptor.py @@ -0,0 +1,481 @@ +from abc import ABCMeta, abstractmethod +from os.path import basename, splitext +import re + +import numpy as np +import pandas as pd +from server_timing import Timing as ServerTiming + +from local_server.common.config.app_config import AppConfig +from local_server.common.constants import Axis +from local_server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError +from local_server.common.utils.utils import jsonify_numpy +from local_server.data_common.fbs.matrix import encode_matrix_fbs + + +class DataAdaptor(metaclass=ABCMeta): + """Base class for loading and accessing matrix data""" + + def __init__(self, data_locator, app_config, dataset_config=None): + if type(app_config) != AppConfig: + raise TypeError("config expected to be of type AppConfig") + + # location to the dataset + self.data_locator = data_locator + + # config is the application configuration + self.app_config = app_config + self.server_config = self.app_config.server_config + self.dataset_config = dataset_config or app_config.dataset_config + + # parameters set by this data adaptor based on the data. + self.parameters = {} + + @staticmethod + @abstractmethod + def pre_load_validation(data_locator): + pass + + @staticmethod + @abstractmethod + def open(data_locator, app_config, dataset_config): + pass + + @staticmethod + @abstractmethod + def file_size(data_locator): + pass + + @abstractmethod + def get_name(self): + """return a string name for this data adaptor""" + pass + + @abstractmethod + def get_library_versions(self): + """return a dictionary of library name to library versions""" + pass + + @abstractmethod + def get_embedding_names(self): + """return a list of pre-computed embedding names""" + pass + + @abstractmethod + def get_embedding_array(self, ename, dims=2): + """return an numpy array for the given pre-computed embedding name.""" + pass + + @abstractmethod + def compute_embedding(self, method, filter): + """compute a new embedding on the specified obs subset, and return the embedding schema. """ + pass + + @abstractmethod + def get_X_array(self, obs_mask=None, var_mask=None): + """return the X array, possibly filtered by obs_mask or var_mask. + the return type is either ndarray or scipy.sparse.spmatrix.""" + pass + + @abstractmethod + def get_shape(self): + pass + + @abstractmethod + def query_var_array(self, term_var): + pass + + @abstractmethod + def query_obs_array(self, term_var): + pass + + @abstractmethod + def get_colors(self): + pass + + @abstractmethod + def get_obs_index(self): + pass + + @abstractmethod + def get_obs_columns(self): + pass + + @abstractmethod + def get_obs_keys(self): + # return list of keys + pass + + @abstractmethod + def get_var_keys(self): + # return list of keys + pass + + @abstractmethod + def cleanup(self): + pass + + def get_data_locator(self): + return self.data_locator + + def get_location(self): + return self.data_locator.uri_or_path + + def get_about(self): + return None + + def get_title(self): + # default to file name + location = self.get_location() + if location.endswith("/"): + location = location[:-1] + return splitext(basename(location))[0] + + def get_corpora_props(self): + return None + + @abstractmethod + def get_schema(self): + """ + Return current schema + """ + pass + + @abstractmethod + def annotation_to_fbs_matrix(self, axis, field=None, uid=None): + """ + Gets annotation value for each observation + :param axis: string obs or var + :param fields: list of keys for annotation to return, returns all annotation values if not set. + :return: flatbuffer: in fbs/matrix.fbs encoding + """ + pass + + def update_parameters(self, parameters): + parameters.update(self.parameters) + + def _index_filter_to_mask(self, filter, count): + mask = np.zeros((count,), dtype=np.bool) + for i in filter: + if type(i) == list: + mask[i[0] : i[1]] = True + else: + mask[i] = True + return mask + + def _axis_filter_to_mask(self, axis, filter, count): + mask = np.ones((count,), dtype=np.bool) + if "index" in filter: + mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count)) + if "annotation_value" in filter: + mask = np.logical_and(mask, self._annotation_filter_to_mask(axis, filter["annotation_value"], count)) + + return mask + + def _annotation_filter_to_mask(self, axis, filter, count): + mask = np.ones((count,), dtype=np.bool) + for v in filter: + name = v["name"] + if axis == Axis.VAR: + anno_data = self.query_var_array(name) + elif axis == Axis.OBS: + anno_data = self.query_obs_array(name) + + if anno_data.dtype.name in ["boolean", "category", "object"]: + values = v.get("values", []) + key_idx = np.in1d(anno_data, values) + mask = np.logical_and(mask, key_idx) + + else: + min_ = v.get("min", None) + max_ = v.get("max", None) + if min_ is not None: + key_idx = (anno_data >= min_).ravel() + mask = np.logical_and(mask, key_idx) + if max_ is not None: + key_idx = (anno_data <= max_).ravel() + mask = np.logical_and(mask, key_idx) + + return mask + + def _filter_to_mask(self, filter): + """ + Return the filter as a row and column selection list. + No filter on a dimension means 'all' + """ + shape = self.get_shape() + var_selector = None + obs_selector = None + if filter is not None: + if Axis.OBS in filter: + obs_selector = self._axis_filter_to_mask(Axis.OBS, filter["obs"], shape[0]) + + if Axis.VAR in filter: + var_selector = self._axis_filter_to_mask(Axis.VAR, filter["var"], shape[1]) + + return (obs_selector, var_selector) + + def check_new_labels(self, labels_df): + """Check the new annotations labels, then set the labels_df index""" + if labels_df is None or labels_df.empty: + return + + labels_df.index = self.get_obs_index() + if labels_df.index.name is None: + labels_df.index.name = "index" + + # all labels must have a name, which must be unique and not used in obs column names + if not labels_df.columns.is_unique: + raise KeyError("All column names specified in user annotations must be unique.") + + # the label index must be unique, and must have same values the anndata obs index + if not labels_df.index.is_unique: + raise KeyError("All row index values specified in user annotations must be unique.") + + obs_columns = self.get_obs_columns() + + duplicate_columns = list(set(labels_df.columns) & set(obs_columns)) + if len(duplicate_columns) > 0: + raise KeyError( + "Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}" + ) + + # labels must have same count as obs annotations + shape = self.get_shape() + if labels_df.shape[0] != shape[0]: + raise ValueError("Labels file must have same number of rows as data file.") + + # This will convert a float column that contains integer data into an integer type. + # This case can occur when a user makes a copy of a category that originally contained integer data. + # The client always copies array data to floats, therefore the copy will contain floats instead of integers. + # float data is not allowed as a categorical type. + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + labels_df = labels_df.convert_dtypes() + for col, dtype in zip(labels_df, labels_df.dtypes): + if isinstance(dtype, pd.Int32Dtype): + labels_df[col] = labels_df[col].astype("int32") + if isinstance(dtype, pd.Int64Dtype): + labels_df[col] = labels_df[col].astype("int64") + + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + raise ValueError("Columns may not have floating point types") + + return labels_df + + def check_new_gene_sets(self, args, context=None): + """ + Check validity of gene sets, return if correct, else raise error. + May also modify the gene set for conditions that should be resolved, + but which do not warrant a hard error. + + Argument 'args' must be a tuple containing (genesets, tid). Genesets + may be either the REST OTA format (list of dicts) or the internal format + (dict of dicts, keyed by the geneset name). + + Rules: + 0. all geneset names must be unique. + 1. All geneset names must be legal, meaning: + * no leading or trailing white space + * no multi-space runs + * character set matches: [A-Z][a-z][0-9][ .()-] + Generates hard error. + 2. Gene symbols must be part of the current var_index. If symbol not in var_index, + will generate a warning and the symbol removed. + 3. Duplicate gene symbols are silently de-duped. + """ + (genesets, tid) = args + messagefn = context["messagefn"] if context else (lambda x: None) + + # accept genesets args as either the internal (dict) or REST (list) format, + # as they are identical except for the dict being keyed by geneset_name. + if type(genesets) not in (dict, list): + raise ValueError("Genesets must be either dict or list.") + genesets = genesets if type(genesets) == list else genesets.values() + + # 0. check for uniqueness of geneset names + geneset_names = [gs["geneset_name"] for gs in genesets] + if len(set(geneset_names)) != len(geneset_names): + raise KeyError("All geneset names must be unique.") + + # 1. check gene set character set and format + legal_name = re.compile(r"^(\w|[ .()-])+$") + for name in geneset_names: + if type(name) != str or len(name) == 0: + raise KeyError("Geneset names must be non-null string.") + if name[0] in " \t\n\r" or name[-1] in " \t\n\r" or not legal_name.match(name) or " " in name: + messagefn( + "Error: " + f"Geneset name {name} is not valid. Only alphanumeric and limited special characters (-_.) " + "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed." + ) + raise KeyError( + "Geneset name is not valid, only alphanumeric and limited special characters (-_.) " + "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed." + ) + + # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will + # generate a warning and be removed. + var_names = set(self.query_var_array(self.parameters.get("var_names"))) + for geneset in genesets: + if type(geneset) != dict: + raise ValueError("Each geneset must be a dict.") + geneset_name = geneset["geneset_name"] + genes = geneset["genes"] + if type(genes) != list: + raise ValueError("Geneset genes field must be a list") + gene_symbol_already_seen = set() + new_genes = [] + for gene in genes: + gene_symbol = gene["gene_symbol"] + if type(gene_symbol) != str or len(gene_symbol) == 0: + raise ValueError("Gene symbol must be non-null string.") + if gene_symbol in gene_symbol_already_seen: + # duplicate check + messagefn( + f"Warning: a duplicate of gene {gene_symbol} was found in geneset {geneset_name}, " + "and will be ignored." + ) + continue + + if gene_symbol not in var_names: + messagefn( + f"Warning: {gene_symbol}, used in geneset {geneset_name}, " + "was not found in the dataset and will be ignored." + ) + continue + + gene_symbol_already_seen.add(gene_symbol) + new_genes.append(gene) + + geneset["genes"] = new_genes + + return args + + def data_frame_to_fbs_matrix(self, filter, axis): + """ + Retrieves data 'X' and returns in a flatbuffer Matrix. + :param filter: filter: dictionary with filter params + :param axis: string obs or var + :return: flatbuffer Matrix + + Caveats: + * currently only supports access on VAR axis + * currently only supports filtering on VAR axis + """ + if axis != Axis.VAR: + raise ValueError("Only VAR dimension access is supported") + + try: + obs_selector, var_selector = self._filter_to_mask(filter) + except (KeyError, IndexError, TypeError, AttributeError): + raise FilterError("Error parsing filter") + + if obs_selector is not None: + raise FilterError("filtering on obs unsupported") + + num_columns = self.get_shape()[1] if var_selector is None else np.count_nonzero(var_selector) + if self.server_config.exceeds_limit("column_request_max", num_columns): + raise ExceedsLimitError("Requested dataframe columns exceed column request limit") + + X = self.get_X_array(obs_selector, var_selector) + col_idx = np.nonzero([] if var_selector is None else var_selector)[0] + return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None) + + def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None): + """ + Computes the top N differentially expressed variables between two observation sets. If mode + is "TOP_N", then stats for the top N + dataframes + contain a subset of variables, then statistics for all variables will be returned, otherwise + only the top N vars will be returned. + :param obsFilterA: filter: dictionary with filter params for first set of observations + :param obsFilterB: filter: dictionary with filter params for second set of observations + :param top_n: Limit results to top N (Top var mode only) + :return: top N genes and corresponding stats + """ + if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB: + raise FilterError("Observation filters may not contain variable conditions") + try: + shape = self.get_shape() + obs_mask_A = self._axis_filter_to_mask(Axis.OBS, obsFilterA["obs"], shape[0]) + obs_mask_B = self._axis_filter_to_mask(Axis.OBS, obsFilterB["obs"], shape[0]) + except (KeyError, IndexError): + raise FilterError("Error parsing filter") + if top_n is None: + top_n = self.dataset_config.diffexp__top_n + + if self.server_config.exceeds_limit( + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + ): + raise ExceedsLimitError("Diffexp request exceeds max cell count limit") + + result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff) + + try: + return jsonify_numpy(result) + except ValueError: + raise JSONEncodingValueError("Error encoding differential expression to JSON") + + @abstractmethod + def compute_diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff): + pass + + @staticmethod + def normalize_embedding(embedding): + """Normalize embedding layout to meet client assumptions. + Embedding is an ndarray, shape (n_obs, n)., where n is normally 2 + """ + + # scale isotropically + try: + min = np.nanmin(embedding, axis=0) + max = np.nanmax(embedding, axis=0) + except RuntimeError: + # indicates entire array was NaN, which should propagate + min = np.NaN + max = np.NaN + + scale = np.amax(max - min) + normalized_layout = (embedding - min) / scale + + # translate to center on both axis + translate = 0.5 - ((max - min) / scale / 2) + normalized_layout = normalized_layout + translate + + normalized_layout = normalized_layout.astype(dtype=np.float32) + return normalized_layout + + def layout_to_fbs_matrix(self, fields): + """ + return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding. + + * returns only first two dimensions, with name {ename}_0 and {ename}_1, + where {ename} is the embedding name. + * client assumes each will be individually centered & scaled (isotropically) + to a [0, 1] range. + * does not support filtering + + """ + embeddings = self.get_embedding_names() if fields is None or len(fields) == 0 else fields + layout_data = [] + with ServerTiming.time("layout.query"): + for ename in embeddings: + embedding = self.get_embedding_array(ename, 2) + normalized_layout = DataAdaptor.normalize_embedding(embedding) + layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"])) + + with ServerTiming.time("layout.encode"): + if layout_data: + df = pd.concat(layout_data, axis=1, copy=False) + else: + df = pd.DataFrame() + fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None) + + return fbs + + def get_last_mod_time(self): + try: + lastmod = self.get_data_locator().lastmodtime() + except RuntimeError: + lastmod = None + return lastmod diff --git a/local_server/data_common/fbs/NetEncoding/Column.py b/local_server/data_common/fbs/NetEncoding/Column.py new file mode 100644 index 00000000..89b786d6 --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Column.py @@ -0,0 +1,41 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Column(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsColumn(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Column() + x.Init(buf, n + offset) + return x + + # Column + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Column + def UType(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return 0 + + # Column + def U(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + from flatbuffers.table import Table + obj = Table(bytearray(), 0) + self._tab.Union(obj, o) + return obj + return None + +def ColumnStart(builder): builder.StartObject(2) +def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0) +def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0) +def ColumnEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Float32Array.py b/local_server/data_common/fbs/NetEncoding/Float32Array.py new file mode 100644 index 00000000..1acc426c --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Float32Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Float32Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsFloat32Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Float32Array() + x.Init(buf, n + offset) + return x + + # Float32Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Float32Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return 0 + + # Float32Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o) + return 0 + + # Float32Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Float32ArrayStart(builder): builder.StartObject(1) +def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def Float32ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Float64Array.py b/local_server/data_common/fbs/NetEncoding/Float64Array.py new file mode 100644 index 00000000..2ec343a2 --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Float64Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Float64Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsFloat64Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Float64Array() + x.Init(buf, n + offset) + return x + + # Float64Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Float64Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8)) + return 0 + + # Float64Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o) + return 0 + + # Float64Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Float64ArrayStart(builder): builder.StartObject(1) +def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8) +def Float64ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Int32Array.py b/local_server/data_common/fbs/NetEncoding/Int32Array.py new file mode 100644 index 00000000..f3f8156f --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Int32Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Int32Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsInt32Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Int32Array() + x.Init(buf, n + offset) + return x + + # Int32Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Int32Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return 0 + + # Int32Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) + return 0 + + # Int32Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Int32ArrayStart(builder): builder.StartObject(1) +def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def Int32ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/JSONEncodedArray.py b/local_server/data_common/fbs/NetEncoding/JSONEncodedArray.py new file mode 100644 index 00000000..366ebadd --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/JSONEncodedArray.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class JSONEncodedArray(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsJSONEncodedArray(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = JSONEncodedArray() + x.Init(buf, n + offset) + return x + + # JSONEncodedArray + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # JSONEncodedArray + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) + return 0 + + # JSONEncodedArray + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o) + return 0 + + # JSONEncodedArray + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def JSONEncodedArrayStart(builder): builder.StartObject(1) +def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1) +def JSONEncodedArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Matrix.py b/local_server/data_common/fbs/NetEncoding/Matrix.py new file mode 100644 index 00000000..8c9d0eb8 --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Matrix.py @@ -0,0 +1,98 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Matrix(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsMatrix(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Matrix() + x.Init(buf, n + offset) + return x + + # Matrix + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Matrix + def NRows(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return 0 + + # Matrix + def NCols(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return 0 + + # Matrix + def Columns(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + x = self._tab.Vector(o) + x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 + x = self._tab.Indirect(x) + from .Column import Column + obj = Column() + obj.Init(self._tab.Bytes, x) + return obj + return None + + # Matrix + def ColumnsLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # Matrix + def ColIndexType(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return 0 + + # Matrix + def ColIndex(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) + if o != 0: + from flatbuffers.table import Table + obj = Table(bytearray(), 0) + self._tab.Union(obj, o) + return obj + return None + + # Matrix + def RowIndexType(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return 0 + + # Matrix + def RowIndex(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) + if o != 0: + from flatbuffers.table import Table + obj = Table(bytearray(), 0) + self._tab.Union(obj, o) + return obj + return None + +def MatrixStart(builder): builder.StartObject(7) +def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0) +def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0) +def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0) +def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0) +def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0) +def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0) +def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0) +def MatrixEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/TypedArray.py b/local_server/data_common/fbs/NetEncoding/TypedArray.py new file mode 100644 index 00000000..e36c4f1b --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/TypedArray.py @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +class TypedArray(object): + NONE = 0 + Float32Array = 1 + Int32Array = 2 + Uint32Array = 3 + Float64Array = 4 + JSONEncodedArray = 5 + diff --git a/local_server/data_common/fbs/NetEncoding/Uint32Array.py b/local_server/data_common/fbs/NetEncoding/Uint32Array.py new file mode 100644 index 00000000..01c7dfdd --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Uint32Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Uint32Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsUint32Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Uint32Array() + x.Init(buf, n + offset) + return x + + # Uint32Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Uint32Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return 0 + + # Uint32Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o) + return 0 + + # Uint32Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Uint32ArrayStart(builder): builder.StartObject(1) +def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def Uint32ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/__init__.py b/local_server/data_common/fbs/NetEncoding/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_common/fbs/__init__.py b/local_server/data_common/fbs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_common/fbs/matrix.py b/local_server/data_common/fbs/matrix.py new file mode 100644 index 00000000..14fa71ba --- /dev/null +++ b/local_server/data_common/fbs/matrix.py @@ -0,0 +1,248 @@ +import json + +import numpy as np +import pandas as pd +from flatbuffers import Builder +from scipy import sparse + +import local_server.data_common.fbs.NetEncoding.Column as Column +import local_server.data_common.fbs.NetEncoding.Float32Array as Float32Array +import local_server.data_common.fbs.NetEncoding.Float64Array as Float64Array +import local_server.data_common.fbs.NetEncoding.Int32Array as Int32Array +import local_server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray +import local_server.data_common.fbs.NetEncoding.Matrix as Matrix +import local_server.data_common.fbs.NetEncoding.TypedArray as TypedArray +import local_server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array + + +# Serialization helper +def serialize_column(builder, typed_arr): + """ Serialize NetEncoding.Column """ + + (u_type, u_value) = typed_arr + Column.ColumnStart(builder) + Column.ColumnAddUType(builder, u_type) + Column.ColumnAddU(builder, u_value) + return Column.ColumnEnd(builder) + + +# Serialization helper +def serialize_matrix(builder, n_rows, n_cols, columns, col_idx): + """ Serialize NetEncoding.Matrix """ + + Matrix.MatrixStart(builder) + Matrix.MatrixAddNRows(builder, n_rows) + Matrix.MatrixAddNCols(builder, n_cols) + Matrix.MatrixAddColumns(builder, columns) + if col_idx is not None: + (u_type, u_val) = col_idx + Matrix.MatrixAddColIndexType(builder, u_type) + Matrix.MatrixAddColIndex(builder, u_val) + return Matrix.MatrixEnd(builder) + + +# Serialization helper +def serialize_typed_array(builder, source_array, encoding_info): + """ + Serialize any of the various typed arrays, eg, Float32Array. Specific means of serialization and type conversion + are provided by type_info. + """ + + arr = source_array + (array_type, as_type) = encoding_info(source_array) + + if isinstance(arr, pd.Index): + arr = arr.to_series() + + # convert to a simple ndarray + if as_type == "json": + as_json = arr.to_json(orient="records") + arr = np.array(bytearray(as_json, "utf-8")) + else: + if sparse.issparse(arr): + arr = arr.toarray() + elif isinstance(arr, pd.Series): + arr = arr.to_numpy() + if arr.dtype != as_type: + arr = arr.astype(as_type) + + # serialize the ndarray into a vector + if arr.ndim == 2: + if arr.shape[0] == 1: + arr = arr[0] + elif arr.shape[1] == 1: + arr = arr.T[0] + + vec = builder.CreateNumpyVector(arr) + + # serialize the typed array table + builder.StartObject(1) + builder.PrependUOffsetTRelativeSlot(0, vec, 0) + array_value = builder.EndObject() + return (array_type, array_value) + + +def column_encoding(arr): + column_encoding_type_map = { + # array protocol string: ( array_type, as_type ) + np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64), + np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), + np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), + np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + } + column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") + + return column_encoding_type_map.get(arr.dtype.str, column_encoding_default) + + +def index_encoding(arr): + index_encoding_type_map = { + # array protocol string: ( array_type, as_type ) + np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + } + index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") + + return index_encoding_type_map.get(arr.dtype.str, index_encoding_default) + + +def guess_at_mem_needed(matrix): + (n_rows, n_cols) = matrix.shape + if isinstance(matrix, np.ndarray) or sparse.issparse(matrix): + guess = (n_rows * n_cols * matrix.dtype.itemsize) + 1024 + elif isinstance(matrix, pd.DataFrame): + # XXX TODO - DataFrame type estimate + guess = 1 + else: + guess = 1 + + # round up to nearest 1024 bytes + guess = (guess + 0x400) & (~0x3FF) + return guess + + +def encode_matrix_fbs(matrix, row_idx=None, col_idx=None): + """ + Given a 2D DataFrame, ndarray or sparse equivalent, create and return a Matrix flatbuffer. + + :param matrix: 2D DataFrame, ndarray or sparse equivalent + :param row_idx: index for row dimension, Index or ndarray + :param col_idx: index for col dimension, Index or ndarray + + NOTE: row indices are (currently) unsupported and must be None + """ + + if row_idx is not None: + raise ValueError("row indexing not supported for FBS Matrix") + if matrix.ndim != 2: + raise ValueError("FBS Matrix must be 2D") + + (n_rows, n_cols) = matrix.shape + + # estimate size needed, so we don't unnecessarily realloc. + builder = Builder(guess_at_mem_needed(matrix)) + + columns = [] + for cidx in range(n_cols - 1, -1, -1): + # serialize the typed array + col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx] + typed_arr = serialize_typed_array(builder, col, column_encoding) + + # serialize the Column union + columns.append(serialize_column(builder, typed_arr)) + + # Serialize Matrix.columns[] + Matrix.MatrixStartColumnsVector(builder, n_cols) + for c in columns: + builder.PrependUOffsetTRelative(c) + matrix_column_vec = builder.EndVector(n_cols) + + # serialize the colIndex if provided + cidx = None + if col_idx is not None: + cidx = serialize_typed_array(builder, col_idx, index_encoding) + + # Serialize Matrix + matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx) + + builder.Finish(matrix) + return builder.Output() + + +def deserialize_typed_array(tarr): + type_map = { + TypedArray.TypedArray.NONE: None, + TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array, + TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, + TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, + TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, + TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray, + } + (u_type, u) = tarr + if u_type is TypedArray.TypedArray.NONE: + return None + + TarType = type_map.get(u_type, None) + if TarType is None: + raise TypeError(f"FBS contains unknown data type: {u_type}") + + arr = TarType() + arr.Init(u.Bytes, u.Pos) + narr = arr.DataAsNumpy() + if u_type == TypedArray.TypedArray.JSONEncodedArray: + narr = json.loads(narr.tostring().decode("utf-8")) + return narr + + +def decode_matrix_fbs(fbs): + """ + Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data and indices. + """ + + matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0) + n_rows = matrix.NRows() + n_cols = matrix.NCols() + if n_rows == 0 or n_cols == 0: + return pd.DataFrame() + + if matrix.RowIndexType() is not TypedArray.TypedArray.NONE: + raise ValueError("row indexing not supported for FBS Matrix") + + columns_length = matrix.ColumnsLength() + + columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex())) + if columns_index is None: + columns_index = range(0, n_cols) + + # sanity checks + if len(columns_index) != n_cols or columns_length != n_cols: + raise ValueError("FBS column count does not match number of columns in underlying matrix") + + columns_data = {} + columns_type = {} + for col_idx in range(0, columns_length): + col = matrix.Columns(col_idx) + tarr = (col.UType(), col.U()) + data = deserialize_typed_array(tarr) + columns_data[columns_index[col_idx]] = data + if len(data) != n_rows: + raise ValueError("FBS column length does not match number of rows") + if col.UType() is TypedArray.TypedArray.JSONEncodedArray: + columns_type[columns_index[col_idx]] = "category" + + df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False) + + # more sanity checks + if not df.columns.is_unique or len(df.columns) != n_cols: + raise KeyError("FBS column indices are not unique") + + return df diff --git a/local_server/data_common/matrix_loader.py b/local_server/data_common/matrix_loader.py new file mode 100644 index 00000000..065e2d91 --- /dev/null +++ b/local_server/data_common/matrix_loader.py @@ -0,0 +1,55 @@ +from enum import Enum +from local_server.common.errors import DatasetAccessError +from local_server.common.data_locator import DataLocator +from http import HTTPStatus + + +class MatrixDataType(Enum): + H5AD = "h5ad" + UNKNOWN = "unknown" + + +class MatrixDataLoader(object): + def __init__(self, location, matrix_data_type=None, app_config=None): + """ location can be a string or DataLocator """ + region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name + self.location = DataLocator(location, region_name=region_name) + if not self.location.exists(): + raise DatasetAccessError("Dataset does not exist.", HTTPStatus.NOT_FOUND) + + # matrix_data_type is an enum value of type MatrixDataType + self.matrix_data_type = matrix_data_type + # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type + self.matrix_type = None + + if matrix_data_type is None: + self.matrix_data_type = self.__matrix_data_type() + + if not self.__matrix_data_type_allowed(app_config): + raise DatasetAccessError("Dataset does not have an allowed type.") + + if self.matrix_data_type == MatrixDataType.H5AD: + from local_server.data_anndata.anndata_adaptor import AnndataAdaptor + + self.matrix_type = AnndataAdaptor + + def __matrix_data_type(self): + if self.location.path.endswith(".h5ad"): + return MatrixDataType.H5AD + else: + return MatrixDataType.UNKNOWN + + def __matrix_data_type_allowed(self, app_config): + return self.matrix_data_type != MatrixDataType.UNKNOWN + + def pre_load_validation(self): + if self.matrix_data_type == MatrixDataType.UNKNOWN: + raise DatasetAccessError("Dataset does not have a recognized type: .h5ad") + self.matrix_type.pre_load_validation(self.location) + + def file_size(self): + return self.matrix_type.file_size(self.location) + + def open(self, app_config, dataset_config=None): + # create and return a DataAdaptor object + return self.matrix_type.open(self.location, app_config, dataset_config) diff --git a/local_server/default_config.py b/local_server/default_config.py new file mode 100644 index 00000000..d77cbdc4 --- /dev/null +++ b/local_server/default_config.py @@ -0,0 +1,138 @@ +import yaml + +default_config = """ +server: + app: + verbose: false + debug: false + host: localhost + port : null + open_browser: false + force_https: false + flask_secret_key: null + + authentication: + # The authentication types may be "none" or "session" + # none: No authentication support, features like user_annotations must not be enabled. + # session: A session based userid is automatically generated. (no params needed) + type: session + insecure_test_environment: false + + single_dataset: + # If datapath is set, then cellxgene with serve a single dataset located at datapath. + datapath: null + obs_names: null + var_names: null + about: null + title: null + + data_locator: + s3: + # s3 region name. + # if true, then the s3 location is automatically determined from the datapath or dataroot. + # if false/null, then do not set. + # if a string, then use that value (e.g. us-east-1). + region_name: true + + adaptor: + anndata_adaptor: + backed: false + + limits: + column_request_max: 32 + diffexp_cellcount_max: null + + +dataset: + app: + # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin. + # these will be injected into the index template as script tags with these attributes set. + scripts: [] + # Inline scripts are a list of file names, where the contents of the file will be injected into the index. + inline_scripts: [] + + # allow authentication support + authentication_enable: true + + presentation: + max_categories: 1000 + custom_colors: true + + user_annotations: + enable: true + type: local_file_csv + local_file_csv: + directory: null + file: null # annotations file name + gene_sets_file: null # gene sets file name + ontology: + enable: false + obo_location: null + gene_sets: + readonly: false # gene sets CRUD enabled/disabled + + embeddings: + names : [] + enable_reembedding: false + + diffexp: + enable: true + lfc_cutoff: 0.01 + top_n: 10 + +external: + # You can retrieve configuration parameters from this config file, the environment, + # the AWS secrets manager, or from the "cellxgene launch" command line arguments. + # They are applied in that order, meaning that if a parameter is defined in more + # than one location, the last one applied takes effect. + + # environment variables: + # This section describes how to map environment variables to configuration parameters. + # The format is a list defining an environment variable. + # Each entry in the list is a dictionary with three entries: + # name: the name of the environment variable + # path: the path within the cellxgene configuration to update. + # required: (default=False) a boolean. If true, then it is an error if the environment variable is not set. + + environment: + - name: CXG_SECRET_KEY + path: [server, app, flask_secret_key] + required: false + + # AWS Secrets Manager + # This section describes how to map aws secrets to configuration parameters. + # The format is the region for the secrets manager, then a list of secrets. + # each secret has a name, and a list of values. + # Each entry in the list of values is a dictionary with three entries: + # key: the key of the aws secret. + # path: the path within the cellxgene configuration to update. + # required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret. + # + # example: + # aws_secrets_manager: + # region: us-west-2 + # - name: my_first_secret + # values: + # - key: flask_secret_key + # path: [server, app, flask_secret_key] + # required: true + # - key: db_uri + # path: [dataset, user_annotations, db_uri] + # required: true + # - name: my_auth_secret + # values: + # - key: client_secret + # path: [server, authentication, client_secret] + # required: true + # - key: client_id + # path: [server, authentication, client_id] + # required: true + + aws_secrets_manager: + region: null + secrets: [] +""" + + +def get_default_config(): + return yaml.load(default_config, Loader=yaml.Loader) diff --git a/local_server/requirements-dev.txt b/local_server/requirements-dev.txt new file mode 100644 index 00000000..9f4fd724 --- /dev/null +++ b/local_server/requirements-dev.txt @@ -0,0 +1,10 @@ +Authlib>=0.14.3 +black +bumpversion>=0.5 +codecov>=2.0.15 +parameterized>=0.7.0 +psycopg2-binary>=2.8.5 +pytest>=3.6.3 +python-jose>=3.2.0 +twine>=1.12.1 +-r requirements.txt diff --git a/local_server/requirements-prepare.txt b/local_server/requirements-prepare.txt new file mode 100644 index 00000000..254f827b --- /dev/null +++ b/local_server/requirements-prepare.txt @@ -0,0 +1,2 @@ +python-igraph +louvain>=0.6 diff --git a/local_server/requirements.txt b/local_server/requirements.txt new file mode 100644 index 00000000..f60c5b13 --- /dev/null +++ b/local_server/requirements.txt @@ -0,0 +1,24 @@ +anndata>=0.7.0 +boto3>=1.12.18 +click>=7.1.2 +fastobo>=0.6.1 +Flask>=1.0.2 +Flask-Compress>=1.4.0 +Flask-Cors>=3.0.6 +Flask-RESTful>=0.3.6 +flask-server-timing>=0.1.2 +flask-talisman>=0.7.0 +flatbuffers>=1.11.0 +flatten-dict>=0.2.0 +fsspec>=0.4.4,<0.8.0 +gunicorn>=20.0.4 +h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5 +numpy>=1.15.0 +packaging>=20.0 +pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446 +PyYAML>=5.3 +scipy>=1.0 +requests>=2.22.0 +s3fs==0.4.2 +scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here +umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn diff --git a/local_server/test/__init__.py b/local_server/test/__init__.py new file mode 100644 index 00000000..7dbd481e --- /dev/null +++ b/local_server/test/__init__.py @@ -0,0 +1,170 @@ +import os +import random +import shutil +import string +import tempfile +import time +from contextlib import contextmanager +from os import path, popen +from subprocess import Popen + +import pandas as pd +import requests + +from local_server.common.annotations.local_file_csv import AnnotationsLocalFile +from local_server.common.config.app_config import AppConfig +from local_server.common.config import DEFAULT_SERVER_PORT +from local_server.common.data_locator import DataLocator +from local_server.common.utils.utils import find_available_port +from local_server.data_common.fbs.matrix import encode_matrix_fbs +from local_server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType + +PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() +FIXTURES_ROOT = PROJECT_ROOT + "/local_server/test/fixtures" +H5AD_FIXTURE = FIXTURES_ROOT + "/pbmc3k-CSC-gz.h5ad" + + +def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): + tmp_dir = tempfile.mkdtemp() + annotations_file = path.join(tmp_dir, "test_annotations.csv") + if annotations_fixture: + shutil.copyfile(f"{PROJECT_ROOT}/local_server/test/fixtures/pbmc3k-annotations.csv", annotations_file) + fname = { + MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + }[ext] + data_locator = DataLocator(fname) + config = AppConfig() + config.update_server_config( + app__flask_secret_key="secret", + single_dataset__obs_names=None, + single_dataset__var_names=None, + single_dataset__datapath=data_locator.path, + ) + config.update_dataset_config( + embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, + ) + + config.complete_config() + data = MatrixDataLoader(data_locator.abspath()).open(config) + anno_config = { + "user-annotations": True, + "genesets-save": False, + } + annotations = AnnotationsLocalFile(anno_config, None, annotations_file, None) + return data, tmp_dir, annotations + + +def make_fbs(data): + df = pd.DataFrame(data) + return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) + + +def skip_if(condition, reason: str): + def decorator(f): + def wraps(self, *args, **kwargs): + if condition(self): + self.skipTest(reason) + else: + f(self, *args, **kwargs) + + return wraps + + return decorator + + +def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}): + config = AppConfig() + config.update_server_config( + app__flask_secret_key="secret", + single_dataset__obs_names=None, + single_dataset__var_names=None, + adaptor__anndata_adaptor__backed=backed, + single_dataset__datapath=data_locator, + limits__diffexp_cellcount_max=None, + limits__column_request_max=None, + ) + config.update_dataset_config( + embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01 + ) + config.update_server_config(**extra_server_config) + config.update_dataset_config(**extra_dataset_config) + config.complete_config() + return config + + +def random_string(n): + return "".join(random.choice(string.ascii_letters) for _ in range(n)) + + +def start_test_server(command_line_args=[], app_config=None, env=None): + """ + Command line arguments can be passed in, as well as an app_config. + This function is meant to be used like this, for example: + + with test_server(...) as server: + r = requests.get(f"{server}/...") + // check r + + where the server can be accessed within the context, and is terminated when + the context is exited. + The port is automatically set using find_available_port, unless passed in as a command line arg. + The verbose flag is automatically set to True. + If an app_config is provided, then this function writes a temporary + yaml config file, which this server will read and parse. + """ + + command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose"] + if "-p" in command_line_args: + port = int(command_line_args[command_line_args.index("-p") + 1]) + elif "--port" in command_line_args: + port = int(command_line_args[command_line_args.index("--port") + 1]) + else: + start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1) + port = int(os.environ.get("CXG_SERVER_PORT", start)) + port = find_available_port("localhost", port) + command += ["--port=%d" % port] + + command += command_line_args + + tempdir = None + if app_config: + tempdir = tempfile.TemporaryDirectory() + config_file = os.path.join(tempdir.name, "config.yaml") + app_config.write_config(config_file) + command.extend(["-c", config_file]) + + server = f"http://localhost:{port}" + ps = Popen(command, env=env) + + for _ in range(10): + try: + requests.get(f"{server}/health") + break + except requests.exceptions.ConnectionError: + time.sleep(1) + + if tempdir: + tempdir.cleanup() + + return ps, server + + +def stop_test_server(ps): + try: + ps.terminate() + except ProcessLookupError: + pass + + +@contextmanager +def test_server(command_line_args=[], app_config=None, env=None): + """A context to run the cellxgene server.""" + + ps, server = start_test_server(command_line_args, app_config, env) + try: + yield server + finally: + try: + stop_test_server(ps) + except ProcessLookupError: + pass diff --git a/local_server/test/fixtures/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad b/local_server/test/fixtures/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad new file mode 100644 index 00000000..eee29036 Binary files /dev/null and b/local_server/test/fixtures/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad differ diff --git a/local_server/test/fixtures/__init__.py b/local_server/test/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/fixtures/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad b/local_server/test/fixtures/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad new file mode 100644 index 00000000..9d7ffcc6 Binary files /dev/null and b/local_server/test/fixtures/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad differ diff --git a/local_server/test/fixtures/dataset_config_outline.py b/local_server/test/fixtures/dataset_config_outline.py new file mode 100644 index 00000000..7555b747 --- /dev/null +++ b/local_server/test/fixtures/dataset_config_outline.py @@ -0,0 +1,34 @@ +f""" +dataset: + app: + scripts: {scripts} #list of strs (filenames) or dicts containing keys + inline_scripts: {inline_scripts} #list of strs (filenames) + + authentication_enable: {authentication_enable} + + presentation: + max_categories: {max_categories} + custom_colors: {custom_colors} + + user_annotations: + enable: {enable_users_annotations} + type: {annotation_type} + local_file_csv: + directory: {local_file_csv_directory} + file: {local_file_csv_file} + gene_sets_file: {local_file_csv_gene_sets_file} + ontology: + enable: {ontology_enabled} + obo_location: {obo_location} + gene_sets: + readonly: {gene_sets_readonly} + + embeddings: + names: {embedding_names} + enable_reembedding: {enable_reembedding} + + diffexp: + enable: {enable_difexp} + lfc_cutoff: {lfc_cutoff} + top_n: {top_n} +""" diff --git a/local_server/test/fixtures/fixtures.py b/local_server/test/fixtures/fixtures.py new file mode 100644 index 00000000..e1a1368d --- /dev/null +++ b/local_server/test/fixtures/fixtures.py @@ -0,0 +1,12 @@ +pbmc3k_colors = { + "louvain": { + "B cells": "#2ca02c", + "CD14+ Monocytes": "#ff7f0e", + "CD4 T cells": "#1f77b4", + "CD8 T cells": "#d62728", + "Dendritic cells": "#e377c2", + "FCGR3A+ Monocytes": "#8c564b", + "Megakaryocytes": "#bcbd22", + "NK cells": "#9467bd", + } +} diff --git a/local_server/test/fixtures/hgnc_example.txt.gz b/local_server/test/fixtures/hgnc_example.txt.gz new file mode 100644 index 00000000..4c7704c4 Binary files /dev/null and b/local_server/test/fixtures/hgnc_example.txt.gz differ diff --git a/local_server/test/fixtures/nan.h5ad b/local_server/test/fixtures/nan.h5ad new file mode 100644 index 00000000..d97078b0 Binary files /dev/null and b/local_server/test/fixtures/nan.h5ad differ diff --git a/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad b/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad new file mode 100644 index 00000000..2c2909c5 Binary files /dev/null and b/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad differ diff --git a/local_server/test/fixtures/pbmc3k-CSR-gz.h5ad b/local_server/test/fixtures/pbmc3k-CSR-gz.h5ad new file mode 100644 index 00000000..3053f0a9 Binary files /dev/null and b/local_server/test/fixtures/pbmc3k-CSR-gz.h5ad differ diff --git a/local_server/test/fixtures/pbmc3k-annotations.csv b/local_server/test/fixtures/pbmc3k-annotations.csv new file mode 100644 index 00000000..8031f7cd --- /dev/null +++ b/local_server/test/fixtures/pbmc3k-annotations.csv @@ -0,0 +1,2641 @@ +# Annotations generated on 2020-02-24T23:19:31 using cellxgene version 0.14.0 +# Input data file was example-dataset/pbmc3k.h5ad, which was last modified on 2019-12-17T17:13:17 +index,cluster-test +AAACATACAACCAC-1,four +AAACATTGAGCTAC-1,three +AAACATTGATCAGC-1,four +AAACCGTGCTTCCG-1,two +AAACCGTGTATGCG-1,five +AAACGCACTGGTAC-1,four +AAACGCTGACCAGT-1,unassigned +AAACGCTGGTTCTT-1,four +AAACGCTGTAGCCA-1,five +AAACGCTGTTTCTG-1,one +AAACTTGAAAAACG-1,three +AAACTTGATCCAGA-1,four +AAAGAGACGAGATA-1,four +AAAGAGACGCGAGA-1,two +AAAGAGACGGACTT-1,five +AAAGAGACGGCATT-1,four +AAAGCAGATATCGG-1,two +AAAGCCTGTATGCG-1,four +AAAGGCCTGTCTAG-1,three +AAAGTTTGATCACG-1,three +AAAGTTTGGGGTGA-1,three +AAAGTTTGTAGAGA-1,four +AAAGTTTGTAGCGT-1,two +AAATCAACAATGCC-1,three +AAATCAACACCAGT-1,four +AAATCAACCAGGAG-1,four +AAATCAACCCTATT-1,seven +AAATCAACGGAAGC-1,four +AAATCAACTCGCAA-1,four +AAATCATGACCACA-1,one +AAATCCCTCCACAA-1,four +AAATCCCTGCTATG-1,three +AAATGTTGAACGAA-1,two +AAATGTTGCCACAA-1,four +AAATGTTGTGGCAT-1,two +AAATTCGAAGGTTC-1,four +AAATTCGAATCACG-1,two +AAATTCGAGCTGAT-1,one +AAATTCGAGGAGTG-1,four +AAATTCGATTCTCA-1,five +AAATTGACACGACT-1,five +AAATTGACTCGCTC-1,unassigned +AACAAACTCATTTC-1,four +AACAAACTTTCGTT-1,four +AACAATACGACGAG-1,four +AACACGTGCAGAGG-1,four +AACACGTGGAAAGT-1,four +AACACGTGGAACCT-1,two +AACACGTGGCTACA-1,four +AACACGTGTACGAC-1,one +AACAGCACAAGAGT-1,two +AACATTGATGGGAG-1,one +AACCAGTGATACCG-1,one +AACCCAGATCGCTC-1,four +AACCGATGCTCCCA-1,four +AACCGATGGTCATG-1,three +AACCGATGTTCTAC-1,two +AACCGCCTAGCGTT-1,four +AACCGCCTCTACGA-1,two +AACCTACTGTGAGG-1,two +AACCTACTGTGTTG-1,four +AACCTTACGAGACG-1,four +AACCTTACGCGAGA-1,five +AACCTTACTAACGC-1,one +AACCTTTGGACGGA-1,four +AACCTTTGTACGCA-1,one +AACGCAACAAGTAG-1,four +AACGCATGACCCAA-1,four +AACGCATGCCTTCG-1,four +AACGCATGTACTTC-1,four +AACGCCCTCGGGAA-1,four +AACGCCCTCGTACA-1,five +AACGCCCTGGCATT-1,four +AACGTCGAGTATCG-1,five +AACGTGTGAAAGCA-1,four +AACGTGTGGCGGAA-1,three +AACGTGTGTCCAAG-1,four +AACGTGTGTGCTTT-1,four +AACTACCTTAGAGA-1,four +AACTCACTCAAGCT-1,two +AACTCACTTGGAGG-1,two +AACTCGGAAAGTGA-1,five +AACTCGGAAGGTCT-1,two +AACTCTTGCAGGAG-1,four +AACTGTCTCCCTTG-1,two +AACTTGCTACGCTA-1,three +AACTTGCTGGGACA-1,four +AAGAACGAGTGTTG-1,four +AAGAAGACGTAGGG-1,three +AAGACAGAAGTCTG-1,three +AAGACAGAGGATCT-1,four +AAGACAGATTACCT-1,two +AAGAGATGGGTAGG-1,three +AAGATGGAAAACAG-1,three +AAGATGGAGAACTC-1,two +AAGATGGAGATAAG-1,two +AAGATTACAACCTG-1,four +AAGATTACAGATCC-1,two +AAGATTACCCGTTC-1,two +AAGATTACCGCCTT-1,unassigned +AAGATTACCTCAAG-1,five +AAGATTACTCCTCG-1,two +AAGCAAGAGCGAGA-1,four +AAGCAAGAGCTTAG-1,five +AAGCAAGAGGTGTT-1,five +AAGCACTGAGCAAA-1,three +AAGCACTGCATACG-1,five +AAGCACTGGTTCTT-1,three +AAGCCAACGTGTTG-1,four +AAGCCATGAACTGC-1,unassigned +AAGCCATGACACGT-1,two +AAGCCATGCGTGAT-1,four +AAGCCATGTCTCGC-1,four +AAGCCTGACATGCA-1,four +AAGCCTGACCGAAT-1,two +AAGCGACTCCTCAC-1,one +AAGCGACTGTGTCA-1,four +AAGCGACTTACAGC-1,three +AAGCGACTTTGACG-1,one +AAGCGTACGTCTTT-1,four +AAGGCTTGCGAACT-1,five +AAGGTCACGGTTAC-1,two +AAGGTCACTGTTTC-1,three +AAGGTCACTTCCCG-1,five +AAGGTCTGACAGTC-1,four +AAGGTCTGCAGATC-1,four +AAGTAACTCTGAAC-1,two +AAGTAACTGAGATA-1,three +AAGTAGGATACAGC-1,five +AAGTATACCGAACT-1,four +AAGTCCGACTTGTT-1,four +AAGTCCGATAGAAG-1,four +AAGTCTCTAGTCGT-1,four +AAGTCTCTCGGAGA-1,five +AAGTGGCTTGGAGG-1,three +AAGTTCCTCATTCT-1,unassigned +AAGTTCCTTCTTAC-1,four +AATAAGCTCGAATC-1,four +AATAAGCTCGTTGA-1,four +AATACCCTGGACGA-1,three +AATACCCTGGCATT-1,four +AATACTGAAAGGGC-1,five +AATACTGAATTGGC-1,five +AATAGGGAACCCTC-1,three +AATAGGGAGAATGA-1,five +AATCAAACTATCGG-1,five +AATCCGGAATGCTG-1,three +AATCCTACCGGTAT-1,four +AATCCTTGACGGGA-1,two +AATCCTTGGTGAGG-1,five +AATCGGTGGAACTC-1,five +AATCGGTGTGCTTT-1,four +AATCTAGAAAAGTG-1,four +AATCTAGAATCGGT-1,four +AATCTCACAGCCTA-1,two +AATCTCACTCTAGG-1,four +AATCTCTGAACAGA-1,four +AATCTCTGCTTTAC-1,five +AATGATACACCAAC-1,five +AATGATACGGTCAT-1,two +AATGCGTGACACCA-1,three +AATGCGTGGACGGA-1,unassigned +AATGCGTGGCTATG-1,three +AATGGAGAATCGTG-1,two +AATGGAGATCCTTA-1,five +AATGGCTGACACCA-1,three +AATGGCTGCGTGAT-1,four +AATGGCTGTAAAGG-1,five +AATGGCTGTACTCT-1,five +AATGGCTGTGAAGA-1,four +AATGTAACGGTGGA-1,four +AATGTCCTCTTCTA-1,four +AATGTTGACAGTCA-1,four +AATGTTGAGTTGAC-1,two +AATGTTGATCTACT-1,four +AATTACGAATTCCT-1,unassigned +AATTACGACTTCTA-1,four +AATTACGAGTGAGG-1,unassigned +AATTACGATTGGCA-1,three +AATTCCTGCTCAGA-1,three +AATTGATGTCGCAA-1,one +AATTGTGACTTGGA-1,three +ACAAAGGAGGGTGA-1,four +ACAAATTGATTCTC-1,one +ACAAATTGCTCAGA-1,two +ACAAATTGTTGCGA-1,five +ACAACCGAGGGATG-1,five +ACAACCGAGTTACG-1,four +ACAAGAGAAGTCGT-1,two +ACAAGAGACTTATC-1,four +ACAAGAGAGTTGAC-1,three +ACAATCCTAACCGT-1,four +ACAATCCTTAGCGT-1,three +ACAATTGACTGACA-1,five +ACAATTGATGACTG-1,five +ACACAGACCATACG-1,five +ACACCAGAGGGCAA-1,five +ACACCCTGGTGTTG-1,five +ACACGAACAGTTCG-1,four +ACACGATGACGCAT-1,five +ACACGATGATGTGC-1,unassigned +ACACGATGTCGTAG-1,five +ACACGATGTGGTCA-1,five +ACAGACACGGCATT-1,four +ACAGACACGTTGTG-1,four +ACAGCAACACCTAG-1,two +ACAGCAACCTCAAG-1,one +ACAGGTACCCCACT-1,four +ACAGGTACGCTGTA-1,five +ACAGGTACTGGTGT-1,five +ACAGTCGACCCAAA-1,four +ACAGTCGACCGATA-1,five +ACAGTGACTCACCC-1,five +ACAGTGACTCTATC-1,five +ACAGTGTGGTCACA-1,four +ACAGTGTGTTGCGA-1,two +ACATCACTCTACTT-1,two +ACATGGTGAAGCCT-1,four +ACATGGTGCAACCA-1,three +ACATGGTGCGAGTT-1,four +ACATTCTGGCATAC-1,four +ACATTCTGGGAACG-1,two +ACCAACGACATGCA-1,four +ACCACAGAAAGTAG-1,four +ACCACAGAGTTGGT-1,four +ACCACCTGTGTGCA-1,four +ACCACGCTACAGCT-1,four +ACCACGCTACCCAA-1,five +ACCACGCTGCGAGA-1,four +ACCACGCTGCTGTA-1,four +ACCAGCCTGACAGG-1,four +ACCAGTGAACGGTT-1,three +ACCAGTGAATACCG-1,one +ACCAGTGAGGGATG-1,four +ACCAGTGATGACTG-1,two +ACCATTACCTTCTA-1,four +ACCATTACGAGATA-1,four +ACCATTTGTCATTC-1,four +ACCCAAGAACTGTG-1,five +ACCCAAGAATTCCT-1,one +ACCCAAGAGGACAG-1,four +ACCCAAGATTCACT-1,three +ACCCACTGCGCCTT-1,two +ACCCACTGGACAGG-1,four +ACCCACTGGTTCAG-1,six +ACCCACTGTCGTAG-1,two +ACCCAGCTCAGAAA-1,four +ACCCAGCTGTTAGC-1,two +ACCCAGCTTGCTTT-1,four +ACCCGTTGATGACC-1,three +ACCCGTTGCTGCAA-1,four +ACCCGTTGCTTCTA-1,unassigned +ACCCTCGACCTATT-1,four +ACCCTCGACGGTAT-1,three +ACCCTCGATAAGGA-1,two +ACCCTCGATCAAGC-1,unassigned +ACCGTGCTACCAGT-1,four +ACCGTGCTGGAACG-1,three +ACCTATTGCTGAGT-1,three +ACCTATTGTGCCCT-1,one +ACCTCCGAGTCCTC-1,four +ACCTCCGATATGCG-1,four +ACCTCCGATGCTGA-1,four +ACCTCGTGAACCAC-1,unassigned +ACCTGAGATATCGG-1,one +ACCTGGCTAAGTAG-1,five +ACCTTTGACTCCCA-1,two +ACCTTTGAGGAACG-1,two +ACCTTTGAGGAAGC-1,one +ACGAACACCTTGTT-1,five +ACGAACTGGCTATG-1,six +ACGAAGCTCTCCAC-1,four +ACGAAGCTCTGAGT-1,three +ACGACCCTATCTCT-1,one +ACGACCCTGATGAA-1,four +ACGACCCTTGACAC-1,three +ACGACCCTTGACCA-1,four +ACGAGGGACAGGAG-1,unassigned +ACGAGGGACGAACT-1,four +ACGAGGGATGTAGC-1,four +ACGAGTACCCTAAG-1,three +ACGAGTACGAATCC-1,three +ACGATCGAGGACTT-1,five +ACGATCGAGTCACA-1,two +ACGATGACAATGCC-1,four +ACGATGACTGGTCA-1,four +ACGATTCTACGGGA-1,four +ACGCAATGGTTCAG-1,five +ACGCACCTGTTAGC-1,three +ACGCCACTGAACTC-1,two +ACGCCGGAAACCAC-1,two +ACGCCGGAAAGCCT-1,four +ACGCCGGAAATGCC-1,four +ACGCCTTGCTCCCA-1,one +ACGCGGTGGCGAGA-1,four +ACGCGGTGTGTGGT-1,three +ACGCGGTGTTTGCT-1,five +ACGCTCACAGTACC-1,four +ACGCTCACCCTTGC-1,four +ACGCTGCTGTTCTT-1,five +ACGGAACTCAGATC-1,five +ACGGAACTGTCGTA-1,one +ACGGAGGACTCTTA-1,two +ACGGATTGGGAGGT-1,three +ACGGATTGGTTAGC-1,four +ACGGCTCTGAGCAG-1,four +ACGGCTCTTGCACA-1,four +ACGGTAACCGCTAA-1,four +ACGGTAACCTTCGC-1,five +ACGGTAACGGTGGA-1,four +ACGGTAACTCGCAA-1,five +ACGGTATGAGTCGT-1,two +ACGGTATGGGTATC-1,three +ACGGTATGGTTGTG-1,three +ACGGTCCTAACGGG-1,four +ACGGTCCTCGGGAA-1,four +ACGTAGACAACCAC-1,four +ACGTAGACTACAGC-1,three +ACGTCAGAAACGAA-1,four +ACGTCAGAGAGCTT-1,two +ACGTCAGAGGGATG-1,four +ACGTCCTGATAAGG-1,two +ACGTCCTGTGAACC-1,unassigned +ACGTCGCTCCTGAA-1,five +ACGTCGCTCTATTC-1,unassigned +ACGTCGCTTCTCAT-1,three +ACGTGATGCCATGA-1,unassigned +ACGTGATGGGTCTA-1,four +ACGTGATGTAACCG-1,one +ACGTGATGTGACAC-1,five +ACGTGCCTCCGTAA-1,four +ACGTGCCTTCTATC-1,five +ACGTTACTTTCCAT-1,four +ACGTTGGAAAAGCA-1,four +ACGTTGGAAACCTG-1,five +ACGTTGGACCGTAA-1,two +ACGTTGGAGCCAAT-1,five +ACGTTGGATATGGC-1,three +ACGTTGGATCAGGT-1,two +ACGTTTACATCAGC-1,four +ACTAAAACCCACAA-1,five +ACTAAAACTCGACA-1,two +ACTACGGAATTTCC-1,four +ACTACGGACCTATT-1,four +ACTACGGATCGCTC-1,four +ACTACTACTAAGGA-1,four +ACTAGGTGGAACCT-1,five +ACTAGGTGGAACTC-1,four +ACTATCACCTTGGA-1,five +ACTATCACTGCCAA-1,four +ACTCAGGACTGAAC-1,four +ACTCAGGATCTATC-1,three +ACTCAGGATTCGTT-1,two +ACTCCTCTCAACTG-1,four +ACTCGCACGAAAGT-1,two +ACTCGCACTACGAC-1,two +ACTCTCCTGACACT-1,four +ACTCTCCTGCATAC-1,four +ACTCTCCTGTTTGG-1,four +ACTGAGACAACCAC-1,five +ACTGAGACCCATAG-1,one +ACTGAGACGTTGGT-1,three +ACTGCCACACACGT-1,five +ACTGCCACTCCGTC-1,five +ACTGGCCTTCAGTG-1,five +ACTGTGGACGTGTA-1,five +ACTGTGGATCTAGG-1,three +ACTGTTACCCACAA-1,two +ACTGTTACTGCAGT-1,four +ACTTAAGAACCACA-1,four +ACTTAAGATTACTC-1,unassigned +ACTTAGCTGCGTAT-1,one +ACTTAGCTGGGAGT-1,four +ACTTCAACAAGCAA-1,unassigned +ACTTCAACGTAGGG-1,five +ACTTCCCTTTCCGC-1,four +ACTTCTGACATGCA-1,three +ACTTGACTCCACAA-1,three +ACTTGGGAGAAAGT-1,two +ACTTGGGAGGTTTG-1,four +ACTTGGGATGTGAC-1,one +ACTTGGGATTGACG-1,two +ACTTGTACCTGTCC-1,four +ACTTTGTGGAAAGT-1,four +ACTTTGTGGATAGA-1,two +AGAAACGAAAGTAG-1,three +AGAAAGTGCGCAAT-1,four +AGAAAGTGGGGATG-1,two +AGAACAGAAATGCC-1,five +AGAACAGACGACTA-1,three +AGAACAGAGACAAA-1,four +AGAACGCTTTGCTT-1,four +AGAAGATGTGACTG-1,four +AGAATGGAAGAAGT-1,five +AGAATTTGTAACCG-1,four +AGAATTTGTAGAGA-1,four +AGACACACTGTAGC-1,four +AGACACTGTCAAGC-1,two +AGACCTGAAGTAGA-1,two +AGACCTGACCAACA-1,four +AGACCTGAGGAAGC-1,five +AGACGTACAGAGGC-1,four +AGACGTACCCCTAC-1,three +AGACGTACCTCTTA-1,unassigned +AGACGTACTCGTGA-1,one +AGACTGACCATCAG-1,five +AGACTGACCCTTTA-1,four +AGACTTCTCATGCA-1,three +AGAGATGACAGTCA-1,three +AGAGATGACTGAAC-1,four +AGAGATGAGGTTTG-1,two +AGAGATGATCTCGC-1,three +AGAGATGATTGTGG-1,four +AGAGCGGAGGCAAG-1,two +AGAGTCTGGTCGTA-1,one +AGAGTGCTCAGCTA-1,four +AGAGTGCTCGAATC-1,four +AGAGTGCTGTCATG-1,four +AGAGTGCTGTCCTC-1,five +AGAGTGCTGTGTTG-1,three +AGATATACCCGTAA-1,five +AGATATACGATGAA-1,four +AGATATACTGTTCT-1,three +AGATATTGCCTACC-1,five +AGATATTGGCCAAT-1,two +AGATCGTGTCTGGA-1,two +AGATCGTGTTTGTC-1,three +AGATCTCTATCACG-1,three +AGATTAACGTTCTT-1,one +AGATTCCTATCGTG-1,four +AGATTCCTCACTTT-1,two +AGATTCCTGACGAG-1,two +AGATTCCTGTTCAG-1,five +AGCAAAGATATGCG-1,four +AGCACAACAGTCTG-1,four +AGCACTGAGGGAGT-1,four +AGCACTGATATGCG-1,four +AGCACTGATGCTTT-1,unassigned +AGCACTGATTGCGA-1,four +AGCATCGAAGATCC-1,four +AGCATCGAAGGGTG-1,three +AGCATCGAGCTTCC-1,two +AGCATCGAGTGAGG-1,four +AGCATCGATAACCG-1,two +AGCATGACGATGAA-1,five +AGCCAATGGGGAGT-1,one +AGCCAATGTATCTC-1,four +AGCCACCTGGATCT-1,five +AGCCGGTGCCAATG-1,four +AGCCGGTGTGTTTC-1,four +AGCCGTCTCAATCG-1,four +AGCCGTCTGAGAGC-1,four +AGCCTCACGTTCGA-1,four +AGCCTCACTGTCAG-1,one +AGCCTCTGCAGTTG-1,three +AGCCTCTGCCAATG-1,five +AGCGAACTGGATCT-1,four +AGCGAACTTACTGG-1,four +AGCGATACGGAGCA-1,four +AGCGATTGAGATCC-1,five +AGCGCCGAATCTCT-1,four +AGCGCCGACAGAGG-1,four +AGCGCTCTACCTTT-1,four +AGCGGCACCGGGAA-1,four +AGCGGCTGATGTGC-1,four +AGCGGGCTTGCCAA-1,four +AGCGTAACATGCTG-1,four +AGCGTAACTGAGAA-1,two +AGCTCGCTACTGGT-1,four +AGCTCGCTCTGCTC-1,four +AGCTGAACCATACG-1,one +AGCTGAACCTCTCG-1,four +AGCTGCCTTGGGAG-1,two +AGCTGCCTTTCATC-1,four +AGCTGCCTTTCTGT-1,four +AGCTGTGATCCAAG-1,four +AGCTTTACAAGTAG-1,four +AGCTTTACACCAAC-1,five +AGCTTTACTCTCAT-1,three +AGGAAATGAGGAGC-1,four +AGGAACCTCTTAGG-1,one +AGGAACCTTGCCTC-1,five +AGGAATGATAACGC-1,four +AGGAATGATTTGTC-1,four +AGGAGTCTGGTTTG-1,four +AGGAGTCTTGTCAG-1,four +AGGATAGACATTTC-1,four +AGGATAGAGGATTC-1,five +AGGATGCTACTAGC-1,three +AGGATGCTTTAGGC-1,five +AGGCAACTGAAGGC-1,four +AGGCAGGAGTACCA-1,five +AGGCCTCTAGTCGT-1,four +AGGCCTCTCGGAGA-1,four +AGGCCTCTCGTAAC-1,four +AGGGACGACGTTGA-1,four +AGGGACGAGTCAAC-1,five +AGGGACGAGTTGTG-1,four +AGGGACGATAGAGA-1,five +AGGGACGATGCATG-1,five +AGGGAGTGAGCCTA-1,four +AGGGCCACCATACG-1,four +AGGGCGCTAACCAC-1,unassigned +AGGGCGCTATGGTC-1,four +AGGGTGGACAGTCA-1,three +AGGGTGGACTCAAG-1,two +AGGGTGGAGTTGCA-1,five +AGGGTTTGTTCATC-1,two +AGGTCATGAGTGTC-1,three +AGGTCATGCTTATC-1,five +AGGTCTGATTCTCA-1,one +AGGTGGGAAGAATG-1,unassigned +AGGTGGGAAGTTCG-1,four +AGGTGTTGGTTACG-1,three +AGGTTCGAACCTCC-1,two +AGGTTCGAACGTAC-1,two +AGGTTCGAGGGTGA-1,four +AGTAAGGAGTTTGG-1,four +AGTAAGGATTCTTG-1,four +AGTAATACATCACG-1,five +AGTAATACCGAACT-1,three +AGTAATTGTCCCAC-1,four +AGTACGTGAGGGTG-1,four +AGTACGTGCTGCAA-1,four +AGTACGTGCTTGGA-1,four +AGTACTCTACGTGT-1,three +AGTACTCTCAACCA-1,five +AGTACTCTCGGTAT-1,five +AGTAGGCTTGCCTC-1,one +AGTATAACTTGTCT-1,one +AGTATCCTAGAACA-1,four +AGTCACGATGAGCT-1,five +AGTCAGACGAATAG-1,four +AGTCAGACGCTTAG-1,four +AGTCAGACTAGAGA-1,five +AGTCAGACTGCACA-1,four +AGTCCAGATATCTC-1,four +AGTCCAGATTTCAC-1,four +AGTCGAACCAACCA-1,four +AGTCGCCTCCGTAA-1,four +AGTCTACTAGGGTG-1,two +AGTCTACTTGCATG-1,one +AGTCTTACACCACA-1,four +AGTCTTACTTCGCC-1,five +AGTCTTACTTCGGA-1,six +AGTGACTGCAACTG-1,one +AGTGTTCTAACCTG-1,four +AGTGTTCTATAAGG-1,four +AGTGTTCTCACTTT-1,two +AGTTAAACCACTTT-1,four +AGTTATGAACAGTC-1,two +AGTTATGACTGAGT-1,four +AGTTATGAGTTCAG-1,four +AGTTCTACCAGCTA-1,two +AGTTCTTGAAGCCT-1,five +AGTTGTCTACTACG-1,four +AGTTTAGATGGTCA-1,four +AGTTTCACGGTCTA-1,five +AGTTTGCTACAGTC-1,four +AGTTTGCTACTGGT-1,five +AGTTTGCTCCAAGT-1,four +ATAAACACAGTGCT-1,three +ATAAACACCACCAA-1,four +ATAACAACATGCTG-1,two +ATAACAACGTCTAG-1,four +ATAACAACTTTGTC-1,four +ATAACATGTACTCT-1,four +ATAACCCTGTTGGT-1,four +ATAACCCTTGGTAC-1,four +ATAAGTACGAATGA-1,three +ATAAGTTGGTACGT-1,one +ATAAGTTGTCTAGG-1,four +ATAATCGAGCTGAT-1,three +ATAATCGATGGTTG-1,four +ATAATGACCTACTT-1,four +ATAATGACTCGTGA-1,two +ATACAATGTTAGGC-1,four +ATACCACTCGTACA-1,two +ATACCACTCTAAGC-1,unassigned +ATACCACTGCCAAT-1,five +ATACCGGAATGCTG-1,unassigned +ATACCGGACATTTC-1,two +ATACCGGACTTCGC-1,four +ATACCGGAGGTGTT-1,two +ATACCGGATCTCGC-1,four +ATACCTACGCATCA-1,unassigned +ATACCTTGGGGCAA-1,four +ATACGGACAGACTC-1,two +ATACGGACCTACTT-1,two +ATACGGACGAGGTG-1,three +ATACGGACTATGCG-1,three +ATACGGACTCTGGA-1,two +ATACGTCTTAACGC-1,two +ATACTCTGCTTCGC-1,two +ATACTCTGGTATGC-1,five +ATAGATACCATGGT-1,four +ATAGATACGACGAG-1,four +ATAGATTGGTGTAC-1,three +ATAGCCGAACGGAG-1,four +ATAGCGTGCAGATC-1,four +ATAGCGTGCCCTTG-1,two +ATAGCGTGGTATCG-1,two +ATAGCGTGTCTCTA-1,four +ATAGCTCTCTGATG-1,four +ATAGCTCTGAGGTG-1,four +ATAGGAGAAACAGA-1,two +ATAGGCTGTCAGAC-1,four +ATAGTCCTAGTGTC-1,two +ATAGTCCTTGCATG-1,five +ATAGTCCTTGTCGA-1,four +ATAGTTGACAACTG-1,four +ATAGTTGACCCTCA-1,one +ATAGTTGAGACGTT-1,four +ATAGTTGATAAGCC-1,four +ATATACGAAGCCAT-1,four +ATATACGAATTGGC-1,four +ATATAGTGGAATGA-1,two +ATATGCCTAGATCC-1,two +ATATGCCTGGACAG-1,four +ATATGCCTTCTCTA-1,four +ATATGCCTTGGTAC-1,four +ATCAAATGAGCCTA-1,two +ATCAAATGGGTAAA-1,one +ATCAACCTAAACGA-1,four +ATCAACCTGAGGAC-1,four +ATCAACCTTCTCTA-1,five +ATCAACCTTTGTCT-1,five +ATCACACTTTGTCT-1,four +ATCACGGATTTCGT-1,three +ATCATCTGACACCA-1,six +ATCATGCTAGAGTA-1,five +ATCATGCTGAACCT-1,three +ATCCAGGACGCTAA-1,one +ATCCAGGATGGAAA-1,four +ATCCATACTCCTTA-1,two +ATCCATACTTCATC-1,one +ATCCCGTGCAGTCA-1,five +ATCCCGTGCATGCA-1,three +ATCCCGTGGCTGAT-1,three +ATCCGCACGCATCA-1,three +ATCCTAACGACGGA-1,three +ATCCTAACGCTACA-1,five +ATCGACGAAACTGC-1,two +ATCGACGAATGACC-1,four +ATCGAGTGGACGTT-1,five +ATCGCAGAATCTCT-1,one +ATCGCAGAGTGTCA-1,three +ATCGCCACTGAGGG-1,four +ATCGCCTGGGTCAT-1,four +ATCGCCTGTGGCAT-1,three +ATCGCGCTCAGAGG-1,two +ATCGCGCTGGGATG-1,four +ATCGCGCTTTTCGT-1,three +ATCGGAACCAGTCA-1,four +ATCGGTGAGTCAAC-1,four +ATCGGTGATTGCAG-1,four +ATCGTTTGCCTACC-1,two +ATCGTTTGGGTACT-1,two +ATCGTTTGTGCCAA-1,two +ATCTACACCCGCTT-1,two +ATCTACACCGGGAA-1,four +ATCTCAACAGGAGC-1,three +ATCTCAACCTCGAA-1,four +ATCTCAACCTTGTT-1,two +ATCTGGGAAACCAC-1,two +ATCTGGGAAGTGTC-1,unassigned +ATCTGGGATTCCGC-1,five +ATCTGTTGAACGGG-1,four +ATCTGTTGACCTCC-1,four +ATCTGTTGCCTTCG-1,one +ATCTGTTGGTTGCA-1,four +ATCTTGACACCAAC-1,four +ATCTTGACCTCCCA-1,two +ATCTTTCTGCATCA-1,unassigned +ATCTTTCTGTTTCT-1,five +ATCTTTCTTGTCCC-1,five +ATGAAACTCTGTGA-1,five +ATGAAACTGAGGCA-1,four +ATGAAGGAACAGCT-1,five +ATGAAGGACCTGTC-1,three +ATGAAGGACCTTAT-1,three +ATGAAGGACTAGTG-1,three +ATGAAGGACTTGCC-1,five +ATGACGTGACGACT-1,three +ATGACGTGATCGGT-1,one +ATGAGAGAAAGTGA-1,four +ATGAGAGAACGCAT-1,four +ATGAGAGAAGTAGA-1,three +ATGAGCACACAGCT-1,four +ATGAGCACATCTTC-1,four +ATGATAACTTCACT-1,five +ATGATATGAAACAG-1,four +ATGATATGACTGGT-1,four +ATGATATGAGCACT-1,four +ATGATATGGTCATG-1,four +ATGATATGGTGCTA-1,five +ATGATATGTTGTCT-1,two +ATGCACGAATGTCG-1,three +ATGCACGACTGTAG-1,four +ATGCACGAGAACCT-1,three +ATGCACGAGTTCGA-1,four +ATGCACGATTGGTG-1,two +ATGCAGTGTTACCT-1,three +ATGCAGTGTTCTAC-1,four +ATGCCAGAACGACT-1,four +ATGCCAGACAGTCA-1,two +ATGCCGCTTGAACC-1,two +ATGCGATGCTATGG-1,two +ATGCGATGCTGAGT-1,four +ATGCGATGGTTACG-1,three +ATGCGCCTTCATTC-1,four +ATGCTTTGCGAATC-1,four +ATGCTTTGGGCGAA-1,five +ATGCTTTGTAGTCG-1,three +ATGGACACATCGGT-1,four +ATGGACACGCATCA-1,five +ATGGGTACAACCTG-1,three +ATGGGTACATCGGT-1,five +ATGGGTACTATTCC-1,four +ATGGGTACTGGGAG-1,four +ATGTAAACACCTCC-1,four +ATGTAAACCCGCTT-1,four +ATGTAAACGGGATG-1,unassigned +ATGTAAACTCTCCG-1,unassigned +ATGTAAACTTCACT-1,two +ATGTACCTCAGTCA-1,four +ATGTACCTTAGTCG-1,four +ATGTACCTTTATCC-1,two +ATGTACCTTTCACT-1,five +ATGTCACTAATGCC-1,two +ATGTCACTCTGCTC-1,four +ATGTCGGAGGTGAG-1,four +ATGTTCACAGTCTG-1,two +ATGTTCACCGTAGT-1,four +ATGTTGCTTTCAGG-1,four +ATTAACGATGAGAA-1,five +ATTAACGATGCAAC-1,five +ATTAAGACTGCAGT-1,four +ATTACCTGCCTTAT-1,two +ATTACCTGGAGGAC-1,one +ATTAGATGTTTCAC-1,four +ATTATGGAATCTCT-1,four +ATTCAAGAACGGGA-1,four +ATTCAAGACCTTTA-1,four +ATTCAGCTCATTGG-1,six +ATTCCAACCATTGG-1,two +ATTCCAACTTAGGC-1,five +ATTCGACTCACTAG-1,four +ATTCGACTGAATAG-1,two +ATTCGACTTTTGTC-1,one +ATTCGGGAAAGGCG-1,two +ATTCGGGATTAGGC-1,four +ATTCTTCTGATACC-1,unassigned +ATTGAATGGACGGA-1,four +ATTGATGAAGGTTC-1,four +ATTGATGACTGAGT-1,four +ATTGATGAGCGAAG-1,four +ATTGATGATCTATC-1,four +ATTGCACTGACGGA-1,four +ATTGCACTGAGAGC-1,one +ATTGCACTGGAGCA-1,two +ATTGCACTTAGCCA-1,four +ATTGCACTTGCTTT-1,one +ATTGCTTGTTACTC-1,three +ATTGGTCTGACTAC-1,three +ATTGGTCTTGTCTT-1,three +ATTGTAGATTCCCG-1,unassigned +ATTGTAGATTGCAG-1,four +ATTGTCTGCGTACA-1,four +ATTTAGGAACCATG-1,one +ATTTAGGACAGAGG-1,four +ATTTCCGAGATGAA-1,four +ATTTCCGAGTGCTA-1,four +ATTTCGTGTATGGC-1,five +ATTTCTCTACTTTC-1,one +ATTTCTCTAGCAAA-1,four +ATTTCTCTCACTTT-1,unassigned +ATTTCTCTTCCCAC-1,four +ATTTGCACAAGATG-1,two +CAAAGCACAGCTCA-1,four +CAAAGCACCGTAAC-1,three +CAAAGCACGGTAAA-1,five +CAAAGCTGAAAGTG-1,four +CAAAGCTGTTGCTT-1,two +CAAATATGTGACAC-1,four +CAAATTGAGGGCAA-1,four +CAAATTGATGGAGG-1,two +CAACCAGAAAAGTG-1,four +CAACCAGAAGTGCT-1,four +CAACCAGAGTTCAG-1,two +CAACCAGATAGAAG-1,two +CAACCGCTGTTCAG-1,four +CAACCGCTTTGAGC-1,four +CAACGATGCGCAAT-1,one +CAACGTGACTCCAC-1,two +CAACGTGAGCCATA-1,five +CAACGTGATCAAGC-1,two +CAAGAAGACCACAA-1,four +CAAGAAGACGTCTC-1,two +CAAGAAGATTCTAC-1,four +CAAGACTGACCTGA-1,three +CAAGACTGAGTAGA-1,three +CAAGCTGACCATAG-1,one +CAAGCTGATCTATC-1,five +CAAGGACTGTTCAG-1,four +CAAGGACTTCTTTG-1,five +CAAGGTTGCTCCAC-1,four +CAAGGTTGTCATTC-1,three +CAAGGTTGTCTGGA-1,five +CAAGTCGAAACAGA-1,three +CAAGTCGATAGCGT-1,two +CAATAAACGCCATA-1,two +CAATAATGAACTGC-1,two +CAATATGACATGGT-1,four +CAATATGACCTTCG-1,five +CAATATGACGTTAG-1,four +CAATATGAGGAGCA-1,four +CAATCGGAGAAACA-1,four +CAATCTACTGACTG-1,five +CAATTCACCCAACA-1,five +CAATTCACGATAGA-1,four +CAATTCACTTGTGG-1,four +CAATTCTGCTTGTT-1,two +CAATTCTGGCGTAT-1,two +CACAACGATACGAC-1,four +CACAATCTTGTTCT-1,four +CACAATCTTTCCAT-1,five +CACACCTGCTTGAG-1,four +CACACCTGTATGGC-1,four +CACAGAACCCTTGC-1,four +CACAGAACCTGATG-1,four +CACAGATGGGATTC-1,four +CACAGATGGTTTCT-1,two +CACAGCCTGATACC-1,unassigned +CACAGCCTTGCCAA-1,one +CACAGCCTTGTAGC-1,four +CACAGTGATGAAGA-1,two +CACATACTACAGCT-1,five +CACATGGAACACGT-1,four +CACATGGAAGTCGT-1,four +CACCACTGCCAACA-1,four +CACCACTGGCGAAG-1,five +CACCCATGTTCTGT-1,four +CACCGGGAATCGAC-1,unassigned +CACCGGGACGAGAG-1,five +CACCGGGACGTGTA-1,four +CACCGGGACTTCTA-1,unassigned +CACCGGGACTTGCC-1,five +CACCGGGATTCGGA-1,five +CACCGTACTAAGGA-1,three +CACCGTACTAGCGT-1,four +CACCTGACACCCAA-1,five +CACCTGACCAGAAA-1,four +CACCTGACCTCAAG-1,five +CACCTGACGAAAGT-1,three +CACCTGACTCGTAG-1,four +CACGAAACTTCCGC-1,three +CACGACCTCGATAC-1,three +CACGCTACAGAAGT-1,four +CACGCTACTGTTCT-1,three +CACGGGACAGAGTA-1,four +CACGGGACATAAGG-1,four +CACGGGACGTAGGG-1,four +CACGGGTGCTTCGC-1,five +CACGGGTGGAGGAC-1,five +CACGGGTGTGTTTC-1,four +CACTAACTCCTAAG-1,three +CACTAACTGAAAGT-1,four +CACTAGGATGATGC-1,three +CACTATACCCCGTT-1,three +CACTATACGTTTGG-1,five +CACTGAGACAGTCA-1,two +CACTGCACTTCATC-1,four +CACTGCTGAGACTC-1,unassigned +CACTGCTGGAAAGT-1,four +CACTTAACCGAATC-1,four +CACTTAACCGTACA-1,three +CACTTTGACTCTAT-1,five +CACTTTGAGCTGTA-1,five +CAGAAGCTCTCAAG-1,four +CAGACATGAACGGG-1,four +CAGACATGTCGACA-1,four +CAGACCCTAAGGTA-1,one +CAGACCCTAATGCC-1,four +CAGACCCTAGGAGC-1,four +CAGACTGAGTATGC-1,four +CAGATCGAATGTCG-1,three +CAGATCGACCTGAA-1,four +CAGATCGATATGGC-1,four +CAGATGACATTCTC-1,five +CAGCAATGCCTTCG-1,four +CAGCAATGGAGGGT-1,five +CAGCAATGGTGCTA-1,four +CAGCAATGTCTACT-1,two +CAGCAATGTGACCA-1,four +CAGCAATGTGAGGG-1,two +CAGCACCTAAGCCT-1,four +CAGCACCTAGGCGA-1,four +CAGCACCTGTAGGG-1,two +CAGCATGACAACCA-1,two +CAGCATGAGACGTT-1,unassigned +CAGCCTACCCAACA-1,four +CAGCCTTGCTACCC-1,two +CAGCCTTGGGGACA-1,four +CAGCGGACACCCTC-1,four +CAGCGGACCTTTAC-1,five +CAGCGTCTAAAGCA-1,four +CAGCGTCTTATCGG-1,four +CAGCTAGATGTGAC-1,two +CAGCTCTGAGGCGA-1,four +CAGCTCTGCAAGCT-1,five +CAGCTCTGTCGTAG-1,two +CAGCTCTGTGTGGT-1,five +CAGGAACTAACTGC-1,five +CAGGAACTCTCAGA-1,two +CAGGCCGAACACCA-1,four +CAGGCCGAACACGT-1,four +CAGGCCGAACGACT-1,four +CAGGCCGAATCTCT-1,unassigned +CAGGCCGACTAGCA-1,three +CAGGGCACCATACG-1,three +CAGGGCACCCAACA-1,four +CAGGGCACTCCCGT-1,four +CAGGTAACAGACTC-1,two +CAGGTATGAGTCGT-1,four +CAGGTATGTGCTTT-1,four +CAGGTTGAGGATCT-1,three +CAGTGATGGACGGA-1,three +CAGTGATGGCTAAC-1,five +CAGTGATGGGACAG-1,four +CAGTGATGTAAGGA-1,three +CAGTGATGTACGCA-1,four +CAGTGTGATGTCAG-1,four +CAGTTACTAAGGTA-1,three +CAGTTACTGATAGA-1,four +CAGTTGGAAAGAGT-1,two +CAGTTGGACATACG-1,four +CAGTTTACACACGT-1,five +CAGTTTACCCCAAA-1,four +CATAAAACGGAGCA-1,two +CATAAATGAACTGC-1,four +CATAACCTTCTCCG-1,four +CATACTACCTCGAA-1,four +CATACTACCTGAAC-1,four +CATACTACGTACCA-1,two +CATACTTGGGTTAC-1,seven +CATAGTCTAATCGC-1,four +CATAGTCTCACTTT-1,four +CATATAGACTAAGC-1,unassigned +CATATAGATCAGGT-1,three +CATCAACTAGAAGT-1,four +CATCAACTCCCTCA-1,four +CATCAGGACTTCCG-1,five +CATCAGGATAGCCA-1,five +CATCAGGATCCTAT-1,one +CATCAGGATGCACA-1,two +CATCAGGATTTCGT-1,two +CATCATACCGCATA-1,two +CATCATACGGAGCA-1,three +CATCATACTCAAGC-1,four +CATCGCTGGGATCT-1,four +CATCGCTGTGGCAT-1,three +CATCGGCTATGCTG-1,four +CATCGGCTTTGGCA-1,four +CATCTCCTATGTGC-1,three +CATCTCCTCGAACT-1,unassigned +CATGAGACACGGGA-1,three +CATGAGACGTTGAC-1,five +CATGAGACTCGCCT-1,four +CATGCCACGGGTGA-1,four +CATGCCACTGCCAA-1,four +CATGCGCTAGTCAC-1,one +CATGCGCTCAGATC-1,two +CATGCGCTTTGCAG-1,one +CATGGCCTAGGGTG-1,two +CATGGCCTGTGCAT-1,four +CATGTACTATCGTG-1,five +CATGTTACAGTCGT-1,five +CATGTTACCTGAGT-1,four +CATGTTTGGGGATG-1,two +CATTACACACGGAG-1,three +CATTACACCAACTG-1,five +CATTACACGGAGTG-1,four +CATTACACTACTCT-1,four +CATTAGCTCCACAA-1,two +CATTGACTAGCGGA-1,two +CATTGGGACTCGAA-1,four +CATTGTACAGCGTT-1,four +CATTGTACTCGATG-1,three +CATTGTACTTATCC-1,four +CATTGTACTTTGCT-1,one +CATTGTTGCTAGTG-1,four +CATTTCGACTCTAT-1,five +CATTTCGAGATACC-1,five +CATTTGACCACACA-1,five +CATTTGACCCTGAA-1,five +CATTTGTGACGACT-1,three +CATTTGTGCATTGG-1,four +CATTTGTGCGGAGA-1,four +CATTTGTGGGATCT-1,three +CCAAAGTGCTACGA-1,two +CCAAAGTGTGAGAA-1,two +CCAACCTGAAGTAG-1,five +CCAACCTGACGTAC-1,two +CCAACCTGTTCGCC-1,unassigned +CCAAGAACCCAATG-1,four +CCAAGAACGTAGCT-1,four +CCAAGAACGTGTCA-1,four +CCAAGAACTACTGG-1,five +CCAAGAACTCCTAT-1,three +CCAAGATGTCATTC-1,four +CCAAGATGTTTCAC-1,four +CCAAGTGAGGAACG-1,four +CCAAGTGATCAAGC-1,four +CCAATTTGAACGTC-1,two +CCACCATGAACGTC-1,four +CCACCATGATCGGT-1,four +CCACCATGGACGAG-1,one +CCACCATGGGGAGT-1,four +CCACCATGTCCTGC-1,five +CCACTGACCCGCTT-1,two +CCACTGTGGGAAGC-1,two +CCACTGTGTGTAGC-1,four +CCACTTCTCGGGAA-1,two +CCAGAAACCCTGTC-1,three +CCAGAAACGAACTC-1,four +CCAGAAACGGTCTA-1,four +CCAGACCTCTGAGT-1,unassigned +CCAGACCTTGTGGT-1,four +CCAGCACTGCGATT-1,one +CCAGCGGAAAGGCG-1,two +CCAGCGGACGACTA-1,two +CCAGCGGATGGGAG-1,four +CCAGCTACACAGTC-1,four +CCAGCTACCAGCTA-1,three +CCAGGTCTACACCA-1,three +CCAGGTCTAGCATC-1,two +CCAGGTCTATGGTC-1,four +CCAGTCACACTGGT-1,unassigned +CCAGTCACACTGTG-1,three +CCAGTCACGTTGTG-1,four +CCAGTGCTAACCAC-1,four +CCAGTGCTCGTAGT-1,four +CCATCCGAAAGCAA-1,four +CCATCCGAACGACT-1,unassigned +CCATCCGAAGGTTC-1,three +CCATCCGATTCGCC-1,two +CCATCGTGAACGGG-1,four +CCATCGTGCTAGAC-1,unassigned +CCCAACACCTCGCT-1,three +CCCAACACGCATCA-1,four +CCCAACACTTTGTC-1,two +CCCAACTGCAATCG-1,two +CCCAGACTGCCTTC-1,two +CCCAGACTGGTTTG-1,three +CCCAGACTTTCGCC-1,one +CCCAGTTGCAGTTG-1,two +CCCAGTTGGGTACT-1,four +CCCAGTTGTCTATC-1,three +CCCGATTGTGTTTC-1,four +CCCGGAGAAGGGTG-1,two +CCCTACGAATTGGC-1,two +CCCTAGTGCAAAGA-1,four +CCCTCAGACACTTT-1,three +CCCTCAGACGAGAG-1,four +CCCTCAGAGGTCAT-1,four +CCCTGAACTAAAGG-1,three +CCCTGATGCAACCA-1,four +CCCTGATGCAAGCT-1,three +CCCTTACTAACCAC-1,two +CCCTTACTGCAGTT-1,four +CCGAAAACCTTGTT-1,unassigned +CCGACACTGGTTTG-1,four +CCGACTACCCAGTA-1,two +CCGACTACCGTGTA-1,four +CCGACTACTGAGGG-1,four +CCGATAGACCTAAG-1,four +CCGATAGAGTTGGT-1,two +CCGCGAGACACACA-1,two +CCGCGAGAGGTTCA-1,five +CCGCTATGGGACGA-1,three +CCGCTATGTGCAAC-1,two +CCGCTATGTGCACA-1,two +CCGGTACTGTCCTC-1,four +CCGTACACAAGCAA-1,four +CCGTACACAGCGTT-1,three +CCGTACACGTCATG-1,five +CCGTACACGTTGGT-1,two +CCGTACACTAACGC-1,five +CCTAAACTTTCGTT-1,four +CCTAAGGACCCAAA-1,four +CCTAAGGACTAGCA-1,two +CCTAAGGAGGGCAA-1,five +CCTAAGGATGATGC-1,four +CCTAAGGATGTCAG-1,four +CCTACCGACTCTTA-1,four +CCTACCGAGGGATG-1,three +CCTAGAGAGGTGAG-1,three +CCTATAACCAAAGA-1,four +CCTATAACGAGACG-1,one +CCTATAACTCAGAC-1,four +CCTATAACTGCATG-1,one +CCTCGAACACTTTC-1,five +CCTCGAACCCGTAA-1,unassigned +CCTCGAACGTATCG-1,four +CCTCGAACTTACTC-1,four +CCTCTACTCTTCGC-1,unassigned +CCTCTACTGGCATT-1,four +CCTGACTGAAGTAG-1,five +CCTGACTGGGGAGT-1,four +CCTGACTGTGTCTT-1,three +CCTGCAACACGTTG-1,two +CCTGGACTCGTGAT-1,four +CCTTAATGCCCAAA-1,three +CCTTAATGTTCTAC-1,one +CCTTCACTACGACT-1,four +CCTTCACTCAGTCA-1,three +CCTTCACTGGAGTG-1,two +CCTTTAGATTCATC-1,two +CGAACATGCCCTAC-1,two +CGAACATGTCAGAC-1,four +CGAAGACTGGAACG-1,five +CGAAGACTGTTACG-1,four +CGAAGGGAAACCTG-1,five +CGAAGGGATCCGAA-1,three +CGAAGTACCAACTG-1,three +CGAATCGAGGAGCA-1,two +CGAATCGAGGAGGT-1,four +CGACAAACCCATAG-1,four +CGACAAACCGACAT-1,three +CGACCACTAAAGTG-1,five +CGACCACTGCCAAT-1,five +CGACCGGAAGGTCT-1,four +CGACCGGATGGAAA-1,four +CGACCTTGCTAGTG-1,four +CGACGTCTATCGTG-1,four +CGACGTCTCGTGTA-1,four +CGACGTCTGAGGCA-1,four +CGACTCACGTCGTA-1,four +CGACTCACGTTGCA-1,four +CGACTCTGTGTGAC-1,unassigned +CGACTGCTTCCTCG-1,two +CGAGAACTAAGGCG-1,five +CGAGAACTACGTTG-1,four +CGAGAACTTGTTCT-1,two +CGAGATTGGACACT-1,five +CGAGATTGGCCATA-1,five +CGAGCCGAACACCA-1,four +CGAGCCGAGGCGAA-1,four +CGAGCGTGCTCCAC-1,four +CGAGCGTGGATACC-1,four +CGAGCGTGTATGCG-1,two +CGAGGAGACCTCCA-1,three +CGAGGAGATGTCGA-1,four +CGAGGCACCTATGG-1,five +CGAGGCACTATGCG-1,two +CGAGGCACTCTTCA-1,four +CGAGGCTGACGCTA-1,five +CGAGGCTGGCAGTT-1,two +CGAGGGCTACGACT-1,three +CGAGGGCTCGAATC-1,four +CGAGTATGTCACCC-1,one +CGATACGAACAGTC-1,two +CGATACGACAGGAG-1,unassigned +CGATACGATTCACT-1,three +CGATAGACCCGTAA-1,unassigned +CGATAGACCGTACA-1,four +CGATAGACGTAGGG-1,two +CGATAGACTGTTCT-1,four +CGATCAGAAGAACA-1,five +CGATCAGAGAGGGT-1,four +CGATCAGAGGTACT-1,four +CGATCAGATGTGAC-1,unassigned +CGATCCACCGGGAA-1,five +CGATCCACTTCCAT-1,three +CGCAAATGCTCGAA-1,four +CGCAACCTCCTTGC-1,four +CGCAACCTGGACGA-1,four +CGCACGGAGGACGA-1,three +CGCACGGATCTTTG-1,four +CGCACTACAGAATG-1,four +CGCACTACAGCCAT-1,four +CGCACTACATTGGC-1,four +CGCACTACTCGCCT-1,three +CGCACTACTCGTGA-1,four +CGCACTTGTCACGA-1,two +CGCAGGACAGATCC-1,five +CGCAGGACCTACTT-1,four +CGCAGGACTTGTCT-1,one +CGCAGGTGCACTGA-1,four +CGCAGGTGCCATAG-1,one +CGCAGGTGGGAACG-1,four +CGCATAGATCACGA-1,two +CGCCATACTGCAAC-1,four +CGCCATTGAGAGGC-1,four +CGCCATTGCTATGG-1,four +CGCCATTGGAGACG-1,four +CGCCATTGGAGCAG-1,three +CGCCATTGTACTGG-1,three +CGCCGAGAGCTTAG-1,five +CGCCTAACGAATGA-1,unassigned +CGCCTAACTGCTCC-1,four +CGCGAGACACAGCT-1,four +CGCGAGACAGGTCT-1,three +CGCGAGACGCTACA-1,unassigned +CGCGATCTCAGTCA-1,two +CGCGATCTGTTGAC-1,two +CGCGATCTTTCTTG-1,three +CGCGGATGGCCAAT-1,four +CGCTAAGAATGTCG-1,four +CGCTAAGACAACTG-1,four +CGCTAAGACCCTTG-1,four +CGCTACTGAACAGA-1,two +CGCTACTGAGAACA-1,four +CGCTACTGTGAGCT-1,four +CGCTACTGTTCCCG-1,five +CGCTCATGCATTTC-1,two +CGGAATTGGTTTGG-1,four +CGGAATTGTGGAGG-1,four +CGGACCGATGCGTA-1,five +CGGACCGATGGGAG-1,four +CGGACTCTAAACAG-1,three +CGGACTCTCCAATG-1,four +CGGACTCTCCTCGT-1,five +CGGAGGCTATTCCT-1,four +CGGAGGCTTGGATC-1,four +CGGATAACAACGAA-1,two +CGGATAACAGCTCA-1,four +CGGATAACTCAGTG-1,two +CGGCACGAACTCAG-1,five +CGGCACGAAGGGTG-1,four +CGGCACGACTACGA-1,one +CGGCATCTTAGAAG-1,five +CGGCATCTTCGTAG-1,two +CGGCCAGAAAGGTA-1,four +CGGCCAGAGAGGCA-1,five +CGGCGAACCAGTCA-1,two +CGGCGAACGACAAA-1,five +CGGCGAACGGTCTA-1,four +CGGCGAACTACTTC-1,five +CGGGACTGCGTGTA-1,four +CGGGACTGGAATAG-1,five +CGGGCATGACCCAA-1,five +CGGGCATGTCTCTA-1,five +CGGGCATGTTGTGG-1,four +CGGTAAACTCGCAA-1,unassigned +CGGTCACTGTTTGG-1,four +CGGTCACTTACTTC-1,four +CGTAACGATCGCCT-1,four +CGTACCACACACAC-1,four +CGTACCACACGTTG-1,four +CGTACCACCTCATT-1,four +CGTACCACGGAGCA-1,four +CGTACCTGGCATCA-1,five +CGTAGCCTCTCTCG-1,two +CGTAGCCTGCGAAG-1,four +CGTAGCCTGTATGC-1,one +CGTCAAGAAAGGTA-1,two +CGTCAAGAACGTGT-1,four +CGTCAAGACAGAGG-1,four +CGTCAAGACAGGAG-1,five +CGTCCATGCTCTTA-1,four +CGTCGACTTTCCGC-1,five +CGTGATGACGCTAA-1,four +CGTGATGAGGTTCA-1,four +CGTGCACTTATGGC-1,two +CGTGTAGAAAAACG-1,four +CGTGTAGACGATAC-1,five +CGTGTAGAGTTACG-1,five +CGTGTAGAGTTCAG-1,two +CGTGTAGATTCGGA-1,five +CGTTAGGAAACCAC-1,four +CGTTAGGATCATTC-1,two +CGTTATACCCTGAA-1,five +CGTTTAACTGGTCA-1,four +CTAAACCTCTGACA-1,five +CTAAACCTGTGCAT-1,three +CTAACACTAACGTC-1,four +CTAACACTAGTGCT-1,one +CTAACGGAACCGAT-1,five +CTAACGGATTTCTG-1,four +CTAACTACGGCAAG-1,four +CTAAGGACACCATG-1,four +CTAAGGACCGTTAG-1,four +CTAAGGACGCCATA-1,two +CTAAGGTGCCTAAG-1,three +CTAAGGTGTTGCAG-1,four +CTAAGGTGTTTCTG-1,four +CTAATAGAGCTATG-1,four +CTAATGCTTGTGGT-1,two +CTACAACTCCCGTT-1,five +CTACCTCTCAACCA-1,unassigned +CTACGCACACCTAG-1,one +CTACGCACTCTCCG-1,four +CTACGCACTGGTCA-1,five +CTACGGCTTTCTTG-1,two +CTACTATGAACCAC-1,two +CTACTATGATGTGC-1,two +CTACTATGCTAAGC-1,unassigned +CTACTATGTAAAGG-1,three +CTACTCCTATGTCG-1,five +CTACTCCTGCCATA-1,two +CTAGAGACACTTTC-1,four +CTAGAGACAGCATC-1,five +CTAGAGACTTTGGG-1,two +CTAGATCTCTCTAT-1,four +CTAGATCTTCGACA-1,four +CTAGGATGAGCCTA-1,three +CTAGGATGATCGTG-1,three +CTAGGCCTCTCAGA-1,four +CTAGGTGATGGTTG-1,one +CTAGTTACCAGAGG-1,four +CTAGTTACCGCATA-1,four +CTAGTTACGAAACA-1,five +CTATAAGATCGTTT-1,three +CTATACTGAGGTTC-1,two +CTATACTGCCAGTA-1,four +CTATACTGCGCTAA-1,two +CTATACTGCTACGA-1,five +CTATACTGTCTCAT-1,five +CTATACTGTTCGTT-1,four +CTATAGCTGTCACA-1,four +CTATAGCTTCGCTC-1,two +CTATAGCTTGCCTC-1,two +CTATCAACGAACTC-1,four +CTATCAACGCAGAG-1,four +CTATCAACTTTGGG-1,four +CTATCCCTCCACCT-1,two +CTATGTACGAGAGC-1,four +CTATGTACGCTTAG-1,four +CTATGTACTGTTTC-1,four +CTATGTTGAAAGCA-1,four +CTATGTTGTCCTCG-1,four +CTATGTTGTCTCGC-1,four +CTATTGACAAACGA-1,two +CTATTGACACTGGT-1,two +CTATTGACGGTGAG-1,one +CTATTGTGGCAAGG-1,four +CTCAATTGGTTCAG-1,four +CTCAATTGGTTGCA-1,two +CTCAGAGATAGAAG-1,four +CTCAGCACTCTAGG-1,four +CTCAGCACTGAACC-1,two +CTCAGCACTTGCAG-1,four +CTCAGCTGAACCTG-1,two +CTCAGCTGCAGTTG-1,four +CTCAGGCTCGTTGA-1,one +CTCAGGCTGCTAAC-1,four +CTCATTGACCTTAT-1,four +CTCATTGATGCTTT-1,one +CTCCACGAGAGATA-1,unassigned +CTCCATCTCTTAGG-1,four +CTCCATCTGACGAG-1,four +CTCCGAACAAGTGA-1,one +CTCCTACTGCCTTC-1,two +CTCGAAGATGTGGT-1,four +CTCGAAGATTAGGC-1,two +CTCGACTGCTCTAT-1,four +CTCGACTGGGTGAG-1,four +CTCGACTGGTTGAC-1,five +CTCGAGCTCTGGAT-1,five +CTCGCATGACTTTC-1,four +CTCGCATGCTTAGG-1,four +CTCTAAACCTCGAA-1,four +CTCTAAACGGCGAA-1,four +CTCTAATGTCCAAG-1,two +CTGAACGACAGTCA-1,four +CTGAACGATGAGGG-1,four +CTGAAGACCCAACA-1,four +CTGAAGACGTGCAT-1,three +CTGAAGTGAAGCCT-1,one +CTGAAGTGCAGCTA-1,three +CTGAAGTGGCTATG-1,four +CTGAAGTGTCCAGA-1,four +CTGAATCTGAATAG-1,four +CTGACAGAATCGTG-1,five +CTGACCACAGCAAA-1,four +CTGAGAACCGGGAA-1,four +CTGAGAACGTAAAG-1,five +CTGATACTAGTAGA-1,four +CTGATTTGGTGTTG-1,two +CTGCAGCTAACCGT-1,four +CTGCAGCTGACACT-1,four +CTGCAGCTGGATTC-1,two +CTGCAGCTTGGCAT-1,unassigned +CTGCCAACAGGAGC-1,five +CTGCCAACCAGCTA-1,four +CTGCCAACTAACCG-1,two +CTGCCAACTGCTCC-1,five +CTGCCAACTTGCAG-1,four +CTGCCAACTTGCTT-1,five +CTGCGACTCCACCT-1,four +CTGGAAACAAACGA-1,five +CTGGAAACATCGAC-1,four +CTGGATGACTGGAT-1,five +CTGGATGACTTGTT-1,four +CTGGATGATGTGAC-1,three +CTGGCACTCAAGCT-1,two +CTGTAACTAACCAC-1,four +CTGTAACTAGCGTT-1,four +CTGTATACGTAAAG-1,four +CTGTATACGTACGT-1,two +CTGTATACGTTGGT-1,four +CTGTGAGACAACCA-1,one +CTGTGAGACCTTGC-1,five +CTGTGAGACGAACT-1,two +CTGTGAGACTGTAG-1,two +CTTAAAGAACCTGA-1,four +CTTAACACCTGTAG-1,four +CTTAACACTATCGG-1,three +CTTAAGCTACCTAG-1,four +CTTAAGCTAGTACC-1,four +CTTAAGCTCATCAG-1,four +CTTAAGCTCCGCTT-1,three +CTTACAACTAACGC-1,four +CTTACAACTCCCGT-1,two +CTTACTGACGTACA-1,four +CTTAGACTAAACGA-1,four +CTTAGGGACTTGCC-1,four +CTTAGGGAGAATCC-1,four +CTTATCGACTCATT-1,four +CTTCACCTACCTGA-1,one +CTTCATGAAGCATC-1,three +CTTCATGAAGTACC-1,four +CTTCATGACCGAAT-1,five +CTTGAACTACGCAT-1,four +CTTGATTGAGGTTC-1,five +CTTGATTGATCTTC-1,two +CTTGATTGCATTCT-1,four +CTTGATTGTTTCGT-1,four +CTTGTATGACACCA-1,three +CTTGTATGCGCAAT-1,three +CTTTACGAGCGAAG-1,four +CTTTAGACCGTGAT-1,four +CTTTAGACGAGACG-1,four +CTTTAGACGATACC-1,four +CTTTAGACGTTGGT-1,four +CTTTAGACTCATTC-1,five +CTTTAGTGACGGGA-1,five +CTTTAGTGGGTGGA-1,four +CTTTCAGAGAAACA-1,four +CTTTGATGAGCACT-1,four +CTTTGATGTCTAGG-1,unassigned +CTTTGATGTGTCCC-1,four +CTTTGATGTGTGGT-1,four +GAAACAGAACTACG-1,four +GAAACAGAATCACG-1,four +GAAACAGACATTCT-1,four +GAAACCTGATCGTG-1,four +GAAACCTGATGCCA-1,four +GAAACCTGCTTATC-1,four +GAAACCTGGACTAC-1,one +GAAACCTGTGCTAG-1,four +GAAAGATGATTTCC-1,four +GAAAGATGCTGATG-1,one +GAAAGATGCTTCGC-1,two +GAAAGATGTAAGGA-1,four +GAAAGCCTACGTTG-1,three +GAAAGTGAAAGTGA-1,unassigned +GAAAGTGACCACAA-1,unassigned +GAAAGTGACTCAAG-1,four +GAAATACTACCAAC-1,two +GAAATACTCTTAGG-1,three +GAAATACTTCCTCG-1,four +GAACACACGTGCAT-1,two +GAACACACTGCCTC-1,four +GAACAGCTAACTGC-1,two +GAACAGCTCTCAGA-1,one +GAACCAACCACAAC-1,four +GAACCAACTTCCGC-1,five +GAACCTGAACGTGT-1,unassigned +GAACCTGAGAGACG-1,four +GAACCTGATGAACC-1,four +GAACGGGATACTTC-1,four +GAACTGTGACCTGA-1,one +GAACTGTGCCAGTA-1,two +GAAGAATGCAATCG-1,four +GAAGCGGACCTATT-1,two +GAAGCTACGAATGA-1,two +GAAGCTACGGTTTG-1,four +GAAGGGTGAAAGTG-1,four +GAAGGGTGCTTAGG-1,two +GAAGGTCTGAAAGT-1,four +GAAGGTCTGTTGCA-1,four +GAAGGTCTTAAAGG-1,unassigned +GAAGTAGACTCCCA-1,five +GAAGTAGATCCAAG-1,one +GAAGTCACCCTCGT-1,two +GAAGTCACCCTGTC-1,three +GAAGTCTGTCGCAA-1,three +GAAGTCTGTTCTGT-1,four +GAAGTGCTAAACGA-1,five +GAAGTGCTCCGCTT-1,two +GAAGTGCTTAACCG-1,four +GAATGCACCCTAAG-1,four +GAATGCACCTTCGC-1,five +GAATGCTGCGGTAT-1,four +GAATTAACGATAAG-1,four +GAATTAACGGTCAT-1,four +GAATTAACGTCGTA-1,five +GAATTAACTGAAGA-1,three +GACAACACAGGCGA-1,four +GACAACACATCGTG-1,four +GACAACACTCGCCT-1,four +GACAACTGAGGTTC-1,four +GACAGGGAAGAGTA-1,three +GACAGGGAATGCCA-1,four +GACAGTACGAGCTT-1,one +GACAGTACTTCGGA-1,one +GACAGTTGAGTAGA-1,three +GACATTCTCCACCT-1,unassigned +GACCAAACGACTAC-1,two +GACCAAACGTATCG-1,two +GACCCTACTAAAGG-1,three +GACCTAGACCTCAC-1,four +GACCTAGACGAGAG-1,four +GACCTCACAAGGTA-1,four +GACCTCACGTACGT-1,two +GACCTCTGCATCAG-1,four +GACGAACTCCCACT-1,four +GACGATTGCCAATG-1,three +GACGCCGACCTTCG-1,four +GACGCTCTCTCTCG-1,six +GACGGCACACGGGA-1,five +GACGGCACGAGATA-1,two +GACGTAACCTATGG-1,four +GACGTAACCTGTGA-1,four +GACGTAACTATGGC-1,four +GACGTATGTTGACG-1,four +GACGTATGTTTGCT-1,four +GACGTCCTACGGAG-1,four +GACGTCCTCTCAAG-1,two +GACGTCCTGATAAG-1,three +GACTACGATGGTCA-1,three +GACTCCTGCTCGCT-1,four +GACTCCTGGGTTAC-1,four +GACTCCTGTTATCC-1,four +GACTCCTGTTGGTG-1,two +GACTGAACCAATCG-1,two +GACTGATGTGATGC-1,four +GACTTTACATGCCA-1,four +GACTTTACGACAGG-1,three +GAGAAATGTTCTCA-1,three +GAGATAGAAAAAGC-1,two +GAGATCACGACAAA-1,one +GAGATGCTCTGGAT-1,five +GAGATGCTGAATGA-1,four +GAGCAGGATTCCCG-1,five +GAGCATACTTTGCT-1,four +GAGCGCACGCGTAT-1,two +GAGCGCACGGTGAG-1,four +GAGCGCTGAAGATG-1,five +GAGCGCTGTCTTAC-1,two +GAGCGGCTGGGAGT-1,three +GAGGACGACTCAGA-1,four +GAGGATCTGAAAGT-1,three +GAGGCAGACTTGCC-1,five +GAGGGAACACCAGT-1,four +GAGGGAACGAGGGT-1,four +GAGGGATGGGAAAT-1,four +GAGGGCCTTCACCC-1,four +GAGGGTGAAGAGTA-1,four +GAGGTACTACGGTT-1,four +GAGGTACTACTCAG-1,two +GAGGTACTGACACT-1,five +GAGGTACTGGGAGT-1,four +GAGGTACTTAGCGT-1,two +GAGGTGGAGTACGT-1,four +GAGGTGGATCCTCG-1,five +GAGGTTACTCGTTT-1,two +GAGGTTTGTAAGCC-1,two +GAGTCAACCATTCT-1,four +GAGTCAACGGGAGT-1,four +GAGTCTGATCGTGA-1,four +GAGTCTGATTTGGG-1,three +GAGTGACTCAGCTA-1,four +GAGTGACTCGGTAT-1,five +GAGTGACTCTTGCC-1,three +GAGTGACTGACTAC-1,three +GAGTGACTGTCTAG-1,four +GAGTGGGAGTCTTT-1,three +GAGTGGGATGCCCT-1,four +GAGTGGGATGCTGA-1,three +GAGTGTTGCTGTAG-1,four +GAGTGTTGTGGTCA-1,four +GAGTTGTGCATGGT-1,unassigned +GAGTTGTGCTGAGT-1,five +GAGTTGTGGCGAGA-1,two +GAGTTGTGGTAGCT-1,six +GAGTTGTGTATGCG-1,one +GATAAGGAGAAACA-1,two +GATAAGGATTCACT-1,two +GATACTCTATCGGT-1,four +GATACTCTTACTTC-1,four +GATACTCTTGACTG-1,two +GATAGAGAAGGGTG-1,five +GATAGAGACTGTGA-1,four +GATAGAGATCACGA-1,unassigned +GATAGCACCCATAG-1,four +GATAGCACGAAGGC-1,two +GATAGCACTTGTCT-1,four +GATATAACAAGGTA-1,four +GATATAACACGCAT-1,four +GATATATGCTGGAT-1,four +GATATATGTCCGTC-1,five +GATATATGTGGAGG-1,two +GATATCCTAGAAGT-1,four +GATATCCTCCCGTT-1,two +GATATTGACAGGAG-1,four +GATATTGACGAGTT-1,one +GATATTGAGCCAAT-1,four +GATCCCTGACCTTT-1,five +GATCCCTGTGTAGC-1,five +GATCCGCTGGTCAT-1,two +GATCGAACCGAGAG-1,unassigned +GATCGATGACTAGC-1,three +GATCGATGGTAAAG-1,three +GATCGATGTAAGGA-1,two +GATCGTGACACTAG-1,four +GATCGTGATTCACT-1,four +GATCTACTGGTGAG-1,four +GATCTACTTTGCAG-1,three +GATCTTACACCCAA-1,one +GATCTTACCCTACC-1,four +GATCTTACGAATAG-1,two +GATCTTACGAGATA-1,two +GATGCAACTCCAGA-1,four +GATGCCCTACGTAC-1,four +GATGCCCTCTCATT-1,four +GATGCCCTGGCAAG-1,two +GATGCCCTTTTGCT-1,four +GATTACCTTGTTCT-1,four +GATTCGGAACGACT-1,four +GATTCGGACAGGAG-1,two +GATTCGGAGAAGGC-1,one +GATTCTTGATTCGG-1,two +GATTCTTGCCGATA-1,four +GATTCTTGCGAGTT-1,three +GATTGGACCCGTTC-1,three +GATTGGACGGTGTT-1,three +GATTGGACTTTCGT-1,three +GATTGGTGTGTCAG-1,four +GATTTAGACACTCC-1,one +GATTTAGACTAAGC-1,three +GATTTAGATTCGTT-1,two +GATTTGCTAACGAA-1,four +GATTTGCTAACGGG-1,two +GCAACCCTCCTCGT-1,two +GCAACTGATTGCGA-1,four +GCAAGACTACTGGT-1,four +GCAAGACTAGGTCT-1,four +GCAAGACTCCCTTG-1,four +GCAATCGACTGCAA-1,four +GCAATCGAGACGTT-1,unassigned +GCAATCGATCCTTA-1,two +GCAATTCTCGTGTA-1,four +GCAATTCTTCTCCG-1,four +GCACAAACAATGCC-1,four +GCACAAACGGTACT-1,unassigned +GCACAATGGTGCAT-1,unassigned +GCACACCTGTGCTA-1,five +GCACCACTCATGAC-1,two +GCACCACTGTTTGG-1,two +GCACCACTTCCTTA-1,five +GCACCACTTTCGGA-1,two +GCACCTACGCGATT-1,four +GCACCTTGGCTGTA-1,four +GCACCTTGGGGAGT-1,two +GCACGGACCAGCTA-1,four +GCACGGTGACCTCC-1,four +GCACGGTGCTATGG-1,two +GCACTAGAACGGGA-1,four +GCACTAGAAGATGA-1,three +GCACTAGACCTTTA-1,one +GCACTAGACGTAAC-1,three +GCACTAGAGTCGTA-1,five +GCACTAGATGCAAC-1,four +GCACTGCTGAGGCA-1,five +GCAGATACAGCGTT-1,five +GCAGATACGACGGA-1,four +GCAGATACGCAGAG-1,four +GCAGCCGACAGTCA-1,four +GCAGCGTGCACTCC-1,four +GCAGCTCTCAATCG-1,four +GCAGCTCTGTTTCT-1,four +GCAGGGCTAAGAAC-1,four +GCAGGGCTAAGGGC-1,four +GCAGGGCTATCGAC-1,five +GCAGGGCTTGGGAG-1,five +GCAGTCCTAACTGC-1,four +GCAGTCCTCTCTTA-1,four +GCATCAGATGCGTA-1,unassigned +GCATGTGACAAGCT-1,four +GCATTGGAGAAGGC-1,four +GCCAAAACGAGGCA-1,five +GCCAAATGATCGAC-1,four +GCCAACCTACGGTT-1,four +GCCAACCTCGCCTT-1,four +GCCACGGAGGCGAA-1,two +GCCACGGATACTGG-1,four +GCCACTACGTCTTT-1,four +GCCCAACTACCGAT-1,four +GCCCAACTATGGTC-1,five +GCCCATACAGCGTT-1,unassigned +GCCGACGAACTCTT-1,two +GCCGAGTGCGTTGA-1,unassigned +GCCGGAACGAACTC-1,five +GCCGGAACGTTCTT-1,five +GCCGGAACTGCACA-1,five +GCCGGAACTTACTC-1,four +GCCGTACTACCTGA-1,four +GCCGTACTGGCAAG-1,two +GCCTACACAGTTCG-1,five +GCCTACACCACTGA-1,one +GCCTACACCTTGAG-1,four +GCCTAGCTACGGAG-1,three +GCCTAGCTCTATTC-1,four +GCCTAGCTTCTCAT-1,four +GCCTAGCTTCTCTA-1,one +GCCTCAACCATGGT-1,four +GCCTCAACTCTTTG-1,five +GCCTCATGTCTTAC-1,five +GCCTGACTCTCAAG-1,two +GCGAAGGAACTCTT-1,four +GCGAAGGATGCCAA-1,four +GCGAGAGAGGGACA-1,four +GCGAGCACTGTCGA-1,three +GCGAGCACTTGACG-1,four +GCGAGCACTTGCTT-1,unassigned +GCGATATGGTACGT-1,four +GCGATATGGTGTTG-1,four +GCGCACGAAGTCGT-1,four +GCGCACGACTTTAC-1,two +GCGCATCTAGGTCT-1,four +GCGCATCTGGTTAC-1,six +GCGCATCTTCGATG-1,four +GCGCATCTTGCTCC-1,five +GCGCATCTTTCTAC-1,four +GCGCGAACGTTCTT-1,three +GCGCGATGAACGGG-1,four +GCGCGATGGTGCAT-1,five +GCGGAGCTCCTGAA-1,four +GCGGCAACCCGATA-1,four +GCGGCAACGGAGGT-1,three +GCGGCAACTGTCGA-1,unassigned +GCGTAAACACGGTT-1,unassigned +GCGTAATGCACCAA-1,one +GCGTATGAACACCA-1,four +GCGTATGATGAGAA-1,two +GCTACAGAAAGGTA-1,four +GCTACAGAATCTTC-1,three +GCTACCTGAGAAGT-1,two +GCTACCTGATCACG-1,four +GCTACGCTAGAATG-1,four +GCTACGCTAGCTAC-1,four +GCTACGCTCCCTAC-1,four +GCTAGAACAGAGGC-1,three +GCTAGAACGGATCT-1,four +GCTAGAACTCCCGT-1,four +GCTAGATGAGCTCA-1,two +GCTAGATGGCGATT-1,five +GCTATACTAAGGCG-1,four +GCTATACTAGCGTT-1,five +GCTATACTCTCTTA-1,four +GCTATACTGGACGA-1,four +GCTCAAGAACCATG-1,three +GCTCAAGAAGTCAC-1,two +GCTCAGCTGTCTAG-1,four +GCTCCATGAGAAGT-1,five +GCTCCATGCCGAAT-1,four +GCTCGACTCTAGTG-1,two +GCTGATGAGGTATC-1,two +GCTTAACTACAGTC-1,two +GCTTAACTACTGGT-1,five +GCTTAACTGCTGAT-1,four +GCTTAACTTAGACC-1,two +GCTTAACTTCAGTG-1,four +GGAACACTCACTTT-1,two +GGAACACTTCAGAC-1,six +GGAACTACTACTTC-1,four +GGAACTTGAAGGTA-1,two +GGAACTTGAGAATG-1,four +GGAACTTGCTCCAC-1,two +GGAACTTGGGTAGG-1,three +GGAAGGACATCGGT-1,two +GGAAGGACCACTAG-1,four +GGAAGGACGAGGGT-1,four +GGAAGGACGCGAAG-1,four +GGAAGGTGGCGAGA-1,five +GGAATCTGAAGGGC-1,four +GGAATCTGAGGAGC-1,four +GGAATCTGCTTAGG-1,three +GGAATCTGCTTGTT-1,two +GGAATCTGGGAGGT-1,four +GGAATGCTTTCTAC-1,four +GGACAGGAAAGGGC-1,five +GGACAGGAGTGCTA-1,three +GGACAGGATCTCGC-1,four +GGACCCGAAGCTAC-1,two +GGACCGTGCTTACT-1,two +GGACCGTGGGAACG-1,unassigned +GGACCGTGTAACGC-1,four +GGACCTCTGTAAGA-1,two +GGACCTCTTTTCTG-1,five +GGACGAGAGTGTCA-1,two +GGACGCTGACGCAT-1,unassigned +GGACGCTGCTAGCA-1,four +GGACGCTGTCCTCG-1,five +GGAGAGACGTGAGG-1,three +GGAGCAGATTCAGG-1,three +GGAGCCACCTTCTA-1,three +GGAGCGCTACGCAT-1,two +GGAGCGCTCCGAAT-1,unassigned +GGAGGATGCCACCT-1,four +GGAGGATGGTTGAC-1,two +GGAGGATGTCAGTG-1,two +GGAGGCCTCGTTGA-1,five +GGAGGCCTTTCTTG-1,four +GGAGGTGATACGCA-1,four +GGAGGTGATCGCTC-1,four +GGATACTGCAGCTA-1,four +GGATACTGTCTAGG-1,four +GGATAGCTCGTCTC-1,four +GGATAGCTCTGAAC-1,five +GGATGTACCAAAGA-1,three +GGATGTACGCGAAG-1,four +GGATGTACGTCTTT-1,two +GGATGTACGTGTCA-1,four +GGATTTCTAGGTTC-1,four +GGATTTCTTTGTCT-1,two +GGCAAGGAAAAAGC-1,five +GGCAAGGAAGAAGT-1,four +GGCAAGGACTTGGA-1,unassigned +GGCAAGGAGGACTT-1,four +GGCAATACGCTAAC-1,three +GGCAATACGGCATT-1,five +GGCAATACGTTTCT-1,three +GGCACGTGGCTTAG-1,one +GGCACTCTTTTGTC-1,four +GGCATATGCTTATC-1,five +GGCATATGGGGAGT-1,six +GGCATATGTGTGAC-1,unassigned +GGCCACGACAGAGG-1,four +GGCCAGACTGGTTG-1,four +GGCCCAGAAAGTAG-1,two +GGCCGAACAACGAA-1,four +GGCCGAACGCAGAG-1,four +GGCCGAACGTAGGG-1,four +GGCCGAACTCTAGG-1,five +GGCCGATGCAGGAG-1,four +GGCCGATGCCGAAT-1,four +GGCCGATGTACTCT-1,five +GGCGACACTGCCCT-1,unassigned +GGCGACTGCGTAAC-1,four +GGCGCATGCCTAAG-1,one +GGCGCATGCTCCAC-1,four +GGCGCATGTGGAAA-1,four +GGCGGACTAGAGGC-1,four +GGCGGACTAGGAGC-1,five +GGCGGACTCTGACA-1,four +GGCGGACTCTTGGA-1,five +GGCGGACTTACTGG-1,four +GGCGGACTTGAACC-1,four +GGCTAAACACCTGA-1,five +GGCTAAACTCTTAC-1,four +GGCTAATGAGCACT-1,five +GGCTAATGGTCTAG-1,four +GGCTCACTACTCAG-1,two +GGGAACGAAGCTCA-1,one +GGGAACGACACAAC-1,three +GGGAACGAGTGTCA-1,two +GGGAAGTGTTGAGC-1,two +GGGACCACACGTTG-1,five +GGGACCACAGAACA-1,four +GGGACCACGAATAG-1,four +GGGACCACGTCATG-1,four +GGGACCACTCAAGC-1,four +GGGACCACTCGTGA-1,four +GGGACCACTGCATG-1,four +GGGACCTGACCCTC-1,two +GGGACCTGCTTGCC-1,four +GGGACCTGTGGAGG-1,three +GGGATGGACGACAT-1,three +GGGATGGATACTTC-1,four +GGGATGGATGGTTG-1,two +GGGATTACGTCTAG-1,four +GGGCAAGATGCATG-1,two +GGGCACACGGTGAG-1,four +GGGCACACGTTGCA-1,four +GGGCAGCTTGGGAG-1,five +GGGCAGCTTTTCTG-1,four +GGGCCAACCTTGGA-1,unassigned +GGGCCAACGCGTTA-1,three +GGGCCAACTACGCA-1,two +GGGCCAACTCCAAG-1,three +GGGCCATGATGGTC-1,unassigned +GGGCCATGTTGACG-1,four +GGGTAACTCAGCTA-1,four +GGGTAACTCTAGTG-1,three +GGGTAACTCTGGAT-1,four +GGGTTAACGTGCAT-1,five +GGTAAAGAGCTAAC-1,five +GGTACAACTGCAAC-1,four +GGTACATGAAAGCA-1,three +GGTACATGAGCTCA-1,four +GGTACATGCGGTAT-1,four +GGTACATGGTTACG-1,four +GGTACATGTGGGAG-1,four +GGTACTGAACTCTT-1,two +GGTAGTACACCACA-1,five +GGTAGTACACTAGC-1,two +GGTAGTACCCTGTC-1,four +GGTAGTACGCCATA-1,two +GGTAGTACTGTCTT-1,unassigned +GGTATCGAGACAAA-1,unassigned +GGTATCGATGAACC-1,three +GGTCAAACCAAAGA-1,four +GGTCTAGAGAAACA-1,two +GGTCTAGATAGCGT-1,two +GGTGATACCGACTA-1,three +GGTGATACGACTAC-1,two +GGTGATACTGTTTC-1,four +GGTGGAGAAACGGG-1,five +GGTGGAGAAGTAGA-1,four +GGTGGAGACAGATC-1,five +GGTGGAGATCGATG-1,four +GGTGGAGATCTCTA-1,two +GGTGGAGATTACTC-1,five +GGTTTACTACGCAT-1,five +GTAACGTGACCTCC-1,two +GTAACGTGATCGGT-1,three +GTAACGTGCAGCTA-1,three +GTAACGTGGTTGAC-1,three +GTAAGCACAACGGG-1,four +GTAAGCACTCATTC-1,three +GTAAGCTGGTACCA-1,two +GTAATAACCTTCTA-1,two +GTAATAACGTTGTG-1,four +GTACCCTGACAGTC-1,one +GTACCCTGGAGCTT-1,four +GTACCCTGTCCTTA-1,five +GTACCCTGTGAACC-1,four +GTACGTGAACGTTG-1,five +GTACTTTGTCGACA-1,five +GTAGACTGAGATGA-1,two +GTAGACTGTATTCC-1,four +GTAGCAACAGTCGT-1,two +GTAGCAACCATTTC-1,three +GTAGCAACGGTAGG-1,two +GTAGCATGCACTCC-1,three +GTAGCATGTAAGCC-1,two +GTAGCCCTGACGTT-1,four +GTAGCTGAAGCTAC-1,one +GTAGCTGAATTCGG-1,four +GTAGGTACACGGGA-1,two +GTAGTGACCTCATT-1,four +GTAGTGTGAGCGGA-1,five +GTAGTGTGAGGCGA-1,two +GTAGTGTGTGGTTG-1,four +GTATCACTGGTAGG-1,two +GTATCTACAGAAGT-1,five +GTATCTACGACGAG-1,five +GTATCTACGTTACG-1,four +GTATTAGAAACAGA-1,four +GTATTAGAGGTCTA-1,four +GTATTCACACAGCT-1,four +GTATTCACGGGTGA-1,five +GTCAACGACACTGA-1,four +GTCAACGAGTGTAC-1,two +GTCAATCTACACCA-1,five +GTCAATCTGTAGCT-1,five +GTCAATCTTGTGGT-1,four +GTCACCTGCCTCCA-1,four +GTCACCTGTCCCGT-1,four +GTCATACTAATCGC-1,three +GTCATACTGCGATT-1,five +GTCATACTTCGCCT-1,six +GTCATACTTTACCT-1,five +GTCATACTTTGACG-1,four +GTCCAAGAAAAACG-1,four +GTCCACTGACCTCC-1,four +GTCCACTGGGTACT-1,one +GTCCAGCTACGGGA-1,four +GTCCCATGTGGTGT-1,two +GTCGAATGAAGGCG-1,two +GTCGACCTGAATGA-1,five +GTCGACCTGTTCAG-1,four +GTCGCACTTGAGAA-1,three +GTCTAACTGGTCTA-1,two +GTCTAGGAGCTTCC-1,two +GTGAACACACTCTT-1,four +GTGAACACTCAGGT-1,four +GTGACCCTTAAGCC-1,unassigned +GTGATGACAAGTGA-1,four +GTGATGACCTGAGT-1,three +GTGATGACGGTTTG-1,five +GTGATTCTCATTTC-1,four +GTGATTCTCTCTCG-1,two +GTGATTCTGGTTCA-1,five +GTGATTCTGTCGAT-1,four +GTGATTCTTAGCGT-1,three +GTGCCACTCAGGAG-1,four +GTGGATTGCACTAG-1,one +GTGGATTGCGGAGA-1,four +GTGGATTGTAACGC-1,four +GTGTACGATCAGTG-1,four +GTGTAGTGGGTACT-1,four +GTGTATCTAGCCTA-1,three +GTGTATCTAGTAGA-1,five +GTGTATCTGTTACG-1,three +GTGTCAGAAGCGTT-1,five +GTTAAAACCGAGAG-1,five +GTTAAATGCTCGAA-1,four +GTTAAATGTCGACA-1,three +GTTAACCTAGCTAC-1,four +GTTAACCTTGCTTT-1,unassigned +GTTAGGTGCACTCC-1,four +GTTAGGTGCCAGTA-1,two +GTTAGGTGCCCAAA-1,one +GTTAGGTGGAACTC-1,three +GTTAGTCTAAGAAC-1,unassigned +GTTATAGAGGACAG-1,four +GTTATGCTTTCATC-1,four +GTTCAACTGGGACA-1,five +GTTCAACTTATGCG-1,four +GTTGACGAGCCCTT-1,five +GTTGACGATATCGG-1,five +GTTGAGTGGTCTTT-1,two +GTTGAGTGTGCTTT-1,four +GTTGATCTGGGACA-1,four +GTTGATCTTTTCAC-1,five +GTTGGATGTTTACC-1,one +GTTGTACTATTCCT-1,two +GTTGTACTTTTGGG-1,four +GTTTAAGACCATGA-1,two +GTTTAAGACTGTCC-1,five +TAAACAACCAACCA-1,four +TAAACAACGAATCC-1,three +TAAAGACTCAGGAG-1,unassigned +TAAATCGATGAGGG-1,four +TAACAATGTGCCCT-1,three +TAACACCTTCGCTC-1,two +TAACACCTTCGTAG-1,two +TAACACCTTGTTTC-1,six +TAACATGACACTAG-1,three +TAACCGGACTTACT-1,four +TAACGTCTCAACCA-1,two +TAACGTCTCATTGG-1,four +TAACTAGAATTTCC-1,four +TAACTAGACTTAGG-1,four +TAACTAGATCTGGA-1,four +TAACTCACGAGGAC-1,two +TAACTCACGTACAC-1,three +TAACTCACGTATCG-1,two +TAACTCACTCTACT-1,five +TAAGAACTGTGTCA-1,two +TAAGAGGACTAAGC-1,three +TAAGAGGACTTGTT-1,five +TAAGATACGGTTCA-1,two +TAAGATTGCGTAGT-1,four +TAAGATTGTTGCTT-1,four +TAAGCGTGAGGTTC-1,two +TAAGCGTGGACAAA-1,two +TAAGCGTGGGAAAT-1,four +TAAGCGTGTGCTCC-1,one +TAAGGCTGCCATGA-1,unassigned +TAAGGCTGCTGCTC-1,four +TAAGGCTGTCTCGC-1,two +TAAGGGCTGCTGTA-1,four +TAAGGGCTTTACTC-1,four +TAAGTAACCGAGAG-1,three +TAAGTAACCTCCAC-1,three +TAAGTAACCTGTAG-1,four +TAAGTAACTTGTCT-1,four +TAATGATGAGCGGA-1,four +TAATGCCTCATGAC-1,four +TAATGCCTCGTCTC-1,five +TAATGTGAAGATGA-1,four +TAATGTGACTGCAA-1,two +TAATGTGATTACTC-1,four +TACAAATGGGTACT-1,four +TACAATGAAAACAG-1,two +TACAATGACTTAGG-1,four +TACAATGATGCTAG-1,five +TACACACTCACACA-1,four +TACACACTCTTACT-1,four +TACATAGAACGCAT-1,five +TACATCACACGGGA-1,four +TACATCACCTGTTT-1,one +TACATCACGCTAAC-1,three +TACATCACTGAACC-1,one +TACCATTGAGGTTC-1,four +TACCATTGCGGGAA-1,two +TACCATTGGGGATG-1,two +TACCATTGTGAGGG-1,two +TACCGGCTGTTGGT-1,four +TACGAGTGATCTCT-1,four +TACGAGTGATGCTG-1,two +TACGAGTGCGGAGA-1,four +TACGAGTGGTTGGT-1,four +TACGATCTAGTGTC-1,four +TACGATCTCACTGA-1,two +TACGATCTTACGAC-1,four +TACGCAGAGAATCC-1,five +TACGCCACATTCCT-1,four +TACGCCACTCCCAC-1,one +TACGCCACTCCGAA-1,three +TACGGAACGCGTTA-1,three +TACGGCCTGGGACA-1,five +TACGTACTACGGAG-1,unassigned +TACGTACTCAGTTG-1,four +TACGTACTCCCGTT-1,three +TACGTTACAGAAGT-1,four +TACGTTACCAAGCT-1,two +TACTAAGAAAGGTA-1,three +TACTAAGAATCACG-1,four +TACTAAGATGATGC-1,two +TACTAAGATTGCGA-1,four +TACTACACGAGAGC-1,four +TACTACACTTACCT-1,four +TACTACTGAACCTG-1,two +TACTACTGATGTCG-1,five +TACTACTGATTCTC-1,four +TACTACTGTATGGC-1,four +TACTCAACGGTCTA-1,two +TACTCAACTGCTAG-1,three +TACTCCCTCAGTTG-1,four +TACTCTGAATCGAC-1,five +TACTCTGACGAGTT-1,four +TACTCTGATTGACG-1,three +TACTGGGATCGATG-1,four +TACTGTTGAAAGCA-1,four +TACTGTTGAGGCGA-1,five +TACTGTTGCTGAAC-1,three +TACTTGACTCCTCG-1,five +TACTTGACTGGTGT-1,three +TACTTTCTTTTGGG-1,four +TAGAAACTAATCGC-1,four +TAGAAACTGCTTCC-1,two +TAGAAACTGGGATG-1,five +TAGAATTGCGACAT-1,four +TAGAATTGTATCGG-1,four +TAGACGTGCTTGAG-1,four +TAGACGTGTCGCTC-1,four +TAGAGCACCTTACT-1,three +TAGATTGACTTGTT-1,three +TAGATTGAGGCATT-1,five +TAGCATCTCAGCTA-1,five +TAGCATCTCCCTCA-1,four +TAGCATCTGCTGTA-1,one +TAGCATCTGGGACA-1,four +TAGCATCTTGTCGA-1,four +TAGCCCACAAAAGC-1,four +TAGCCCACAGCCAT-1,five +TAGCCCACAGCTAC-1,five +TAGCCCACCCACAA-1,four +TAGCCCTGCGGAGA-1,five +TAGCCGCTTACGAC-1,four +TAGCCGCTTACTTC-1,five +TAGCCGCTTTCCAT-1,two +TAGCTACTGAATAG-1,three +TAGCTACTGTAGCT-1,four +TAGCTACTTTTGCT-1,five +TAGGACTGTGCTGA-1,four +TAGGAGCTAAGGCG-1,three +TAGGAGCTGAGGGT-1,four +TAGGAGCTTGCATG-1,three +TAGGCAACCGTCTC-1,three +TAGGCATGCTCTCG-1,unassigned +TAGGCATGGCGAGA-1,five +TAGGCTGATGCCTC-1,two +TAGGGACTGAACTC-1,five +TAGGTCGACACTGA-1,four +TAGGTCGAGGATCT-1,three +TAGGTGACACACTG-1,four +TAGGTGACACGTTG-1,three +TAGGTGTGTTCTGT-1,four +TAGGTTCTGAAGGC-1,four +TAGGTTCTTCTTAC-1,one +TAGGTTCTTGCTGA-1,two +TAGTAAACCTCGCT-1,four +TAGTAAACGTCACA-1,four +TAGTAATGAGATCC-1,four +TAGTACCTAAGAAC-1,two +TAGTATGATCTTAC-1,three +TAGTATGATTCTCA-1,four +TAGTCTTGGCTGTA-1,four +TAGTCTTGGGACTT-1,four +TAGTCTTGTGGAAA-1,five +TAGTGGTGAAGTGA-1,five +TAGTTAGAACCACA-1,five +TAGTTAGATGAACC-1,four +TATAAGACAACAGA-1,four +TATAAGACAGCTCA-1,one +TATAAGTGACACCA-1,four +TATAAGTGTATCGG-1,two +TATAAGTGTGGTGT-1,one +TATACAGAACCCTC-1,five +TATACAGAAGAACA-1,four +TATACAGATCCAGA-1,four +TATACCACCTGATG-1,three +TATACGCTACCAAC-1,four +TATAGATGGACGGA-1,four +TATAGATGTTCCGC-1,one +TATCCAACCAGCTA-1,two +TATCCAACTCTCTA-1,unassigned +TATCGACTACTAGC-1,five +TATCGACTCGATAC-1,two +TATCGTACAGATGA-1,unassigned +TATCGTACATTCCT-1,two +TATCTCGAGAGATA-1,four +TATCTGACAGGTTC-1,five +TATCTGACTGTTTC-1,five +TATCTTCTAAACAG-1,four +TATGAATGGAGGAC-1,five +TATGAATGTTTGCT-1,one +TATGCGGATAACCG-1,four +TATGGGTGCATCAG-1,five +TATGGGTGCTAGCA-1,three +TATGGTCTCTACCC-1,four +TATGTCACGGAACG-1,three +TATGTCACTAACCG-1,unassigned +TATGTCACTTCTCA-1,three +TATGTGCTCCGATA-1,two +TATGTGCTGGATTC-1,four +TATTGCTGAAGAAC-1,two +TATTGCTGCCGTTC-1,four +TATTGCTGTCTGGA-1,four +TATTGCTGTGCACA-1,five +TATTTCCTATTGGC-1,two +TATTTCCTGGAGGT-1,five +TATTTCCTGGTGTT-1,four +TCAACACTGTTTGG-1,five +TCAAGGACAGCGTT-1,four +TCAAGGACATTCTC-1,five +TCAAGGACGGTGTT-1,two +TCAATCACACTCTT-1,five +TCAATCACAGTCGT-1,one +TCACAACTATGTGC-1,four +TCACAACTTTGCTT-1,two +TCACATACACTTTC-1,four +TCACATACAGGGTG-1,four +TCACCCGAGACGGA-1,two +TCACCGTGCTCGCT-1,one +TCACCTCTACGACT-1,five +TCACCTCTTCCAAG-1,one +TCACGAGAGGAGGT-1,three +TCACTATGGGGCAA-1,five +TCACTATGGTTGTG-1,three +TCAGACGACGCTAA-1,four +TCAGACGACGTTAG-1,five +TCAGAGACTCCAGA-1,four +TCAGCAGACTCCAC-1,four +TCAGCGCTCTAGTG-1,five +TCAGCGCTGGATCT-1,three +TCAGCGCTGGTATC-1,five +TCAGGATGAAGTAG-1,four +TCAGGATGCCTTTA-1,one +TCAGTGGAAGATCC-1,five +TCAGTTACCTACGA-1,two +TCAGTTACTAGAAG-1,four +TCATCAACCCGATA-1,four +TCATCAACTGTTCT-1,two +TCATCATGCAGTTG-1,four +TCATCCCTTACTGG-1,unassigned +TCATTCGATACAGC-1,one +TCCACGTGGAAACA-1,unassigned +TCCACTCTACACTG-1,four +TCCACTCTGAGCTT-1,two +TCCACTCTTACTTC-1,three +TCCATAACAAAGTG-1,four +TCCATAACCGTAGT-1,three +TCCATAACGATGAA-1,four +TCCATAACTACGCA-1,four +TCCATCCTCCCTAC-1,five +TCCCACGATCATTC-1,four +TCCCATCTCAAAGA-1,two +TCCCGAACACAGTC-1,five +TCCCGAACTTCGCC-1,one +TCCCGATGAGATCC-1,four +TCCCGATGCCTGAA-1,two +TCCCTACTCAACTG-1,three +TCCGAAGACAATCG-1,two +TCCGAAGACGTTAG-1,four +TCCGGACTGAGGTG-1,four +TCCGGACTGTACGT-1,five +TCCTAAACATCGAC-1,four +TCCTAAACCGAGAG-1,four +TCCTAAACCGCATA-1,five +TCCTAATGGTTTGG-1,five +TCCTACCTGTCGTA-1,four +TCCTATGAAAAGCA-1,five +TCGAATCTCTGGTA-1,five +TCGACCTGCCGATA-1,one +TCGACGCTTCTATC-1,two +TCGACGCTTTGACG-1,four +TCGAGAACGACAGG-1,four +TCGAGAACGTTAGC-1,four +TCGAGCCTATCAGC-1,three +TCGAGCCTGCGAGA-1,five +TCGAGCCTTGTGAC-1,four +TCGATACTATTCCT-1,unassigned +TCGATACTTGCACA-1,four +TCGATTTGATGCCA-1,two +TCGATTTGCACTCC-1,four +TCGATTTGCAGCTA-1,five +TCGATTTGCCTACC-1,two +TCGATTTGTCGTGA-1,three +TCGCAGCTAGATCC-1,five +TCGCCATGAGACTC-1,four +TCGCCATGTGGTCA-1,three +TCGGACCTAACAGA-1,four +TCGGACCTATAAGG-1,three +TCGGACCTGTACAC-1,four +TCGGTAGAGTAGGG-1,four +TCGGTAGATCCCAC-1,three +TCGTAGGATCGACA-1,four +TCGTTATGGACAAA-1,four +TCTAACACCAGTTG-1,five +TCTAACACGAGCAG-1,unassigned +TCTAACTGAACCAC-1,four +TCTAAGCTAATGCC-1,four +TCTAAGCTTAGTCG-1,five +TCTAAGCTTCTAGG-1,one +TCTAAGCTTGTTCT-1,two +TCTAAGCTTTCGCC-1,three +TCTACAACGACTAC-1,two +TCTAGACTTAGAAG-1,two +TCTAGTTGCACCAA-1,two +TCTATGTGAAGAGT-1,two +TCTATGTGAGTCTG-1,five +TCTCAAACCTAAGC-1,three +TCTCTAGAATTTCC-1,four +TCTGATACACGTGT-1,four +TCTGATACTCGCCT-1,four +TCTTACGAACCTGA-1,two +TCTTCAGAGCTACA-1,two +TCTTGATGCGGAGA-1,five +TGAAATTGGTGAGG-1,two +TGAACCGAAAACGA-1,four +TGAACCGACTACTT-1,one +TGAACCGATTCGGA-1,one +TGAAGCACTCACGA-1,five +TGAAGCTGAACGAA-1,four +TGAAGCTGCATGGT-1,three +TGAAGCTGCGTAAC-1,four +TGAATAACCACTTT-1,four +TGAATAACTCCCAC-1,five +TGACACGACCTTAT-1,three +TGACCAGACAACCA-1,four +TGACCAGAGGATTC-1,four +TGACCGCTAAAAGC-1,four +TGACCGCTCTGCAA-1,four +TGACGATGCAAAGA-1,two +TGACGCCTGTACCA-1,unassigned +TGACGCCTTTACTC-1,four +TGACTGGAAGAGAT-1,two +TGACTGGACCGTAA-1,four +TGACTGGACGCAAT-1,four +TGACTGGAGGACAG-1,three +TGACTGGATTCTCA-1,four +TGACTTACACACCA-1,four +TGACTTACAGTCTG-1,two +TGACTTTGCGCATA-1,four +TGACTTTGTTTGTC-1,four +TGAGACACAAGGTA-1,unassigned +TGAGACACTCAAGC-1,four +TGAGCTGAATGCTG-1,one +TGAGCTGACTGGAT-1,two +TGAGCTGATGCTAG-1,unassigned +TGAGGACTCTCATT-1,two +TGAGGACTTCATTC-1,four +TGAGGTACGAACCT-1,one +TGAGTCGAGTTACG-1,three +TGAGTGACTGAGCT-1,four +TGATAAACGAATCC-1,four +TGATAAACTCCGTC-1,five +TGATAAACTTTCAC-1,four +TGATACCTCACTAG-1,two +TGATACCTGTTGGT-1,four +TGATACCTTATGCG-1,four +TGATACCTTGAAGA-1,four +TGATATGAACCTTT-1,four +TGATCACTAGCATC-1,five +TGATCACTCTCGCT-1,four +TGATCACTTCTACT-1,three +TGATCGGACTGACA-1,one +TGATCGGAGGAGCA-1,three +TGATCGGATATGCG-1,one +TGATTAGACATTGG-1,four +TGATTAGATGACTG-1,three +TGATTAGATGCTAG-1,four +TGATTCACTATGCG-1,four +TGATTCACTGTCAG-1,three +TGATTCTGCCGAAT-1,four +TGATTCTGCTCTTA-1,four +TGCAAGTGAGAACA-1,four +TGCAAGTGGGTAGG-1,two +TGCAATCTTCAGGT-1,three +TGCACAGACGACAT-1,five +TGCCAAGAGCAGTT-1,one +TGCCAAGATCTCTA-1,four +TGCCACTGAACGTC-1,four +TGCCACTGCGATAC-1,five +TGCCAGCTTGGCAT-1,four +TGCCCAACAGCAAA-1,four +TGCCCAACCGCATA-1,one +TGCCGACTCTCCCA-1,three +TGCGAAACAGTCAC-1,three +TGCGAAACGTTGCA-1,three +TGCGATGAACGGTT-1,four +TGCGATGACCTCGT-1,four +TGCGATGACTAGTG-1,three +TGCGATGACTGCTC-1,three +TGCGATGACTTGCC-1,three +TGCGATGAGTGCTA-1,two +TGCGCACTCTTGAG-1,two +TGCGTAGAATAAGG-1,two +TGCGTAGACGGGAA-1,three +TGCGTAGATGGTCA-1,one +TGCTAGGAAACCGT-1,five +TGCTAGGATAGTCG-1,four +TGCTATACGGTTCA-1,two +TGCTATACTGCTGA-1,five +TGCTGAGAGAGCAG-1,five +TGCTGAGATTATCC-1,four +TGGAAAGACTCTCG-1,three +TGGAAAGAGCGATT-1,four +TGGAAAGAGGTCAT-1,one +TGGAAAGATATGGC-1,four +TGGAACACAAACAG-1,five +TGGAACACGCTAAC-1,four +TGGAAGCTCAGATC-1,four +TGGACCCTACACTG-1,two +TGGACCCTCATGGT-1,five +TGGACCCTGGTACT-1,two +TGGACTGAGTATGC-1,four +TGGAGACTATCAGC-1,two +TGGAGACTTCAAGC-1,four +TGGAGACTTGACCA-1,five +TGGAGGGACGGAGA-1,two +TGGAGGGAGCTATG-1,four +TGGATCGATAAAGG-1,four +TGGATGTGACCTAG-1,three +TGGATGTGTGAAGA-1,three +TGGATTCTCATACG-1,five +TGGCAATGCTTGTT-1,three +TGGCACCTTCACGA-1,four +TGGCACCTTCAGTG-1,two +TGGGTATGAAGAGT-1,four +TGGGTATGCACAAC-1,four +TGGGTATGGTACGT-1,three +TGGGTATGTTTGGG-1,two +TGGTAGACATGCCA-1,four +TGGTAGACCCTCAC-1,five +TGGTAGACCTGATG-1,four +TGGTAGTGCACTGA-1,unassigned +TGGTATCTAAACAG-1,four +TGGTATCTCTTCCG-1,four +TGGTCAGACCCAAA-1,four +TGGTTACTGACGTT-1,two +TGGTTACTGTTCTT-1,four +TGTAACCTAGAGGC-1,four +TGTAACCTTGCCTC-1,two +TGTAATGACACAAC-1,five +TGTAATGAGGTAAA-1,five +TGTACTTGCTCTAT-1,one +TGTAGGTGCGAGAG-1,three +TGTAGGTGCTATGG-1,four +TGTAGGTGCTCTAT-1,unassigned +TGTAGGTGTGCTGA-1,four +TGTAGTCTTCCAGA-1,four +TGTAGTCTTGCACA-1,five +TGTATCTGTTAGGC-1,four +TGTATGCTCATGGT-1,three +TGTATGCTGTAGGG-1,four +TGTATGCTTTCATC-1,four +TGTCAGGAATACCG-1,four +TGTCAGGAGATGAA-1,one +TGTCTAACCCCTTG-1,four +TGTGACGATTCTCA-1,five +TGTGAGACTGTCAG-1,one +TGTGAGACTTGAGC-1,four +TGTGAGTGACCACA-1,four +TGTGAGTGAGTGCT-1,unassigned +TGTGAGTGGAGATA-1,five +TGTGATCTCTCTAT-1,three +TGTGATCTGACACT-1,four +TGTGGATGGCCAAT-1,three +TGTTAAGACAAAGA-1,two +TGTTAAGATAAGGA-1,two +TGTTACACCGCATA-1,unassigned +TGTTACACGACTAC-1,one +TGTTACTGGCTACA-1,five +TGTTACTGTAGTCG-1,five +TTAACCACCGTAAC-1,four +TTAACCACTAAGGA-1,five +TTAACCACTCAGAC-1,five +TTACACACGTGTTG-1,four +TTACACACTCCTAT-1,four +TTACCATGAATCGC-1,two +TTACCATGGTTGAC-1,four +TTACCATGTGTCTT-1,four +TTACCATGTTGTGG-1,three +TTACGACTGAGAGC-1,four +TTACGACTTGACAC-1,two +TTACGTACGTTCAG-1,six +TTACTCGAAGAATG-1,two +TTACTCGACGCAAT-1,unassigned +TTACTCGAGGGTGA-1,four +TTACTCGATCTACT-1,five +TTAGAATGTGGTGT-1,four +TTAGAATGTGTAGC-1,four +TTAGACCTCCTACC-1,four +TTAGACCTCCTTTA-1,two +TTAGCTACAACCGT-1,four +TTAGCTACTGTCCC-1,unassigned +TTAGCTACTTTCGT-1,four +TTAGGGACGCGAAG-1,five +TTAGGGTGCTGGAT-1,three +TTAGGGTGTCCTGC-1,two +TTAGGTCTACTTTC-1,two +TTAGTCACCAGTTG-1,four +TTAGTCTGAAAGCA-1,three +TTAGTCTGCCAACA-1,five +TTAGTCTGTGCACA-1,four +TTATCCGACTAGTG-1,three +TTATCCGAGAAAGT-1,one +TTATGAGAGATAAG-1,four +TTATGCACGTCACA-1,three +TTATGGCTTATGGC-1,unassigned +TTATTCCTATGCTG-1,two +TTATTCCTGGACAG-1,three +TTATTCCTGGTACT-1,three +TTATTCCTTCGTGA-1,four +TTCAAAGATAAAGG-1,five +TTCAACACAACAGA-1,four +TTCAACACCCCAAA-1,two +TTCAACACGGACGA-1,one +TTCAAGCTAAGAAC-1,two +TTCAAGCTAGATGA-1,four +TTCAAGCTGTTGAC-1,unassigned +TTCAAGCTTGATGC-1,four +TTCAAGCTTTCGCC-1,four +TTCACAACCCGTTC-1,two +TTCACAACGTCTGA-1,one +TTCAGACTACCCAA-1,four +TTCAGACTCTCGAA-1,four +TTCAGTACCGACTA-1,four +TTCAGTACTCAAGC-1,four +TTCAGTACTCCTAT-1,two +TTCAGTTGCCAAGT-1,four +TTCAGTTGTCCTTA-1,unassigned +TTCAGTTGTCTAGG-1,four +TTCAGTTGTCTCGC-1,three +TTCATCGAGGTGGA-1,two +TTCATGTGTGGTGT-1,four +TTCATTCTATGTCG-1,two +TTCATTCTTCTCTA-1,four +TTCCAAACCTATGG-1,two +TTCCAAACCTCCCA-1,three +TTCCAAACTCCCAC-1,five +TTCCAAACTTGACG-1,five +TTCCATGACGAGAG-1,two +TTCCATGACTGTCC-1,four +TTCCCACTTGAGGG-1,five +TTCCCACTTGTCTT-1,two +TTCCTAGAAAGTGA-1,four +TTCCTAGACTAGTG-1,one +TTCGAGGACTCTAT-1,four +TTCGAGGAGGGCAA-1,four +TTCGAGGATAGAAG-1,one +TTCGATTGAGCATC-1,four +TTCGGAGAATGCCA-1,two +TTCGGAGATGTGCA-1,four +TTCGTATGAAAAGC-1,five +TTCGTATGGATAGA-1,five +TTCGTATGGTCTGA-1,four +TTCGTATGTCCTTA-1,two +TTCTACGAACGTAC-1,four +TTCTACGAGTTGGT-1,four +TTCTAGTGACACGT-1,four +TTCTAGTGCATGAC-1,two +TTCTAGTGGAGAGC-1,five +TTCTAGTGGTCACA-1,unassigned +TTCTCAGAAGAGAT-1,four +TTCTCAGAAGCATC-1,three +TTCTCAGATGGAGG-1,four +TTCTGATGGAGACG-1,five +TTCTTACTCTGGAT-1,one +TTGAACCTCCTTGC-1,three +TTGAATGAACTACG-1,one +TTGAATGACTTACT-1,four +TTGAATGATCTCAT-1,three +TTGACACTCTGTAG-1,four +TTGACACTGATAAG-1,three +TTGAGGACAGAACA-1,two +TTGAGGACTACGCA-1,unassigned +TTGAGGTGGACGGA-1,four +TTGCATTGAGCTAC-1,five +TTGCATTGCTAAGC-1,two +TTGCATTGTGACTG-1,four +TTGCTAACACCAAC-1,five +TTGCTAACACGCTA-1,one +TTGCTAACCACTCC-1,four +TTGCTATGGTACGT-1,two +TTGCTATGGTAGGG-1,one +TTGGAGACCAATCG-1,unassigned +TTGGAGACGCTATG-1,two +TTGGAGACTATGGC-1,three +TTGGGAACTGAACC-1,four +TTGGTACTACTGGT-1,one +TTGGTACTCTTAGG-1,four +TTGGTACTGAATCC-1,three +TTGGTACTGGATTC-1,three +TTGTACACGTTGTG-1,three +TTGTACACTTGCAG-1,four +TTGTAGCTAGCTCA-1,four +TTGTAGCTCTCTTA-1,three +TTGTCATGGACGGA-1,five +TTTAGAGATCCTCG-1,three +TTTAGCTGATACCG-1,four +TTTAGCTGGATACC-1,two +TTTAGCTGTACTCT-1,unassigned +TTTAGGCTCCTTTA-1,two +TTTATCCTGTTGTG-1,two +TTTCACGAGGTTCA-1,four +TTTCAGTGGAAGGC-1,five +TTTCAGTGTCACGA-1,three +TTTCAGTGTCTATC-1,two +TTTCAGTGTGCAGT-1,three +TTTCCAGAGGTGAG-1,four +TTTCGAACACCTGA-1,unassigned +TTTCGAACTCTCAT-1,two +TTTCTACTGAGGCA-1,three +TTTCTACTTCCTCG-1,three +TTTGCATGAGAGGC-1,three +TTTGCATGCCTCAC-1,four diff --git a/local_server/test/fixtures/pbmc3k-genesets.csv b/local_server/test/fixtures/pbmc3k-genesets.csv new file mode 100644 index 00000000..2b8e0c3d --- /dev/null +++ b/local_server/test/fixtures/pbmc3k-genesets.csv @@ -0,0 +1,12 @@ +# Test fixture +gene_set_name, gene_set_description, gene_symbol, gene_description +first gene set name,,F5, a gene_description +first gene set name,a description, NO_SUCH_GENE, non-existent gene +first gene set name,a description, F5, duplicate gene +first gene set name, a description, SUMO3, +first gene set name,, SRM, +second gene set,,RER1 +second gene set,,SIK1 +third gene set,,NO_SUCH_GENE +fourth_gene_set,fourth description,,gene intentionally missing +fifth_dataset,,, diff --git a/local_server/test/fixtures/schema.json b/local_server/test/fixtures/schema.json new file mode 100644 index 00000000..eb3cc981 --- /dev/null +++ b/local_server/test/fixtures/schema.json @@ -0,0 +1,83 @@ +{ + "dataframe": { + "nObs": 2638, + "nVar": 1838, + "type": "float32" + }, + "annotations": { + "obs": { + "index": "name_0", + "columns": [ + { + "name": "name_0", + "type": "string", + "writable": false + }, + { + "name": "n_genes", + "type": "int32", + "writable": false + }, + { + "name": "percent_mito", + "type": "float32", + "writable": false + }, + { + "name": "n_counts", + "type": "float32", + "writable": false + }, + { + "name": "louvain", + "type": "categorical", + "categories": [ + "CD4 T cells", + "CD14+ Monocytes", + "B cells", + "CD8 T cells", + "NK cells", + "FCGR3A+ Monocytes", + "Dendritic cells", + "Megakaryocytes" + ], + "writable": false + } + ] + }, + "var": { + "index": "name_0", + "columns": [ + { + "name": "name_0", + "type": "string", + "writable": false + }, + { + "name": "n_cells", + "type": "int32", + "writable": false + } + ] + } + }, + "layout": { + "obs": [ + { + "name": "umap", + "type": "float32", + "dims": ["umap_0", "umap_1"] + }, + { + "name": "tsne", + "type": "float32", + "dims": ["tsne_0", "tsne_1"] + }, + { + "name": "pca", + "type": "float32", + "dims": ["pca_0", "pca_1"] + } + ] + } +} diff --git a/local_server/test/fixtures/schema_test_data/generate_test_data.sh b/local_server/test/fixtures/schema_test_data/generate_test_data.sh new file mode 100755 index 00000000..bb33d5a5 --- /dev/null +++ b/local_server/test/fixtures/schema_test_data/generate_test_data.sh @@ -0,0 +1,139 @@ +#!/bin/bash +wget "https://s3-us-west-2.amazonaws.com/10x.files/samples/cell/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz" +tar xf "pbmc3k_filtered_gene_bc_matrices.tar.gz" + +python3 - < genes_tmp.tsv; mv genes_tmp.tsv merged/genes.tsv + +echo -e "\n\n\nRunning tutorial on original\n\n\n" +Rscript - < + -- Does initial render -- + 4. Concurrently load all /annotations/obs and all /layouts/obs + -- Fully initialized -- + """ + + # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency. + wait_time = between(0.01, 0.1) + + def on_start(self): + self.dataset = self.parent.dataset + self.client.verify = False + self.api_less_client = HttpSession( + base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""), + request_success=self.client.request_success, + request_failure=self.client.request_failure, + ) + + @task + def index(self): + self.api_less_client.get(f"{self.dataset}", stream=True) + + @task + def loadConfigAndSchema(self): + self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True) + self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True) + + @task + def loadBootstrapData(self): + self.client.get( + f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True + ) + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}", + headers={"Accept": "application/octet-stream"}, + catch_response=True, + ) + + @task + def loadObsAnnotationsAndLayouts(self): + obs_names = self.parent.obs_annotation_names() + for name in obs_names: + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) + + layouts = self.parent.layout_names() + for name in layouts: + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) + + @task + def done(self): + self.interrupt() + + @task(1) + def load_expression(self): + """ + Simulate user occasionally loading some expression data for a gene + """ + + gene_name = random.choice(self.gene_names) + filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}} + self.client.put( + f"{self.dataset}/{API_SUFFIX}/data/var", + data=json.dumps(filter), + headers={"Content-Type": "application/json", "Accept": "application/octet-stream"}, + stream=True, + ).close() + + +class CellxgeneUser(HttpUser): + tasks = [CellXGeneTasks] + + # Most ops do not require back-end interaction, so slow cadence for users + wait_time = between(10, 60) diff --git a/local_server/test/locust/requirements-locust.txt b/local_server/test/locust/requirements-locust.txt new file mode 100644 index 00000000..e8a5c450 --- /dev/null +++ b/local_server/test/locust/requirements-locust.txt @@ -0,0 +1,2 @@ +locust +-r ../../requirements.txt diff --git a/local_server/test/performance/__init__.py b/local_server/test/performance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/performance/create_test_matrix.py b/local_server/test/performance/create_test_matrix.py new file mode 100644 index 00000000..476b6387 --- /dev/null +++ b/local_server/test/performance/create_test_matrix.py @@ -0,0 +1,44 @@ +import anndata +import argparse +import random +import scipy +import numpy as np + + +def main(): + parser = argparse.ArgumentParser("A command to generate test h5ad files") + parser.add_argument("output", help="Name of the output file") + parser.add_argument("nobs", type=int, help="Number of observations (rows)") + parser.add_argument("nvar", type=int, help="Number of variables (columns)") + parser.add_argument("-n", "--nnz-percent", type=float, default=100, help="percent of non-zeros") + parser.add_argument("-c", "--col-shift", action="store_true", help="add a random value to each column") + parser.add_argument("--seed", type=int, default=None, help="add a random value to each column") + + args = parser.parse_args() + create_test_h5ad(args.output, args.nobs, args.nvar, args.nnz_percent, args.col_shift, args.seed) + + +def create_test_h5ad(outfile, nobs, nvar, nnz_percent=100, apply_col_shift=False, seed=None): + random.seed(seed) + np.random.seed(seed) + x = create_X_array(nobs, nvar, nnz_percent, apply_col_shift) + obsm = {"X_random": np.random.rand(nobs, 2).astype(np.float32)} + adata = anndata.AnnData(x, obsm=obsm) + adata.write(outfile) + + +def create_X_array(nobs, nvar, nnz_percent, apply_col_shift): + if nnz_percent < 100: + array = scipy.sparse.random(nobs, nvar, nnz_percent * 0.01, dtype=np.float32, format="csc") + else: + array = np.random.rand(nobs, nvar).astype(np.float32) + + if apply_col_shift: + col_shift = np.random.rand((nvar)) + array += col_shift + + return array + + +if __name__ == "__main__": + main() diff --git a/local_server/test/performance/run_diffexp.py b/local_server/test/performance/run_diffexp.py new file mode 100644 index 00000000..cd2634bd --- /dev/null +++ b/local_server/test/performance/run_diffexp.py @@ -0,0 +1,106 @@ +import sys +import argparse +import random +import time +import numpy as np + +import local_server.compute.diffexp_generic as diffexp_generic + +from local_server.common.config.app_config import AppConfig +from local_server.data_common.matrix_loader import MatrixDataLoader + + +def main(): + parser = argparse.ArgumentParser("A command to test diffexp") + parser.add_argument("dataset", help="name of a dataset to load") + parser.add_argument("-na", "--numA", type=int, help="number of rows in group A") + parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B") + parser.add_argument("-va", "--varA", help="obs variable:value to use for group A") + parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B") + parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials") + parser.add_argument( + "-a", "--alg", choices=("default", "generic"), default="default", help="algorithm to use" + ) + parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results") + parser.add_argument( + "-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial" + ) + parser.add_argument("--seed", default=1, type=int, help="set the random seed") + + args = parser.parse_args() + + app_config = AppConfig() + app_config.update_server_config(single_dataset__datapath=args.dataset) + app_config.update_server_config(app__verbose=True) + app_config.complete_config() + + loader = MatrixDataLoader(args.dataset) + adaptor = loader.open(app_config) + + random.seed(args.seed) + np.random.seed(args.seed) + rows = adaptor.get_shape()[0] + + if args.numA: + filterA = random.sample(range(rows), args.numA) + elif args.varA: + vname, vval = args.varA.split(":") + filterA = get_filter_from_obs(adaptor, vname, vval) + else: + print("must supply numA or varA") + sys.exit(1) + + if args.numB: + filterB = random.sample(range(rows), args.numB) + elif args.varB: + vname, vval = args.varB.split(":") + filterB = get_filter_from_obs(adaptor, vname, vval) + else: + print("must supply numB or varB") + sys.exit(1) + + for i in range(args.trials): + if args.new_selection: + if args.numA: + filterA = random.sample(range(rows), args.numA) + if args.numB: + filterB = random.sample(range(rows), args.numB) + + maskA = np.zeros(rows, dtype=bool) + maskA[filterA] = True + maskB = np.zeros(rows, dtype=bool) + maskB[filterB] = True + + t1 = time.time() + if args.alg == "default": + results = adaptor.compute_diffexp_ttest(maskA, maskB) + elif args.alg == "generic": + results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB) + + t2 = time.time() + print("TIME=", t2 - t1) + + if args.show: + for res in results: + print(res) + + +def get_filter_from_obs(adaptor, obsname, obsval): + attrs = adaptor.get_obs_columns() + if obsname not in attrs: + print(f"Unknown obs attr {obsname}: expected on of {attrs}") + sys.exit(1) + obsvals = adaptor.query_obs_array(obsname)[:] + obsval = type(obsvals[0])(obsval) + + vfilter = np.where(obsvals == obsval)[0] + if len(vfilter) == 0: + u = np.unique(obsvals) + print(f"Unknown value in variable {obsname}:{obsval}: expected one of {list(u)}") + sys.exit(1) + + return vfilter + + +if __name__ == "__main__": + main() diff --git a/local_server/test/unit/__init__.py b/local_server/test/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/auth/__init__.py b/local_server/test/unit/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/auth/test_auth.py b/local_server/test/unit/auth/test_auth.py new file mode 100644 index 00000000..f5075dd3 --- /dev/null +++ b/local_server/test/unit/auth/test_auth.py @@ -0,0 +1,89 @@ +import unittest + +import requests + +from local_server.common.config.app_config import AppConfig +from local_server.test import H5AD_FIXTURE, test_server + + +class AuthTest(unittest.TestCase): + def setUp(self): + self.dataset_datapath = H5AD_FIXTURE + + def test_auth_none(self): + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__type=None, single_dataset__datapath=self.dataset_datapath) + app_config.update_dataset_config(user_annotations__enable=False, user_annotations__gene_sets__readonly=True) + + app_config.complete_config() + + with test_server(app_config=app_config) as server: + session = requests.Session() + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertNotIn("authentication", config["config"]) + self.assertIsNone(userinfo) + + def test_auth_session(self): + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__type="session", single_dataset__datapath=self.dataset_datapath) + app_config.update_dataset_config(user_annotations__enable=True) + app_config.complete_config() + + with test_server(app_config=app_config) as server: + session = requests.Session() + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + + self.assertFalse(config["config"]["authentication"]["requires_client_login"]) + self.assertTrue(userinfo["userinfo"]["is_authenticated"]) + self.assertEqual(userinfo["userinfo"]["username"], "anonymous") + + def test_auth_test_single(self): + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config( + authentication__type="test", + single_dataset__datapath=self.dataset_datapath, + authentication__insecure_test_environment=True, + ) + + app_config.complete_config() + + with test_server(app_config=app_config) as server: + session = requests.Session() + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertFalse(userinfo["userinfo"]["is_authenticated"]) + self.assertIsNone(userinfo["userinfo"]["username"]) + self.assertTrue(config["config"]["authentication"]["requires_client_login"]) + self.assertTrue(config["config"]["parameters"]["annotations"]) + + login_uri = config["config"]["authentication"]["login"] + logout_uri = config["config"]["authentication"]["logout"] + + self.assertEqual(login_uri, "/login") + self.assertEqual(logout_uri, "/logout") + + response = session.get(f"{server}/{login_uri}") + # check that the login redirect worked + self.assertEqual(response.history[0].status_code, 302) + self.assertEqual(response.url, f"{server}/") + + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertTrue(userinfo["userinfo"]["is_authenticated"]) + self.assertEqual(userinfo["userinfo"]["username"], "test_account") + self.assertTrue(config["config"]["parameters"]["annotations"]) + + response = session.get(f"{server}/{logout_uri}") + # check that the logout redirect worked + self.assertEqual(response.history[0].status_code, 302) + self.assertEqual(response.url, f"{server}/") + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertFalse(userinfo["userinfo"]["is_authenticated"]) + self.assertIsNone(userinfo["userinfo"]["username"]) + self.assertTrue(config["config"]["parameters"]["annotations"]) diff --git a/local_server/test/unit/cli/__init__.py b/local_server/test/unit/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/cli/test_launch.py b/local_server/test/unit/cli/test_launch.py new file mode 100644 index 00000000..d58638d8 --- /dev/null +++ b/local_server/test/unit/cli/test_launch.py @@ -0,0 +1,27 @@ +import filecmp +import os +import shutil +import unittest + +import yaml + +from local_server.default_config import default_config +from local_server.test import FIXTURES_ROOT + + +class CLIPLaunchTests(unittest.TestCase): + tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs") + + @classmethod + def setUpClass(cls) -> None: + os.mkdir(cls.tmp_dir) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_dir) + + def test_dump_default_config(self): + os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt") + with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config: + expected_config.write(yaml.dump(default_config)) + filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt") diff --git a/local_server/test/unit/cli/test_prepare.py b/local_server/test/unit/cli/test_prepare.py new file mode 100644 index 00000000..85ea4bd6 --- /dev/null +++ b/local_server/test/unit/cli/test_prepare.py @@ -0,0 +1,15 @@ +import unittest + +import pandas as pd + +from local_server.cli.prepare import make_index_unique + + +class CLIPrepareTests(unittest.TestCase): + """ Test cases for CLI prepare logic """ + + def test_make_index_unique(self): + index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"]) + result = make_index_unique(index) + expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"]) + self.assertTrue(all(left == right for left, right in zip(result.values, expected.values))) diff --git a/local_server/test/unit/cli/test_upgrade.py b/local_server/test/unit/cli/test_upgrade.py new file mode 100644 index 00000000..9e5dae43 --- /dev/null +++ b/local_server/test/unit/cli/test_upgrade.py @@ -0,0 +1,28 @@ +import unittest + +from local_server.cli.upgrade import validate_version_str, split_version, version_gt + + +class CLIUpgradeTests(unittest.TestCase): + """ Test cases for CLI logic """ + + def test_validate_version_str(self): + self.assertTrue(validate_version_str("0.1.2")) + self.assertTrue(validate_version_str("0.1.2-RC", release_only=False)) + self.assertFalse(validate_version_str("0.1")) + self.assertFalse(validate_version_str("0.1.2.3")) + self.assertFalse(validate_version_str("0.1.2-RC")) + + def test_split_version_str(self): + self.assertEqual(split_version("0.1.2"), [0, 1, 2]) + with self.assertRaises(AttributeError): + split_version("0.1") + + def test_assert_verstion_gt(self): + self.assertTrue(version_gt("1.0.0", "0.1.1")) + self.assertTrue(version_gt("0.1.0", "0.0.1")) + self.assertTrue(version_gt("0.0.1", "0.0.0")) + self.assertFalse(version_gt("0.0.0", "0.0.0")) + self.assertFalse(version_gt("0.0.0", "0.0.1")) + self.assertFalse(version_gt("0.0.1", "0.1.0")) + self.assertFalse(version_gt("0.1.1", "1.0.0")) diff --git a/local_server/test/unit/common/__init__.py b/local_server/test/unit/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/common/config/__init__.py b/local_server/test/unit/common/config/__init__.py new file mode 100644 index 00000000..577d2e16 --- /dev/null +++ b/local_server/test/unit/common/config/__init__.py @@ -0,0 +1,228 @@ +import os +import shutil +import unittest +import random +from unittest import mock +import yaml + +from local_server.test import FIXTURES_ROOT + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class ConfigTests(unittest.TestCase): + tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir") + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_fixtures_directory) + + @classmethod + def setUpClass(cls) -> None: + os.makedirs(cls.tmp_fixtures_directory) + + def custom_server_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="secret", + auth_type="session", + insecure_test_environment="false", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + data_locater_region_name="us-east-1", + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + config_file_name="server_config.yaml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py") + with open(server_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + server_config = eval(config) + with open(configfile, "w") as server_config_file: + server_config_file.write(server_config) + return configfile + + def custom_app_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="secret", + auth_type="session", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + data_locater_region_name="us-east-1", + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + scripts=[], + inline_scripts=[], + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + local_file_csv_gene_sets_file="null", + ontology_enabled="false", + obo_location="null", + gene_sets_readonly="false", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + environment=None, + aws_secrets_manager_region=None, + aws_secrets_manager_secrets=[], + config_file_name="app_config.yml", + ): + random_num = random.randrange(999999) + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config = self.custom_server_config( + verbose=verbose, + debug=debug, + host=host, + port=port, + open_browser=open_browser, + force_https=force_https, + flask_secret_key=flask_secret_key, + auth_type=auth_type, + index=index, + allowed_matrix_types=allowed_matrix_types, + max_cached_datasets=max_cached_datasets, + timelimit_s=timelimit_s, + dataset_datapath=dataset_datapath, + obs_names=obs_names, + var_names=var_names, + about=about, + title=title, + data_locater_region_name=data_locater_region_name, + anndata_backed=anndata_backed, + column_request_max=column_request_max, + diffexp_cellcount_max=diffexp_cellcount_max, + config_file_name=f"temp_server_config_{random_num}.yml", + ) + dataset_config = self.custom_dataset_config( + scripts=scripts, + inline_scripts=inline_scripts, + authentication_enable=authentication_enable, + max_categories=max_categories, + custom_colors=custom_colors, + enable_users_annotations=enable_users_annotations, + annotation_type=annotation_type, + db_uri=db_uri, + hosted_file_directory=hosted_file_directory, + local_file_csv_directory=local_file_csv_directory, + local_file_csv_file=local_file_csv_file, + local_file_csv_gene_sets_file=local_file_csv_gene_sets_file, + ontology_enabled=ontology_enabled, + obo_location=obo_location, + gene_sets_readonly=gene_sets_readonly, + embedding_names=embedding_names, + enable_reembedding=enable_reembedding, + enable_difexp=enable_difexp, + lfc_cutoff=lfc_cutoff, + top_n=top_n, + config_file_name=f"temp_dataset_config_{random_num}.yml", + ) + external_config = self.custom_external_config( + environment=environment, + aws_secrets_manager_region=aws_secrets_manager_region, + aws_secrets_manager_secrets=aws_secrets_manager_secrets, + config_file_name=f"temp_external_config_{random_num}.yml", + ) + + with open(configfile, "w") as app_config_file: + app_config_file.write(open(server_config).read()) + app_config_file.write(open(dataset_config).read()) + app_config_file.write(open(external_config).read()) + + return configfile + + def custom_dataset_config( + self, + scripts=[], + inline_scripts=[], + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + local_file_csv_gene_sets_file="null", + ontology_enabled="false", + obo_location="null", + gene_sets_readonly="false", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + config_file_name="dataset_config.yml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py") + with open(dataset_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + dataset_config = eval(config) + with open(configfile, "w") as dataset_config_file: + dataset_config_file.write(dataset_config) + + return configfile + + def custom_external_config( + self, + environment=None, + aws_secrets_manager_region=None, + aws_secrets_manager_secrets=[], + config_file_name="external_config.yaml", + ): + # set to the default if environment is None + if environment is None: + environment = [ + dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False), + ] + external_config = { + "external": { + "environment": environment, + "aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets}, + } + } + + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + with open(configfile, "w") as external_config_file: + yaml.dump(external_config, external_config_file) + return configfile diff --git a/local_server/test/unit/common/config/test_app_config.py b/local_server/test/unit/common/config/test_app_config.py new file mode 100644 index 00000000..b2280f86 --- /dev/null +++ b/local_server/test/unit/common/config/test_app_config.py @@ -0,0 +1,153 @@ +import os +import tempfile +import unittest + +import yaml + +from local_server.default_config import default_config +from local_server.common.config.app_config import AppConfig +from local_server.test.unit.common.config import ConfigTests +from local_server.common.errors import ConfigurationError +from local_server.test import FIXTURES_ROOT, H5AD_FIXTURE + + +class AppConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataset_datapath=H5AD_FIXTURE, config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_get_default_config_correctly_reads_default_config_file(self): + app_default_config = AppConfig().default_config + + expected_config = yaml.load(default_config, Loader=yaml.Loader) + + server_config = app_default_config["server"] + dataset_config = app_default_config["dataset"] + + expected_server_config = expected_config["server"] + expected_dataset_config = expected_config["dataset"] + + self.assertDictEqual(app_default_config, expected_config) + self.assertDictEqual(server_config, expected_server_config) + self.assertDictEqual(dataset_config, expected_dataset_config) + + def test_get_dataset_config_returns_dataset_config_for_single_datasets(self): + datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" + file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name) + config = AppConfig() + config.update_from_config_file(file_name) + + self.assertEqual(config.get_dataset_config(), config.dataset_config) + + def test_update_server_config_updates_server_config_and_config_status(self): + config = self.get_config() + config.complete_config() + config.check_config() + config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + with self.assertRaises(ConfigurationError): + config.server_config.check_config() + + def test_write_config_outputs_yaml_with_all_config_vars(self): + config = self.get_config() + config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml") + with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config: + default_config_yml = yaml.safe_load(default_config) + + with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config: + output_config_yml = yaml.safe_load(output_config) + self.maxDiff = None + self.assertEqual(default_config_yml, output_config_yml) + + def test_update_app_config(self): + config = AppConfig() + config.update_server_config(app__verbose=True, single_dataset__datapath="datapath") + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, [("app__verbose", True, False), ("single_dataset__datapath", "datapath", None)]) + + config = AppConfig() + config.update_dataset_config(app__scripts=(), app__inline_scripts=()) + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_dataset_config(app__scripts=[], app__inline_scripts=[]) + vars = config.dataset_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) + vars = config.dataset_config.changes_from_default() + self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) + + def test_configfile_no_server_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + dataset: + user_annotations: + enable: false + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.dataset_config.changes_from_default() + self.assertEqual(server_changes, []) + self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) + + def test_simple_update_single_config_from_path_and_value(self): + """Update a simple config parameter""" + + config = AppConfig() + config.server_config.single_dataset__datapath = "my/data/path" + + # test simple value in server + config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret") + self.assertEqual(config.server_config.app__flask_secret_key, "mysecret") + + # test simple value in default dataset + config.update_single_config_from_path_and_value( + ["dataset", "user_annotations", "ontology", "obo_location"], "dummy_location", + ) + self.assertEqual(config.dataset_config.user_annotations__ontology__obo_location, "dummy_location") + + # error checking + bad_paths = [ + ( + ["dataset", "does", "not", "exist"], + "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'", + ), + (["does", "not", "exist"], "path must start with 'server', or 'dataset'"), + ([], "path must start with 'server', or 'dataset'"), + ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"), + ("string", "path must be a list of strings, got 'string'"), + ] + for bad_path, error_message in bad_paths: + with self.assertRaises(ConfigurationError) as config_error: + config.update_single_config_from_path_and_value(bad_path, "value") + + self.assertEqual(config_error.exception.message, error_message) diff --git a/local_server/test/unit/common/config/test_base_config.py b/local_server/test/unit/common/config/test_base_config.py new file mode 100644 index 00000000..c289106a --- /dev/null +++ b/local_server/test/unit/common/config/test_base_config.py @@ -0,0 +1,63 @@ +import unittest + +from local_server.common.config.app_config import AppConfig +from local_server.test import H5AD_FIXTURE +from local_server.test.unit.common.config import ConfigTests +from local_server.common.errors import ConfigurationError + + +class BaseConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_mapping_creation_returns_map_of_server_and_dataset_config(self): + config = AppConfig() + mapping = config.dataset_config.create_mapping(config.default_config) + self.assertIsNotNone(mapping["server__app__verbose"]) + self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) + self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"]) + + def test_changes_from_default_returns_list_of_nondefault_config_values(self): + config = self.get_config(verbose="true", lfc_cutoff=0.05) + server_changes = config.server_config.changes_from_default() + dataset_changes = config.dataset_config.changes_from_default() + + self.assertEqual( + server_changes, + [ + ("app__verbose", True, False), + ("app__flask_secret_key", "secret", None), + ("single_dataset__datapath", H5AD_FIXTURE, None), + ('data_locator__s3__region_name', 'us-east-1', True) + ], + ) + self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)]) + + def test_check_config_throws_error_if_attr_has_not_been_checked(self): + config = self.get_config(verbose="true") + config.complete_config() + config.check_config() + config.update_server_config(app__verbose=False) + with self.assertRaises(ConfigurationError): + config.check_config() diff --git a/local_server/test/unit/common/config/test_dataset_config.py b/local_server/test/unit/common/config/test_dataset_config.py new file mode 100644 index 00000000..4a66a80a --- /dev/null +++ b/local_server/test/unit/common/config/test_dataset_config.py @@ -0,0 +1,157 @@ +import os +import tempfile + +import unittest +from unittest.mock import patch + +from local_server.common.annotations.local_file_csv import AnnotationsLocalFile +from local_server.common.config.app_config import AppConfig +from local_server.common.config.base_config import BaseConfig +from local_server.test import FIXTURES_ROOT, H5AD_FIXTURE + +from local_server.common.errors import ConfigurationError +from local_server.test.unit.common.config import ConfigTests + + +class TestDatasetConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.dataset_config = self.config.dataset_config + self.config.complete_config() + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_datatset_config_sets_vars_from_config(self): + config = AppConfig() + self.assertEqual(config.dataset_config.presentation__max_categories, 1000) + self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv") + self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01) + self.assertIsNone(config.dataset_config.user_annotations__ontology__obo_location) + + @patch("local_server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.dataset_config.complete_config(self.context) + self.assertIsNotNone(self.config.server_config.data_adaptor) + self.assertEqual(mock_check_attrs.call_count, 19) + + def test_app_sets_script_vars(self): + config = self.get_config(scripts=["path/to/script"]) + config.dataset_config.handle_app() + + self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}]) + + config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}]) + config.dataset_config.handle_app() + self.assertEqual( + config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}] + ) + + config = self.get_config(scripts=["path/to/script", "different/script/path"]) + config.dataset_config.handle_app() + # TODO @madison -- is this the desired functionality? + self.assertEqual( + config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}] + ) + + config = self.get_config(scripts=[{"more": "different/script/path"}]) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_app() + + def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self): + config = self.get_config(enable_users_annotations="true", authentication_enable="false") + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_user_annotations(self.context) + + config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend") + with self.assertRaises(ConfigurationError): + config.server_config.complete_config(self.context) + + def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self): + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile) + + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL" + ) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_user_annotations(self.context) + + def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self): + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.dataset_config.handle_local_file_csv_annotations(self.context) + self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile) + cwd = os.getcwd() + self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd) + + def test_handle_embeddings__checks_data_file_types(self): + file_name = self.custom_app_config( + embedding_names=["name1", "name2"], + enable_reembedding="true", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + anndata_backed="true", + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_embeddings() + + def test_handle_diffexp__raises_warning_for_large_datasets(self): + config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15) + config.server_config.complete_config(self.context) + config.dataset_config.handle_diffexp(self.context) + self.assertEqual(len(self.context["messages"]), 1) + + def test_configfile_with_specialization(self): + # test that per_dataset_config config load the default config, then the specialized config + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + single_dataset: + datapath: fake_datapath + dataset: + user_annotations: + enable: false + type: local_file_csv + local_file_csv: + file: fake_file + directory: fake_dir + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + + test_config = app_config.dataset_config + + # test config from default + self.assertEqual(test_config.user_annotations__type, "local_file_csv") + self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file") diff --git a/local_server/test/unit/common/config/test_external_config.py b/local_server/test/unit/common/config/test_external_config.py new file mode 100644 index 00000000..b0751c44 --- /dev/null +++ b/local_server/test/unit/common/config/test_external_config.py @@ -0,0 +1,215 @@ +import os +from unittest.mock import patch + +import requests + +from local_server.common.errors import ConfigurationError +from local_server.common.config.app_config import AppConfig +from local_server.test import test_server, FIXTURES_ROOT +from local_server.common.utils.type_conversion_utils import convert_string_to_value +from local_server.test.unit.common.config import ConfigTests + + +class TestExternalConfig(ConfigTests): + def test_type_convert(self): + # The values from environment variables and aws secrets are returned as strings. + # These values need to be converted to the proper types. + + self.assertEqual(convert_string_to_value("1"), int(1)) + self.assertEqual(convert_string_to_value("1.1"), float(1.1)) + self.assertEqual(convert_string_to_value("string"), "string") + self.assertEqual(convert_string_to_value("true"), True) + self.assertEqual(convert_string_to_value("True"), True) + self.assertEqual(convert_string_to_value("false"), False) + self.assertEqual(convert_string_to_value("False"), False) + self.assertEqual(convert_string_to_value("null"), None) + self.assertEqual(convert_string_to_value("None"), None) + self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string")) + + def test_environment_variable(self): + configfile = self.custom_external_config( + environment=[ + dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True), + dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True), + ], + config_file_name="environment_external_config.yaml", + ) + + env = os.environ + env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad" + env["DIFFEXP"] = "False" + with test_server(command_line_args=["-c", configfile], env=env) as server: + session = requests.Session() + response = session.get(f"{server}/api/v0.2/config") + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k-CSC-gz") + self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"]) + + env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad" + env["DIFFEXP"] = "True" + with test_server(command_line_args=["-c", configfile], env=env) as server: + session = requests.Session() + response = session.get(f"{server}/api/v0.2/config") + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b") + self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"]) + + def test_environment_variable_errors(self): + + # no name + app_config = AppConfig() + app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'name' is missing") + + # required has wrong type + app_config = AppConfig() + app_config.external_config.environment = [ + dict(name="myenvar", required="optional", path=["this", "is", "a", "path"]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool") + + # no path + app_config = AppConfig() + app_config.external_config.environment = [dict(name="myenvar", required=True)] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'path' is missing") + + # required environment variable is not set + app_config = AppConfig() + app_config.external_config.environment = [ + dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set") + + @patch("local_server.common.config.external_config.get_secret_key") + def test_aws_secrets_manager(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "flask_secret_key": "mock_flask_secret_key", + } + configfile = self.custom_external_config( + aws_secrets_manager_region="us-west-2", + aws_secrets_manager_secrets=[ + dict( + name="my_secret", + values=[ + dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=True), + ], + ) + ], + config_file_name="secret_external_config.yaml", + ) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad" + + app_config.complete_config() + + self.assertEqual(app_config.server_config.app__flask_secret_key, "mock_flask_secret_key") + + @patch("local_server.common.config.external_config.get_secret_key") + def test_aws_secrets_manager_error(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "db_uri": "mock_db_uri", + } + + # no region + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = None + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual( + config_error.exception.message, + "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType", + ) + + # missing secret name + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing") + + # secret name wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string") + + # missing values name + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing") + + # values wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"])) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list") + + # entry missing key + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret") + + # entry required is wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret") + + # entry missing path + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(key="db_uri", required=True)]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret") + + # secret missing required key + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict( + name="mysecret", + values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])], + ) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set") diff --git a/local_server/test/unit/common/config/test_server_config.py b/local_server/test/unit/common/config/test_server_config.py new file mode 100644 index 00000000..f50430a5 --- /dev/null +++ b/local_server/test/unit/common/config/test_server_config.py @@ -0,0 +1,122 @@ +import os +import unittest +from unittest import mock +from unittest.mock import patch + +from local_server.common.config.base_config import BaseConfig +from local_server.test import H5AD_FIXTURE + +from local_server.common.config.app_config import AppConfig +from local_server.common.errors import ConfigurationError +from local_server.test.unit.common.config import ConfigTests + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class TestServerConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_raises_error_if_default_config_is_invalid(self): + invalid_config = self.get_config(port="not_valid") + with self.assertRaises(ConfigurationError): + invalid_config.complete_config() + + @patch("local_server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.server_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 20) + + def test_handle_app__throws_error_if_port_doesnt_exist(self): + config = self.get_config(port=99999999) + with self.assertRaises(ConfigurationError): + config.server_config.handle_app(self.context) + + @patch("local_server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name): + mock_discover_region_name.return_value = None + # Default config + self.assertEqual(self.config.server_config.data_locator__s3__region_name, None) + # hard coded + config = self.get_config() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") + # incorrectly formatted + datapath = "s3://shouldnt/work" + file_name = self.custom_app_config( + dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_locator() + + def test_handle_app___can_use_envar_port(self): + config = self.get_config(port=24) + self.assertEqual(config.server_config.app__port, 24) + + # Note if the port is set in the config file it will NOT be overwritten by a different envvar + os.environ["CXG_SERVER_PORT"] = "4008" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.server_config.handle_app(self.context) + self.assertEqual(self.config.server_config.app__port, 4008) + del os.environ["CXG_SERVER_PORT"] + + def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self): + config = self.get_config(flask_secret_key="KEY_FROM_FILE") + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE") + + os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV" + config.external_config.handle_environment(self.context) + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV") + + def test_config_for_single_dataset(self): + file_name = self.custom_app_config( + config_file_name="single_dataset.yml", dataset_datapath=f"{H5AD_FIXTURE}" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_single_dataset(self.context) + + file_name = self.custom_app_config( + config_file_name="single_dataset_with_about.yml", + about="www.cziscience.com", + dataset_datapath=f"{H5AD_FIXTURE}", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_single_dataset(self.context) + + def test_test_auth_only_in_insecure(self): + + config = self.get_config(auth_type="test") + with self.assertRaises(ConfigurationError): + config.complete_config() + + config.update_server_config(authentication__insecure_test_environment=True) + config.complete_config() diff --git a/local_server/test/unit/common/test_api.py b/local_server/test/unit/common/test_api.py new file mode 100644 index 00000000..41bedbeb --- /dev/null +++ b/local_server/test/unit/common/test_api.py @@ -0,0 +1,684 @@ +import shutil +import time +import unittest +import zlib +from http import HTTPStatus +import tempfile +from os import path + +import pandas as pd +import requests + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.data_common.matrix_loader import MatrixDataType +from local_server.test import ( + data_with_tmp_annotations, + make_fbs, + PROJECT_ROOT, + FIXTURES_ROOT, + start_test_server, + stop_test_server, +) +from local_server.test.fixtures.fixtures import pbmc3k_colors + +BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} + + +# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental + + +class EndPoints(object): + ANNOTATIONS_ENABLED = True + GENESETS_READONLY = False + + def test_initialize(self): + endpoint = "schema" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638) + self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2) + self.assertEqual( + len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5 + ) + + def test_config(self): + endpoint = "config" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIn("library_versions", result_data["config"]) + self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") + self.assertIsNotNone(result_data["config"]["parameters"]) + + def test_get_layout_fbs(self): + endpoint = "layout/obs" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 8) + self.assertIsNotNone(df["columns"]) + self.assertSetEqual( + set(df["col_idx"]), + {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"}, + ) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + + def test_put_layout_fbs(self): + # first check that re-embedding is turned on + result = self.session.get(f"{self.URL_BASE}config") + config_data = result.json() + re_embed = config_data["config"]["parameters"]["enable-reembedding"] + if not re_embed: + return + # attempt to reembed with umap over 100 cells. + endpoint = "layout/obs" + url = f"{self.URL_BASE}{endpoint}" + data = {} + data["filter"] = {} + data["filter"]["obs"] = {} + data["filter"]["obs"]["index"] = list(range(100)) + data["method"] = "umap" + result = self.session.put(url, json=data) + + self.assertEqual(result.status_code, HTTPStatus.OK) + result_data = result.json() + self.assertIsInstance(result_data, dict) + self.assertEqual(result_data["type"], "float32") + self.assertTrue(result_data["name"].startswith("reembed:umap_")) + self.assertIsInstance(result_data["dims"], list) + self.assertEqual(len(result_data["dims"]), 2) + dims = result_data["dims"] + self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0")) + self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1")) + + def test_bad_filter(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.put(url, json=BAD_FILTER) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_get_annotations_obs_fbs(self): + endpoint = "annotations/obs" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] + self.assertCountEqual( + df["col_idx"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] + + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []), + ) + + def test_get_annotations_obs_keys_fbs(self): + endpoint = "annotations/obs" + query = "annotation-name=n_genes&annotation-name=percent_mito" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"]) + + def test_get_annotations_obs_error(self): + endpoint = "annotations/obs" + query = "annotation-name=notakey" + url = f"{self.URL_BASE}{endpoint}?{query}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_get_annotations_var_fbs(self): + endpoint = "annotations/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"]) + + def test_get_annotations_var_keys_fbs(self): + endpoint = "annotations/var" + query = "annotation-name=n_cells" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 1) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertCountEqual(df["col_idx"], ["n_cells"]) + + def test_get_annotations_var_error(self): + endpoint = "annotations/var" + query = "annotation-name=notakey" + url = f"{self.URL_BASE}{endpoint}?{query}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_data_mimetype_error(self): + endpoint = "data/var" + header = {"Accept": "xxx"} + url = f"{self.URL_BASE}{endpoint}" + result = self.session.put(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE) + + def test_fbs_default(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.put(url) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + filter = {"filter": {"var": {"index": [0, 1, 4]}}} + result = self.session.put(url, json=filter) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + + def test_data_put_fbs(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.put(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_data_get_fbs(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_data_put_filter_fbs(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + filter = {"filter": {"var": {"index": [0, 1, 4]}}} + result = self.session.put(url, headers=header, json=filter) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 3) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4]) + + def test_data_get_filter_fbs(self): + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + endpoint = "data/var" + query = f"var:{index_col_name}=SIK1" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + + def test_data_get_unknown_filter_fbs(self): + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + endpoint = "data/var" + query = f"var:{index_col_name}=UNKNOWN" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 0) + + def test_data_put_single_var(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}} + result = self.session.put(url, headers=header, json=var_filter) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + + def test_colors(self): + endpoint = "colors" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(result_data, pbmc3k_colors) + + def test_static(self): + endpoint = "static" + file = "assets/favicon.ico" + url = f"{self.server}/{endpoint}/{file}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + + def test_genesets_config(self): + result = self.session.get(f"{self.URL_BASE}config") + config_data = result.json() + params = config_data["config"]["parameters"] + annotations_genesets = params["annotations_genesets"] + annotations_genesets_readonly = params["annotations_genesets_readonly"] + annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"] + self.assertTrue(annotations_genesets) + self.assertEqual(annotations_genesets_readonly, self.GENESETS_READONLY) + self.assertEqual(annotations_genesets_summary_methods, ["mean"]) + + def test_get_genesets(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIsNotNone(result_data["genesets"]) + + def _setupClass(child_class, command_line): + child_class.ps, child_class.server = start_test_server(command_line) + child_class.URL_BASE = f"{child_class.server}/api/v0.2/" + child_class.session = requests.Session() + for i in range(90): + try: + result = child_class.session.get(f"{child_class.URL_BASE}schema") + child_class.schema = result.json() + except requests.exceptions.ConnectionError: + time.sleep(1) + + +class EndPointsAnnotations(EndPoints): + def test_get_schema_existing_writable(self): + self._test_get_schema_writable("cluster-test") + + def test_get_user_annotations_existing_obs_keys_fbs(self): + self._test_get_user_annotations_obs_keys_fbs( + "cluster-test", + {"unassigned", "one", "two", "three", "four", "five", "six", "seven"}, + ) + + def test_put_user_annotations_obs_fbs(self): + endpoint = "annotations/obs" + query = "annotation-collection-name=test_annotations" + url = f"{self.URL_BASE}{endpoint}?{query}" + n_rows = self.data.get_shape()[0] + fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")}) + result = self.session.put(url, data=zlib.compress(fbs)) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + self.assertEqual(result.json(), {"status": "OK"}) + self._test_get_schema_writable("cat_A") + self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"}) + + def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns): + endpoint = "annotations/obs" + query = f"annotation-name={annotation_name}" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + self.assertListEqual(df["col_idx"], [annotation_name]) + self.assertEqual(set(df["columns"][0]), columns) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + + def _test_get_schema_writable(self, cluster_name): + endpoint = "schema" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + columns = result_data["schema"]["annotations"]["obs"]["columns"] + matching_columns = [c for c in columns if c["name"] == cluster_name] + self.assertEqual(len(matching_columns), 1) + self.assertTrue(matching_columns[0]["writable"]) + + +class EndPointsAnndata(unittest.TestCase, EndPoints): + """Test Case for endpoints""" + + ANNOTATIONS_ENABLED = False + GENESETS_READONLY = True + + @classmethod + def setUpClass(cls): + cls._setupClass( + cls, + [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--disable-gene-sets-save", + "--experimental-enable-reembedding", + ], + ) + + @classmethod + def tearDownClass(cls): + stop_test_server(cls.ps) + + @property + def annotations_enabled(self): + return False + + def test_diff_exp(self): + endpoint = "diffexp/obs" + url = f"{self.URL_BASE}{endpoint}" + params = { + "mode": "topN", + "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}}, + "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}}, + "count": 7, + } + result = self.session.post(url, json=params) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(len(result_data), 7) + + def test_diff_exp_indices(self): + endpoint = "diffexp/obs" + url = f"{self.URL_BASE}{endpoint}" + params = { + "mode": "topN", + "count": 10, + "set1": {"filter": {"obs": {"index": [[0, 500]]}}}, + "set2": {"filter": {"obs": {"index": [[500, 1000]]}}}, + } + result = self.session.post(url, json=params) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(len(result_data), 10) + + +class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): + """Test Case for endpoints""" + + ANNOTATIONS_ENABLED = True + GENESETS_READONLY = False + + @classmethod + def setUpClass(cls): + cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations( + MatrixDataType.H5AD, annotations_fixture=True + ) + cls._setupClass(cls, ["--annotations-file", cls.annotations.label_output_file, cls.data.get_location()]) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp_dir) + stop_test_server(cls.ps) + + +class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints): + ANNOTATIONS_ENABLED = False + GENESETS_READONLY = False + + @classmethod + def setUpClass(cls): + cls.tmp_dir = tempfile.mkdtemp() + genesets_file = path.join(cls.tmp_dir, "test_genesets.csv") + shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-genesets.csv", genesets_file) + cls._setupClass( + cls, + [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--gene-sets-file", + genesets_file, + ], + ) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp_dir) + stop_test_server(cls.ps) + + def test_get_genesets_json(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIsNotNone(result_data["genesets"]) + self.assertIsNotNone(result_data["tid"]) + + self.assertEqual( + result_data, + { + "genesets": [ + { + "genes": [ + {"gene_description": "a gene_description", "gene_symbol": "F5"}, + {"gene_description": "", "gene_symbol": "SUMO3"}, + {"gene_description": "", "gene_symbol": "SRM"}, + ], + "geneset_description": "a description", + "geneset_name": "first gene set name", + }, + { + "genes": [ + {"gene_description": "", "gene_symbol": "RER1"}, + {"gene_description": "", "gene_symbol": "SIK1"}, + ], + "geneset_description": "", + "geneset_name": "second gene set", + }, + {"genes": [], "geneset_description": "", "geneset_name": "third gene set"}, + {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"}, + {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"}, + ], + "tid": 0, + }, + ) + + def test_get_genesets_csv(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "text/csv"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "text/csv") + self.assertEqual( + result.text, + """gene_set_name,gene_set_description,gene_symbol,gene_description\r +first gene set name,a description,F5,a gene_description\r +first gene set name,a description,SUMO3,\r +first gene set name,a description,SRM,\r +second gene set,,RER1,\r +second gene set,,SIK1,\r +third gene set,,,\r +fourth_gene_set,fourth description,,\r +fifth_dataset,,,\r +""", + ) + + def test_put_genesets(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + + # assume we start with TID 0 + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json()["tid"], 0) + + test1 = {"tid": 3, "genesets": []} + result = self.session.put(url, json=test1) + self.assertEqual(result.status_code, HTTPStatus.OK) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), test1) + + # stale TID + result = self.session.put(url, json=test1) + self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND) + + test2 = {"tid": 4, "genesets": [{"geneset_name": "foobar", "genes": []}]} + test2_response = {"tid": 4, "genesets": [{"geneset_name": "foobar", "geneset_description": "", "genes": []}]} + result = self.session.put(url, json=test2) + self.assertEqual(result.status_code, HTTPStatus.OK) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), test2_response) + + test3 = { + "tid": 5, + "genesets": [ + { + "geneset_name": "foobar", + "geneset_description": "", + "genes": [ + { + "gene_symbol": "F5", + "gene_description": "", + } + ], + } + ], + } + result = self.session.put(url, json=test3) + self.assertEqual(result.status_code, HTTPStatus.OK) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), test3) + + def test_put_genesets_malformed(self): + """ test malformed submissions that we expect the backend to catch/tolerate """ + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + original_data = result.json() + tid = original_data["tid"] + + def test_case(test, expected_code, original_data): + """ check for expected error AND that no change was made to the original state """ + result = self.session.put(url, json=test) + self.assertEqual(result.status_code, expected_code) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), original_data) + + # missing or malformed genesets + test_case( + {"tid": tid + 1}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + test_case( + {"tid": tid + 1, "genesets": 99}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # illegal geneset_name + test_case( + {"tid": tid + 1, "genesets": [{"geneset_name": """, "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # duplicate geneset_name + test_case( + { + "tid": tid + 1, + "genesets": [ + {"geneset_name": "foo", "genes": []}, + {"geneset_name": "foo", "genes": []}, + ], + }, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # missing geneset_name + test_case( + {"tid": tid + 1, "genesets": [{"genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # non-numeric TID + test_case( + {"tid": [], "genesets": [{"geneset_name": "foo", "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + test_case( + {"tid": None, "genesets": [{"geneset_name": "foo", "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + test_case( + {"tid": "not a number", "genesets": [{"geneset_name": "foo", "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # duplicate gene_symbol + test_case( + { + "tid": "not a number", + "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": "SIK1"}, {"gene_symbol": "SIK1"}]}], + }, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # gene_symbol is not a string + test_case( + { + "tid": "not a number", + "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": 99}]}], + }, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + """ + TODO once we have some code to support it: + 1. GET genesets_summary + 2. genesets_summary obeys tid + """ diff --git a/local_server/test/unit/common/test_colors.py b/local_server/test/unit/common/test_colors.py new file mode 100644 index 00000000..6e14637e --- /dev/null +++ b/local_server/test/unit/common/test_colors.py @@ -0,0 +1,40 @@ +import unittest + +import anndata +from local_server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors +from local_server.common.errors import ColorFormatException +from local_server.test import PROJECT_ROOT +from local_server.test.fixtures.fixtures import pbmc3k_colors + + +class ColorsTest(unittest.TestCase): + """ Test color helper functions """ + + def test_convert_color_to_hex_format(self): + self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3") + self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3") + self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3") + self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3") + self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3") + self.assertEqual( + convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3" + ) + for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]: + with self.assertRaises(ColorFormatException): + convert_color_to_hex_format(bad_input) + + def test_anndata_colors_to_cxg_colors(self): + # test standard behavior + adata = self._get_h5ad() + self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors) + # test that invalid color formats raise an exception + adata.uns["louvain_colors"][0] = "#NOTCOOL" + with self.assertRaises(ColorFormatException): + convert_anndata_category_colors_to_cxg_category_colors(adata) + # test that colors without a matching obs category are skipped + adata = self._get_h5ad() + del adata.obs["louvain"] + self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {}) + + def _get_h5ad(self): + return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") diff --git a/local_server/test/unit/common/test_corpora.py b/local_server/test/unit/common/test_corpora.py new file mode 100644 index 00000000..589795e4 --- /dev/null +++ b/local_server/test/unit/common/test_corpora.py @@ -0,0 +1,164 @@ +import json +import shutil +import tempfile +import unittest +from http import HTTPStatus + +import anndata +import requests + +from local_server.common.corpora import ( + corpora_get_versions_from_anndata, + corpora_is_version_supported, + corpora_get_props_from_anndata, +) +from local_server.test import PROJECT_ROOT, start_test_server, stop_test_server + +VERSION = "v0.2" + + +class CorporaAPITest(unittest.TestCase): + def test_corpora_get_versions_from_anndata(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_versions_from_anndata(adata)) + + # something bogus + adata.uns["version"] = 99 + self.assertIsNone(corpora_get_versions_from_anndata(adata)) + + # something legit + adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"} + self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"]) + + def test_corpora_is_version_supported(self): + self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0")) + self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0")) + self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0")) + + def test_corpora_get_props_from_anndata(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # something bogus + adata.uns["version"] = 99 + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # unsupported version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"} + with self.assertRaises(ValueError): + corpora_get_props_from_anndata(adata) + + # legit version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"} + with self.assertRaises(KeyError): + corpora_get_props_from_anndata(adata) + + some_fields = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "title", + "layer_descriptions": "layer_descriptions", + "organism": "organism", + "organism_ontology_term_id": "organism_ontology_term_id", + "project_name": "project_name", + "project_description": "project_description", + "contributors": json.dumps([{"contributors": "contributors"}]), + "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]), + } + for k in some_fields: + adata.uns[k] = some_fields[k] + some_fields["contributors"] = json.loads(some_fields["contributors"]) + some_fields["project_links"] = json.loads(some_fields["project_links"]) + self.assertEqual(corpora_get_props_from_anndata(adata), some_fields) + + def test_corpora_get_props_from_anndata_v110(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # legit version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"} + with self.assertRaises(KeyError): + corpora_get_props_from_anndata(adata) + + # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0 + some_110_fields = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "title", + "layer_descriptions": "layer_descriptions", + "organism": "organism", + "organism_ontology_term_id": "organism_ontology_term_id", + } + for k in some_110_fields: + adata.uns[k] = some_110_fields[k] + self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields) + + def _get_h5ad(self): + return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + + +class CorporaRESTAPITest(unittest.TestCase): + """ Confirm endpoints reflect Corpora-specific features """ + + @classmethod + def setCorporaFields(cls, path): + adata = anndata.read_h5ad(path) + corpora_props = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "PBMC3K", + "contributors": json.dumps([{"name": "name"}]), + "layer_descriptions": {"X": "raw counts"}, + "organism": "human", + "organism_ontology_term_id": "unknown", + "project_name": "test project", + "project_description": "test description", + "project_links": json.dumps( + [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}] + ), + "default_embedding": "X_tsne", + } + adata.uns.update(corpora_props) + adata.write(path) + + @classmethod + def setUpClass(cls): + cls.tmp_dir = tempfile.TemporaryDirectory() + src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad" + dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad" + shutil.copyfile(src, dst) + cls.setCorporaFields(dst) + cls.ps, cls.server = start_test_server([dst]) + + @classmethod + def tearDownClass(cls): + stop_test_server(cls.ps) + cls.tmp_dir.cleanup() + + def setUp(self): + self.session = requests.Session() + self.url_base = f"{self.server}/api/{VERSION}/" + + def test_config(self): + endpoint = "config" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + + result_data = result.json() + self.assertIsInstance(result_data["config"]["corpora_props"], dict) + self.assertIsInstance(result_data["config"]["parameters"], dict) + + corpora_props = result_data["config"]["corpora_props"] + parameters = result_data["config"]["parameters"] + + self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0") + self.assertEqual(corpora_props["organism"], "human") + self.assertEqual(parameters["default_embedding"], "tsne") diff --git a/local_server/test/unit/common/test_nan_rest.py b/local_server/test/unit/common/test_nan_rest.py new file mode 100644 index 00000000..7ec01464 --- /dev/null +++ b/local_server/test/unit/common/test_nan_rest.py @@ -0,0 +1,61 @@ +from http import HTTPStatus +import unittest +import math +from local_server.test import start_test_server, stop_test_server + +import local_server.test.unit.decode_fbs as decode_fbs + +import requests + +VERSION = "v0.2" +BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} + + +class WithNaNs(unittest.TestCase): + """Test Case for endpoints""" + + @classmethod + def setUpClass(cls): + cls.ps, cls.server = start_test_server(["test/fixtures/nan.h5ad"]) + + @classmethod + def tearDownClass(cls): + stop_test_server(cls.ps) + + def setUp(self): + self.session = requests.Session() + self.url_base = f"{self.server}/api/{VERSION}/" + + def test_initialize(self): + endpoint = "schema" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + + def test_data(self): + endpoint = "data/var" + url = f"{self.url_base}{endpoint}" + filter = {"filter": {"var": {"index": [[0, 20]]}}} + result = self.session.put(url, json=filter) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][3][3])) + + def test_annotation_obs(self): + endpoint = "annotations/obs" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][2][0])) + + def test_annotation_var(self): + endpoint = "annotations/var" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][2][0])) diff --git a/local_server/test/unit/common/test_rest.py b/local_server/test/unit/common/test_rest.py new file mode 100644 index 00000000..82ae3ad5 --- /dev/null +++ b/local_server/test/unit/common/test_rest.py @@ -0,0 +1,79 @@ +import unittest +from urllib.parse import parse_qs +from werkzeug.datastructures import MultiDict +from local_server.common.rest import _query_parameter_to_filter +from local_server.common.errors import FilterError + + +def _qsparse(qs): + """ emulate what Flask/Werkzeug do to our QS """ + return MultiDict(parse_qs(qs)) + + +class FilterParseTests(unittest.TestCase): + """ Test cases for various filter parsing """ + + def test_queryparam_to_filter_parse(self): + # categories + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")), + { + "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]}, + "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]}, + }, + ) + + # ranges + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")), + { + "obs": { + "annotation_value": [ + {"name": "A", "min": 1, "max": 99.0}, + {"name": "B", "max": 100.0}, + {"name": "C", "min": 0.0}, + ] + }, + }, + ) + + # combo + self.assertEqual( + _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")), + { + "var": { + "annotation_value": [ + {"name": "B", "values": ["YES", "NO"]}, + {"name": "A", "min": 1.0, "max": 99.0}, + ] + }, + }, + ) + + def test_queryparam_to_filter_escaping(self): + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")), + {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}}, + ) + + def test_queryparam_to_filter_errors(self): + + # should raise FilterError + filter_errors = [ + "foo=bar", # no axis + "X=&Y=3", # no value + "X&Y=3", # no value + "moo:foo=bar", # bad axis + "obs:x=1,A", # non-numeric range + "var:X=1,2&var:X=3,4", # duplicate ranges + "var:Y=,", + "var:Y=2,", + "var:Y=,5", + "var:Y=*,", + "var:Y=,*", + "var:Y=*,*", + ] + + for qs in filter_errors: + with self.assertRaises(FilterError): + _query_parameter_to_filter(_qsparse(qs)) diff --git a/local_server/test/unit/common/test_writable_annotation.py b/local_server/test/unit/common/test_writable_annotation.py new file mode 100644 index 00000000..2d26e630 --- /dev/null +++ b/local_server/test/unit/common/test_writable_annotation.py @@ -0,0 +1,168 @@ +import json +import shutil +import unittest +from os import path, listdir + +import numpy as np +import pandas as pd + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.common.rest import schema_get_helper, annotations_put_fbs_helper +from local_server.data_common.matrix_loader import MatrixDataType +from local_server.test import data_with_tmp_annotations, make_fbs + + +class WritableAnnotationTest(unittest.TestCase): + def setUp(self): + self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD) + self.data.dataset_config.user_annotations = self.annotations + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def annotation_put_fbs(self, fbs): + annotations_put_fbs_helper(self.data, fbs) + res = json.dumps({"status": "OK"}) + return res + + def test_error_checks(self): + # verify that the expected errors are generated + n_rows = self.data.get_shape()[0] + fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")}) + + # ensure we catch attempt to overwrite non-writable data + with self.assertRaises(KeyError): + self.annotation_put_fbs(fbs_bad) + + def test_write_to_file(self): + # verify the file is written as expected + n_rows = self.data.get_shape()[0] + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"), + "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"), + } + ) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + self.assertTrue(path.exists(self.annotations.label_output_file)) + df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#") + self.assertEqual(df.shape, (n_rows, 2)) + self.assertEqual(set(df.columns), {"cat_A", "cat_B"}) + self.assertTrue(self.data.original_obs_index.equals(df.index)) + self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows)) + self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows)) + + # verify complete overwrite on second attempt, AND rotation occurs + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"), + "cat_C": pd.Series(["label_C"] * n_rows, dtype="category"), + } + ) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + self.assertTrue(path.exists(self.annotations.label_output_file)) + df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#") + self.assertEqual(set(df.columns), {"cat_A", "cat_C"}) + self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows)) + self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows)) + + # rotation + name, ext = path.splitext(self.annotations.label_output_file) + backup_dir = f"{name}-backups" + self.assertTrue(path.isdir(backup_dir)) + found_files = listdir(backup_dir) + self.assertEqual(len(found_files), 1) + + def test_file_rotation_to_max_9(self): + # verify we stop rotation at 9 + n_rows = self.data.get_shape()[0] + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"), + "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"), + } + ) + for i in range(0, 11): + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + name, ext = path.splitext(self.annotations.label_output_file) + backup_dir = f"{name}-backups" + self.assertTrue(path.isdir(backup_dir)) + found_files = listdir(backup_dir) + self.assertTrue(len(found_files) <= 9) + + def test_put_get_roundtrip(self): + # verify that OBS PUTs (annotation_put_fbs) are accessible via + # GET (annotation_to_fbs_matrix) + + n_rows = self.data.get_shape()[0] + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"), + "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"), + } + ) + + # put + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + # get + labels = self.annotations.read_labels(None) + fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels) + schema = schema_get_helper(self.data) + annotations = decode_fbs.decode_matrix_FBS(fbsAll) + obs_index_col_name = schema["annotations"]["obs"]["index"] + self.assertEqual(annotations["n_rows"], n_rows) + self.assertEqual(annotations["n_cols"], 7) + self.assertIsNone(annotations["row_idx"]) + self.assertEqual( + annotations["col_idx"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"], + ) + col_idx = annotations["col_idx"] + self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows) + self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows) + + # verify the schema was updated + all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} + self.assertEqual( + all_col_schema["cat_A"], + {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True}, + ) + self.assertEqual( + all_col_schema["cat_B"], + {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True}, + ) + + def test_put_float_data(self): + # verify that OBS PUTs (annotation_put_fbs) are accessible via + # GET (annotation_to_fbs_matrix) + + n_rows = self.data.get_shape()[0] + + # verifies that floating point with decimals fail. + fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))}) + with self.assertRaises(ValueError) as exception_context: + res = self.annotation_put_fbs(fbs) + self.assertEqual(str(exception_context.exception), "Columns may not have floating point types") + + # verifies that floating point that can be converted to int passes + fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")}) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + # check read_labels + labels = self.annotations.read_labels(None) + fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels) + schema = schema_get_helper(self.data) + annotations = decode_fbs.decode_matrix_FBS(fbsAll) + self.assertEqual(annotations["n_rows"], n_rows) + all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} + self.assertEqual( + all_col_schema["cat_F_PASS"], + {"name": "cat_F_PASS", "type": "int32", "writable": True}, + ) diff --git a/local_server/test/unit/common/utils/__init__.py b/local_server/test/unit/common/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/common/utils/test_type_conversion_utils.py b/local_server/test/unit/common/utils/test_type_conversion_utils.py new file mode 100644 index 00000000..179f6a70 --- /dev/null +++ b/local_server/test/unit/common/utils/test_type_conversion_utils.py @@ -0,0 +1,217 @@ +import unittest +from time import time +from unittest.mock import patch + +import numpy as np +from pandas import Series, DataFrame + +from local_server.common.utils.type_conversion_utils import ( + can_cast_to_float32, + can_cast_to_int32, + get_dtype_of_array, + get_schema_type_hint_of_array, + get_dtypes_and_schemas_of_dataframe, + convert_pandas_series_to_numpy, +) + + +class TestTypeConversionUtils(unittest.TestCase): + def test__can_cast_to_float32__string_is_false(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=str) + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_float32__float64_is_true_warning_outputted(self): + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64)) + + with self.assertLogs(level="WARN") as logger: + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + self.assertIn("may lose precision", logger.output[0]) + + self.assertTrue(can_cast) + + @patch("logging.warning") + def test__can_cast_to_float32__float32_is_false(self, mock_log_warning): + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32)) + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + assert not mock_log_warning.called + + def test__can_cast_to_float32__categorical_float64_is_false(self): + array_to_convert = Series(data=[1.1, 2.2, 3.3], dtype="category") + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_float32__categorical_int64_with_nans_is_true(self): + array_to_convert = Series(data=[1, 2, np.NaN], dtype="category") + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_float_32__float_32_with_nans_is_true(self): + array_to_convert = Series(data=[1, 2, np.NaN], dtype=np.dtype(np.float32)) + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__string_is_false(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=str) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_int32__int64_is_true(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__int16_is_true(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__int64_with_large_value_is_false(self): + array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_int32__int64_with_nans_is_false(self): + array_to_convert = Series(data=[np.NaN, "2", "3"], dtype="category") + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__get_dtype_of_array__supported_dtypes_return_as_expected(self): + types = [np.float32, np.int32, np.bool_, str] + expected_dtypes = [np.float32, np.int32, np.uint8, str] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) + + def test__get_dtype_of_array__categories_return_as_expected(self): + array = Series(data=["a", "b", "c"], dtype="category") + expected_dtype = str + + actual_dtype = get_dtype_of_array(array) + + self.assertEqual(expected_dtype, actual_dtype) + + def test__get_dtype_of_array__unordered_integer_categories_return_as_expected(self): + array = Series(data=[2, 3, 1, 3, 1, 2], dtype="category") + expected_dtype = np.int32 + + actual_dtype = get_dtype_of_array(array) + + self.assertEqual(expected_dtype, actual_dtype) + + def test__get_dtype_of_array__castable_dtypes_return_as_expected(self): + types = [np.float64, np.int64] + expected_dtypes = [np.float32, np.int32] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) + + def test__get_dtype_of_array__unsupported_type_raises_exception(self): + unsupported_array = Series(list([time() for _ in range(2)]), dtype="datetime64[ns]") + + with self.assertRaises(TypeError) as exception_context: + get_dtype_of_array(unsupported_array) + + self.assertIn("unsupported", str(exception_context.exception)) + + def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self): + types = [np.float32, np.int32, np.bool_, str] + expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) + + def test__get_schema_type_hint_of_array__categories_return_as_expected(self): + array = Series(data=["a", "b", "b"], dtype="category") + expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]} + + actual_schema_hint = get_schema_type_hint_of_array(array) + + self.assertEqual(expected_schema_hint, actual_schema_hint) + + def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self): + types = [np.float64, np.int64] + expected_schema_hints = [{"type": "float32"}, {"type": "int32"}] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", + i=test_type_index, + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) + + def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self): + float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64)) + category_array = Series(data=["a", "b", "b"], dtype="category") + dataframe = DataFrame({"float_array": float_array, "category_array": category_array}) + + expected_data_types_dict = {"float_array": np.float32, "category_array": str} + expected_schema_type_hints_dict = { + "float_array": {"type": "float32"}, + "category_array": {"type": "categorical", "categories": ["a", "b"]}, + } + + actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe) + + self.assertEqual(expected_data_types_dict, actual_dataframe_data_types) + self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints) + + def test__convert_pandas_series_to_numpy__categorical_float64_to_float64_with_nans(self): + expected_float_array = np.array([1.1, 2.2, np.NaN], dtype=np.float64) + float_series = Series(data=[1.1, 2.2, np.NaN], dtype="category") + + actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64) + + np.testing.assert_equal(expected_float_array, actual_float_array) + + def test__convert_pandas_series_to_numpy__float64_to_float64(self): + expected_float_array = np.array([1.1, 2.2], dtype=np.float64) + float_series = Series(data=[1.1, 2.2], dtype=np.dtype(np.float64)) + + actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64) + + np.testing.assert_equal(expected_float_array, actual_float_array) + + def test__convert_pandas_series_to_numpy__int64_to_int32_with_nans_throws_error(self): + int_series = Series(data=[1, 2, np.NaN], dtype="category") + + with self.assertLogs(level="ERROR") as logger: + convert_pandas_series_to_numpy(int_series, np.int32) + + self.assertIn( + "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0] + ) diff --git a/local_server/test/unit/common/utils/test_utils.py b/local_server/test/unit/common/utils/test_utils.py new file mode 100644 index 00000000..852c0969 --- /dev/null +++ b/local_server/test/unit/common/utils/test_utils.py @@ -0,0 +1,34 @@ +import os +import shutil +import unittest + +from local_server.common.utils.utils import import_plugins +from local_server.test import PROJECT_ROOT, random_string + + +class TestPlugins(unittest.TestCase): + """ Test plugin import functionality """ + + plugins_dir = f"{PROJECT_ROOT}/local_server/test/plugins" + test_plugin_path = f"{plugins_dir}/foo.py" + secret = random_string(8) + + @classmethod + def setUpClass(cls) -> None: + if not os.path.isdir(cls.plugins_dir): + os.mkdir(cls.plugins_dir) + with open(cls.test_plugin_path, "w") as fh: + fh.write(f'SECRET = "{cls.secret}"\n') + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(cls.plugins_dir): + shutil.rmtree(cls.plugins_dir) + + def test_import_plugins(self): + self.assertTrue(os.path.isfile(self.test_plugin_path)) + loaded_modules = import_plugins("local_server.test.plugins") + # test that import plugins found the file + self.assertEqual(["local_server.test.plugins.foo"], [ele.__name__ for ele in loaded_modules]) + # test that the module was properly executed + self.assertEqual(self.secret, loaded_modules[0].SECRET) diff --git a/local_server/test/unit/compute/__init__.py b/local_server/test/unit/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/compute/test_diffexp_cxg.py b/local_server/test/unit/compute/test_diffexp_cxg.py new file mode 100644 index 00000000..dd5020b4 --- /dev/null +++ b/local_server/test/unit/compute/test_diffexp_cxg.py @@ -0,0 +1,62 @@ +import unittest + +import numpy as np + +from local_server.data_common.matrix_loader import MatrixDataLoader +from local_server.test import PROJECT_ROOT, app_config + + +class DiffExpTest(unittest.TestCase): + """Tests the diffexp returns the expected results for one test case, using different + adaptor types and different algorithms.""" + + def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}): + config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config) + loader = MatrixDataLoader(path) + adaptor = loader.open(config) + return adaptor + + def get_mask(self, adaptor, start, stride): + """Simple function to return a mask or rows""" + rows = adaptor.get_shape()[0] + sel = list(range(start, rows, stride)) + mask = np.zeros(rows, dtype=bool) + mask[sel] = True + return mask + + def compare_diffexp_results(self, results, expects): + self.assertEqual(len(results), len(expects)) + for result, expect in zip(results, expects): + self.assertEqual(result[0], expect[0]) + self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4)) + self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4)) + self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4)) + + def check_1_10_2_10(self, results): + """Checks the results for a specific set of rows selections""" + expects = [ + [956, 0.016060986, 0.0008649321884808977, 1.0], + [1124, 0.96602094, 0.0011717216548271284, 1.0], + [1809, 1.1110606, 0.0019304405196777848, 1.0], + [1712, -0.5525154, 0.0051788902660723345, 1.0], + [1754, 0.5201581, 0.005691734062127954, 1.0], + [948, 1.6390722, 0.006622111055981219, 1.0], + [1810, 0.78618884, 0.007055917428377063, 1.0], + [779, 1.5241305, 0.007202934422407284, 1.0], + [1575, 1.0317602, 0.007830310753043345, 1.0], + [576, 0.97873515, 0.008272092578813124, 1.0], + ] + self.compare_diffexp_results(results, expects) + + def get_X_col(self, adaptor, cols): + varmask = np.zeros(adaptor.get_shape()[1], dtype=bool) + varmask[cols] = True + return adaptor.get_X_array(None, varmask) + + def test_anndata_default(self): + """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)""" + adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + maskA = self.get_mask(adaptor, 1, 10) + maskB = self.get_mask(adaptor, 2, 10) + results = adaptor.compute_diffexp_ttest(maskA, maskB, 10) + self.check_1_10_2_10(results) diff --git a/local_server/test/unit/converters/__init__.py b/local_server/test/unit/converters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/converters/schema/__init__.py b/local_server/test/unit/converters/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/converters/schema/test_gene_symbol.py b/local_server/test/unit/converters/schema/test_gene_symbol.py new file mode 100644 index 00000000..fcd400c8 --- /dev/null +++ b/local_server/test/unit/converters/schema/test_gene_symbol.py @@ -0,0 +1,61 @@ +import os +import unittest + +import pandas as pd + +from local_server.test import FIXTURES_ROOT +from local_server.converters.schema import gene_symbol + + +class TestHGNCSymbolChecker(unittest.TestCase): + + def setUp(self): + self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz") + self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path) + + def test_symbol_upgrade(self): + self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2") + + # Strip off seurat endings when appropriate + self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2") + + # DIFF6 is ambiguous so don't upgrade it + self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6") + self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6") + + # ARG1 is approved + self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1") + + # HAP1 is both approved and withdrawn + self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1") + + # Leave unknown symbols alone + self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL") + self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol") + + # Upgrade HGNC ids unless you can't find it + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456") + + def test_check_symbol(self): + self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE) + self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS) + self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN) + + # HAP1 is one of the approved and withdrawn symbols + self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED) + + def test_upgrade_index(self): + index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"]) + var_df = pd.DataFrame([[0] * len(index)], index=index) + upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path) + self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"]) diff --git a/local_server/test/unit/converters/schema/test_ontology.py b/local_server/test/unit/converters/schema/test_ontology.py new file mode 100644 index 00000000..25aa7f1e --- /dev/null +++ b/local_server/test/unit/converters/schema/test_ontology.py @@ -0,0 +1,129 @@ +import json + +import unittest +import unittest.mock + +from local_server.converters.schema import ontology + + +class TestOntologyParsing(unittest.TestCase): + def setUp(self): + + self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"] + + self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"] + + self.values = ["0002048", "0000174", "9606", "0008995"] + + self.iris = [ + "http://purl.obolibrary.org/obo/UBERON_0002048", + "http://purl.obolibrary.org/obo/HsapDv_0000174", + "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "http://www.ebi.ac.uk/efo/EFO_0008995", + ] + + URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/" + self.urls = [ + URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048", + URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174", + URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606", + URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995", + ] + + self.responses = { + "UBERON:0002048": { + "iri": "http://purl.obolibrary.org/obo/UBERON_0002048", + "description": ["Respiration organ that develops as an outpocketing of the esophagus."], + "label": "lung", + }, + "HsapDv:0000174": { + "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174", + "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."], + "label": "1-month-old human stage", + }, + "NCBITaxon:9606": { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "description": None, + "label": "Homo sapiens", + }, + "EFO:0008995": { + "iri": "http://www.ebi.ac.uk/efo/EFO_0008995", + "description": [ + ( + '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated ' + "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent " + "of 1 million pipetting steps. Successive versions of the 10x chemistry use different " + "barcode locations to improve the sequencing yield and quality of 10x experiments." + ) + ], + "label": "10X sequencing", + }, + } + + def test_ontololgy_name(self): + for curie, expected_name in zip(self.curies, self.names): + self.assertEqual(ontology._ontology_name(curie), expected_name) + + def test_ontololgy_value(self): + for curie, expected_value in zip(self.curies, self.values): + self.assertEqual(ontology._ontology_value(curie), expected_value) + + def test_iri(self): + for curie, expected_iri in zip(self.curies, self.iris): + self.assertEqual(ontology._iri(curie), expected_iri) + + def test_ontology_info_url(self): + for curie, expected_url in zip(self.curies, self.urls): + self.assertEqual(ontology._ontology_info_url(curie), expected_url) + + def test_empty_ontology_info_url(self): + self.assertEqual(ontology._ontology_info_url(""), "") + + +class TestOntologyLookup(unittest.TestCase): + def setUp(self): + self.responses = { + "UBERON:0002048": { + "iri": "http://purl.obolibrary.org/obo/UBERON_0002048", + "description": ["Respiration organ that develops as an outpocketing of the esophagus."], + "label": "lung", + }, + "HsapDv:0000174": { + "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174", + "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."], + "label": "1-month-old human stage", + }, + "NCBITaxon:9606": { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "description": None, + "label": "Homo sapiens", + }, + "EFO:0008995": { + "iri": "http://www.ebi.ac.uk/efo/EFO_0008995", + "description": [ + ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated ' + 'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent ' + 'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode ' + 'locations to improve the sequencing yield and quality of 10x experiments.') + ], + "label": "10X sequencing", + }, + } + + self.labels = { + "UBERON:0002048": "lung", + "HsapDv:0000174": "1-month-old human stage", + "NCBITaxon:9606": "Homo sapiens", + "EFO:0008995": "10X sequencing", + } + + @unittest.mock.patch("requests.get") + def test_lookup_label(self, mock_get): + + for curie, response in self.responses.items(): + mock_get.return_value.content = json.dumps(response) + mock_get.return_value.json.return_value = response + mock_get.return_value.status_code = 200 + + label = ontology.get_ontology_label(curie) + self.assertEqual(label, self.labels[curie]) diff --git a/local_server/test/unit/converters/schema/test_remix.py b/local_server/test/unit/converters/schema/test_remix.py new file mode 100644 index 00000000..49713d29 --- /dev/null +++ b/local_server/test/unit/converters/schema/test_remix.py @@ -0,0 +1,257 @@ +import json +import os +import unittest +import unittest.mock + +import anndata +import numpy +import pandas as pd +import scanpy as sc + +from local_server.converters.schema import remix + +PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip() + + +class TestApplySchema(unittest.TestCase): + + def setUp(self): + self.source_h5ad_path = f"{PROJECT_ROOT}/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad" + self.output_h5ad_path = f"{PROJECT_ROOT}/local_server/test/fixtures/test_remix.h5ad" + self.config_path = f"{PROJECT_ROOT}/local_server/test/fixtures/test_config.yaml" + self.bad_config_path = f"{PROJECT_ROOT}/local_server/test/fixtures/test_bad_config.yaml" + + def tearDown(self): + try: + os.remove(self.output_h5ad_path) + except OSError: + pass + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_apply_schema(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path) + new_adata = sc.read_h5ad(self.output_h5ad_path) + + self.assertIn("cell_type", new_adata.obs.columns) + self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist()) + self.assertListEqual( + ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"], + sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist()) + ) + + self.assertIn("version", new_adata.uns_keys()) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_apply_bad_schema(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path) + new_adata = sc.read_h5ad(self.output_h5ad_path) + + # Should refuse to write the version + self.assertNotIn("version", new_adata.uns_keys()) + +class TestFieldParsing(unittest.TestCase): + + def test_is_curie(self): + self.assertTrue(remix.is_curie("EFO:00001")) + self.assertTrue(remix.is_curie("UBERON:123456")) + self.assertTrue(remix.is_curie("HsapDv:0001")) + self.assertFalse(remix.is_curie("UBERON")) + self.assertFalse(remix.is_curie("UBERON:")) + self.assertFalse(remix.is_curie("123456")) + + def test_is_ontology_field(self): + self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id")) + self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id")) + self.assertFalse(remix.is_ontology_field("cell_ontology")) + self.assertFalse(remix.is_ontology_field("method")) + + def test_get_label_field_name(self): + self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id")) + self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id")) + + def test_split_suffix(self): + self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)")) + self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)")) + self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234")) + self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)")) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_get_curie_and_label(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + self.assertEqual( + remix.get_curie_and_label("UBERON:1234"), + ("UBERON:1234", "test label") + ) + self.assertEqual( + remix.get_curie_and_label("UBERON:1234 (cell culture)"), + ("UBERON:1234 (cell culture)", "test label (cell culture)") + ) + self.assertEqual( + remix.get_curie_and_label("whatever"), + ("", "whatever") + ) + + +class TestManipulateAnndata(unittest.TestCase): + + def setUp(self): + + self.cell_count = 20 + self.gene_count = 200 + X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count)) + uns = {"organism": "monkey", "experiment": "monkey experiment"} + obs = pd.DataFrame( + index=[f"Cell{d}" for d in range(self.cell_count)], + columns=["tissue", "CellType"], + data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2) + ) + var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)]) + + self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns) + + def test_safe_add_field(self): + + remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count) + self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count) + self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count) + + remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}]) + self.assertEqual( + self.adata.uns["contributors"], + json.dumps([{"name": "contributor1"}, {"name": "contributor2"}]) + ) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_remix_uns(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "Pan troglodytes" + uns_config = { + "version": { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + }, + "organism_ontology_term_id": "NCBITaxon:9598", + "contributors": [ + { + "name": "scientist", + "email": "scientist@science.com" + } + ] + } + + remix.remix_uns(self.adata, uns_config) + + self.assertEqual( + sorted(self.adata.uns_keys()), + sorted(["organism_original", "organism", "organism_ontology_term_id", + "contributors", "version", "experiment"]) + ) + + self.assertEqual(self.adata.uns['organism'], "Pan troglodytes") + self.assertEqual(self.adata.uns['organism_original'], "monkey") + self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598") + self.assertEqual(self.adata.uns['contributors'], + json.dumps([{"name": "scientist", "email": "scientist@science.com"}])) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_remix_obs(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "lung (in a monkey)" + obs_config = { + "tissue_ontology_term_id": { + "tissue": { + "lung": "UBERON:00000" + } + }, + "cell_color": { + "CellType": { + "epithelial": "fuschia", + "endothelial": "khaki" + } + }, + "sex": "male" + } + + remix.remix_obs(self.adata, obs_config) + self.assertEqual( + sorted(self.adata.obs_keys()), + sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"]) + ) + + self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original)) + self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id)) + self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue)) + self.assertTrue(all(v == "male" for v in self.adata.obs.sex)) + self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki")) + for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color))) + + +class TestFixupGeneSymbols(unittest.TestCase): + + def setUp(self): + self.seurat_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/seurat_tutorial.h5ad" + self.seurat_merged_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad" + self.sctransform_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/sctransform.h5ad" + self.sctransform_merged_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/sctransform_merged.h5ad" + + # There's lots of MALAT1, but it doesn't collide with any other names, + # so it shouldn't change during merging. + self.stable_gene = "MALAT1" + + def test_fixup_gene_symbols_seurat(self): + + if not os.path.isfile(self.seurat_path): + return unittest.skip( + "Skipping gene symbol conversion tests because test h5ads are not present. To create them, " + "run local_server/test/fixtures/schema_test_data/generate_test_data.sh" + ) + + original_adata = sc.read_h5ad(self.seurat_path) + merged_adata = sc.read_h5ad(self.seurat_merged_path) + + fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"} + + fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config) + + self.assertEqual( + merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum() + ) + self.assertAlmostEqual( + merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum() + ) + + self.assertAlmostEqual( + merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum() + ) + + def test_fixup_gene_symbols_sctransform(self): + + if not os.path.isfile(self.sctransform_path): + return unittest.skip( + "Skipping gene symbol conversion tests because test h5ads are not present. To create them, " + "run local_server/test/fixtures/schema_test_data/generate_test_data.sh" + ) + + original_adata = sc.read_h5ad(self.sctransform_path) + merged_adata = sc.read_h5ad(self.sctransform_merged_path) + + fixup_config = {"X": "log1p", "counts": "raw"} + + fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config) + + # sctransform does a bunch of stuff, including slightly modifying the + # raw counts. So we can't assert for exact equality the way we do with + # the vanilla seurat tutorial. But, the results should still be very + # close. + merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum() + fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum() + self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable) + + self.assertAlmostEqual( + merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(), + 0 + ) diff --git a/local_server/test/unit/converters/schema/test_validate.py b/local_server/test/unit/converters/schema/test_validate.py new file mode 100644 index 00000000..269e629b --- /dev/null +++ b/local_server/test/unit/converters/schema/test_validate.py @@ -0,0 +1,435 @@ +import json +import os +import unittest + +import pandas as pd +import scanpy as sc + +from local_server.converters.schema import validate + +PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip() + + +class TestFieldValidation(unittest.TestCase): + + def test_validate_stringified_list_of_dicts(self): + + good = json.dumps([{"a": 1}, {2: "x", "z": "y"}]) + not_stringified = [{"a": 1}, {2: "x", "z": "y"}] + not_a_list = json.dumps({"bad": "dict"}) + not_json = "oh hey!" + + self.assertTrue(validate._validate_stringified_list_of_dicts(good)) + + self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified)) + self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list)) + self.assertFalse(validate._validate_stringified_list_of_dicts(not_json)) + + def test_validate_human_readable_string(self): + + good = "oh hey!" + curie = "EFO:0001" + ensg = "ENSG000001234" + enst = "ENST000005678" + + self.assertTrue(validate._validate_human_readable_string(good)) + + self.assertFalse(validate._validate_human_readable_string(curie)) + self.assertFalse(validate._validate_human_readable_string(ensg)) + self.assertFalse(validate._validate_human_readable_string(enst)) + + def test_validate_curie(self): + + self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"])) + self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"])) + + self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"])) + + def test_validate_suffixed_curie(self): + + self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"])) + self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"])) + + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"])) + + +class TestColumnValidation(unittest.TestCase): + + def test_validate_unique(self): + unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]], + index=["X", "Y", "Z"], columns=["col1", "col2"]) + duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]], + index=["X", "Y", "X"], columns=["col1", "col2"]) + + schema_def = {"unique": True} + + errors = validate._validate_column(unique.index, "index", "unique_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(duped.index, "index", "duped_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + schema_def = {"unique": False} + errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def) + self.assertFalse(errors) + + def test_validate_nullable(self): + non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]], + index=["X", "Y", "Z"], columns=["col1", "col2"]) + has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]], + index=["X", "Y", "Z"], columns=["col1", "col2", "col3"]) + + schema_def = {"nullable": False} + errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def) + self.assertFalse(errors) + errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("contains empty values", errors[0]) + errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("contains empty values", errors[0]) + + schema_def = {"nullable": True} + errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def) + self.assertFalse(errors) + + def test_human_readable(self): + hr_df = pd.DataFrame( + [["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"], + ["hope you're well", "bit of lungs", "brain"]], + index=["ENSG00001", "ENSG00002"], + columns=["good", "curie", "suffixed_curie"]) + + schema_def = {"type": "human-readable string"} + errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + def test_curie(self): + + curie_df = pd.DataFrame( + [["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"], + ["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"], + ["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]], + index=["X", "Y", "Z"], + columns=["good", "good_suffix", "bad", "bad_suffix"]) + + # Good + schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]} + errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def) + self.assertFalse(errors) + + # Good suffix + schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]} + errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def) + self.assertFalse(errors) + + # Bad prefix + schema_def = {"type": "curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + self.assertIn("must be curies from one of these", errors[0]) + + # Bad curies + schema_def = {"type": "curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + # Bad suffixes + schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + def test_enum(self): + enum_df = pd.DataFrame( + [["abc", "ghi"], + ["def", "jkl"]], + index=["X", "Y"], + columns=["col1", "col2"]) + + # All match + schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]} + errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def) + self.assertFalse(errors) + + # Missing value + schema_def = {"type": "string", "enum": ["abc", "xyz"]} + errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("unpermitted values", errors[0]) + + +class TestDictValidations(unittest.TestCase): + + + def test_key_presence(self): + + schema_def = {"keys": {"abc": None, "def": None}} + + dict_ = {"abc": "123", "def": "456"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + # Missing keys are bad + dict_ = {"abc": "123"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing key", errors[0]) + + # Extra keys are okay + dict_ = {"abc": "123", "def": "456", "xyz": "789"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + # Better not be empty come on + dict_ = {} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 2) + + def test_nullable(self): + + schema_def = {"keys": {"abc": {"type": "string", "nullable": False}, + "def": {"type": "string", "nullable": True}}} + + dict_ = {"abc": "xyz", "def": ""} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + dict_ = {"abc": "", "def": ""} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("empty value", errors[0]) + + def test_recurse(self): + + schema_def = { + "keys": { + "subdict": { + "type": "dict", + "keys": { + "subdict_key1": None, + "subdict_key2": None + } + }, + "ontology": { + "type": "curie", + "prefixes": ["ONTOLOGY"] + }, + "blob": { + "type": "stringified list of dicts" + } + } + } + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + dict_ = { + "subdict": {"subdict_key1": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing key", errors[0]) + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "oh no not an ontology term", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": [{"abc": 123}, {"def": 456}] + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("JSON-encoded list of dicts", errors[0]) + + # Multiple errors + dict_ = { + "subdict": {"subdict_key1": "any"}, + "ontology": "oh no not an ontology term", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 2) + + +class TestDataframeValidation(unittest.TestCase): + + def test_column_presence(self): + df = pd.DataFrame( + [["abc", "EFO:123"], + ["def", "UBERON:456"]], + columns=["hr_string", "ontology"], + index=["X", "Y"] + ) + + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "another_hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing column", errors[0]) + + # Extra is okay + df = pd.DataFrame( + [["abc", "EFO:123", "extra"], + ["def", "UBERON:456", "extra"]], + columns=["hr_string", "ontology", "extra"], + index=["X", "Y"] + ) + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + + def test_index(self): + df = pd.DataFrame( + [["abc", "123"], + ["def", "456"]], + columns=["col1", "col2"], + index=["ENSG0001", "ENSG0002"] + ) + + schema_def = {"index": {"unique": True}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + schema_def = {"index": {"type": "human-readable string"}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + df = pd.DataFrame( + [["abc", "123"], + ["def", "456"]], + columns=["col1", "col2"], + index=["ENSG0001", "ENSG0001"] + ) + schema_def = {"index": {"unique": True}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + def test_recurse(self): + + df = pd.DataFrame( + [["abc", "HsapDv:0001"], + ["EFO:123", "UBERON:456"]], + columns=["hr_string", "ontology"], + index=["X", "Y"] + ) + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 2) + self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1) + self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1) + + +class TestGetSchema(unittest.TestCase): + + def test_get_schema(self): + self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict) + + with self.assertRaises(ValueError): + validate.get_schema_definition("10.1.5") + + +class TestValidate(unittest.TestCase): + + def setUp(self): + self.source_h5ad_path = f"{PROJECT_ROOT}/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad" + + def test_shallow(self): + + adata = sc.read_h5ad(self.source_h5ad_path) + self.assertFalse(validate.validate_adata(adata, True)) + + adata.uns["version"] = { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + } + self.assertTrue(validate.validate_adata(adata, True)) + + def test_deep(self): + adata = sc.read_h5ad(self.source_h5ad_path) + self.assertFalse(validate.validate_adata(adata, False)) + + adata.uns["version"] = { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + } + self.assertFalse(validate.validate_adata(adata, False)) diff --git a/local_server/test/unit/data_anndata/__init__.py b/local_server/test/unit/data_anndata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/data_anndata/test_anndata_adaptor.py b/local_server/test/unit/data_anndata/test_anndata_adaptor.py new file mode 100644 index 00000000..7aeae152 --- /dev/null +++ b/local_server/test/unit/data_anndata/test_anndata_adaptor.py @@ -0,0 +1,232 @@ +import json +import sys +import time +import unittest + +import numpy as np +import pandas as pd +import pytest +from parameterized import parameterized_class + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.common.data_locator import DataLocator +from local_server.common.errors import FilterError +from local_server.data_anndata.anndata_adaptor import AnndataAdaptor +from local_server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT +from local_server.test.fixtures.fixtures import pbmc3k_colors + +""" +Test the anndata adaptor using the pbmc3k data set. +""" + + +@parameterized_class( + ("data_locator", "backed"), + [ + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True), + ], +) +class AdaptorTest(unittest.TestCase): + def setUp(self): + config = app_config(self.data_locator, self.backed) + self.data = AnndataAdaptor(DataLocator(self.data_locator), config) + + def test_init(self): + self.assertEqual(self.data.cell_count, 2638) + self.assertEqual(self.data.gene_count, 1838) + epsilon = 0.000_005 + self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) + + def test_mandatory_annotations(self): + obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] + self.assertIn(obs_index_col_name, self.data.data.obs) + self.assertEqual(list(self.data.data.obs.index), list(range(2638))) + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + self.assertIn(var_index_col_name, self.data.data.var) + self.assertEqual(list(self.data.data.var.index), list(range(1838))) + + @pytest.mark.filterwarnings("ignore:Anndata data matrix") + def test_data_type(self): + # don't run the test on the more exotic data types, as they don't + # support the astype() interface (used by this test, but not underlying app) + if isinstance(self.data.data.X, np.ndarray): + self.data.data.X = self.data.data.X.astype("float64") + with self.assertWarns(UserWarning): + self.data._validate_data_types() + + def test_filter_idx(self): + filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}} + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 102) + + def test_filter_complex(self): + filter_ = { + "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}} + } + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 91) + + def test_obs_and_var_names(self): + self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0) + self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0) + + def test_get_colors(self): + self.assertEqual(self.data.get_colors(), pbmc3k_colors) + + def test_get_schema(self): + with open(f"{FIXTURES_ROOT}/schema.json") as fh: + schema = json.load(fh) + self.assertDictEqual(self.data.get_schema(), schema) + + def test_schema_produces_error(self): + self.data.data.obs["time"] = pd.Series( + list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]", + ) + with pytest.raises(TypeError): + self.data._create_schema() + + def test_layout(self): + fbs = self.data.layout_to_fbs_matrix(fields=None) + layout = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(layout["n_cols"], 6) + self.assertEqual(layout["n_rows"], 2638) + + X = layout["columns"][0] + self.assertTrue((X >= 0).all() and (X <= 1).all()) + Y = layout["columns"][1] + self.assertTrue((Y >= 0).all() and (Y <= 1).all()) + + def test_layout_fields(self): + """ X_pca, X_tsne, X_umap are available """ + fbs = self.data.layout_to_fbs_matrix(["pca"]) + layout = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(layout["n_cols"], 2) + self.assertEqual(layout["n_rows"], 2638) + self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"]) + + fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"]) + layout = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(layout["n_cols"], 4) + self.assertEqual(layout["n_rows"], 2638) + self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"]) + + def test_annotations(self): + fbs = self.data.annotation_to_fbs_matrix("obs") + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 2638) + self.assertEqual(annotations["n_cols"], 5) + obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] + self.assertEqual( + annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], + ) + + fbs = self.data.annotation_to_fbs_matrix("var") + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 2) + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) + + def test_annotation_fields(self): + fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 2638) + self.assertEqual(annotations["n_cols"], 2) + + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 1) + + def test_diffexp_topN(self): + f1 = {"filter": {"obs": {"index": [[0, 500]]}}} + f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) + self.assertEqual(len(result), 10) + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) + self.assertEqual(len(result), 20) + + def test_data_frame(self): + f1 = {"var": {"index": [[0, 10]]}} + fbs = self.data.data_frame_to_fbs_matrix(f1, "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 10) + + with self.assertRaises(ValueError): + self.data.data_frame_to_fbs_matrix(None, "obs") + + def test_filtered_data_frame(self): + filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}} + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 1040) + + filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}} + with self.assertRaises(FilterError): + self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + + def test_data_named_gene(self): + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}} + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 1) + self.assertEqual(data["col_idx"], [4]) + + filter_ = { + "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}} + } + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 3) + self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all()) + + def test_compute_embedding(self): + filter = {"obs": {"index": [[0, 100]]}} + + # Verify that we correctly handle the case where we lack scanpy + import unittest.mock + + with unittest.mock.patch.dict(sys.modules, {"scanpy": None}): + with self.assertRaises(NotImplementedError): + self.data.compute_embedding("umap", filter) + + # if we happen to have scanpy, test the full API, else punt + import importlib + + scanpy_spec = importlib.util.find_spec("scanpy") + if scanpy_spec is None: + print("Skipping compute_embedding test as ScanPy not installed") + return + + # this feature is unsupported in backed mode, and we expect an error + if self.data.data.isbacked: + with self.assertRaises(NotImplementedError): + self.data.compute_embedding("umap", filter) + return + + schema = self.data.compute_embedding("umap", filter) + + self.assertIsInstance(schema["name"], str) + name = schema["name"] + self.assertEqual(schema["type"], "float32") + self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"]) + + emb = self.data.data.obsm[f"X_{name}"] + self.assertEqual(emb.shape, (2638, 2)) + self.assertTrue(np.isfinite(emb[0:100]).all()) + self.assertTrue(np.isnan(emb[100:]).all()) diff --git a/local_server/test/unit/data_anndata/test_anndata_adaptor_data_load.py b/local_server/test/unit/data_anndata/test_anndata_adaptor_data_load.py new file mode 100644 index 00000000..dfef08da --- /dev/null +++ b/local_server/test/unit/data_anndata/test_anndata_adaptor_data_load.py @@ -0,0 +1,81 @@ +import unittest +import json + +from local_server.data_anndata.anndata_adaptor import AnndataAdaptor +from local_server.common.data_locator import DataLocator +from local_server.common.config.app_config import AppConfig +from local_server.test import PROJECT_ROOT + + +class DataLoadAdaptorTest(unittest.TestCase): + """ + Test file loading, including deferred loading/update. + """ + + def setUp(self): + self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + config = AppConfig() + config.update_server_config(single_dataset__datapath=self.data_file.path) + config.update_server_config(app__flask_secret_key="secret") + config.complete_config() + self.data = AnndataAdaptor(self.data_file, config) + + def test_delayed_load_data(self): + self.data._create_schema() + self.assertEqual(self.data.cell_count, 2638) + self.assertEqual(self.data.gene_count, 1838) + epsilon = 0.000_005 + self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) + + def test_diffexp_topN(self): + f1 = {"filter": {"obs": {"index": [[0, 500]]}}} + f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) + self.assertEqual(len(result), 10) + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) + self.assertEqual(len(result), 20) + + +class DataLocatorAdaptorTest(unittest.TestCase): + """ + Test various types of data locators we expect to consume + """ + + def get_basic_config(self): + config = AppConfig() + config.update_server_config( + single_dataset__obs_names=None, single_dataset__var_names=None, + ) + config.update_server_config(app__flask_secret_key="secret") + config.update_dataset_config( + embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, + ) + return config + + def stdAsserts(self, data): + """ run these each time we load the data """ + self.assertIsNotNone(data) + self.assertEqual(data.cell_count, 2638) + self.assertEqual(data.gene_count, 1838) + + def test_posix_file(self): + locator = DataLocator("../example-dataset/pbmc3k.h5ad") + config = self.get_basic_config() + config.update_server_config(single_dataset__datapath=locator.path) + config.complete_config() + data = AnndataAdaptor(locator, config) + self.stdAsserts(data) + + def test_url_https(self): + url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad" + locator = DataLocator(url) + config = self.get_basic_config() + data = AnndataAdaptor(locator, config) + self.stdAsserts(data) + + def test_url_http(self): + url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad" + locator = DataLocator(url) + config = self.get_basic_config() + data = AnndataAdaptor(locator, config) + self.stdAsserts(data) diff --git a/local_server/test/unit/data_anndata/test_nan_anndata_adaptor.py b/local_server/test/unit/data_anndata/test_nan_anndata_adaptor.py new file mode 100644 index 00000000..2ab1c559 --- /dev/null +++ b/local_server/test/unit/data_anndata/test_nan_anndata_adaptor.py @@ -0,0 +1,64 @@ +import math +import unittest +import warnings + +import pytest + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.common.data_locator import DataLocator +from local_server.common.errors import FilterError +from local_server.data_anndata.anndata_adaptor import AnndataAdaptor +from local_server.test import app_config, FIXTURES_ROOT + + +class NaNTest(unittest.TestCase): + def setUp(self): + self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad") + self.config = app_config(self.data_locator.path) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=UserWarning) + self.data = AnndataAdaptor(self.data_locator, self.config) + self.data._create_schema() + + def test_load(self): + with self.assertLogs(level="WARN") as logger: + self.data = AnndataAdaptor(self.data_locator, self.config) + self.assertTrue(logger.output) + + def test_init(self): + self.assertEqual(self.data.cell_count, 100) + self.assertEqual(self.data.gene_count, 100) + epsilon = 0.000_005 + self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) + + def test_dataframe(self): + data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var")) + self.assertIsNotNone(data_frame_var) + self.assertEqual(data_frame_var["n_rows"], 100) + self.assertEqual(data_frame_var["n_cols"], 100) + self.assertTrue(math.isnan(data_frame_var["columns"][3][3])) + + with pytest.raises(FilterError): + self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") + with pytest.raises(FilterError): + filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}} + self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + + def test_dataframe_obs_not_implemented(self): + with self.assertRaises(ValueError) as cm: + decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs")) + self.assertIsNotNone(cm.exception) + + def test_annotation(self): + annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) + obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] + self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]) + self.assertEqual(annotations["n_rows"], 100) + self.assertTrue(math.isnan(annotations["columns"][2][0])) + + annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var")) + var_index_col_name = self.data.schema["annotations"]["var"]["index"] + self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"]) + self.assertEqual(annotations["n_rows"], 100) + self.assertTrue(math.isnan(annotations["columns"][2][0])) diff --git a/local_server/test/unit/data_common/__init__.py b/local_server/test/unit/data_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/data_common/fbs/__init__.py b/local_server/test/unit/data_common/fbs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/data_common/fbs/test_matrix.py b/local_server/test/unit/data_common/fbs/test_matrix.py new file mode 100644 index 00000000..aaaf12aa --- /dev/null +++ b/local_server/test/unit/data_common/fbs/test_matrix.py @@ -0,0 +1,82 @@ +import unittest +import pandas as pd +import numpy as np +from scipy import sparse + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs + + +class FbsTests(unittest.TestCase): + """Test Case for Matrix FBS data encode/decode """ + + def test_encode_boundary(self): + """ test various boundary checks """ + + # row indexing is unsupported + with self.assertRaises(ValueError): + encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[]) + + # matrix must be 2D + with self.assertRaises(ValueError): + encode_matrix_fbs(matrix=np.zeros((3, 2, 1))) + with self.assertRaises(ValueError): + encode_matrix_fbs(matrix=np.ones((10,))) + + def fbs_checks(self, fbs, dims, expected_types, expected_column_idx): + d = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(d["n_rows"], dims[0]) + self.assertEqual(d["n_cols"], dims[1]) + self.assertIsNone(d["row_idx"]) + self.assertEqual(len(d["columns"]), dims[1]) + for i in range(0, len(d["columns"])): + self.assertEqual(len(d["columns"][i]), dims[0]) + self.assertIsInstance(d["columns"][i], expected_types[i][0]) + if expected_types[i][1] is not None: + self.assertEqual(d["columns"][i].dtype, expected_types[i][1]) + if expected_column_idx is not None: + self.assertSetEqual(set(expected_column_idx), set(d["col_idx"])) + + def test_encode_DataFrame(self): + df = pd.DataFrame( + data={ + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } + ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None)) + fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) + self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"]) + + def test_encode_ndarray(self): + arr = np.zeros((3, 2), dtype=np.float32) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32)) + fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) + self.fbs_checks(fbs, (3, 2), expected_types, None) + + def test_encode_sparse(self): + csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) + expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32)) + fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) + self.fbs_checks(fbs, (2, 3), expected_types, None) + + def test_roundtrip(self): + dfSrc = pd.DataFrame( + data={ + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } + ) + dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) + self.assertEqual(dfSrc.shape, dfDst.shape) + self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) + for c in dfSrc.columns: + self.assertTrue(c in dfDst.columns) + if isinstance(dfSrc[c], pd.Series): + self.assertTrue(np.all(dfSrc[c] == dfDst[c])) + else: + self.assertEqual(dfSrc[c], dfDst[c]) diff --git a/local_server/test/unit/decode_fbs.py b/local_server/test/unit/decode_fbs.py new file mode 100644 index 00000000..0d5681fa --- /dev/null +++ b/local_server/test/unit/decode_fbs.py @@ -0,0 +1,31 @@ +""" +Code to decode, for testing purposes, the flatbuffer encoded blobs. + +This code will need to be updated if fbs/matrix.fbs changes. For more information, see fbs/matrix.fbs and +local_server/data_common/fbs/ +""" + +import local_server.data_common.fbs.NetEncoding.Matrix as Matrix +from local_server.data_common.fbs.matrix import deserialize_typed_array + + +def decode_matrix_FBS(buf): + """ + Given a FBS Matrix, return an decoded Python dict containing same info in native format. + NOTE / TODO: row_idx not currently implemented + """ + df = Matrix.Matrix.GetRootAsMatrix(buf, 0) + n_rows = df.NRows() + n_cols = df.NCols() + + columns_length = df.ColumnsLength() + + decoded_columns = [] + for col_idx in range(0, columns_length): + col = df.Columns(col_idx) + tarr = (col.UType(), col.U()) + decoded_columns.append(deserialize_typed_array(tarr)) + + cidx = deserialize_typed_array((df.ColIndexType(), df.ColIndex())) + + return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None} diff --git a/requirements.txt b/requirements.txt index 815cac2f..58b2e504 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ --r ./server/requirements.txt +-r ./local_server/requirements.txt diff --git a/server/__init__.py b/server/__init__.py index 100b1874..2c855aaa 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -2,7 +2,7 @@ import logging import sys from server.common.utils.utils import import_plugins -__version__ = "0.16.0" +__version__ = "0.16.7" display_version = "cellxgene v" + __version__ try: diff --git a/server/app/app.py b/server/app/app.py index a2fe26b7..e9b9e14a 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -240,7 +240,7 @@ class DatasetResource(Resource): class SchemaAPI(DatasetResource): # TODO @mdunitz separate dataset schema and user schema - @cache_control(no_store=True) + @cache_control(public=True, max_age=ONE_WEEK) @rest_get_data_adaptor def get(self, data_adaptor): return common_rest.schema_get(data_adaptor) @@ -261,7 +261,7 @@ class UserInfoAPI(DatasetResource): class AnnotationsObsAPI(DatasetResource): - @cache_control(public=True, no_store=True) + @cache_control(public=True, max_age=ONE_WEEK) @rest_get_data_adaptor def get(self, data_adaptor): return common_rest.annotations_obs_get(request, data_adaptor) @@ -434,11 +434,17 @@ class Server: self.app.register_blueprint(dataroot_resources.blueprint) self.app.add_url_rule( - f"/{url_dataroot}//", + f"/{url_dataroot}/", f"dataset_index_{url_dataroot}", lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset), methods=["GET"], ) + self.app.add_url_rule( + f"/{url_dataroot}//", + f"dataset_index_{url_dataroot}/", + lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset), + methods=["GET"], + ) self.app.add_url_rule( f"/{url_dataroot}//static/", f"static_assets_{url_dataroot}", diff --git a/server/common/annotations/annotations.py b/server/common/annotations/annotations.py index d57cb270..f5e48f66 100644 --- a/server/common/annotations/annotations.py +++ b/server/common/annotations/annotations.py @@ -33,10 +33,10 @@ class Annotations(metaclass=ABCMeta): raise OntologyLoadFailure("Unable to find OBO ontology path") from e except SyntaxError as e: - raise OntologyLoadFailure("Syntax error loading OBO ontology") from e + raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e except Exception as e: - raise OntologyLoadFailure("Error loading OBO file") from e + raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e def get_schema(self, data_adaptor): schema = [] diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py index 14eac613..48112b86 100644 --- a/server/common/config/server_config.py +++ b/server/common/config/server_config.py @@ -42,6 +42,9 @@ class ServerConfig(BaseConfig): self.app__web_base_url = default_config["app"]["web_base_url"] self.authentication__type = default_config["authentication"]["type"] + self.authentication__insecure_test_environment = default_config["authentication"][ + "insecure_test_environment" + ] self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][ "oauth_api_base_url" ] @@ -168,6 +171,10 @@ class ServerConfig(BaseConfig): def handle_authentication(self): self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool) + + if self.authentication__type == "test" and not self.authentication__insecure_test_environment: + raise ConfigurationError("Test auth can only be used in an insecure test environment") # oauth ptypes = str if self.authentication__type == "oauth" else (type(None), str) diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py index 90ceabbb..971bf641 100644 --- a/server/common/utils/type_conversion_utils.py +++ b/server/common/utils/type_conversion_utils.py @@ -44,7 +44,7 @@ def get_dtype_from_dtype(dtype, array_values=None): if dtype_name == "bool": return np.uint8 if dtype_name == "object" and dtype_kind == "O": - return np.unicode + return str if dtype_name == "category": return get_dtype_from_dtype(dtype.categories.dtype, array_values) diff --git a/server/default_config.py b/server/default_config.py index d000141f..51384ca8 100644 --- a/server/default_config.py +++ b/server/default_config.py @@ -34,6 +34,7 @@ server: # session: A session based userid is automatically generated. (no params needed) # oauth: oauth2 is used for authentication; parameters are defined in params_oauth. type: session + insecure_test_environment: false params_oauth: # url to the oauth server diff --git a/server/requirements-dev.txt b/server/requirements-dev.txt index 9f4fd724..76370edf 100644 --- a/server/requirements-dev.txt +++ b/server/requirements-dev.txt @@ -8,3 +8,4 @@ pytest>=3.6.3 python-jose>=3.2.0 twine>=1.12.1 -r requirements.txt +rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/server/requirements.txt b/server/requirements.txt index 1cd9a0e2..1cfdf7d5 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -24,3 +24,4 @@ tiledb>=0.5.9,>=0.6.2,!=0.7.2 s3fs==0.4.2 scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here sqlalchemy>=1.3.18 +umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn diff --git a/server/test/__init__.py b/server/test/__init__.py index 1b96f7c1..ad8e7eef 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -34,7 +34,10 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( - app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test", + app__flask_secret_key="secret", + multi_dataset__dataroot=data_locator.path, + authentication__type="test", + authentication__insecure_test_environment=True, ) config.update_default_dataset_config( embeddings__names=["umap"], diff --git a/server/test/fixtures/server_config_outline.py b/server/test/fixtures/server_config_outline.py index 4bbb4675..a23f7d24 100644 --- a/server/test/fixtures/server_config_outline.py +++ b/server/test/fixtures/server_config_outline.py @@ -14,6 +14,7 @@ f"""server: web_base_url: {web_base_url} authentication: type: {auth_type} + insecure_test_environment: {insecure_test_environment} params_oauth: oauth_api_base_url: {oauth_api_base_url} client_id: {client_id} diff --git a/server/test/locust/requirements-locust.txt b/server/test/locust/requirements-locust.txt index 8eaebfd6..e8a5c450 100644 --- a/server/test/locust/requirements-locust.txt +++ b/server/test/locust/requirements-locust.txt @@ -1 +1,2 @@ locust +-r ../../requirements.txt diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py index 2caba6b1..12f062e6 100644 --- a/server/test/unit/auth/test_auth.py +++ b/server/test/unit/auth/test_auth.py @@ -45,6 +45,7 @@ class AuthTest(unittest.TestCase): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config(authentication__type="test") + app_config.update_server_config(authentication__insecure_test_environment=True) app_config.update_server_config( multi_dataset__dataroot=dict( a1=dict(dataroot=self.dataset_dataroot, base_url="auth"), @@ -79,7 +80,7 @@ class AuthTest(unittest.TestCase): r = session.get(f"{server}/{login_uri}") # check that the login redirect worked self.assertEqual(r.history[0].status_code, 302) - self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/") + self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg") config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() @@ -91,7 +92,7 @@ class AuthTest(unittest.TestCase): r = session.get(f"{server}/{logout_uri}") # check that the logout redirect worked self.assertEqual(r.history[0].status_code, 302) - self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/") + self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg") config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) @@ -116,6 +117,7 @@ class AuthTest(unittest.TestCase): app_config.update_server_config( authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg" ) + app_config.update_server_config(authentication__insecure_test_environment=True) app_config.complete_config() diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py index 8a48be5e..12fc7656 100644 --- a/server/test/unit/common/config/__init__.py +++ b/server/test/unit/common/config/__init__.py @@ -38,6 +38,7 @@ class ConfigTests(unittest.TestCase): api_base_url="null", web_base_url="null", auth_type="session", + insecure_test_environment="false", oauth_api_base_url="null", client_id="null", client_secret="null", diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py index e7598b7f..932a5d33 100644 --- a/server/test/unit/common/config/test_server_config.py +++ b/server/test/unit/common/config/test_server_config.py @@ -53,7 +53,7 @@ class TestServerConfig(ConfigTests): def test_complete_config_checks_all_attr(self, mock_check_attrs): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() self.server_config.complete_config(self.context) - self.assertEqual(mock_check_attrs.call_count, 40) + self.assertEqual(mock_check_attrs.call_count, 41) def test_handle_app__throws_error_if_port_doesnt_exist(self): config = self.get_config(port=99999999) @@ -278,6 +278,14 @@ class TestServerConfig(ConfigTests): response = session.get(f"{server}/health") assert response.json()["status"] == "pass" + # access a dataset (no slash) + response = session.get(f"{server}/set2/pbmc3k.cxg") + self.assertEqual(response.status_code, 200) + + # access a dataset (with slash) + response = session.get(f"{server}/set2/pbmc3k.cxg/") + self.assertEqual(response.status_code, 200) + @patch("server.common.config.server_config.diffexp_tiledb.set_config") def test_handle_diffexp(self, mock_tiledb_config): custom_config_file = self.custom_app_config( @@ -304,3 +312,12 @@ class TestServerConfig(ConfigTests): mock_tiledb_context.assert_called_once_with( {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} ) + + def test_test_auth_only_in_insecure(self): + + config = self.get_config(auth_type="test") + with self.assertRaises(ConfigurationError): + config.complete_config() + + config.update_server_config(authentication__insecure_test_environment=True) + config.complete_config() diff --git a/server/test/unit/common/utils/test_type_conversion_utils.py b/server/test/unit/common/utils/test_type_conversion_utils.py index ade2d999..0d5319fa 100644 --- a/server/test/unit/common/utils/test_type_conversion_utils.py +++ b/server/test/unit/common/utils/test_type_conversion_utils.py @@ -99,7 +99,7 @@ class TestTypeConversionUtils(unittest.TestCase): def test__get_dtype_of_array__supported_dtypes_return_as_expected(self): types = [np.float32, np.int32, np.bool_, str] - expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode] + expected_dtypes = [np.float32, np.int32, np.uint8, str] for test_type_index in range(len(types)): with self.subTest( @@ -110,7 +110,7 @@ class TestTypeConversionUtils(unittest.TestCase): def test__get_dtype_of_array__categories_return_as_expected(self): array = Series(data=["a", "b", "c"], dtype="category") - expected_dtype = np.unicode + expected_dtype = str actual_dtype = get_dtype_of_array(array) @@ -179,7 +179,7 @@ class TestTypeConversionUtils(unittest.TestCase): category_array = Series(data=["a", "b", "b"], dtype="category") dataframe = DataFrame({"float_array": float_array, "category_array": category_array}) - expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode} + expected_data_types_dict = {"float_array": np.float32, "category_array": str} expected_schema_type_hints_dict = { "float_array": {"type": "float32"}, "category_array": {"type": "categorical", "categories": ["a", "b"]}, diff --git a/setup.cfg b/setup.cfg index b240487d..3b07f60b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ [flake8] max-line-length = 120 ignore = E203, W503 -exclude = server/data_common/fbs/NetEncoding/,.git,__pycache__,venv,server/venv,old,build,dist,server/eb/artifact.dir/ +exclude = server/data_common/fbs/NetEncoding/,.git,__pycache__,venv,server/venv,old,build,dist,server/eb/artifact.dir/,local_server/data_common/fbs/NetEncoding diff --git a/setup.py b/setup.py index 56a2bd28..2efc7d24 100644 --- a/setup.py +++ b/setup.py @@ -3,15 +3,15 @@ from setuptools import setup, find_packages with open("README.md", "rb") as fh: long_description = fh.read().decode() -with open("server/requirements.txt") as fh: +with open("local_server/requirements.txt") as fh: requirements = fh.read().splitlines() -with open("server/requirements-prepare.txt") as fh: +with open("local_server/requirements-prepare.txt") as fh: requirements_prepare = fh.read().splitlines() setup( name="cellxgene", - version="0.16.0", + version="0.16.7", packages=find_packages(), url="https://github.com/chanzuckerberg/cellxgene", license="MIT", @@ -38,6 +38,6 @@ setup( "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering :: Bio-Informatics", ], - entry_points={"console_scripts": ["cellxgene = server.cli.cli:cli"]}, + entry_points={"console_scripts": ["cellxgene = local_server.cli.cli:cli"]}, extras_require=dict(prepare=requirements_prepare), ) diff --git a/setup_hosted.py b/setup_hosted.py new file mode 100644 index 00000000..56a2bd28 --- /dev/null +++ b/setup_hosted.py @@ -0,0 +1,43 @@ +from setuptools import setup, find_packages + +with open("README.md", "rb") as fh: + long_description = fh.read().decode() + +with open("server/requirements.txt") as fh: + requirements = fh.read().splitlines() + +with open("server/requirements-prepare.txt") as fh: + requirements_prepare = fh.read().splitlines() + +setup( + name="cellxgene", + version="0.16.0", + packages=find_packages(), + url="https://github.com/chanzuckerberg/cellxgene", + license="MIT", + author="Chan Zuckerberg Initiative", + author_email="cellxgene@chanzuckerberg.com", + description="Web application for exploration of large scale scRNA-seq datasets", + long_description=long_description, + long_description_content_type="text/markdown", + install_requires=requirements, + include_package_data=True, + zip_safe=False, + classifiers=[ + "Framework :: Flask", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: POSIX", + "Operating System :: Unix", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: JavaScript", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering :: Bio-Informatics", + ], + entry_points={"console_scripts": ["cellxgene = server.cli.cli:cli"]}, + extras_require=dict(prepare=requirements_prepare), +)