diff --git a/.gitignore b/.gitignore index cd52f01f..755f2a8b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,39 @@ npm-debug.log .vscode data + + +*.idea* + +__pycache__ +*.DS_Store* + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml + +GBM +venv +extesting + +Dockerfile-* +*-data/* +data/* +runServer.py +templates/favicon.png +templates/index.html +templates/service-worker.js +templates/static/* +Graph.dot* + +server/app/web/static/css/ +server/app/web/static/img/ +server/app/web/static/js/ +server/app/web/templates/index\.html + + +*.egg-info +dist/* +build/* + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..6fb222c1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +language: python +python: + - "3.6" +node_js: + - "8" +cache: + pip: true +install: + - set -eo pipefail + - pip install flake8 httpie + - ./bin/build-client + - python setup.py install +script: + - set -eo pipefail + - flake8 server/app/ + - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py + - cellxgene & + - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done + - pytest server/test/test_api.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..b881e298 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include server/app/web/templates * +recursive-include server/app/web/static * + diff --git a/README.md b/README.md index 6a645a93..43c249ed 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,37 @@ A React + Redux web application for exploring large scale single cell RNA sequence data. -##### Quickstart: +### Requirements +- OS: OSX, Windows, Linux +- python 3.6 +- npm +- Google Chrome -* `npm install` -* `npm start` -* `localhost:3000` + +## Installation + +#### clone project + + git clone https://github.com/chanzuckerberg/cellxgene.git + +#### install client + + cd cellxgene + ./bin/build-client + +#### To use with virtual env for python (optional, but recommended) + + ENV_NAME=cellxgene + python3 -m venv ${ENV_NAME} + source ${ENV_NAME}/bin/activate + +#### install server + + python3 setup.py install + +#### run (with demo data) + + cellxgene + + +*Thanks to Alex Wolf his help with test data* diff --git a/bin/build-client b/bin/build-client new file mode 100755 index 00000000..8e51b8db --- /dev/null +++ b/bin/build-client @@ -0,0 +1,10 @@ +#!/bin/bash + +npm install --prefix client/ client +npm run --prefix client build +mkdir -p server/app/web/static/img +cp client/build/index.html server/app/web/templates/ +cp -r client/build/static server/app/web/ + +cp client/build/favicon.png server/app/web/static/img +cp client/build/service-worker.js server/app/web/static/js/ diff --git a/__tests__/util/bitArray.test.js b/client/__tests__/util/bitArray.test.js similarity index 100% rename from __tests__/util/bitArray.test.js rename to client/__tests__/util/bitArray.test.js diff --git a/__tests__/util/positiveInterval.test.js b/client/__tests__/util/positiveInterval.test.js similarity index 100% rename from __tests__/util/positiveInterval.test.js rename to client/__tests__/util/positiveInterval.test.js diff --git a/__tests__/util/typedCrossfilter.test.js b/client/__tests__/util/typedCrossfilter.test.js similarity index 100% rename from __tests__/util/typedCrossfilter.test.js rename to client/__tests__/util/typedCrossfilter.test.js diff --git a/configuration/babel/babel.dev.js b/client/configuration/babel/babel.dev.js similarity index 100% rename from configuration/babel/babel.dev.js rename to client/configuration/babel/babel.dev.js diff --git a/configuration/babel/babel.prod.js b/client/configuration/babel/babel.prod.js similarity index 100% rename from configuration/babel/babel.prod.js rename to client/configuration/babel/babel.prod.js diff --git a/configuration/babel/babel.test.js b/client/configuration/babel/babel.test.js similarity index 100% rename from configuration/babel/babel.test.js rename to client/configuration/babel/babel.test.js diff --git a/configuration/eslint/eslint.js b/client/configuration/eslint/eslint.js similarity index 100% rename from configuration/eslint/eslint.js rename to client/configuration/eslint/eslint.js diff --git a/configuration/polyfills/polyfills.js b/client/configuration/polyfills/polyfills.js similarity index 100% rename from configuration/polyfills/polyfills.js rename to client/configuration/polyfills/polyfills.js diff --git a/configuration/webpack/webpack.config.dev.js b/client/configuration/webpack/webpack.config.dev.js similarity index 100% rename from configuration/webpack/webpack.config.dev.js rename to client/configuration/webpack/webpack.config.dev.js diff --git a/configuration/webpack/webpack.config.prod.js b/client/configuration/webpack/webpack.config.prod.js similarity index 100% rename from configuration/webpack/webpack.config.prod.js rename to client/configuration/webpack/webpack.config.prod.js diff --git a/favicon.png b/client/favicon.png similarity index 100% rename from favicon.png rename to client/favicon.png diff --git a/index.html b/client/index.html similarity index 97% rename from index.html rename to client/index.html index db192aec..332616c9 100644 --- a/index.html +++ b/client/index.html @@ -3,7 +3,7 @@ - Cellx + cellxgene + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + diff --git a/server/app/web/webapp.py b/server/app/web/webapp.py new file mode 100644 index 00000000..4717f155 --- /dev/null +++ b/server/app/web/webapp.py @@ -0,0 +1,24 @@ +from flask import ( + Blueprint, render_template, url_for, current_app +) + +bp = Blueprint("webapp", __name__, template_folder="templates") + + +@bp.route("/") +def index(): + url_base = current_app.config["CXG_API_BASE"] + dataset_title = current_app.config["DATASET_TITLE"] + return render_template("index.html", prefix=url_base, datasetTitle=dataset_title) + + +# renders swagger documentation +@bp.route("/swagger") +def swag(): + return render_template("swagger.html") + + +# renders swagger documentation +@bp.route("/favicon.png") +def favicon(): + return url_for("static", filename="img/favicon.png") diff --git a/server/requirements.txt b/server/requirements.txt new file mode 100644 index 00000000..7ab96abb --- /dev/null +++ b/server/requirements.txt @@ -0,0 +1,41 @@ +aniso8601==3.0.2 +anndata==0.6.1 +certifi==2018.4.16 +chardet==3.0.4 +click==6.7 +cycler==0.10.0 +decorator==4.3.0 +Flask==0.12.4 +Flask-Compress==1.4.0 +Flask-Cors==3.0.6 +Flask-RESTful==0.3.6 +flask-restful-swagger-2==0.35 +h5py==2.8.0 +idna==2.7 +itsdangerous==0.24 +Jinja2==2.10 +joblib==0.12.0 +kiwisolver==1.0.1 +llvmlite==0.23.2 +MarkupSafe==1.0 +matplotlib==2.2.2 +natsort==5.3.2 +networkx==2.1 +numba==0.38.1 +numexpr==2.6.5 +numpy==1.14.5 +pandas==0.23.1 +patsy==0.5.0 +pyparsing==2.2.0 +python-dateutil==2.7.3 +pytz==2018.4 +requests==2.19.1 +scanpy==1.0.4 +scikit-learn==0.19.1 +scipy==1.1.0 +seaborn==0.8.1 +six==1.11.0 +statsmodels==0.9.0 +tables==3.4.4 +urllib3==1.23 +Werkzeug==0.14.1 \ No newline at end of file diff --git a/server/run.py b/server/run.py new file mode 100644 index 00000000..dd4feb33 --- /dev/null +++ b/server/run.py @@ -0,0 +1,3 @@ +from app.app import app + +app.run(host="0.0.0.0", debug=True, port=5005) diff --git a/server/test/test_api.py b/server/test/test_api.py new file mode 100644 index 00000000..b004740a --- /dev/null +++ b/server/test/test_api.py @@ -0,0 +1,54 @@ +import unittest +import requests +import json + + +class EndPoints(unittest.TestCase): + """Test Case for endpoints""" + + def setUp(self): + # Local + self.url_base = "http://0.0.0.0:5005/api/" + "v0.1/" + self.session = requests.Session() + + def test_cells(self): + url = "{base}{endpoint}?{params}".format(base=self.url_base, endpoint="cells", params="&".join( + ["louvain=B cells"])) + result = self.session.get(url) + assert result.status_code == 200 + result_data = result.json() + assert "B cells" in result_data["data"]["ranges"]["louvain"]["options"] + url = "{base}{endpoint}?{params}".format(base=self.url_base, endpoint="cells", params="&".join( + ["louvain=B cells", "louvain=Megakaryocytes"])) + result = self.session.get(url) + assert result.status_code == 200 + result_data = result.json() + assert "Megakaryocytes" in result_data["data"]["ranges"]["louvain"]["options"] + + def test_initialize(self): + url = "{base}{endpoint}".format(base=self.url_base, endpoint="initialize") + result = self.session.get(url) + assert result.status_code == 200 + result_data = result.json() + assert result_data["data"]["cellcount"] == 2638 + assert len(result_data["data"]['ranges']['CellName']['options']) == 2638 + + + def test_expression_get(self): + url = "{base}{endpoint}".format(base=self.url_base, endpoint="expression") + result = self.session.get(url) + assert result.status_code == 200 + + def test_expression_post(self): + url = "{base}{endpoint}".format(base=self.url_base, endpoint="expression") + result = self.session.post(url, data=json.dumps({"celllist": ["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], "genelist": ["BACH1", "MIS18A", "ATP5O"]}), headers={'content-type': 'application/json'}) + assert result.status_code == 200 + result_data = result.json() + assert len(result_data["data"]["cells"]) == 2 + assert len(result_data["data"]["cells"][0]['e']) == 3 + + def test_diffexp(self): + url = "{base}{endpoint}".format(base=self.url_base, endpoint="diffexpression") + result = self.session.post(url, data=json.dumps({"celllist1": ["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], "celllist2": ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"]}), headers={'content-type': 'application/json'}) + assert result.status_code == 200 + \ No newline at end of file diff --git a/server/test/test_filter.py b/server/test/test_filter.py new file mode 100644 index 00000000..3641fee1 --- /dev/null +++ b/server/test/test_filter.py @@ -0,0 +1,75 @@ +import unittest +from unittest.mock import MagicMock + +from server.app.util.filter import _convert_variable, parse_filter + + +class UtilTest(unittest.TestCase): + """Test Case for endpoints""" + + def setUp(self): + self.schema = { + "cluster": { + "displayname": "Cluster", + "include": True, + "type": "int", + "variabletype": "categorical" + }, + "louvain": { + "displayname": "Louvain Cluster", + "include": True, + "type": "string", + "variabletype": "categorical" + }, + "n_genes": { + "displayname": "Num Genes", + "include": True, + "type": "int", + "variabletype": "continuous" + } + } + + def test_convert(self): + five = _convert_variable("int", "5") + assert five == 5 + + def test_convert_zero(self): + zero = _convert_variable("int", "0") + assert zero == 0 + + def test_empty_convert(self): + empty = _convert_variable("int", None) + assert empty is None + + def test_bad_convert(self): + with self.assertRaises(ValueError): + _convert_variable("int", "5.5") + + def test_filter_categorical(self): + filterMock = MagicMock() + filterMock.__iter__.return_value = iter(["louvain"]) + filterMock.getlist.return_value = ["B cells", "T cells"] + query = parse_filter(filterMock, self.schema) + assert query == {"louvain": {"variable_type": "categorical", "value_type": "string", "query": ["B cells", "T cells"]}} + filterMock.__iter__.return_value = iter(["cluster"]) + filterMock.getlist.return_value = ["1", "2"] + query = parse_filter(filterMock, self.schema) + assert query == {"cluster": {"variable_type": "categorical", "value_type": "int", "query": [1, 2]}} + + def test_filter_contiunous(self): + filterMock = MagicMock() + filterMock.__iter__.return_value = iter(["n_genes"]) + filterMock.getlist.return_value = ["0,100"] + query = parse_filter(filterMock, self.schema) + assert query == {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 0, "max": 100}}} + filterMock.__iter__.return_value = iter(["n_genes"]) + filterMock.getlist.return_value = ["*,100"] + query = parse_filter(filterMock, self.schema) + assert query == {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": None, "max": 100}}} + filterMock.__iter__.return_value = iter(["n_genes"]) + filterMock.getlist.return_value = ["0,*"] + query = parse_filter(filterMock, self.schema) + assert query == {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 0, "max": None}}} + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py new file mode 100644 index 00000000..e346fb6c --- /dev/null +++ b/server/test/test_scanpy_engine.py @@ -0,0 +1,69 @@ +import unittest + +from server.app.scanpy_engine.scanpy_engine import ScanpyEngine + + +class UtilTest(unittest.TestCase): + def setUp(self): + self.data = ScanpyEngine("example-dataset/", schema="data_schema.json") + + def test_init(self): + self.assertEqual(self.data.cell_count, 2638) + self.assertEqual(self.data.gene_count, 1838) + epsilon = 0.000005 + self.assertTrue(self.data.data.X[0,0] - -0.17146951 < epsilon) + + def test_schema(self): + self.assertEqual(self.data.schema, {'CellName': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Name', 'include': True}, 'n_genes': {'type': 'int', 'variabletype': 'continuous', 'displayname': 'Num Genes', 'include': True}, 'percent_mito': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Mitochondrial Percentage', 'include': True}, 'n_counts': {'type': 'float', 'variabletype': 'continuous', 'displayname': 'Num Counts', 'include': True}, 'louvain': {'type': 'string', 'variabletype': 'categorical', 'displayname': 'Louvain Cluster', 'include': True}}) + + def test_cells(self): + cells = self.data.cells() + self.assertIn("AAACATACAACCAC-1", cells) + self.assertEqual(len(cells), 2638) + + def test_genes(self): + genes = self.data.genes() + self.assertIn("SEPT4", genes) + self.assertEqual(len(genes), 1838) + + def test_filter_categorical(self): + filter = {"louvain": {"variable_type": "categorical", "value_type": "string", "query": ["B cells"]}} + filtered_data = self.data.filter_cells(filter) + self.assertEqual(filtered_data.shape, (342, 1838)) + louvain_vals = filtered_data.obs['louvain'].tolist() + self.assertIn("B cells", louvain_vals) + self.assertNotIn("NK cells", louvain_vals) + + def test_filter_continuous(self): + # print(self.data.data.obs["n_genes"].tolist()) + filter = {"n_genes": {"variable_type": "continuous", "value_type": "int", "query": {"min": 300, "max": 400}}} + filtered_data = self.data.filter_cells(filter) + self.assertEqual(filtered_data.shape, (71, 1838)) + n_genes_vals = filtered_data.obs['n_genes'].tolist() + for val in n_genes_vals: + self.assertTrue(300 <= val <= 400) + + def test_metadata(self): + metadata = self.data.metadata(df=self.data.data) + self.assertEqual(len(metadata), 2638) + self.assertIn('louvain', metadata[0]) + + @unittest.skip("Umap not producing the same graph on different systems, even with the same seed. Skipping for now") + def test_create_graph(self): + graph = self.data.create_graph(df=self.data.data) + self.assertEqual(graph[0][1], 0.5545382653143183) + self.assertEqual(graph[0][2], 0.6021833809031731) + + def test_diffexp(self): + diffexp = self.data.diffexp(["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"], 0.5, 7) + self.assertEqual(diffexp["celllist1"]["topgenes"], ['EBNA1BP2', 'DIAPH1', 'SLC25A11', 'SNRNP27', 'COMMD8', 'COTL1', 'GTF3A']) + + def test_expression(self): + expression = self.data.expression(cells=["AAACATACAACCAC-1"]) + data_exp = self.data.data[["AAACATACAACCAC-1"], :].X + for idx in range(len(expression["cells"][0]["e"])): + self.assertEqual(expression["cells"][0]["e"][idx], data_exp[idx]) + + +if __name__ == '__main__': + unittest.main() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..6deafc26 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..f591a66f --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +from setuptools import setup, find_packages + +with open("README.md", "r") as fh: + long_description = fh.read() + +with open('server/requirements.txt') as fh: + requirements = fh.read().splitlines() + +setup( + name='cellxgene', + version='0.0.1', + packages=find_packages(), + url='https://github.com/chanzuckerberg/cellxgene', + license='MIT', + author='Colin Megill, Charlotte Weaver', + author_email='cweaver@chanzuckerberg.com', + description='Web application for exploration of large scale scRNA-seq datasets', + long_description=long_description, + install_requires=requirements, + include_package_data=True, + zip_safe=False, + classifiers=( + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + ), + entry_points={ + 'console_scripts': + ['cellxgene = server.app.app:main'] + } +)