diff --git a/.gitignore b/.gitignore index 24dd510..e1c196c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,138 @@ +# Custom .DS_Store __pycache__ *.pyc run.sh .cellxgene-gateway + + +# Created by https://www.gitignore.io/api/python +# Edit at https://www.gitignore.io/?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# End of https://www.gitignore.io/api/python diff --git a/Readme.md b/Readme.md index f0a9de2..9cb4179 100644 --- a/Readme.md +++ b/Readme.md @@ -1,50 +1,85 @@ -# Overview # +# Overview Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server. -# Running locally # +## Running locally + +We assume your current working directory is the directory into which you've cloned this repository. 0. This project requires python 3.6 or higher. Please check your version with - python --version +```bash +$ python --version +``` 1. Set up a venv with -``` + +```bash python -m venv .cellxgene-gateway source .cellxgene-gateway/bin/activate ``` 2. Install requirements with -``` + +```bash pip install -r requirements.txt ``` -3. Prepare a folder with .h5ad files, for example +3. Install the gateway: + +_To install in development mode:_ + +```bash +python setup.py develop ``` -mkdir cellxgene_data + +_To install from GitHub:_ + +```bash +pip install git+https://github.com/Novartis/cellxgene-gateway +``` + +_To install from PyPI:_ + +```bash +# NOT YET DONE, COMING! STAY TUNED +``` + +4. Prepare a folder with .h5ad files, for example + +```bash +mkdir ../cellxgene_data wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad ``` -4. Copy run.sh.example to run.sh: -``` -cp run.sh.example run.sh -``` -`run.sh` defines various environment variables: +5. Set your environment variables correctly: -* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd' -* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene -* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data -* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally -* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. - -The defaults should be fine if you set up a venv and cellxgene_data folder as above. - -5. Finally, execute run.sh: -``` -source run.sh +```bash +export CELLXGENE_LOCATION=`which cellxgene` +export CELLXGENE_DATA=../cellxgene_data # change this directory if you put data in a different place. +export GATEWAY_HOST=localhost:5005 +export GATEWAY_PROTOCOL=http +export GATEWAY_IP=127.0.0.1 ``` -# Customization # +6. Now, execute the cellxgene gateway: + +```bash +cellxgene-gateway +``` + +For convenience, you can also change `run.sh.example` and execute it. + +Here's what the environment variables mean: + +* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene` +* `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data` +* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically `localhost:5005` if running locally +* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. + +The defaults should be fine if you set up a venv and cellxgene_data folder as above. + +# Customization The current paradigm for customization is to modify files during a build or deployment phase: @@ -55,29 +90,23 @@ The current paradigm for customization is to modify files during a build or depl Currently we use a build.sh that copies the gateway to a "build" directory before modifying with sed and the like. -# Development # +# Development -## Running Linters ## +## Running Linters pip install isort flake8 black -``` +```bash isort -rc . -``` - -``` flake8 . +black -l 79 . ``` -``` -black . -``` - -# Getting Help # +# Getting Help If you need help for any reason, please make a github ticket. One of the contributors should help you out. -# Contributors # +# Contributors * Niket Patel - https://github.com/NiketPatel9 * Alok Saldanha - https://github.com/alokito diff --git a/__init__.py b/cellxgene_gateway/__init__.py similarity index 99% rename from __init__.py rename to cellxgene_gateway/__init__.py index 8605fcc..447b57c 100755 --- a/__init__.py +++ b/cellxgene_gateway/__init__.py @@ -6,4 +6,3 @@ # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. - diff --git a/backend_cache.py b/cellxgene_gateway/backend_cache.py similarity index 84% rename from backend_cache.py rename to cellxgene_gateway/backend_cache.py index 8293c80..08d0235 100644 --- a/backend_cache.py +++ b/cellxgene_gateway/backend_cache.py @@ -12,10 +12,10 @@ from threading import Thread from flask_api import status -import env -from cache_entry import CacheEntry -from cellxgene_exception import CellxgeneException -from subprocess_backend import SubprocessBackend +from cellxgene_gateway import env +from cellxgene_gateway.cache_entry import CacheEntry +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.subprocess_backend import SubprocessBackend process_backend = SubprocessBackend() @@ -51,7 +51,8 @@ class BackendCache: entry = CacheEntry.for_dataset(dataset, file_path, port) background_thread = Thread( - target=process_backend.launch, args=(env.cellxgene_location, scripts, entry) + target=process_backend.launch, + args=(env.cellxgene_location, scripts, entry), ) background_thread.start() diff --git a/cache_entry.py b/cellxgene_gateway/cache_entry.py similarity index 77% rename from cache_entry.py rename to cellxgene_gateway/cache_entry.py index 6f9e297..f818040 100644 --- a/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -9,9 +9,9 @@ from flask import make_response, request from requests import get, post, put -import env -from cellxgene_exception import CellxgeneException -from util import current_time_stamp +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.util import current_time_stamp class CacheEntry: @@ -92,14 +92,26 @@ class CacheEntry: if "content-type" in request.headers: headers["content-type"] = request.headers["content-type"] - if request.method in ['GET', 'HEAD', 'OPTIONS']: - cellxgene_response = get(cellxgene_basepath + subpath, headers=headers) - elif request.method == 'PUT': - cellxgene_response = put(cellxgene_basepath + subpath, headers=headers, data=request.data.decode()) - elif request.method == 'POST': - cellxgene_response = post(cellxgene_basepath + subpath, headers=headers, data=request.data.decode()) + if request.method in ["GET", "HEAD", "OPTIONS"]: + cellxgene_response = get( + cellxgene_basepath + subpath, headers=headers + ) + elif request.method == "PUT": + cellxgene_response = put( + cellxgene_basepath + subpath, + headers=headers, + data=request.data.decode(), + ) + elif request.method == "POST": + cellxgene_response = post( + cellxgene_basepath + subpath, + headers=headers, + data=request.data.decode(), + ) else: - raise CellxgeneException(f"Unexpected method {request.method}", 400) + raise CellxgeneException( + f"Unexpected method {request.method}", 400 + ) content_type = cellxgene_response.headers["content-type"] if "text" in content_type: cellxgene_content = cellxgene_response.content.decode() @@ -108,11 +120,11 @@ class CacheEntry: ).replace(cellxgene_basepath, gateway_basepath) else: gateway_content = cellxgene_response.content - + gateway_response = make_response( - gateway_content, + gateway_content, cellxgene_response.status_code, - {"Content-Type": content_type } + {"Content-Type": content_type}, ) return gateway_response diff --git a/cellxgene_exception.py b/cellxgene_gateway/cellxgene_exception.py similarity index 99% rename from cellxgene_exception.py rename to cellxgene_gateway/cellxgene_exception.py index ae645c3..d46d58c 100644 --- a/cellxgene_exception.py +++ b/cellxgene_gateway/cellxgene_exception.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + class CellxgeneException(Exception): def __init__(self, message, http_status): Exception.__init__(self) diff --git a/dir_util.py b/cellxgene_gateway/dir_util.py similarity index 90% rename from dir_util.py rename to cellxgene_gateway/dir_util.py index 15d1370..d0a9af8 100644 --- a/dir_util.py +++ b/cellxgene_gateway/dir_util.py @@ -11,8 +11,8 @@ import os from flask_api import status -import env -from cellxgene_exception import CellxgeneException +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException def is_subdir(full_path, parent_path): @@ -31,7 +31,8 @@ def create_dir(parent_path, dir_name): ) elif not os.path.exists(parent_path): raise CellxgeneException( - "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST + "The selected User directory does not exist.", + status.HTTP_400_BAD_REQUEST, ) elif os.path.exists(full_path): raise CellxgeneException( @@ -74,7 +75,8 @@ def recurse_dir(path): } else: raise CellxgeneException( - "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST + "Given path is neither file nor directory.", + status.HTTP_400_BAD_REQUEST, ) return [make_entry(x) for x in os.listdir(path)] diff --git a/env.py b/cellxgene_gateway/env.py similarity index 55% rename from env.py rename to cellxgene_gateway/env.py index 686bd4f..5967718 100644 --- a/env.py +++ b/cellxgene_gateway/env.py @@ -9,9 +9,35 @@ import os -deployment_env = os.environ.get("DEPLOYMENT_ENV") cellxgene_location = os.environ.get("CELLXGENE_LOCATION") cellxgene_data = os.environ.get("CELLXGENE_DATA") gateway_host = os.environ.get("GATEWAY_HOST") gateway_protocol = os.environ.get("GATEWAY_PROTOCOL") ip = os.environ.get("GATEWAY_IP") + +env_vars = { + "CELLXGENE_LOCATION": cellxgene_location, + "CELLXGENE_DATA": cellxgene_data, + "GATEWAY_HOST": gateway_host, + "GATEWAY_PROTOCOL": gateway_protocol, + "GATEWAY_IP": ip, +} + +if not all(env_vars.values()): + raise ValueError( + f""" +Please ensure that environment variables are set correctly. +The ones with None below are missing and need to be set. + +{env_vars} + +Set them at the terminal before running the gateway. +An example is: + + export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene + export CELLXGENE_DATA=../cellxgene_data + export GATEWAY_HOST=localhost:5005 + export GATEWAY_PROTOCOL=http + export GATEWAY_IP=127.0.0.1 +""" + ) diff --git a/extra_scripts.py b/cellxgene_gateway/extra_scripts.py similarity index 99% rename from extra_scripts.py rename to cellxgene_gateway/extra_scripts.py index 7c05b97..439c92e 100644 --- a/extra_scripts.py +++ b/cellxgene_gateway/extra_scripts.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + def get_extra_scripts(): # can be array of script tags to inject on every page, e.g. for google analytics could be # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2', diff --git a/gateway.py b/cellxgene_gateway/gateway.py similarity index 76% rename from gateway.py rename to cellxgene_gateway/gateway.py index c92c35b..958a731 100644 --- a/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -12,19 +12,25 @@ import datetime import os from threading import Thread -from flask import Flask, redirect, render_template, request, send_from_directory +from flask import ( + Flask, + redirect, + render_template, + request, + send_from_directory, +) from flask_api import status from werkzeug import secure_filename -import env -from backend_cache import BackendCache -from cellxgene_exception import CellxgeneException -from dir_util import create_dir, recurse_dir, render_entries -from extra_scripts import get_extra_scripts -from path_util import get_dataset, get_file_path -from process_exception import ProcessException -from prune_process_cache import PruneProcessCache -from util import current_time_stamp +from cellxgene_gateway import env +from cellxgene_gateway.backend_cache import BackendCache +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries +from cellxgene_gateway.extra_scripts import get_extra_scripts +from cellxgene_gateway.path_util import get_dataset, get_file_path +from cellxgene_gateway.process_exception import ProcessException +from cellxgene_gateway.prune_process_cache import PruneProcessCache +from cellxgene_gateway.util import current_time_stamp app = Flask(__name__) cache = BackendCache() @@ -38,7 +44,9 @@ def handle_invalid_usage(error): return ( render_template( - "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message + "cellxgene_error.html", + extra_scripts=get_extra_scripts(), + message=message, ), error.http_status, ) @@ -56,7 +64,9 @@ def handle_invalid_process(error): return ( render_template( - "process_error.html", extra_scripts=get_extra_scripts(), message=message + "process_error.html", + extra_scripts=get_extra_scripts(), + message=message, ), error.http_status, ) @@ -116,7 +126,9 @@ def upload_file(): if "file" in request.files: f = request.files["file"] if f and f.filename.endswith(".h5ad"): - f.save(full_upload_path + "/" + secure_filename(f.filename)) + f.save( + full_upload_path + "/" + secure_filename(f.filename) + ) return redirect("/filecrawl.html", code=302) else: raise CellxgeneException( @@ -125,10 +137,13 @@ def upload_file(): ) else: raise CellxgeneException( - "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST + "A file must be chosen to upload.", + status.HTTP_400_BAD_REQUEST, ) else: - raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST) + raise CellxgeneException( + "Invalid directory.", status.HTTP_400_BAD_REQUEST + ) return redirect(env.location, code=302) @@ -139,7 +154,9 @@ def filecrawl(): entries = recurse_dir(env.cellxgene_data) rendered_html = render_entries(entries) return render_template( - "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html + "filecrawl.html", + extra_scripts=get_extra_scripts(), + rendered_html=rendered_html, ) @@ -166,8 +183,12 @@ def do_GET(path): raise ProcessException.from_pid_object(match) -if __name__ == "__main__": +def main(): background_thread = Thread(target=PruneProcessCache(cache)) background_thread.start() app.run(host="0.0.0.0", port=5005, debug=False) + + +if __name__ == "__main__": + main() diff --git a/path_util.py b/cellxgene_gateway/path_util.py similarity index 93% rename from path_util.py rename to cellxgene_gateway/path_util.py index 0672184..6dc4fb6 100644 --- a/path_util.py +++ b/cellxgene_gateway/path_util.py @@ -11,8 +11,8 @@ import os from flask_api import status -import env -from cellxgene_exception import CellxgeneException +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException def get_dataset(path): diff --git a/process_exception.py b/cellxgene_gateway/process_exception.py similarity index 99% rename from process_exception.py rename to cellxgene_gateway/process_exception.py index e54d842..5afdb00 100644 --- a/process_exception.py +++ b/cellxgene_gateway/process_exception.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + class ProcessException(Exception): def __init__(self, message, stdout, stderr, http_status): Exception.__init__(self) diff --git a/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py similarity index 96% rename from prune_process_cache.py rename to cellxgene_gateway/prune_process_cache.py index 71bfcfa..f7240a0 100644 --- a/prune_process_cache.py +++ b/cellxgene_gateway/prune_process_cache.py @@ -11,7 +11,7 @@ import time import psutil -from util import current_time_stamp +from cellxgene_gateway.util import current_time_stamp class PruneProcessCache: diff --git a/static/css/homepagestyle.css b/cellxgene_gateway/static/css/homepagestyle.css similarity index 100% rename from static/css/homepagestyle.css rename to cellxgene_gateway/static/css/homepagestyle.css diff --git a/static/nibr.ico b/cellxgene_gateway/static/nibr.ico similarity index 100% rename from static/nibr.ico rename to cellxgene_gateway/static/nibr.ico diff --git a/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py similarity index 97% rename from subprocess_backend.py rename to cellxgene_gateway/subprocess_backend.py index 10aabdc..a49f6e1 100644 --- a/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -12,7 +12,7 @@ import subprocess from flask_api import status -from process_exception import ProcessException +from cellxgene_gateway.process_exception import ProcessException class SubprocessBackend: diff --git a/templates/cellxgene_error.html b/cellxgene_gateway/templates/cellxgene_error.html similarity index 100% rename from templates/cellxgene_error.html rename to cellxgene_gateway/templates/cellxgene_error.html diff --git a/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html similarity index 100% rename from templates/filecrawl.html rename to cellxgene_gateway/templates/filecrawl.html diff --git a/templates/index.html b/cellxgene_gateway/templates/index.html similarity index 99% rename from templates/index.html rename to cellxgene_gateway/templates/index.html index 7aef319..2fcfd23 100644 --- a/templates/index.html +++ b/cellxgene_gateway/templates/index.html @@ -39,7 +39,6 @@ File Crawler: Allows you to view all uploaded data. -

diff --git a/templates/loading.html b/cellxgene_gateway/templates/loading.html similarity index 100% rename from templates/loading.html rename to cellxgene_gateway/templates/loading.html diff --git a/templates/process_error.html b/cellxgene_gateway/templates/process_error.html similarity index 100% rename from templates/process_error.html rename to cellxgene_gateway/templates/process_error.html diff --git a/util.py b/cellxgene_gateway/util.py similarity index 100% rename from util.py rename to cellxgene_gateway/util.py diff --git a/environment-dev.yml b/environment-dev.yml new file mode 100644 index 0000000..f67bdda --- /dev/null +++ b/environment-dev.yml @@ -0,0 +1,11 @@ +name: cellxgene-dev +channels: +- conda-forge +dependencies: +- python=3.7 +- requests +- flask +- psutil +- pip: + - flask-api + - cellxgene diff --git a/requirements.txt b/requirements.txt index c89a503..19bb2ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ flask flask_api psutil requests - diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5381bee --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +import os +from setuptools import setup + + +def parse_requirements(): + reqs = [] + with open("requirements.txt", "r") as f: + for l in f.readlines(): + reqs.append(l.strip("\n")) + return reqs + + +install_reqs = parse_requirements() + +setup( + # mandatory + name="cellxgene-gateway", + # mandatory + version="0.1", + # mandatory + author="Niket Patel, Yohann Potier, Alok Saldanha", + author_email="alok.saldanha@novartis.com", + description=("Cell-by-gene Gateway"), + license="MIT", + keywords="visualization, genomics", + url="http://github.com/Novartis/cellxgene-gateway", + packages=["cellxgene_gateway"], + package_data={"": ["README.md", "LICENSE.txt"]}, + install_requires=install_reqs, + entry_points={ + "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"] + }, + classifiers=["Topic :: Scientific/Engineering :: Visualization"], +)