diff --git a/Readme.md b/Readme.md index f0a9de2..7ee0843 100644 --- a/Readme.md +++ b/Readme.md @@ -1,50 +1,58 @@ -# Overview # +# Overview Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server. -# Running locally # +## Running locally 0. This project requires python 3.6 or higher. Please check your version with - python --version +```bash +$ python --version +``` 1. Set up a venv with -``` + +```bash python -m venv .cellxgene-gateway source .cellxgene-gateway/bin/activate ``` -2. Install requirements with -``` +1. Install requirements with + +```bash pip install -r requirements.txt ``` -3. Prepare a folder with .h5ad files, for example -``` -mkdir cellxgene_data +1. Prepare a folder with .h5ad files, for example + +```bash +mkdir ../cellxgene_data wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad ``` -4. Copy run.sh.example to run.sh: -``` +1. Copy run.sh.example to run.sh: + +```bash cp run.sh.example run.sh ``` + `run.sh` defines various environment variables: -* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd' -* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene -* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data -* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally -* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. +* `DEPLOYMENT_ENV` - expects 'dev', 'tst' or 'prd' +* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene +* `CELLXGENE_DATA` - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data +* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically localhost:5005 if running locally +* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. The defaults should be fine if you set up a venv and cellxgene_data folder as above. -5. Finally, execute run.sh: +1. Finally, execute run.sh: + ``` source run.sh ``` -# Customization # +# Customization The current paradigm for customization is to modify files during a build or deployment phase: diff --git a/cellxgene_gateway.egg-info/PKG-INFO b/cellxgene_gateway.egg-info/PKG-INFO new file mode 100644 index 0000000..7c8dfff --- /dev/null +++ b/cellxgene_gateway.egg-info/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 1.1 +Name: cellxgene-gateway +Version: 0.1 +Summary: Cell-by-gene Gateway +Home-page: http://github.com/Novartis/cellxgene-gateway +Author: Niket Patel, Yohann Potier, Alok Saldanha +Author-email: alok.saldanha@novartis.com +License: MIT +Description: UNKNOWN +Keywords: visualization,genomics +Platform: UNKNOWN +Classifier: Topic :: Scientific/Engineering :: Visualization diff --git a/cellxgene_gateway.egg-info/SOURCES.txt b/cellxgene_gateway.egg-info/SOURCES.txt new file mode 100644 index 0000000..5c52134 --- /dev/null +++ b/cellxgene_gateway.egg-info/SOURCES.txt @@ -0,0 +1,20 @@ +setup.py +cellxgene_gateway/__init__.py +cellxgene_gateway/backend_cache.py +cellxgene_gateway/cache_entry.py +cellxgene_gateway/cellxgene_exception.py +cellxgene_gateway/dir_util.py +cellxgene_gateway/env.py +cellxgene_gateway/extra_scripts.py +cellxgene_gateway/gateway.py +cellxgene_gateway/path_util.py +cellxgene_gateway/process_exception.py +cellxgene_gateway/prune_process_cache.py +cellxgene_gateway/subprocess_backend.py +cellxgene_gateway/util.py +cellxgene_gateway.egg-info/PKG-INFO +cellxgene_gateway.egg-info/SOURCES.txt +cellxgene_gateway.egg-info/dependency_links.txt +cellxgene_gateway.egg-info/entry_points.txt +cellxgene_gateway.egg-info/requires.txt +cellxgene_gateway.egg-info/top_level.txt \ No newline at end of file diff --git a/cellxgene_gateway.egg-info/dependency_links.txt b/cellxgene_gateway.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/cellxgene_gateway.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/cellxgene_gateway.egg-info/entry_points.txt b/cellxgene_gateway.egg-info/entry_points.txt new file mode 100644 index 0000000..6bb9dea --- /dev/null +++ b/cellxgene_gateway.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +cellxgene-gateway = cellxgene_gateway.gateway:main + diff --git a/cellxgene_gateway.egg-info/requires.txt b/cellxgene_gateway.egg-info/requires.txt new file mode 100644 index 0000000..19bb2ef --- /dev/null +++ b/cellxgene_gateway.egg-info/requires.txt @@ -0,0 +1,5 @@ +cellxgene +flask +flask_api +psutil +requests diff --git a/cellxgene_gateway.egg-info/top_level.txt b/cellxgene_gateway.egg-info/top_level.txt new file mode 100644 index 0000000..0224ebb --- /dev/null +++ b/cellxgene_gateway.egg-info/top_level.txt @@ -0,0 +1 @@ +cellxgene_gateway diff --git a/__init__.py b/cellxgene_gateway/__init__.py similarity index 99% rename from __init__.py rename to cellxgene_gateway/__init__.py index 8605fcc..447b57c 100755 --- a/__init__.py +++ b/cellxgene_gateway/__init__.py @@ -6,4 +6,3 @@ # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. - diff --git a/backend_cache.py b/cellxgene_gateway/backend_cache.py similarity index 84% rename from backend_cache.py rename to cellxgene_gateway/backend_cache.py index 8293c80..08d0235 100644 --- a/backend_cache.py +++ b/cellxgene_gateway/backend_cache.py @@ -12,10 +12,10 @@ from threading import Thread from flask_api import status -import env -from cache_entry import CacheEntry -from cellxgene_exception import CellxgeneException -from subprocess_backend import SubprocessBackend +from cellxgene_gateway import env +from cellxgene_gateway.cache_entry import CacheEntry +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.subprocess_backend import SubprocessBackend process_backend = SubprocessBackend() @@ -51,7 +51,8 @@ class BackendCache: entry = CacheEntry.for_dataset(dataset, file_path, port) background_thread = Thread( - target=process_backend.launch, args=(env.cellxgene_location, scripts, entry) + target=process_backend.launch, + args=(env.cellxgene_location, scripts, entry), ) background_thread.start() diff --git a/cache_entry.py b/cellxgene_gateway/cache_entry.py similarity index 77% rename from cache_entry.py rename to cellxgene_gateway/cache_entry.py index 6f9e297..f818040 100644 --- a/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -9,9 +9,9 @@ from flask import make_response, request from requests import get, post, put -import env -from cellxgene_exception import CellxgeneException -from util import current_time_stamp +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.util import current_time_stamp class CacheEntry: @@ -92,14 +92,26 @@ class CacheEntry: if "content-type" in request.headers: headers["content-type"] = request.headers["content-type"] - if request.method in ['GET', 'HEAD', 'OPTIONS']: - cellxgene_response = get(cellxgene_basepath + subpath, headers=headers) - elif request.method == 'PUT': - cellxgene_response = put(cellxgene_basepath + subpath, headers=headers, data=request.data.decode()) - elif request.method == 'POST': - cellxgene_response = post(cellxgene_basepath + subpath, headers=headers, data=request.data.decode()) + if request.method in ["GET", "HEAD", "OPTIONS"]: + cellxgene_response = get( + cellxgene_basepath + subpath, headers=headers + ) + elif request.method == "PUT": + cellxgene_response = put( + cellxgene_basepath + subpath, + headers=headers, + data=request.data.decode(), + ) + elif request.method == "POST": + cellxgene_response = post( + cellxgene_basepath + subpath, + headers=headers, + data=request.data.decode(), + ) else: - raise CellxgeneException(f"Unexpected method {request.method}", 400) + raise CellxgeneException( + f"Unexpected method {request.method}", 400 + ) content_type = cellxgene_response.headers["content-type"] if "text" in content_type: cellxgene_content = cellxgene_response.content.decode() @@ -108,11 +120,11 @@ class CacheEntry: ).replace(cellxgene_basepath, gateway_basepath) else: gateway_content = cellxgene_response.content - + gateway_response = make_response( - gateway_content, + gateway_content, cellxgene_response.status_code, - {"Content-Type": content_type } + {"Content-Type": content_type}, ) return gateway_response diff --git a/cellxgene_exception.py b/cellxgene_gateway/cellxgene_exception.py similarity index 99% rename from cellxgene_exception.py rename to cellxgene_gateway/cellxgene_exception.py index ae645c3..d46d58c 100644 --- a/cellxgene_exception.py +++ b/cellxgene_gateway/cellxgene_exception.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + class CellxgeneException(Exception): def __init__(self, message, http_status): Exception.__init__(self) diff --git a/dir_util.py b/cellxgene_gateway/dir_util.py similarity index 90% rename from dir_util.py rename to cellxgene_gateway/dir_util.py index 15d1370..d0a9af8 100644 --- a/dir_util.py +++ b/cellxgene_gateway/dir_util.py @@ -11,8 +11,8 @@ import os from flask_api import status -import env -from cellxgene_exception import CellxgeneException +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException def is_subdir(full_path, parent_path): @@ -31,7 +31,8 @@ def create_dir(parent_path, dir_name): ) elif not os.path.exists(parent_path): raise CellxgeneException( - "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST + "The selected User directory does not exist.", + status.HTTP_400_BAD_REQUEST, ) elif os.path.exists(full_path): raise CellxgeneException( @@ -74,7 +75,8 @@ def recurse_dir(path): } else: raise CellxgeneException( - "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST + "Given path is neither file nor directory.", + status.HTTP_400_BAD_REQUEST, ) return [make_entry(x) for x in os.listdir(path)] diff --git a/env.py b/cellxgene_gateway/env.py similarity index 54% rename from env.py rename to cellxgene_gateway/env.py index 686bd4f..98bb912 100644 --- a/env.py +++ b/cellxgene_gateway/env.py @@ -15,3 +15,32 @@ cellxgene_data = os.environ.get("CELLXGENE_DATA") gateway_host = os.environ.get("GATEWAY_HOST") gateway_protocol = os.environ.get("GATEWAY_PROTOCOL") ip = os.environ.get("GATEWAY_IP") + +env_vars = { + "DEPLOYMENT_ENV": deployment_env, + "CELLXGENE_LOCATION": cellxgene_location, + "CELLXGENE_DATA": cellxgene_data, + "GATEWAY_HOST": gateway_host, + "GATEWAY_PROTOCOL": gateway_protocol, + "GATEWAY_IP": ip, +} + +if not all(env_vars.values()): + raise ValueError( + f""" +Please ensure that environment variables are set correctly. +The ones with None below are missing and need to be set. + +{env_vars} + +Set them at the terminal before running the gateway. +An example is: + + export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene + export CELLXGENE_DATA=../cellxgene_data + export DEPLOYMENT_ENV=dev + export GATEWAY_HOST=localhost:5005 + export GATEWAY_PROTOCOL=http + export GATEWAY_IP=127.0.0.1 +""" + ) diff --git a/extra_scripts.py b/cellxgene_gateway/extra_scripts.py similarity index 99% rename from extra_scripts.py rename to cellxgene_gateway/extra_scripts.py index 7c05b97..439c92e 100644 --- a/extra_scripts.py +++ b/cellxgene_gateway/extra_scripts.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + def get_extra_scripts(): # can be array of script tags to inject on every page, e.g. for google analytics could be # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2', diff --git a/gateway.py b/cellxgene_gateway/gateway.py similarity index 76% rename from gateway.py rename to cellxgene_gateway/gateway.py index c92c35b..958a731 100644 --- a/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -12,19 +12,25 @@ import datetime import os from threading import Thread -from flask import Flask, redirect, render_template, request, send_from_directory +from flask import ( + Flask, + redirect, + render_template, + request, + send_from_directory, +) from flask_api import status from werkzeug import secure_filename -import env -from backend_cache import BackendCache -from cellxgene_exception import CellxgeneException -from dir_util import create_dir, recurse_dir, render_entries -from extra_scripts import get_extra_scripts -from path_util import get_dataset, get_file_path -from process_exception import ProcessException -from prune_process_cache import PruneProcessCache -from util import current_time_stamp +from cellxgene_gateway import env +from cellxgene_gateway.backend_cache import BackendCache +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries +from cellxgene_gateway.extra_scripts import get_extra_scripts +from cellxgene_gateway.path_util import get_dataset, get_file_path +from cellxgene_gateway.process_exception import ProcessException +from cellxgene_gateway.prune_process_cache import PruneProcessCache +from cellxgene_gateway.util import current_time_stamp app = Flask(__name__) cache = BackendCache() @@ -38,7 +44,9 @@ def handle_invalid_usage(error): return ( render_template( - "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message + "cellxgene_error.html", + extra_scripts=get_extra_scripts(), + message=message, ), error.http_status, ) @@ -56,7 +64,9 @@ def handle_invalid_process(error): return ( render_template( - "process_error.html", extra_scripts=get_extra_scripts(), message=message + "process_error.html", + extra_scripts=get_extra_scripts(), + message=message, ), error.http_status, ) @@ -116,7 +126,9 @@ def upload_file(): if "file" in request.files: f = request.files["file"] if f and f.filename.endswith(".h5ad"): - f.save(full_upload_path + "/" + secure_filename(f.filename)) + f.save( + full_upload_path + "/" + secure_filename(f.filename) + ) return redirect("/filecrawl.html", code=302) else: raise CellxgeneException( @@ -125,10 +137,13 @@ def upload_file(): ) else: raise CellxgeneException( - "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST + "A file must be chosen to upload.", + status.HTTP_400_BAD_REQUEST, ) else: - raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST) + raise CellxgeneException( + "Invalid directory.", status.HTTP_400_BAD_REQUEST + ) return redirect(env.location, code=302) @@ -139,7 +154,9 @@ def filecrawl(): entries = recurse_dir(env.cellxgene_data) rendered_html = render_entries(entries) return render_template( - "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html + "filecrawl.html", + extra_scripts=get_extra_scripts(), + rendered_html=rendered_html, ) @@ -166,8 +183,12 @@ def do_GET(path): raise ProcessException.from_pid_object(match) -if __name__ == "__main__": +def main(): background_thread = Thread(target=PruneProcessCache(cache)) background_thread.start() app.run(host="0.0.0.0", port=5005, debug=False) + + +if __name__ == "__main__": + main() diff --git a/path_util.py b/cellxgene_gateway/path_util.py similarity index 93% rename from path_util.py rename to cellxgene_gateway/path_util.py index 0672184..6dc4fb6 100644 --- a/path_util.py +++ b/cellxgene_gateway/path_util.py @@ -11,8 +11,8 @@ import os from flask_api import status -import env -from cellxgene_exception import CellxgeneException +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException def get_dataset(path): diff --git a/process_exception.py b/cellxgene_gateway/process_exception.py similarity index 99% rename from process_exception.py rename to cellxgene_gateway/process_exception.py index e54d842..5afdb00 100644 --- a/process_exception.py +++ b/cellxgene_gateway/process_exception.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + class ProcessException(Exception): def __init__(self, message, stdout, stderr, http_status): Exception.__init__(self) diff --git a/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py similarity index 96% rename from prune_process_cache.py rename to cellxgene_gateway/prune_process_cache.py index 71bfcfa..f7240a0 100644 --- a/prune_process_cache.py +++ b/cellxgene_gateway/prune_process_cache.py @@ -11,7 +11,7 @@ import time import psutil -from util import current_time_stamp +from cellxgene_gateway.util import current_time_stamp class PruneProcessCache: diff --git a/static/css/homepagestyle.css b/cellxgene_gateway/static/css/homepagestyle.css similarity index 100% rename from static/css/homepagestyle.css rename to cellxgene_gateway/static/css/homepagestyle.css diff --git a/static/nibr.ico b/cellxgene_gateway/static/nibr.ico similarity index 100% rename from static/nibr.ico rename to cellxgene_gateway/static/nibr.ico diff --git a/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py similarity index 97% rename from subprocess_backend.py rename to cellxgene_gateway/subprocess_backend.py index 10aabdc..a49f6e1 100644 --- a/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -12,7 +12,7 @@ import subprocess from flask_api import status -from process_exception import ProcessException +from cellxgene_gateway.process_exception import ProcessException class SubprocessBackend: diff --git a/templates/cellxgene_error.html b/cellxgene_gateway/templates/cellxgene_error.html similarity index 100% rename from templates/cellxgene_error.html rename to cellxgene_gateway/templates/cellxgene_error.html diff --git a/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html similarity index 100% rename from templates/filecrawl.html rename to cellxgene_gateway/templates/filecrawl.html diff --git a/templates/index.html b/cellxgene_gateway/templates/index.html similarity index 99% rename from templates/index.html rename to cellxgene_gateway/templates/index.html index 7aef319..2fcfd23 100644 --- a/templates/index.html +++ b/cellxgene_gateway/templates/index.html @@ -39,7 +39,6 @@ File Crawler: Allows you to view all uploaded data. -

diff --git a/templates/loading.html b/cellxgene_gateway/templates/loading.html similarity index 100% rename from templates/loading.html rename to cellxgene_gateway/templates/loading.html diff --git a/templates/process_error.html b/cellxgene_gateway/templates/process_error.html similarity index 100% rename from templates/process_error.html rename to cellxgene_gateway/templates/process_error.html diff --git a/util.py b/cellxgene_gateway/util.py similarity index 100% rename from util.py rename to cellxgene_gateway/util.py diff --git a/environment-dev.yml b/environment-dev.yml new file mode 100644 index 0000000..f67bdda --- /dev/null +++ b/environment-dev.yml @@ -0,0 +1,11 @@ +name: cellxgene-dev +channels: +- conda-forge +dependencies: +- python=3.7 +- requests +- flask +- psutil +- pip: + - flask-api + - cellxgene diff --git a/requirements.txt b/requirements.txt index c89a503..19bb2ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ flask flask_api psutil requests - diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5381bee --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +import os +from setuptools import setup + + +def parse_requirements(): + reqs = [] + with open("requirements.txt", "r") as f: + for l in f.readlines(): + reqs.append(l.strip("\n")) + return reqs + + +install_reqs = parse_requirements() + +setup( + # mandatory + name="cellxgene-gateway", + # mandatory + version="0.1", + # mandatory + author="Niket Patel, Yohann Potier, Alok Saldanha", + author_email="alok.saldanha@novartis.com", + description=("Cell-by-gene Gateway"), + license="MIT", + keywords="visualization, genomics", + url="http://github.com/Novartis/cellxgene-gateway", + packages=["cellxgene_gateway"], + package_data={"": ["README.md", "LICENSE.txt"]}, + install_requires=install_reqs, + entry_points={ + "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"] + }, + classifiers=["Topic :: Scientific/Engineering :: Visualization"], +)