From 952cb1e8e1d651a337170ea679dfe8c76de18ed8 Mon Sep 17 00:00:00 2001 From: Eric Ma Date: Thu, 5 Sep 2019 11:54:34 -0400 Subject: [PATCH 1/6] refactor to allow use with console entry points --- Readme.md | 44 ++++++++------ cellxgene_gateway.egg-info/PKG-INFO | 12 ++++ cellxgene_gateway.egg-info/SOURCES.txt | 20 +++++++ .../dependency_links.txt | 1 + cellxgene_gateway.egg-info/entry_points.txt | 3 + cellxgene_gateway.egg-info/requires.txt | 5 ++ cellxgene_gateway.egg-info/top_level.txt | 1 + __init__.py => cellxgene_gateway/__init__.py | 1 - .../backend_cache.py | 11 ++-- .../cache_entry.py | 38 +++++++----- .../cellxgene_exception.py | 1 + dir_util.py => cellxgene_gateway/dir_util.py | 10 ++-- env.py => cellxgene_gateway/env.py | 29 +++++++++ .../extra_scripts.py | 1 + gateway.py => cellxgene_gateway/gateway.py | 55 ++++++++++++------ .../path_util.py | 4 +- .../process_exception.py | 1 + .../prune_process_cache.py | 2 +- .../static}/css/homepagestyle.css | 0 {static => cellxgene_gateway/static}/nibr.ico | Bin .../subprocess_backend.py | 2 +- .../templates}/cellxgene_error.html | 0 .../templates}/filecrawl.html | 0 .../templates}/index.html | 1 - .../templates}/loading.html | 0 .../templates}/process_error.html | 0 util.py => cellxgene_gateway/util.py | 0 environment-dev.yml | 11 ++++ requirements.txt | 1 - setup.py | 34 +++++++++++ 30 files changed, 224 insertions(+), 64 deletions(-) create mode 100644 cellxgene_gateway.egg-info/PKG-INFO create mode 100644 cellxgene_gateway.egg-info/SOURCES.txt create mode 100644 cellxgene_gateway.egg-info/dependency_links.txt create mode 100644 cellxgene_gateway.egg-info/entry_points.txt create mode 100644 cellxgene_gateway.egg-info/requires.txt create mode 100644 cellxgene_gateway.egg-info/top_level.txt rename __init__.py => cellxgene_gateway/__init__.py (99%) rename backend_cache.py => cellxgene_gateway/backend_cache.py (84%) rename cache_entry.py => cellxgene_gateway/cache_entry.py (77%) rename cellxgene_exception.py => cellxgene_gateway/cellxgene_exception.py (99%) rename dir_util.py => cellxgene_gateway/dir_util.py (90%) rename env.py => cellxgene_gateway/env.py (54%) rename extra_scripts.py => cellxgene_gateway/extra_scripts.py (99%) rename gateway.py => cellxgene_gateway/gateway.py (76%) rename path_util.py => cellxgene_gateway/path_util.py (93%) rename process_exception.py => cellxgene_gateway/process_exception.py (99%) rename prune_process_cache.py => cellxgene_gateway/prune_process_cache.py (96%) rename {static => cellxgene_gateway/static}/css/homepagestyle.css (100%) rename {static => cellxgene_gateway/static}/nibr.ico (100%) rename subprocess_backend.py => cellxgene_gateway/subprocess_backend.py (97%) rename {templates => cellxgene_gateway/templates}/cellxgene_error.html (100%) rename {templates => cellxgene_gateway/templates}/filecrawl.html (100%) rename {templates => cellxgene_gateway/templates}/index.html (99%) rename {templates => cellxgene_gateway/templates}/loading.html (100%) rename {templates => cellxgene_gateway/templates}/process_error.html (100%) rename util.py => cellxgene_gateway/util.py (100%) create mode 100644 environment-dev.yml create mode 100644 setup.py diff --git a/Readme.md b/Readme.md index f0a9de2..7ee0843 100644 --- a/Readme.md +++ b/Readme.md @@ -1,50 +1,58 @@ -# Overview # +# Overview Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server. -# Running locally # +## Running locally 0. This project requires python 3.6 or higher. Please check your version with - python --version +```bash +$ python --version +``` 1. Set up a venv with -``` + +```bash python -m venv .cellxgene-gateway source .cellxgene-gateway/bin/activate ``` -2. Install requirements with -``` +1. Install requirements with + +```bash pip install -r requirements.txt ``` -3. Prepare a folder with .h5ad files, for example -``` -mkdir cellxgene_data +1. Prepare a folder with .h5ad files, for example + +```bash +mkdir ../cellxgene_data wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad ``` -4. Copy run.sh.example to run.sh: -``` +1. Copy run.sh.example to run.sh: + +```bash cp run.sh.example run.sh ``` + `run.sh` defines various environment variables: -* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd' -* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene -* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data -* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally -* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. +* `DEPLOYMENT_ENV` - expects 'dev', 'tst' or 'prd' +* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene +* `CELLXGENE_DATA` - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data +* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically localhost:5005 if running locally +* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. The defaults should be fine if you set up a venv and cellxgene_data folder as above. -5. Finally, execute run.sh: +1. Finally, execute run.sh: + ``` source run.sh ``` -# Customization # +# Customization The current paradigm for customization is to modify files during a build or deployment phase: diff --git a/cellxgene_gateway.egg-info/PKG-INFO b/cellxgene_gateway.egg-info/PKG-INFO new file mode 100644 index 0000000..7c8dfff --- /dev/null +++ b/cellxgene_gateway.egg-info/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 1.1 +Name: cellxgene-gateway +Version: 0.1 +Summary: Cell-by-gene Gateway +Home-page: http://github.com/Novartis/cellxgene-gateway +Author: Niket Patel, Yohann Potier, Alok Saldanha +Author-email: alok.saldanha@novartis.com +License: MIT +Description: UNKNOWN +Keywords: visualization,genomics +Platform: UNKNOWN +Classifier: Topic :: Scientific/Engineering :: Visualization diff --git a/cellxgene_gateway.egg-info/SOURCES.txt b/cellxgene_gateway.egg-info/SOURCES.txt new file mode 100644 index 0000000..5c52134 --- /dev/null +++ b/cellxgene_gateway.egg-info/SOURCES.txt @@ -0,0 +1,20 @@ +setup.py +cellxgene_gateway/__init__.py +cellxgene_gateway/backend_cache.py +cellxgene_gateway/cache_entry.py +cellxgene_gateway/cellxgene_exception.py +cellxgene_gateway/dir_util.py +cellxgene_gateway/env.py +cellxgene_gateway/extra_scripts.py +cellxgene_gateway/gateway.py +cellxgene_gateway/path_util.py +cellxgene_gateway/process_exception.py +cellxgene_gateway/prune_process_cache.py +cellxgene_gateway/subprocess_backend.py +cellxgene_gateway/util.py +cellxgene_gateway.egg-info/PKG-INFO +cellxgene_gateway.egg-info/SOURCES.txt +cellxgene_gateway.egg-info/dependency_links.txt +cellxgene_gateway.egg-info/entry_points.txt +cellxgene_gateway.egg-info/requires.txt +cellxgene_gateway.egg-info/top_level.txt \ No newline at end of file diff --git a/cellxgene_gateway.egg-info/dependency_links.txt b/cellxgene_gateway.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/cellxgene_gateway.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/cellxgene_gateway.egg-info/entry_points.txt b/cellxgene_gateway.egg-info/entry_points.txt new file mode 100644 index 0000000..6bb9dea --- /dev/null +++ b/cellxgene_gateway.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +cellxgene-gateway = cellxgene_gateway.gateway:main + diff --git a/cellxgene_gateway.egg-info/requires.txt b/cellxgene_gateway.egg-info/requires.txt new file mode 100644 index 0000000..19bb2ef --- /dev/null +++ b/cellxgene_gateway.egg-info/requires.txt @@ -0,0 +1,5 @@ +cellxgene +flask +flask_api +psutil +requests diff --git a/cellxgene_gateway.egg-info/top_level.txt b/cellxgene_gateway.egg-info/top_level.txt new file mode 100644 index 0000000..0224ebb --- /dev/null +++ b/cellxgene_gateway.egg-info/top_level.txt @@ -0,0 +1 @@ +cellxgene_gateway diff --git a/__init__.py b/cellxgene_gateway/__init__.py similarity index 99% rename from __init__.py rename to cellxgene_gateway/__init__.py index 8605fcc..447b57c 100755 --- a/__init__.py +++ b/cellxgene_gateway/__init__.py @@ -6,4 +6,3 @@ # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. - diff --git a/backend_cache.py b/cellxgene_gateway/backend_cache.py similarity index 84% rename from backend_cache.py rename to cellxgene_gateway/backend_cache.py index 8293c80..08d0235 100644 --- a/backend_cache.py +++ b/cellxgene_gateway/backend_cache.py @@ -12,10 +12,10 @@ from threading import Thread from flask_api import status -import env -from cache_entry import CacheEntry -from cellxgene_exception import CellxgeneException -from subprocess_backend import SubprocessBackend +from cellxgene_gateway import env +from cellxgene_gateway.cache_entry import CacheEntry +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.subprocess_backend import SubprocessBackend process_backend = SubprocessBackend() @@ -51,7 +51,8 @@ class BackendCache: entry = CacheEntry.for_dataset(dataset, file_path, port) background_thread = Thread( - target=process_backend.launch, args=(env.cellxgene_location, scripts, entry) + target=process_backend.launch, + args=(env.cellxgene_location, scripts, entry), ) background_thread.start() diff --git a/cache_entry.py b/cellxgene_gateway/cache_entry.py similarity index 77% rename from cache_entry.py rename to cellxgene_gateway/cache_entry.py index 6f9e297..f818040 100644 --- a/cache_entry.py +++ b/cellxgene_gateway/cache_entry.py @@ -9,9 +9,9 @@ from flask import make_response, request from requests import get, post, put -import env -from cellxgene_exception import CellxgeneException -from util import current_time_stamp +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.util import current_time_stamp class CacheEntry: @@ -92,14 +92,26 @@ class CacheEntry: if "content-type" in request.headers: headers["content-type"] = request.headers["content-type"] - if request.method in ['GET', 'HEAD', 'OPTIONS']: - cellxgene_response = get(cellxgene_basepath + subpath, headers=headers) - elif request.method == 'PUT': - cellxgene_response = put(cellxgene_basepath + subpath, headers=headers, data=request.data.decode()) - elif request.method == 'POST': - cellxgene_response = post(cellxgene_basepath + subpath, headers=headers, data=request.data.decode()) + if request.method in ["GET", "HEAD", "OPTIONS"]: + cellxgene_response = get( + cellxgene_basepath + subpath, headers=headers + ) + elif request.method == "PUT": + cellxgene_response = put( + cellxgene_basepath + subpath, + headers=headers, + data=request.data.decode(), + ) + elif request.method == "POST": + cellxgene_response = post( + cellxgene_basepath + subpath, + headers=headers, + data=request.data.decode(), + ) else: - raise CellxgeneException(f"Unexpected method {request.method}", 400) + raise CellxgeneException( + f"Unexpected method {request.method}", 400 + ) content_type = cellxgene_response.headers["content-type"] if "text" in content_type: cellxgene_content = cellxgene_response.content.decode() @@ -108,11 +120,11 @@ class CacheEntry: ).replace(cellxgene_basepath, gateway_basepath) else: gateway_content = cellxgene_response.content - + gateway_response = make_response( - gateway_content, + gateway_content, cellxgene_response.status_code, - {"Content-Type": content_type } + {"Content-Type": content_type}, ) return gateway_response diff --git a/cellxgene_exception.py b/cellxgene_gateway/cellxgene_exception.py similarity index 99% rename from cellxgene_exception.py rename to cellxgene_gateway/cellxgene_exception.py index ae645c3..d46d58c 100644 --- a/cellxgene_exception.py +++ b/cellxgene_gateway/cellxgene_exception.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + class CellxgeneException(Exception): def __init__(self, message, http_status): Exception.__init__(self) diff --git a/dir_util.py b/cellxgene_gateway/dir_util.py similarity index 90% rename from dir_util.py rename to cellxgene_gateway/dir_util.py index 15d1370..d0a9af8 100644 --- a/dir_util.py +++ b/cellxgene_gateway/dir_util.py @@ -11,8 +11,8 @@ import os from flask_api import status -import env -from cellxgene_exception import CellxgeneException +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException def is_subdir(full_path, parent_path): @@ -31,7 +31,8 @@ def create_dir(parent_path, dir_name): ) elif not os.path.exists(parent_path): raise CellxgeneException( - "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST + "The selected User directory does not exist.", + status.HTTP_400_BAD_REQUEST, ) elif os.path.exists(full_path): raise CellxgeneException( @@ -74,7 +75,8 @@ def recurse_dir(path): } else: raise CellxgeneException( - "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST + "Given path is neither file nor directory.", + status.HTTP_400_BAD_REQUEST, ) return [make_entry(x) for x in os.listdir(path)] diff --git a/env.py b/cellxgene_gateway/env.py similarity index 54% rename from env.py rename to cellxgene_gateway/env.py index 686bd4f..98bb912 100644 --- a/env.py +++ b/cellxgene_gateway/env.py @@ -15,3 +15,32 @@ cellxgene_data = os.environ.get("CELLXGENE_DATA") gateway_host = os.environ.get("GATEWAY_HOST") gateway_protocol = os.environ.get("GATEWAY_PROTOCOL") ip = os.environ.get("GATEWAY_IP") + +env_vars = { + "DEPLOYMENT_ENV": deployment_env, + "CELLXGENE_LOCATION": cellxgene_location, + "CELLXGENE_DATA": cellxgene_data, + "GATEWAY_HOST": gateway_host, + "GATEWAY_PROTOCOL": gateway_protocol, + "GATEWAY_IP": ip, +} + +if not all(env_vars.values()): + raise ValueError( + f""" +Please ensure that environment variables are set correctly. +The ones with None below are missing and need to be set. + +{env_vars} + +Set them at the terminal before running the gateway. +An example is: + + export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene + export CELLXGENE_DATA=../cellxgene_data + export DEPLOYMENT_ENV=dev + export GATEWAY_HOST=localhost:5005 + export GATEWAY_PROTOCOL=http + export GATEWAY_IP=127.0.0.1 +""" + ) diff --git a/extra_scripts.py b/cellxgene_gateway/extra_scripts.py similarity index 99% rename from extra_scripts.py rename to cellxgene_gateway/extra_scripts.py index 7c05b97..439c92e 100644 --- a/extra_scripts.py +++ b/cellxgene_gateway/extra_scripts.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + def get_extra_scripts(): # can be array of script tags to inject on every page, e.g. for google analytics could be # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2', diff --git a/gateway.py b/cellxgene_gateway/gateway.py similarity index 76% rename from gateway.py rename to cellxgene_gateway/gateway.py index c92c35b..958a731 100644 --- a/gateway.py +++ b/cellxgene_gateway/gateway.py @@ -12,19 +12,25 @@ import datetime import os from threading import Thread -from flask import Flask, redirect, render_template, request, send_from_directory +from flask import ( + Flask, + redirect, + render_template, + request, + send_from_directory, +) from flask_api import status from werkzeug import secure_filename -import env -from backend_cache import BackendCache -from cellxgene_exception import CellxgeneException -from dir_util import create_dir, recurse_dir, render_entries -from extra_scripts import get_extra_scripts -from path_util import get_dataset, get_file_path -from process_exception import ProcessException -from prune_process_cache import PruneProcessCache -from util import current_time_stamp +from cellxgene_gateway import env +from cellxgene_gateway.backend_cache import BackendCache +from cellxgene_gateway.cellxgene_exception import CellxgeneException +from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries +from cellxgene_gateway.extra_scripts import get_extra_scripts +from cellxgene_gateway.path_util import get_dataset, get_file_path +from cellxgene_gateway.process_exception import ProcessException +from cellxgene_gateway.prune_process_cache import PruneProcessCache +from cellxgene_gateway.util import current_time_stamp app = Flask(__name__) cache = BackendCache() @@ -38,7 +44,9 @@ def handle_invalid_usage(error): return ( render_template( - "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message + "cellxgene_error.html", + extra_scripts=get_extra_scripts(), + message=message, ), error.http_status, ) @@ -56,7 +64,9 @@ def handle_invalid_process(error): return ( render_template( - "process_error.html", extra_scripts=get_extra_scripts(), message=message + "process_error.html", + extra_scripts=get_extra_scripts(), + message=message, ), error.http_status, ) @@ -116,7 +126,9 @@ def upload_file(): if "file" in request.files: f = request.files["file"] if f and f.filename.endswith(".h5ad"): - f.save(full_upload_path + "/" + secure_filename(f.filename)) + f.save( + full_upload_path + "/" + secure_filename(f.filename) + ) return redirect("/filecrawl.html", code=302) else: raise CellxgeneException( @@ -125,10 +137,13 @@ def upload_file(): ) else: raise CellxgeneException( - "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST + "A file must be chosen to upload.", + status.HTTP_400_BAD_REQUEST, ) else: - raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST) + raise CellxgeneException( + "Invalid directory.", status.HTTP_400_BAD_REQUEST + ) return redirect(env.location, code=302) @@ -139,7 +154,9 @@ def filecrawl(): entries = recurse_dir(env.cellxgene_data) rendered_html = render_entries(entries) return render_template( - "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html + "filecrawl.html", + extra_scripts=get_extra_scripts(), + rendered_html=rendered_html, ) @@ -166,8 +183,12 @@ def do_GET(path): raise ProcessException.from_pid_object(match) -if __name__ == "__main__": +def main(): background_thread = Thread(target=PruneProcessCache(cache)) background_thread.start() app.run(host="0.0.0.0", port=5005, debug=False) + + +if __name__ == "__main__": + main() diff --git a/path_util.py b/cellxgene_gateway/path_util.py similarity index 93% rename from path_util.py rename to cellxgene_gateway/path_util.py index 0672184..6dc4fb6 100644 --- a/path_util.py +++ b/cellxgene_gateway/path_util.py @@ -11,8 +11,8 @@ import os from flask_api import status -import env -from cellxgene_exception import CellxgeneException +from cellxgene_gateway import env +from cellxgene_gateway.cellxgene_exception import CellxgeneException def get_dataset(path): diff --git a/process_exception.py b/cellxgene_gateway/process_exception.py similarity index 99% rename from process_exception.py rename to cellxgene_gateway/process_exception.py index e54d842..5afdb00 100644 --- a/process_exception.py +++ b/cellxgene_gateway/process_exception.py @@ -7,6 +7,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the License for # the specific language governing permissions and limitations under the License. + class ProcessException(Exception): def __init__(self, message, stdout, stderr, http_status): Exception.__init__(self) diff --git a/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py similarity index 96% rename from prune_process_cache.py rename to cellxgene_gateway/prune_process_cache.py index 71bfcfa..f7240a0 100644 --- a/prune_process_cache.py +++ b/cellxgene_gateway/prune_process_cache.py @@ -11,7 +11,7 @@ import time import psutil -from util import current_time_stamp +from cellxgene_gateway.util import current_time_stamp class PruneProcessCache: diff --git a/static/css/homepagestyle.css b/cellxgene_gateway/static/css/homepagestyle.css similarity index 100% rename from static/css/homepagestyle.css rename to cellxgene_gateway/static/css/homepagestyle.css diff --git a/static/nibr.ico b/cellxgene_gateway/static/nibr.ico similarity index 100% rename from static/nibr.ico rename to cellxgene_gateway/static/nibr.ico diff --git a/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py similarity index 97% rename from subprocess_backend.py rename to cellxgene_gateway/subprocess_backend.py index 10aabdc..a49f6e1 100644 --- a/subprocess_backend.py +++ b/cellxgene_gateway/subprocess_backend.py @@ -12,7 +12,7 @@ import subprocess from flask_api import status -from process_exception import ProcessException +from cellxgene_gateway.process_exception import ProcessException class SubprocessBackend: diff --git a/templates/cellxgene_error.html b/cellxgene_gateway/templates/cellxgene_error.html similarity index 100% rename from templates/cellxgene_error.html rename to cellxgene_gateway/templates/cellxgene_error.html diff --git a/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html similarity index 100% rename from templates/filecrawl.html rename to cellxgene_gateway/templates/filecrawl.html diff --git a/templates/index.html b/cellxgene_gateway/templates/index.html similarity index 99% rename from templates/index.html rename to cellxgene_gateway/templates/index.html index 7aef319..2fcfd23 100644 --- a/templates/index.html +++ b/cellxgene_gateway/templates/index.html @@ -39,7 +39,6 @@ File Crawler: Allows you to view all uploaded data. -

diff --git a/templates/loading.html b/cellxgene_gateway/templates/loading.html similarity index 100% rename from templates/loading.html rename to cellxgene_gateway/templates/loading.html diff --git a/templates/process_error.html b/cellxgene_gateway/templates/process_error.html similarity index 100% rename from templates/process_error.html rename to cellxgene_gateway/templates/process_error.html diff --git a/util.py b/cellxgene_gateway/util.py similarity index 100% rename from util.py rename to cellxgene_gateway/util.py diff --git a/environment-dev.yml b/environment-dev.yml new file mode 100644 index 0000000..f67bdda --- /dev/null +++ b/environment-dev.yml @@ -0,0 +1,11 @@ +name: cellxgene-dev +channels: +- conda-forge +dependencies: +- python=3.7 +- requests +- flask +- psutil +- pip: + - flask-api + - cellxgene diff --git a/requirements.txt b/requirements.txt index c89a503..19bb2ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ flask flask_api psutil requests - diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5381bee --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +import os +from setuptools import setup + + +def parse_requirements(): + reqs = [] + with open("requirements.txt", "r") as f: + for l in f.readlines(): + reqs.append(l.strip("\n")) + return reqs + + +install_reqs = parse_requirements() + +setup( + # mandatory + name="cellxgene-gateway", + # mandatory + version="0.1", + # mandatory + author="Niket Patel, Yohann Potier, Alok Saldanha", + author_email="alok.saldanha@novartis.com", + description=("Cell-by-gene Gateway"), + license="MIT", + keywords="visualization, genomics", + url="http://github.com/Novartis/cellxgene-gateway", + packages=["cellxgene_gateway"], + package_data={"": ["README.md", "LICENSE.txt"]}, + install_requires=install_reqs, + entry_points={ + "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"] + }, + classifiers=["Topic :: Scientific/Engineering :: Visualization"], +) From f2a8f44f3887fc34ea0e41176d98f02cdaacd067 Mon Sep 17 00:00:00 2001 From: Eric Ma Date: Thu, 5 Sep 2019 11:55:44 -0400 Subject: [PATCH 2/6] update readme --- Readme.md | 72 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/Readme.md b/Readme.md index 7ee0843..08dd79f 100644 --- a/Readme.md +++ b/Readme.md @@ -4,6 +4,8 @@ Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zu ## Running locally +We assume your current working directory is the directory into which you've cloned this repository. + 0. This project requires python 3.6 or higher. Please check your version with ```bash @@ -23,6 +25,26 @@ source .cellxgene-gateway/bin/activate pip install -r requirements.txt ``` +1. Install the gateway: + +_To install in development mode:_ + +```bash +python setup.py develop +``` + +_To install from GitHub:_ + +```bash +pip install git+https://github.com/Novartis/cellxgene-gateway +``` + +_To install from PyPI:_ + +```bash +# NOT YET DONE, COMING! STAY TUNED +``` + 1. Prepare a folder with .h5ad files, for example ```bash @@ -30,27 +52,33 @@ mkdir ../cellxgene_data wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad ``` -1. Copy run.sh.example to run.sh: +1. Set your environment variables correctly: ```bash -cp run.sh.example run.sh +export CELLXGENE_LOCATION=`which cellxgene` +export CELLXGENE_DATA=../cellxgene_data # change this directory if you put data in a different place. +export GATEWAY_HOST=localhost:5005 +export GATEWAY_PROTOCOL=http +export GATEWAY_IP=127.0.0.1 ``` -`run.sh` defines various environment variables: +1. Now, execute the cellxgene gateway: -* `DEPLOYMENT_ENV` - expects 'dev', 'tst' or 'prd' -* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene -* `CELLXGENE_DATA` - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data -* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically localhost:5005 if running locally +```bash +cellxgene-gateway +``` + +For convenience, you can also change `run.sh.example` and execute it. + +Here's what the environment variables mean: + +* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene` +* `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data` +* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically `localhost:5005` if running locally * `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. -The defaults should be fine if you set up a venv and cellxgene_data folder as above. +The defaults should be fine if you set up a venv and cellxgene_data folder as above. -1. Finally, execute run.sh: - -``` -source run.sh -``` # Customization @@ -63,29 +91,23 @@ The current paradigm for customization is to modify files during a build or depl Currently we use a build.sh that copies the gateway to a "build" directory before modifying with sed and the like. -# Development # +# Development -## Running Linters ## +## Running Linters pip install isort flake8 black -``` +```bash isort -rc . -``` - -``` flake8 . +black -l 79 . ``` -``` -black . -``` - -# Getting Help # +# Getting Help If you need help for any reason, please make a github ticket. One of the contributors should help you out. -# Contributors # +# Contributors * Niket Patel - https://github.com/NiketPatel9 * Alok Saldanha - https://github.com/alokito From f0de7fbfc3d5982bec29a31a088c95fd167aeded Mon Sep 17 00:00:00 2001 From: Eric Ma Date: Thu, 5 Sep 2019 11:55:44 -0400 Subject: [PATCH 3/6] update readme --- Readme.md | 72 ++++++++++++------- cellxgene_gateway.egg-info/PKG-INFO | 12 ---- cellxgene_gateway.egg-info/SOURCES.txt | 20 ------ .../dependency_links.txt | 1 - cellxgene_gateway.egg-info/entry_points.txt | 3 - cellxgene_gateway.egg-info/requires.txt | 5 -- cellxgene_gateway.egg-info/top_level.txt | 1 - 7 files changed, 47 insertions(+), 67 deletions(-) delete mode 100644 cellxgene_gateway.egg-info/PKG-INFO delete mode 100644 cellxgene_gateway.egg-info/SOURCES.txt delete mode 100644 cellxgene_gateway.egg-info/dependency_links.txt delete mode 100644 cellxgene_gateway.egg-info/entry_points.txt delete mode 100644 cellxgene_gateway.egg-info/requires.txt delete mode 100644 cellxgene_gateway.egg-info/top_level.txt diff --git a/Readme.md b/Readme.md index 7ee0843..08dd79f 100644 --- a/Readme.md +++ b/Readme.md @@ -4,6 +4,8 @@ Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zu ## Running locally +We assume your current working directory is the directory into which you've cloned this repository. + 0. This project requires python 3.6 or higher. Please check your version with ```bash @@ -23,6 +25,26 @@ source .cellxgene-gateway/bin/activate pip install -r requirements.txt ``` +1. Install the gateway: + +_To install in development mode:_ + +```bash +python setup.py develop +``` + +_To install from GitHub:_ + +```bash +pip install git+https://github.com/Novartis/cellxgene-gateway +``` + +_To install from PyPI:_ + +```bash +# NOT YET DONE, COMING! STAY TUNED +``` + 1. Prepare a folder with .h5ad files, for example ```bash @@ -30,27 +52,33 @@ mkdir ../cellxgene_data wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad ``` -1. Copy run.sh.example to run.sh: +1. Set your environment variables correctly: ```bash -cp run.sh.example run.sh +export CELLXGENE_LOCATION=`which cellxgene` +export CELLXGENE_DATA=../cellxgene_data # change this directory if you put data in a different place. +export GATEWAY_HOST=localhost:5005 +export GATEWAY_PROTOCOL=http +export GATEWAY_IP=127.0.0.1 ``` -`run.sh` defines various environment variables: +1. Now, execute the cellxgene gateway: -* `DEPLOYMENT_ENV` - expects 'dev', 'tst' or 'prd' -* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene -* `CELLXGENE_DATA` - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data -* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically localhost:5005 if running locally +```bash +cellxgene-gateway +``` + +For convenience, you can also change `run.sh.example` and execute it. + +Here's what the environment variables mean: + +* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene` +* `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data` +* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically `localhost:5005` if running locally * `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. -The defaults should be fine if you set up a venv and cellxgene_data folder as above. +The defaults should be fine if you set up a venv and cellxgene_data folder as above. -1. Finally, execute run.sh: - -``` -source run.sh -``` # Customization @@ -63,29 +91,23 @@ The current paradigm for customization is to modify files during a build or depl Currently we use a build.sh that copies the gateway to a "build" directory before modifying with sed and the like. -# Development # +# Development -## Running Linters ## +## Running Linters pip install isort flake8 black -``` +```bash isort -rc . -``` - -``` flake8 . +black -l 79 . ``` -``` -black . -``` - -# Getting Help # +# Getting Help If you need help for any reason, please make a github ticket. One of the contributors should help you out. -# Contributors # +# Contributors * Niket Patel - https://github.com/NiketPatel9 * Alok Saldanha - https://github.com/alokito diff --git a/cellxgene_gateway.egg-info/PKG-INFO b/cellxgene_gateway.egg-info/PKG-INFO deleted file mode 100644 index 7c8dfff..0000000 --- a/cellxgene_gateway.egg-info/PKG-INFO +++ /dev/null @@ -1,12 +0,0 @@ -Metadata-Version: 1.1 -Name: cellxgene-gateway -Version: 0.1 -Summary: Cell-by-gene Gateway -Home-page: http://github.com/Novartis/cellxgene-gateway -Author: Niket Patel, Yohann Potier, Alok Saldanha -Author-email: alok.saldanha@novartis.com -License: MIT -Description: UNKNOWN -Keywords: visualization,genomics -Platform: UNKNOWN -Classifier: Topic :: Scientific/Engineering :: Visualization diff --git a/cellxgene_gateway.egg-info/SOURCES.txt b/cellxgene_gateway.egg-info/SOURCES.txt deleted file mode 100644 index 5c52134..0000000 --- a/cellxgene_gateway.egg-info/SOURCES.txt +++ /dev/null @@ -1,20 +0,0 @@ -setup.py -cellxgene_gateway/__init__.py -cellxgene_gateway/backend_cache.py -cellxgene_gateway/cache_entry.py -cellxgene_gateway/cellxgene_exception.py -cellxgene_gateway/dir_util.py -cellxgene_gateway/env.py -cellxgene_gateway/extra_scripts.py -cellxgene_gateway/gateway.py -cellxgene_gateway/path_util.py -cellxgene_gateway/process_exception.py -cellxgene_gateway/prune_process_cache.py -cellxgene_gateway/subprocess_backend.py -cellxgene_gateway/util.py -cellxgene_gateway.egg-info/PKG-INFO -cellxgene_gateway.egg-info/SOURCES.txt -cellxgene_gateway.egg-info/dependency_links.txt -cellxgene_gateway.egg-info/entry_points.txt -cellxgene_gateway.egg-info/requires.txt -cellxgene_gateway.egg-info/top_level.txt \ No newline at end of file diff --git a/cellxgene_gateway.egg-info/dependency_links.txt b/cellxgene_gateway.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/cellxgene_gateway.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/cellxgene_gateway.egg-info/entry_points.txt b/cellxgene_gateway.egg-info/entry_points.txt deleted file mode 100644 index 6bb9dea..0000000 --- a/cellxgene_gateway.egg-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -cellxgene-gateway = cellxgene_gateway.gateway:main - diff --git a/cellxgene_gateway.egg-info/requires.txt b/cellxgene_gateway.egg-info/requires.txt deleted file mode 100644 index 19bb2ef..0000000 --- a/cellxgene_gateway.egg-info/requires.txt +++ /dev/null @@ -1,5 +0,0 @@ -cellxgene -flask -flask_api -psutil -requests diff --git a/cellxgene_gateway.egg-info/top_level.txt b/cellxgene_gateway.egg-info/top_level.txt deleted file mode 100644 index 0224ebb..0000000 --- a/cellxgene_gateway.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -cellxgene_gateway From 985634a974845da2272b711bd85c7e3180072e22 Mon Sep 17 00:00:00 2001 From: Eric Ma Date: Thu, 5 Sep 2019 21:36:59 -0400 Subject: [PATCH 4/6] Added standard gitignore from Python --- .gitignore | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/.gitignore b/.gitignore index 24dd510..e1c196c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,138 @@ +# Custom .DS_Store __pycache__ *.pyc run.sh .cellxgene-gateway + + +# Created by https://www.gitignore.io/api/python +# Edit at https://www.gitignore.io/?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# End of https://www.gitignore.io/api/python From e43b39687746d3bf71ec4790ea705ad465939f07 Mon Sep 17 00:00:00 2001 From: Eric Ma Date: Fri, 6 Sep 2019 13:58:39 -0400 Subject: [PATCH 5/6] fix numbering --- Readme.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Readme.md b/Readme.md index 08dd79f..9cb4179 100644 --- a/Readme.md +++ b/Readme.md @@ -19,13 +19,13 @@ python -m venv .cellxgene-gateway source .cellxgene-gateway/bin/activate ``` -1. Install requirements with +2. Install requirements with ```bash pip install -r requirements.txt ``` -1. Install the gateway: +3. Install the gateway: _To install in development mode:_ @@ -45,14 +45,14 @@ _To install from PyPI:_ # NOT YET DONE, COMING! STAY TUNED ``` -1. Prepare a folder with .h5ad files, for example +4. Prepare a folder with .h5ad files, for example ```bash mkdir ../cellxgene_data wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad ``` -1. Set your environment variables correctly: +5. Set your environment variables correctly: ```bash export CELLXGENE_LOCATION=`which cellxgene` @@ -62,7 +62,7 @@ export GATEWAY_PROTOCOL=http export GATEWAY_IP=127.0.0.1 ``` -1. Now, execute the cellxgene gateway: +6. Now, execute the cellxgene gateway: ```bash cellxgene-gateway @@ -79,7 +79,6 @@ Here's what the environment variables mean: The defaults should be fine if you set up a venv and cellxgene_data folder as above. - # Customization The current paradigm for customization is to modify files during a build or deployment phase: From 7e9611e3b60a18707d5db63da6777f05d37807c8 Mon Sep 17 00:00:00 2001 From: Eric Ma Date: Fri, 6 Sep 2019 13:59:52 -0400 Subject: [PATCH 6/6] take out deployemnt_env --- cellxgene_gateway/env.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cellxgene_gateway/env.py b/cellxgene_gateway/env.py index 98bb912..5967718 100644 --- a/cellxgene_gateway/env.py +++ b/cellxgene_gateway/env.py @@ -9,7 +9,6 @@ import os -deployment_env = os.environ.get("DEPLOYMENT_ENV") cellxgene_location = os.environ.get("CELLXGENE_LOCATION") cellxgene_data = os.environ.get("CELLXGENE_DATA") gateway_host = os.environ.get("GATEWAY_HOST") @@ -17,7 +16,6 @@ gateway_protocol = os.environ.get("GATEWAY_PROTOCOL") ip = os.environ.get("GATEWAY_IP") env_vars = { - "DEPLOYMENT_ENV": deployment_env, "CELLXGENE_LOCATION": cellxgene_location, "CELLXGENE_DATA": cellxgene_data, "GATEWAY_HOST": gateway_host, @@ -38,7 +36,6 @@ An example is: export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene export CELLXGENE_DATA=../cellxgene_data - export DEPLOYMENT_ENV=dev export GATEWAY_HOST=localhost:5005 export GATEWAY_PROTOCOL=http export GATEWAY_IP=127.0.0.1