diff --git a/.gitignore b/.gitignore
index 24dd510..e1c196c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,138 @@
+# Custom
.DS_Store
__pycache__
*.pyc
run.sh
.cellxgene-gateway
+
+
+# Created by https://www.gitignore.io/api/python
+# Edit at https://www.gitignore.io/?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# End of https://www.gitignore.io/api/python
diff --git a/Readme.md b/Readme.md
index f0a9de2..9cb4179 100644
--- a/Readme.md
+++ b/Readme.md
@@ -1,50 +1,85 @@
-# Overview #
+# Overview
Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server.
-# Running locally #
+## Running locally
+
+We assume your current working directory is the directory into which you've cloned this repository.
0. This project requires python 3.6 or higher. Please check your version with
- python --version
+```bash
+$ python --version
+```
1. Set up a venv with
-```
+
+```bash
python -m venv .cellxgene-gateway
source .cellxgene-gateway/bin/activate
```
2. Install requirements with
-```
+
+```bash
pip install -r requirements.txt
```
-3. Prepare a folder with .h5ad files, for example
+3. Install the gateway:
+
+_To install in development mode:_
+
+```bash
+python setup.py develop
```
-mkdir cellxgene_data
+
+_To install from GitHub:_
+
+```bash
+pip install git+https://github.com/Novartis/cellxgene-gateway
+```
+
+_To install from PyPI:_
+
+```bash
+# NOT YET DONE, COMING! STAY TUNED
+```
+
+4. Prepare a folder with .h5ad files, for example
+
+```bash
+mkdir ../cellxgene_data
wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
```
-4. Copy run.sh.example to run.sh:
-```
-cp run.sh.example run.sh
-```
-`run.sh` defines various environment variables:
+5. Set your environment variables correctly:
-* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd'
-* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene
-* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data
-* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally
-* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
-
-The defaults should be fine if you set up a venv and cellxgene_data folder as above.
-
-5. Finally, execute run.sh:
-```
-source run.sh
+```bash
+export CELLXGENE_LOCATION=`which cellxgene`
+export CELLXGENE_DATA=../cellxgene_data # change this directory if you put data in a different place.
+export GATEWAY_HOST=localhost:5005
+export GATEWAY_PROTOCOL=http
+export GATEWAY_IP=127.0.0.1
```
-# Customization #
+6. Now, execute the cellxgene gateway:
+
+```bash
+cellxgene-gateway
+```
+
+For convenience, you can also change `run.sh.example` and execute it.
+
+Here's what the environment variables mean:
+
+* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
+* `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
+* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically `localhost:5005` if running locally
+* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
+
+The defaults should be fine if you set up a venv and cellxgene_data folder as above.
+
+# Customization
The current paradigm for customization is to modify files during a build or deployment phase:
@@ -55,29 +90,23 @@ The current paradigm for customization is to modify files during a build or depl
Currently we use a build.sh that copies the gateway to a "build" directory before modifying with sed and the like.
-# Development #
+# Development
-## Running Linters ##
+## Running Linters
pip install isort flake8 black
-```
+```bash
isort -rc .
-```
-
-```
flake8 .
+black -l 79 .
```
-```
-black .
-```
-
-# Getting Help #
+# Getting Help
If you need help for any reason, please make a github ticket. One of the contributors should help you out.
-# Contributors #
+# Contributors
* Niket Patel - https://github.com/NiketPatel9
* Alok Saldanha - https://github.com/alokito
diff --git a/__init__.py b/cellxgene_gateway/__init__.py
similarity index 99%
rename from __init__.py
rename to cellxgene_gateway/__init__.py
index 8605fcc..447b57c 100755
--- a/__init__.py
+++ b/cellxgene_gateway/__init__.py
@@ -6,4 +6,3 @@
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
-
diff --git a/backend_cache.py b/cellxgene_gateway/backend_cache.py
similarity index 84%
rename from backend_cache.py
rename to cellxgene_gateway/backend_cache.py
index 8293c80..08d0235 100644
--- a/backend_cache.py
+++ b/cellxgene_gateway/backend_cache.py
@@ -12,10 +12,10 @@ from threading import Thread
from flask_api import status
-import env
-from cache_entry import CacheEntry
-from cellxgene_exception import CellxgeneException
-from subprocess_backend import SubprocessBackend
+from cellxgene_gateway import env
+from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.subprocess_backend import SubprocessBackend
process_backend = SubprocessBackend()
@@ -51,7 +51,8 @@ class BackendCache:
entry = CacheEntry.for_dataset(dataset, file_path, port)
background_thread = Thread(
- target=process_backend.launch, args=(env.cellxgene_location, scripts, entry)
+ target=process_backend.launch,
+ args=(env.cellxgene_location, scripts, entry),
)
background_thread.start()
diff --git a/cache_entry.py b/cellxgene_gateway/cache_entry.py
similarity index 77%
rename from cache_entry.py
rename to cellxgene_gateway/cache_entry.py
index 6f9e297..f818040 100644
--- a/cache_entry.py
+++ b/cellxgene_gateway/cache_entry.py
@@ -9,9 +9,9 @@
from flask import make_response, request
from requests import get, post, put
-import env
-from cellxgene_exception import CellxgeneException
-from util import current_time_stamp
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.util import current_time_stamp
class CacheEntry:
@@ -92,14 +92,26 @@ class CacheEntry:
if "content-type" in request.headers:
headers["content-type"] = request.headers["content-type"]
- if request.method in ['GET', 'HEAD', 'OPTIONS']:
- cellxgene_response = get(cellxgene_basepath + subpath, headers=headers)
- elif request.method == 'PUT':
- cellxgene_response = put(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
- elif request.method == 'POST':
- cellxgene_response = post(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
+ if request.method in ["GET", "HEAD", "OPTIONS"]:
+ cellxgene_response = get(
+ cellxgene_basepath + subpath, headers=headers
+ )
+ elif request.method == "PUT":
+ cellxgene_response = put(
+ cellxgene_basepath + subpath,
+ headers=headers,
+ data=request.data.decode(),
+ )
+ elif request.method == "POST":
+ cellxgene_response = post(
+ cellxgene_basepath + subpath,
+ headers=headers,
+ data=request.data.decode(),
+ )
else:
- raise CellxgeneException(f"Unexpected method {request.method}", 400)
+ raise CellxgeneException(
+ f"Unexpected method {request.method}", 400
+ )
content_type = cellxgene_response.headers["content-type"]
if "text" in content_type:
cellxgene_content = cellxgene_response.content.decode()
@@ -108,11 +120,11 @@ class CacheEntry:
).replace(cellxgene_basepath, gateway_basepath)
else:
gateway_content = cellxgene_response.content
-
+
gateway_response = make_response(
- gateway_content,
+ gateway_content,
cellxgene_response.status_code,
- {"Content-Type": content_type }
+ {"Content-Type": content_type},
)
return gateway_response
diff --git a/cellxgene_exception.py b/cellxgene_gateway/cellxgene_exception.py
similarity index 99%
rename from cellxgene_exception.py
rename to cellxgene_gateway/cellxgene_exception.py
index ae645c3..d46d58c 100644
--- a/cellxgene_exception.py
+++ b/cellxgene_gateway/cellxgene_exception.py
@@ -7,6 +7,7 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
+
class CellxgeneException(Exception):
def __init__(self, message, http_status):
Exception.__init__(self)
diff --git a/dir_util.py b/cellxgene_gateway/dir_util.py
similarity index 90%
rename from dir_util.py
rename to cellxgene_gateway/dir_util.py
index 15d1370..d0a9af8 100644
--- a/dir_util.py
+++ b/cellxgene_gateway/dir_util.py
@@ -11,8 +11,8 @@ import os
from flask_api import status
-import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
def is_subdir(full_path, parent_path):
@@ -31,7 +31,8 @@ def create_dir(parent_path, dir_name):
)
elif not os.path.exists(parent_path):
raise CellxgeneException(
- "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST
+ "The selected User directory does not exist.",
+ status.HTTP_400_BAD_REQUEST,
)
elif os.path.exists(full_path):
raise CellxgeneException(
@@ -74,7 +75,8 @@ def recurse_dir(path):
}
else:
raise CellxgeneException(
- "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST
+ "Given path is neither file nor directory.",
+ status.HTTP_400_BAD_REQUEST,
)
return [make_entry(x) for x in os.listdir(path)]
diff --git a/env.py b/cellxgene_gateway/env.py
similarity index 55%
rename from env.py
rename to cellxgene_gateway/env.py
index 686bd4f..5967718 100644
--- a/env.py
+++ b/cellxgene_gateway/env.py
@@ -9,9 +9,35 @@
import os
-deployment_env = os.environ.get("DEPLOYMENT_ENV")
cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
cellxgene_data = os.environ.get("CELLXGENE_DATA")
gateway_host = os.environ.get("GATEWAY_HOST")
gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
ip = os.environ.get("GATEWAY_IP")
+
+env_vars = {
+ "CELLXGENE_LOCATION": cellxgene_location,
+ "CELLXGENE_DATA": cellxgene_data,
+ "GATEWAY_HOST": gateway_host,
+ "GATEWAY_PROTOCOL": gateway_protocol,
+ "GATEWAY_IP": ip,
+}
+
+if not all(env_vars.values()):
+ raise ValueError(
+ f"""
+Please ensure that environment variables are set correctly.
+The ones with None below are missing and need to be set.
+
+{env_vars}
+
+Set them at the terminal before running the gateway.
+An example is:
+
+ export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
+ export CELLXGENE_DATA=../cellxgene_data
+ export GATEWAY_HOST=localhost:5005
+ export GATEWAY_PROTOCOL=http
+ export GATEWAY_IP=127.0.0.1
+"""
+ )
diff --git a/extra_scripts.py b/cellxgene_gateway/extra_scripts.py
similarity index 99%
rename from extra_scripts.py
rename to cellxgene_gateway/extra_scripts.py
index 7c05b97..439c92e 100644
--- a/extra_scripts.py
+++ b/cellxgene_gateway/extra_scripts.py
@@ -7,6 +7,7 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
+
def get_extra_scripts():
# can be array of script tags to inject on every page, e.g. for google analytics could be
# ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
diff --git a/gateway.py b/cellxgene_gateway/gateway.py
similarity index 76%
rename from gateway.py
rename to cellxgene_gateway/gateway.py
index c92c35b..958a731 100644
--- a/gateway.py
+++ b/cellxgene_gateway/gateway.py
@@ -12,19 +12,25 @@ import datetime
import os
from threading import Thread
-from flask import Flask, redirect, render_template, request, send_from_directory
+from flask import (
+ Flask,
+ redirect,
+ render_template,
+ request,
+ send_from_directory,
+)
from flask_api import status
from werkzeug import secure_filename
-import env
-from backend_cache import BackendCache
-from cellxgene_exception import CellxgeneException
-from dir_util import create_dir, recurse_dir, render_entries
-from extra_scripts import get_extra_scripts
-from path_util import get_dataset, get_file_path
-from process_exception import ProcessException
-from prune_process_cache import PruneProcessCache
-from util import current_time_stamp
+from cellxgene_gateway import env
+from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries
+from cellxgene_gateway.extra_scripts import get_extra_scripts
+from cellxgene_gateway.path_util import get_dataset, get_file_path
+from cellxgene_gateway.process_exception import ProcessException
+from cellxgene_gateway.prune_process_cache import PruneProcessCache
+from cellxgene_gateway.util import current_time_stamp
app = Flask(__name__)
cache = BackendCache()
@@ -38,7 +44,9 @@ def handle_invalid_usage(error):
return (
render_template(
- "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message
+ "cellxgene_error.html",
+ extra_scripts=get_extra_scripts(),
+ message=message,
),
error.http_status,
)
@@ -56,7 +64,9 @@ def handle_invalid_process(error):
return (
render_template(
- "process_error.html", extra_scripts=get_extra_scripts(), message=message
+ "process_error.html",
+ extra_scripts=get_extra_scripts(),
+ message=message,
),
error.http_status,
)
@@ -116,7 +126,9 @@ def upload_file():
if "file" in request.files:
f = request.files["file"]
if f and f.filename.endswith(".h5ad"):
- f.save(full_upload_path + "/" + secure_filename(f.filename))
+ f.save(
+ full_upload_path + "/" + secure_filename(f.filename)
+ )
return redirect("/filecrawl.html", code=302)
else:
raise CellxgeneException(
@@ -125,10 +137,13 @@ def upload_file():
)
else:
raise CellxgeneException(
- "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST
+ "A file must be chosen to upload.",
+ status.HTTP_400_BAD_REQUEST,
)
else:
- raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST)
+ raise CellxgeneException(
+ "Invalid directory.", status.HTTP_400_BAD_REQUEST
+ )
return redirect(env.location, code=302)
@@ -139,7 +154,9 @@ def filecrawl():
entries = recurse_dir(env.cellxgene_data)
rendered_html = render_entries(entries)
return render_template(
- "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html
+ "filecrawl.html",
+ extra_scripts=get_extra_scripts(),
+ rendered_html=rendered_html,
)
@@ -166,8 +183,12 @@ def do_GET(path):
raise ProcessException.from_pid_object(match)
-if __name__ == "__main__":
+def main():
background_thread = Thread(target=PruneProcessCache(cache))
background_thread.start()
app.run(host="0.0.0.0", port=5005, debug=False)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/path_util.py b/cellxgene_gateway/path_util.py
similarity index 93%
rename from path_util.py
rename to cellxgene_gateway/path_util.py
index 0672184..6dc4fb6 100644
--- a/path_util.py
+++ b/cellxgene_gateway/path_util.py
@@ -11,8 +11,8 @@ import os
from flask_api import status
-import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
def get_dataset(path):
diff --git a/process_exception.py b/cellxgene_gateway/process_exception.py
similarity index 99%
rename from process_exception.py
rename to cellxgene_gateway/process_exception.py
index e54d842..5afdb00 100644
--- a/process_exception.py
+++ b/cellxgene_gateway/process_exception.py
@@ -7,6 +7,7 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
+
class ProcessException(Exception):
def __init__(self, message, stdout, stderr, http_status):
Exception.__init__(self)
diff --git a/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py
similarity index 96%
rename from prune_process_cache.py
rename to cellxgene_gateway/prune_process_cache.py
index 71bfcfa..f7240a0 100644
--- a/prune_process_cache.py
+++ b/cellxgene_gateway/prune_process_cache.py
@@ -11,7 +11,7 @@ import time
import psutil
-from util import current_time_stamp
+from cellxgene_gateway.util import current_time_stamp
class PruneProcessCache:
diff --git a/static/css/homepagestyle.css b/cellxgene_gateway/static/css/homepagestyle.css
similarity index 100%
rename from static/css/homepagestyle.css
rename to cellxgene_gateway/static/css/homepagestyle.css
diff --git a/static/nibr.ico b/cellxgene_gateway/static/nibr.ico
similarity index 100%
rename from static/nibr.ico
rename to cellxgene_gateway/static/nibr.ico
diff --git a/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py
similarity index 97%
rename from subprocess_backend.py
rename to cellxgene_gateway/subprocess_backend.py
index 10aabdc..a49f6e1 100644
--- a/subprocess_backend.py
+++ b/cellxgene_gateway/subprocess_backend.py
@@ -12,7 +12,7 @@ import subprocess
from flask_api import status
-from process_exception import ProcessException
+from cellxgene_gateway.process_exception import ProcessException
class SubprocessBackend:
diff --git a/templates/cellxgene_error.html b/cellxgene_gateway/templates/cellxgene_error.html
similarity index 100%
rename from templates/cellxgene_error.html
rename to cellxgene_gateway/templates/cellxgene_error.html
diff --git a/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html
similarity index 100%
rename from templates/filecrawl.html
rename to cellxgene_gateway/templates/filecrawl.html
diff --git a/templates/index.html b/cellxgene_gateway/templates/index.html
similarity index 99%
rename from templates/index.html
rename to cellxgene_gateway/templates/index.html
index 7aef319..2fcfd23 100644
--- a/templates/index.html
+++ b/cellxgene_gateway/templates/index.html
@@ -39,7 +39,6 @@
File Crawler: Allows you to view all uploaded data.
-