Merge pull request #1 from ericmjl/conda

Package-ified

.gitignore

@@ -1,5 +1,138 @@
+# Custom
 .DS_Store
 __pycache__
 *.pyc
 run.sh
 .cellxgene-gateway
+
+
+# Created by https://www.gitignore.io/api/python
+# Edit at https://www.gitignore.io/?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# End of https://www.gitignore.io/api/python

Readme.md

@@ -1,50 +1,85 @@
-# Overview #
+# Overview
 
 Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available  proxies requests to that server.
 
-# Running locally #
+## Running locally
+
+We assume your current working directory is the directory into which you've cloned this repository.
 
 0. This project requires python 3.6 or higher. Please check your version with
 
-    python --version
+```bash
+$ python --version
+```
 
 1. Set up a venv with
-```
+
+```bash
 python -m venv .cellxgene-gateway
 source .cellxgene-gateway/bin/activate
 ```
 
 2. Install requirements with
-```
+
+```bash
 pip install -r requirements.txt
 ```
-3. Prepare a folder with .h5ad files, for example
 
+3. Install the gateway:
+
+_To install in development mode:_
+
+```bash
+python setup.py develop
 ```
-mkdir cellxgene_data
+
+_To install from GitHub:_
+
+```bash
+pip install git+https://github.com/Novartis/cellxgene-gateway
+```
+
+_To install from PyPI:_
+
+```bash
+# NOT YET DONE, COMING! STAY TUNED
+```
+
+4. Prepare a folder with .h5ad files, for example
+
+```bash
+mkdir ../cellxgene_data
 wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
 ```
 
-4. Copy run.sh.example to run.sh:
+5. Set your environment variables correctly:
-```
-cp run.sh.example run.sh
-```
-`run.sh` defines various environment variables:
 
-* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd'
+```bash
-* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene
+export CELLXGENE_LOCATION=`which cellxgene`
-* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data
+export CELLXGENE_DATA=../cellxgene_data  # change this directory if you put data in a different place.
-* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally
+export GATEWAY_HOST=localhost:5005
-* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
+export GATEWAY_PROTOCOL=http
-
+export GATEWAY_IP=127.0.0.1
-The defaults should be fine if you set up  a venv and cellxgene_data folder as above.
-
-5. Finally, execute run.sh:
-```
-source run.sh
 ```
 
-# Customization #
+6. Now, execute the cellxgene gateway:
+
+```bash
+cellxgene-gateway
+```
+
+For convenience, you can also change `run.sh.example` and execute it.
+
+Here's what the environment variables mean:
+
+* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
+* `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
+* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically `localhost:5005` if running locally
+* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
+
+The defaults should be fine if you set up a venv and cellxgene_data folder as above.
+
+# Customization
 
 The current paradigm for customization is to modify files during a build or deployment phase:
 
@@ -55,29 +90,23 @@ The current paradigm for customization is to modify files during a build or depl
 
 Currently we use a build.sh that copies the gateway to a "build" directory before modifying with sed and the like.
 
-# Development #
+# Development
 
-## Running Linters ##
+## Running Linters
 
 pip install isort flake8 black
 
-```
+```bash
 isort -rc .
-```
-
-```
 flake8 .
+black -l 79 .
 ```
 
-```
+# Getting Help
-black .
-```
-
-# Getting Help #
 
 If you need help for any reason, please make a github ticket. One of the contributors should help you out.
 
-# Contributors #
+# Contributors
 
 * Niket Patel - https://github.com/NiketPatel9
 * Alok Saldanha - https://github.com/alokito

cellxgene_gateway/__init__.py

@@ -6,4 +6,3 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
-

cellxgene_gateway/backend_cache.py

@@ -12,10 +12,10 @@ from threading import Thread
 
 from flask_api import status
 
-import env
+from cellxgene_gateway import env
-from cache_entry import CacheEntry
+from cellxgene_gateway.cache_entry import CacheEntry
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from subprocess_backend import SubprocessBackend
+from cellxgene_gateway.subprocess_backend import SubprocessBackend
 
 process_backend = SubprocessBackend()
 
@@ -51,7 +51,8 @@ class BackendCache:
         entry = CacheEntry.for_dataset(dataset, file_path, port)
 
         background_thread = Thread(
-            target=process_backend.launch, args=(env.cellxgene_location, scripts, entry)
+            target=process_backend.launch,
+            args=(env.cellxgene_location, scripts, entry),
         )
         background_thread.start()
 

cellxgene_gateway/cache_entry.py

@@ -9,9 +9,9 @@
 from flask import make_response, request
 from requests import get, post, put
 
-import env
+from cellxgene_gateway import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from util import current_time_stamp
+from cellxgene_gateway.util import current_time_stamp
 
 
 class CacheEntry:
@@ -92,14 +92,26 @@ class CacheEntry:
         if "content-type" in request.headers:
             headers["content-type"] = request.headers["content-type"]
 
-        if request.method in ['GET', 'HEAD', 'OPTIONS']:
+        if request.method in ["GET", "HEAD", "OPTIONS"]:
-            cellxgene_response = get(cellxgene_basepath + subpath, headers=headers)
+            cellxgene_response = get(
-        elif request.method == 'PUT':
+                cellxgene_basepath + subpath, headers=headers
-            cellxgene_response = put(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
+            )
-        elif request.method == 'POST':
+        elif request.method == "PUT":
-            cellxgene_response = post(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
+            cellxgene_response = put(
+                cellxgene_basepath + subpath,
+                headers=headers,
+                data=request.data.decode(),
+            )
+        elif request.method == "POST":
+            cellxgene_response = post(
+                cellxgene_basepath + subpath,
+                headers=headers,
+                data=request.data.decode(),
+            )
         else:
-            raise CellxgeneException(f"Unexpected method {request.method}", 400)
+            raise CellxgeneException(
+                f"Unexpected method {request.method}", 400
+            )
         content_type = cellxgene_response.headers["content-type"]
         if "text" in content_type:
             cellxgene_content = cellxgene_response.content.decode()
@@ -108,11 +120,11 @@ class CacheEntry:
             ).replace(cellxgene_basepath, gateway_basepath)
         else:
             gateway_content = cellxgene_response.content
-        
+
         gateway_response = make_response(
-            gateway_content, 
+            gateway_content,
             cellxgene_response.status_code,
-            {"Content-Type": content_type }
+            {"Content-Type": content_type},
         )
 
         return gateway_response

cellxgene_gateway/cellxgene_exception.py

@@ -7,6 +7,7 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+
 class CellxgeneException(Exception):
     def __init__(self, message, http_status):
         Exception.__init__(self)

cellxgene_gateway/dir_util.py

@@ -11,8 +11,8 @@ import os
 
 from flask_api import status
 
-import env
+from cellxgene_gateway import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
 
 
 def is_subdir(full_path, parent_path):
@@ -31,7 +31,8 @@ def create_dir(parent_path, dir_name):
         )
     elif not os.path.exists(parent_path):
         raise CellxgeneException(
-            "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST
+            "The selected User directory does not exist.",
+            status.HTTP_400_BAD_REQUEST,
         )
     elif os.path.exists(full_path):
         raise CellxgeneException(
@@ -74,7 +75,8 @@ def recurse_dir(path):
             }
         else:
             raise CellxgeneException(
-                "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST
+                "Given path is neither file nor directory.",
+                status.HTTP_400_BAD_REQUEST,
             )
 
     return [make_entry(x) for x in os.listdir(path)]

cellxgene_gateway/env.py

@@ -9,9 +9,35 @@
 
 import os
 
-deployment_env = os.environ.get("DEPLOYMENT_ENV")
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
 cellxgene_data = os.environ.get("CELLXGENE_DATA")
 gateway_host = os.environ.get("GATEWAY_HOST")
 gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
 ip = os.environ.get("GATEWAY_IP")
+
+env_vars = {
+    "CELLXGENE_LOCATION": cellxgene_location,
+    "CELLXGENE_DATA": cellxgene_data,
+    "GATEWAY_HOST": gateway_host,
+    "GATEWAY_PROTOCOL": gateway_protocol,
+    "GATEWAY_IP": ip,
+}
+
+if not all(env_vars.values()):
+    raise ValueError(
+        f"""
+Please ensure that environment variables are set correctly.
+The ones with None below are missing and need to be set.
+
+{env_vars}
+
+Set them at the terminal before running the gateway.
+An example is:
+
+    export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
+    export CELLXGENE_DATA=../cellxgene_data
+    export GATEWAY_HOST=localhost:5005
+    export GATEWAY_PROTOCOL=http
+    export GATEWAY_IP=127.0.0.1
+"""
+    )

cellxgene_gateway/extra_scripts.py

@@ -7,6 +7,7 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+
 def get_extra_scripts():
     # can be array of script tags to inject on every page, e.g. for google analytics could be
     # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',

cellxgene_gateway/gateway.py

@@ -12,19 +12,25 @@ import datetime
 import os
 from threading import Thread
 
-from flask import Flask, redirect, render_template, request, send_from_directory
+from flask import (
+    Flask,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+)
 from flask_api import status
 from werkzeug import secure_filename
 
-import env
+from cellxgene_gateway import env
-from backend_cache import BackendCache
+from cellxgene_gateway.backend_cache import BackendCache
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
-from dir_util import create_dir, recurse_dir, render_entries
+from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries
-from extra_scripts import get_extra_scripts
+from cellxgene_gateway.extra_scripts import get_extra_scripts
-from path_util import get_dataset, get_file_path
+from cellxgene_gateway.path_util import get_dataset, get_file_path
-from process_exception import ProcessException
+from cellxgene_gateway.process_exception import ProcessException
-from prune_process_cache import PruneProcessCache
+from cellxgene_gateway.prune_process_cache import PruneProcessCache
-from util import current_time_stamp
+from cellxgene_gateway.util import current_time_stamp
 
 app = Flask(__name__)
 cache = BackendCache()
@@ -38,7 +44,9 @@ def handle_invalid_usage(error):
 
     return (
         render_template(
-            "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message
+            "cellxgene_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=message,
         ),
         error.http_status,
     )
@@ -56,7 +64,9 @@ def handle_invalid_process(error):
 
     return (
         render_template(
-            "process_error.html", extra_scripts=get_extra_scripts(), message=message
+            "process_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=message,
         ),
         error.http_status,
     )
@@ -116,7 +126,9 @@ def upload_file():
             if "file" in request.files:
                 f = request.files["file"]
                 if f and f.filename.endswith(".h5ad"):
-                    f.save(full_upload_path + "/" + secure_filename(f.filename))
+                    f.save(
+                        full_upload_path + "/" + secure_filename(f.filename)
+                    )
                     return redirect("/filecrawl.html", code=302)
                 else:
                     raise CellxgeneException(
@@ -125,10 +137,13 @@ def upload_file():
                     )
             else:
                 raise CellxgeneException(
-                    "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST
+                    "A file must be chosen to upload.",
+                    status.HTTP_400_BAD_REQUEST,
                 )
     else:
-        raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST)
+        raise CellxgeneException(
+            "Invalid directory.", status.HTTP_400_BAD_REQUEST
+        )
 
     return redirect(env.location, code=302)
 
@@ -139,7 +154,9 @@ def filecrawl():
     entries = recurse_dir(env.cellxgene_data)
     rendered_html = render_entries(entries)
     return render_template(
-        "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html
+        "filecrawl.html",
+        extra_scripts=get_extra_scripts(),
+        rendered_html=rendered_html,
     )
 
 
@@ -166,8 +183,12 @@ def do_GET(path):
         raise ProcessException.from_pid_object(match)
 
 
-if __name__ == "__main__":
+def main():
     background_thread = Thread(target=PruneProcessCache(cache))
     background_thread.start()
 
     app.run(host="0.0.0.0", port=5005, debug=False)
+
+
+if __name__ == "__main__":
+    main()

cellxgene_gateway/path_util.py

@@ -11,8 +11,8 @@ import os
 
 from flask_api import status
 
-import env
+from cellxgene_gateway import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
 
 
 def get_dataset(path):

cellxgene_gateway/process_exception.py

@@ -7,6 +7,7 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+
 class ProcessException(Exception):
     def __init__(self, message, stdout, stderr, http_status):
         Exception.__init__(self)

cellxgene_gateway/prune_process_cache.py

@@ -11,7 +11,7 @@ import time
 
 import psutil
 
-from util import current_time_stamp
+from cellxgene_gateway.util import current_time_stamp
 
 
 class PruneProcessCache:

cellxgene_gateway/subprocess_backend.py

@@ -12,7 +12,7 @@ import subprocess
 
 from flask_api import status
 
-from process_exception import ProcessException
+from cellxgene_gateway.process_exception import ProcessException
 
 
 class SubprocessBackend:

cellxgene_gateway/templates/index.html

@@ -39,7 +39,6 @@
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
-
 	<br>
 
 	<h1 style="padding-left:35px">

environment-dev.yml

@@ -0,0 +1,11 @@
+name: cellxgene-dev
+channels:
+- conda-forge
+dependencies:
+- python=3.7
+- requests
+- flask
+- psutil
+- pip:
+  - flask-api
+  - cellxgene

requirements.txt

@@ -3,4 +3,3 @@ flask
 flask_api
 psutil
 requests
-

setup.py

@@ -0,0 +1,34 @@
+import os
+from setuptools import setup
+
+
+def parse_requirements():
+    reqs = []
+    with open("requirements.txt", "r") as f:
+        for l in f.readlines():
+            reqs.append(l.strip("\n"))
+    return reqs
+
+
+install_reqs = parse_requirements()
+
+setup(
+    # mandatory
+    name="cellxgene-gateway",
+    # mandatory
+    version="0.1",
+    # mandatory
+    author="Niket Patel, Yohann Potier, Alok Saldanha",
+    author_email="alok.saldanha@novartis.com",
+    description=("Cell-by-gene Gateway"),
+    license="MIT",
+    keywords="visualization, genomics",
+    url="http://github.com/Novartis/cellxgene-gateway",
+    packages=["cellxgene_gateway"],
+    package_data={"": ["README.md", "LICENSE.txt"]},
+    install_requires=install_reqs,
+    entry_points={
+        "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
+    },
+    classifiers=["Topic :: Scientific/Engineering :: Visualization"],
+)