refactor to allow use with console entry points

Readme.md

@@ -1,50 +1,58 @@
-# Overview #
+# Overview
 
 Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available  proxies requests to that server.
 
-# Running locally #
+## Running locally
 
 0. This project requires python 3.6 or higher. Please check your version with
 
-    python --version
+```bash
+$ python --version
+```
 
 1. Set up a venv with
-```
+
+```bash
 python -m venv .cellxgene-gateway
 source .cellxgene-gateway/bin/activate
 ```
 
-2. Install requirements with
-```
+1. Install requirements with
+
+```bash
 pip install -r requirements.txt
 ```
-3. Prepare a folder with .h5ad files, for example
 
-```
-mkdir cellxgene_data
+1. Prepare a folder with .h5ad files, for example
+
+```bash
+mkdir ../cellxgene_data
 wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
 ```
 
-4. Copy run.sh.example to run.sh:
-```
+1. Copy run.sh.example to run.sh:
+
+```bash
 cp run.sh.example run.sh
 ```
+
 `run.sh` defines various environment variables:
 
-* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd'
-* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene
-* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data
-* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally
-* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
+* `DEPLOYMENT_ENV` - expects 'dev', 'tst' or 'prd'
+* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene
+* `CELLXGENE_DATA` - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data
+* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically localhost:5005 if running locally
+* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
 
 The defaults should be fine if you set up  a venv and cellxgene_data folder as above.
 
-5. Finally, execute run.sh:
+1. Finally, execute run.sh:
+
 ```
 source run.sh
 ```
 
-# Customization #
+# Customization
 
 The current paradigm for customization is to modify files during a build or deployment phase:
 

cellxgene_gateway.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 1.1
+Name: cellxgene-gateway
+Version: 0.1
+Summary: Cell-by-gene Gateway
+Home-page: http://github.com/Novartis/cellxgene-gateway
+Author: Niket Patel, Yohann Potier, Alok Saldanha
+Author-email: alok.saldanha@novartis.com
+License: MIT
+Description: UNKNOWN
+Keywords: visualization,genomics
+Platform: UNKNOWN
+Classifier: Topic :: Scientific/Engineering :: Visualization

cellxgene_gateway.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+setup.py
+cellxgene_gateway/__init__.py
+cellxgene_gateway/backend_cache.py
+cellxgene_gateway/cache_entry.py
+cellxgene_gateway/cellxgene_exception.py
+cellxgene_gateway/dir_util.py
+cellxgene_gateway/env.py
+cellxgene_gateway/extra_scripts.py
+cellxgene_gateway/gateway.py
+cellxgene_gateway/path_util.py
+cellxgene_gateway/process_exception.py
+cellxgene_gateway/prune_process_cache.py
+cellxgene_gateway/subprocess_backend.py
+cellxgene_gateway/util.py
+cellxgene_gateway.egg-info/PKG-INFO
+cellxgene_gateway.egg-info/SOURCES.txt
+cellxgene_gateway.egg-info/dependency_links.txt
+cellxgene_gateway.egg-info/entry_points.txt
+cellxgene_gateway.egg-info/requires.txt
+cellxgene_gateway.egg-info/top_level.txt

cellxgene_gateway.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

cellxgene_gateway.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+cellxgene-gateway = cellxgene_gateway.gateway:main
+

cellxgene_gateway.egg-info/requires.txt

@@ -0,0 +1,5 @@
+cellxgene
+flask
+flask_api
+psutil
+requests

cellxgene_gateway.egg-info/top_level.txt

@@ -0,0 +1 @@
+cellxgene_gateway

cellxgene_gateway/__init__.py

@@ -6,4 +6,3 @@
 # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
-

cellxgene_gateway/backend_cache.py

@@ -12,10 +12,10 @@ from threading import Thread
 
 from flask_api import status
 
-import env
-from cache_entry import CacheEntry
-from cellxgene_exception import CellxgeneException
-from subprocess_backend import SubprocessBackend
+from cellxgene_gateway import env
+from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.subprocess_backend import SubprocessBackend
 
 process_backend = SubprocessBackend()
 
@@ -51,7 +51,8 @@ class BackendCache:
         entry = CacheEntry.for_dataset(dataset, file_path, port)
 
         background_thread = Thread(
-            target=process_backend.launch, args=(env.cellxgene_location, scripts, entry)
+            target=process_backend.launch,
+            args=(env.cellxgene_location, scripts, entry),
         )
         background_thread.start()
 

cellxgene_gateway/cache_entry.py

@@ -9,9 +9,9 @@
 from flask import make_response, request
 from requests import get, post, put
 
-import env
-from cellxgene_exception import CellxgeneException
-from util import current_time_stamp
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.util import current_time_stamp
 
 
 class CacheEntry:
@@ -92,14 +92,26 @@ class CacheEntry:
         if "content-type" in request.headers:
             headers["content-type"] = request.headers["content-type"]
 
-        if request.method in ['GET', 'HEAD', 'OPTIONS']:
-            cellxgene_response = get(cellxgene_basepath + subpath, headers=headers)
-        elif request.method == 'PUT':
-            cellxgene_response = put(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
-        elif request.method == 'POST':
-            cellxgene_response = post(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
+        if request.method in ["GET", "HEAD", "OPTIONS"]:
+            cellxgene_response = get(
+                cellxgene_basepath + subpath, headers=headers
+            )
+        elif request.method == "PUT":
+            cellxgene_response = put(
+                cellxgene_basepath + subpath,
+                headers=headers,
+                data=request.data.decode(),
+            )
+        elif request.method == "POST":
+            cellxgene_response = post(
+                cellxgene_basepath + subpath,
+                headers=headers,
+                data=request.data.decode(),
+            )
         else:
-            raise CellxgeneException(f"Unexpected method {request.method}", 400)
+            raise CellxgeneException(
+                f"Unexpected method {request.method}", 400
+            )
         content_type = cellxgene_response.headers["content-type"]
         if "text" in content_type:
             cellxgene_content = cellxgene_response.content.decode()
@@ -108,11 +120,11 @@ class CacheEntry:
             ).replace(cellxgene_basepath, gateway_basepath)
         else:
             gateway_content = cellxgene_response.content
-        
+
         gateway_response = make_response(
-            gateway_content, 
+            gateway_content,
             cellxgene_response.status_code,
-            {"Content-Type": content_type }
+            {"Content-Type": content_type},
         )
 
         return gateway_response

cellxgene_gateway/cellxgene_exception.py

@@ -7,6 +7,7 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+
 class CellxgeneException(Exception):
     def __init__(self, message, http_status):
         Exception.__init__(self)

cellxgene_gateway/dir_util.py

@@ -11,8 +11,8 @@ import os
 
 from flask_api import status
 
-import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
 
 
 def is_subdir(full_path, parent_path):
@@ -31,7 +31,8 @@ def create_dir(parent_path, dir_name):
         )
     elif not os.path.exists(parent_path):
         raise CellxgeneException(
-            "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST
+            "The selected User directory does not exist.",
+            status.HTTP_400_BAD_REQUEST,
         )
     elif os.path.exists(full_path):
         raise CellxgeneException(
@@ -74,7 +75,8 @@ def recurse_dir(path):
             }
         else:
             raise CellxgeneException(
-                "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST
+                "Given path is neither file nor directory.",
+                status.HTTP_400_BAD_REQUEST,
             )
 
     return [make_entry(x) for x in os.listdir(path)]

cellxgene_gateway/env.py

@@ -15,3 +15,32 @@ cellxgene_data = os.environ.get("CELLXGENE_DATA")
 gateway_host = os.environ.get("GATEWAY_HOST")
 gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
 ip = os.environ.get("GATEWAY_IP")
+
+env_vars = {
+    "DEPLOYMENT_ENV": deployment_env,
+    "CELLXGENE_LOCATION": cellxgene_location,
+    "CELLXGENE_DATA": cellxgene_data,
+    "GATEWAY_HOST": gateway_host,
+    "GATEWAY_PROTOCOL": gateway_protocol,
+    "GATEWAY_IP": ip,
+}
+
+if not all(env_vars.values()):
+    raise ValueError(
+        f"""
+Please ensure that environment variables are set correctly.
+The ones with None below are missing and need to be set.
+
+{env_vars}
+
+Set them at the terminal before running the gateway.
+An example is:
+
+    export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
+    export CELLXGENE_DATA=../cellxgene_data
+    export DEPLOYMENT_ENV=dev
+    export GATEWAY_HOST=localhost:5005
+    export GATEWAY_PROTOCOL=http
+    export GATEWAY_IP=127.0.0.1
+"""
+    )

cellxgene_gateway/extra_scripts.py

@@ -7,6 +7,7 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+
 def get_extra_scripts():
     # can be array of script tags to inject on every page, e.g. for google analytics could be
     # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',

cellxgene_gateway/gateway.py

@@ -12,19 +12,25 @@ import datetime
 import os
 from threading import Thread
 
-from flask import Flask, redirect, render_template, request, send_from_directory
+from flask import (
+    Flask,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+)
 from flask_api import status
 from werkzeug import secure_filename
 
-import env
-from backend_cache import BackendCache
-from cellxgene_exception import CellxgeneException
-from dir_util import create_dir, recurse_dir, render_entries
-from extra_scripts import get_extra_scripts
-from path_util import get_dataset, get_file_path
-from process_exception import ProcessException
-from prune_process_cache import PruneProcessCache
-from util import current_time_stamp
+from cellxgene_gateway import env
+from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries
+from cellxgene_gateway.extra_scripts import get_extra_scripts
+from cellxgene_gateway.path_util import get_dataset, get_file_path
+from cellxgene_gateway.process_exception import ProcessException
+from cellxgene_gateway.prune_process_cache import PruneProcessCache
+from cellxgene_gateway.util import current_time_stamp
 
 app = Flask(__name__)
 cache = BackendCache()
@@ -38,7 +44,9 @@ def handle_invalid_usage(error):
 
     return (
         render_template(
-            "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message
+            "cellxgene_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=message,
         ),
         error.http_status,
     )
@@ -56,7 +64,9 @@ def handle_invalid_process(error):
 
     return (
         render_template(
-            "process_error.html", extra_scripts=get_extra_scripts(), message=message
+            "process_error.html",
+            extra_scripts=get_extra_scripts(),
+            message=message,
         ),
         error.http_status,
     )
@@ -116,7 +126,9 @@ def upload_file():
             if "file" in request.files:
                 f = request.files["file"]
                 if f and f.filename.endswith(".h5ad"):
-                    f.save(full_upload_path + "/" + secure_filename(f.filename))
+                    f.save(
+                        full_upload_path + "/" + secure_filename(f.filename)
+                    )
                     return redirect("/filecrawl.html", code=302)
                 else:
                     raise CellxgeneException(
@@ -125,10 +137,13 @@ def upload_file():
                     )
             else:
                 raise CellxgeneException(
-                    "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST
+                    "A file must be chosen to upload.",
+                    status.HTTP_400_BAD_REQUEST,
                 )
     else:
-        raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST)
+        raise CellxgeneException(
+            "Invalid directory.", status.HTTP_400_BAD_REQUEST
+        )
 
     return redirect(env.location, code=302)
 
@@ -139,7 +154,9 @@ def filecrawl():
     entries = recurse_dir(env.cellxgene_data)
     rendered_html = render_entries(entries)
     return render_template(
-        "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html
+        "filecrawl.html",
+        extra_scripts=get_extra_scripts(),
+        rendered_html=rendered_html,
     )
 
 
@@ -166,8 +183,12 @@ def do_GET(path):
         raise ProcessException.from_pid_object(match)
 
 
-if __name__ == "__main__":
+def main():
     background_thread = Thread(target=PruneProcessCache(cache))
     background_thread.start()
 
     app.run(host="0.0.0.0", port=5005, debug=False)
+
+
+if __name__ == "__main__":
+    main()

cellxgene_gateway/path_util.py

@@ -11,8 +11,8 @@ import os
 
 from flask_api import status
 
-import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
 
 
 def get_dataset(path):

cellxgene_gateway/process_exception.py

@@ -7,6 +7,7 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
+
 class ProcessException(Exception):
     def __init__(self, message, stdout, stderr, http_status):
         Exception.__init__(self)

cellxgene_gateway/prune_process_cache.py

@@ -11,7 +11,7 @@ import time
 
 import psutil
 
-from util import current_time_stamp
+from cellxgene_gateway.util import current_time_stamp
 
 
 class PruneProcessCache:

cellxgene_gateway/subprocess_backend.py

@@ -12,7 +12,7 @@ import subprocess
 
 from flask_api import status
 
-from process_exception import ProcessException
+from cellxgene_gateway.process_exception import ProcessException
 
 
 class SubprocessBackend:

cellxgene_gateway/templates/index.html

@@ -39,7 +39,6 @@
 		  <u>File Crawler: Allows you to view all uploaded data.</u></a>
 		  
 	</div>
-
 	<br>
 
 	<h1 style="padding-left:35px">

environment-dev.yml

@@ -0,0 +1,11 @@
+name: cellxgene-dev
+channels:
+- conda-forge
+dependencies:
+- python=3.7
+- requests
+- flask
+- psutil
+- pip:
+  - flask-api
+  - cellxgene

requirements.txt

@@ -3,4 +3,3 @@ flask
 flask_api
 psutil
 requests
-

setup.py

@@ -0,0 +1,34 @@
+import os
+from setuptools import setup
+
+
+def parse_requirements():
+    reqs = []
+    with open("requirements.txt", "r") as f:
+        for l in f.readlines():
+            reqs.append(l.strip("\n"))
+    return reqs
+
+
+install_reqs = parse_requirements()
+
+setup(
+    # mandatory
+    name="cellxgene-gateway",
+    # mandatory
+    version="0.1",
+    # mandatory
+    author="Niket Patel, Yohann Potier, Alok Saldanha",
+    author_email="alok.saldanha@novartis.com",
+    description=("Cell-by-gene Gateway"),
+    license="MIT",
+    keywords="visualization, genomics",
+    url="http://github.com/Novartis/cellxgene-gateway",
+    packages=["cellxgene_gateway"],
+    package_data={"": ["README.md", "LICENSE.txt"]},
+    install_requires=install_reqs,
+    entry_points={
+        "console_scripts": ["cellxgene-gateway=cellxgene_gateway.gateway:main"]
+    },
+    classifiers=["Topic :: Scientific/Engineering :: Visualization"],
+)