diff --git a/Readme.md b/Readme.md
index f0a9de2..7ee0843 100644
--- a/Readme.md
+++ b/Readme.md
@@ -1,50 +1,58 @@
-# Overview #
+# Overview
Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server.
-# Running locally #
+## Running locally
0. This project requires python 3.6 or higher. Please check your version with
- python --version
+```bash
+$ python --version
+```
1. Set up a venv with
-```
+
+```bash
python -m venv .cellxgene-gateway
source .cellxgene-gateway/bin/activate
```
-2. Install requirements with
-```
+1. Install requirements with
+
+```bash
pip install -r requirements.txt
```
-3. Prepare a folder with .h5ad files, for example
-```
-mkdir cellxgene_data
+1. Prepare a folder with .h5ad files, for example
+
+```bash
+mkdir ../cellxgene_data
wget https://github.com/chanzuckerberg/cellxgene/raw/master/example-dataset/pbmc3k.h5ad -O ../cellxgene_data/pbmc3k.h5ad
```
-4. Copy run.sh.example to run.sh:
-```
+1. Copy run.sh.example to run.sh:
+
+```bash
cp run.sh.example run.sh
```
+
`run.sh` defines various environment variables:
-* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd'
-* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene
-* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data
-* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally
-* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
+* `DEPLOYMENT_ENV` - expects 'dev', 'tst' or 'prd'
+* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene
+* `CELLXGENE_DATA` - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data
+* `GATEWAY_HOST` - the hostname and port that the gateway will run on, typically localhost:5005 if running locally
+* `GATEWAY_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy.
The defaults should be fine if you set up a venv and cellxgene_data folder as above.
-5. Finally, execute run.sh:
+1. Finally, execute run.sh:
+
```
source run.sh
```
-# Customization #
+# Customization
The current paradigm for customization is to modify files during a build or deployment phase:
diff --git a/cellxgene_gateway.egg-info/PKG-INFO b/cellxgene_gateway.egg-info/PKG-INFO
new file mode 100644
index 0000000..7c8dfff
--- /dev/null
+++ b/cellxgene_gateway.egg-info/PKG-INFO
@@ -0,0 +1,12 @@
+Metadata-Version: 1.1
+Name: cellxgene-gateway
+Version: 0.1
+Summary: Cell-by-gene Gateway
+Home-page: http://github.com/Novartis/cellxgene-gateway
+Author: Niket Patel, Yohann Potier, Alok Saldanha
+Author-email: alok.saldanha@novartis.com
+License: MIT
+Description: UNKNOWN
+Keywords: visualization,genomics
+Platform: UNKNOWN
+Classifier: Topic :: Scientific/Engineering :: Visualization
diff --git a/cellxgene_gateway.egg-info/SOURCES.txt b/cellxgene_gateway.egg-info/SOURCES.txt
new file mode 100644
index 0000000..5c52134
--- /dev/null
+++ b/cellxgene_gateway.egg-info/SOURCES.txt
@@ -0,0 +1,20 @@
+setup.py
+cellxgene_gateway/__init__.py
+cellxgene_gateway/backend_cache.py
+cellxgene_gateway/cache_entry.py
+cellxgene_gateway/cellxgene_exception.py
+cellxgene_gateway/dir_util.py
+cellxgene_gateway/env.py
+cellxgene_gateway/extra_scripts.py
+cellxgene_gateway/gateway.py
+cellxgene_gateway/path_util.py
+cellxgene_gateway/process_exception.py
+cellxgene_gateway/prune_process_cache.py
+cellxgene_gateway/subprocess_backend.py
+cellxgene_gateway/util.py
+cellxgene_gateway.egg-info/PKG-INFO
+cellxgene_gateway.egg-info/SOURCES.txt
+cellxgene_gateway.egg-info/dependency_links.txt
+cellxgene_gateway.egg-info/entry_points.txt
+cellxgene_gateway.egg-info/requires.txt
+cellxgene_gateway.egg-info/top_level.txt
\ No newline at end of file
diff --git a/cellxgene_gateway.egg-info/dependency_links.txt b/cellxgene_gateway.egg-info/dependency_links.txt
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/cellxgene_gateway.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/cellxgene_gateway.egg-info/entry_points.txt b/cellxgene_gateway.egg-info/entry_points.txt
new file mode 100644
index 0000000..6bb9dea
--- /dev/null
+++ b/cellxgene_gateway.egg-info/entry_points.txt
@@ -0,0 +1,3 @@
+[console_scripts]
+cellxgene-gateway = cellxgene_gateway.gateway:main
+
diff --git a/cellxgene_gateway.egg-info/requires.txt b/cellxgene_gateway.egg-info/requires.txt
new file mode 100644
index 0000000..19bb2ef
--- /dev/null
+++ b/cellxgene_gateway.egg-info/requires.txt
@@ -0,0 +1,5 @@
+cellxgene
+flask
+flask_api
+psutil
+requests
diff --git a/cellxgene_gateway.egg-info/top_level.txt b/cellxgene_gateway.egg-info/top_level.txt
new file mode 100644
index 0000000..0224ebb
--- /dev/null
+++ b/cellxgene_gateway.egg-info/top_level.txt
@@ -0,0 +1 @@
+cellxgene_gateway
diff --git a/__init__.py b/cellxgene_gateway/__init__.py
similarity index 99%
rename from __init__.py
rename to cellxgene_gateway/__init__.py
index 8605fcc..447b57c 100755
--- a/__init__.py
+++ b/cellxgene_gateway/__init__.py
@@ -6,4 +6,3 @@
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
-
diff --git a/backend_cache.py b/cellxgene_gateway/backend_cache.py
similarity index 84%
rename from backend_cache.py
rename to cellxgene_gateway/backend_cache.py
index 8293c80..08d0235 100644
--- a/backend_cache.py
+++ b/cellxgene_gateway/backend_cache.py
@@ -12,10 +12,10 @@ from threading import Thread
from flask_api import status
-import env
-from cache_entry import CacheEntry
-from cellxgene_exception import CellxgeneException
-from subprocess_backend import SubprocessBackend
+from cellxgene_gateway import env
+from cellxgene_gateway.cache_entry import CacheEntry
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.subprocess_backend import SubprocessBackend
process_backend = SubprocessBackend()
@@ -51,7 +51,8 @@ class BackendCache:
entry = CacheEntry.for_dataset(dataset, file_path, port)
background_thread = Thread(
- target=process_backend.launch, args=(env.cellxgene_location, scripts, entry)
+ target=process_backend.launch,
+ args=(env.cellxgene_location, scripts, entry),
)
background_thread.start()
diff --git a/cache_entry.py b/cellxgene_gateway/cache_entry.py
similarity index 77%
rename from cache_entry.py
rename to cellxgene_gateway/cache_entry.py
index 6f9e297..f818040 100644
--- a/cache_entry.py
+++ b/cellxgene_gateway/cache_entry.py
@@ -9,9 +9,9 @@
from flask import make_response, request
from requests import get, post, put
-import env
-from cellxgene_exception import CellxgeneException
-from util import current_time_stamp
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.util import current_time_stamp
class CacheEntry:
@@ -92,14 +92,26 @@ class CacheEntry:
if "content-type" in request.headers:
headers["content-type"] = request.headers["content-type"]
- if request.method in ['GET', 'HEAD', 'OPTIONS']:
- cellxgene_response = get(cellxgene_basepath + subpath, headers=headers)
- elif request.method == 'PUT':
- cellxgene_response = put(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
- elif request.method == 'POST':
- cellxgene_response = post(cellxgene_basepath + subpath, headers=headers, data=request.data.decode())
+ if request.method in ["GET", "HEAD", "OPTIONS"]:
+ cellxgene_response = get(
+ cellxgene_basepath + subpath, headers=headers
+ )
+ elif request.method == "PUT":
+ cellxgene_response = put(
+ cellxgene_basepath + subpath,
+ headers=headers,
+ data=request.data.decode(),
+ )
+ elif request.method == "POST":
+ cellxgene_response = post(
+ cellxgene_basepath + subpath,
+ headers=headers,
+ data=request.data.decode(),
+ )
else:
- raise CellxgeneException(f"Unexpected method {request.method}", 400)
+ raise CellxgeneException(
+ f"Unexpected method {request.method}", 400
+ )
content_type = cellxgene_response.headers["content-type"]
if "text" in content_type:
cellxgene_content = cellxgene_response.content.decode()
@@ -108,11 +120,11 @@ class CacheEntry:
).replace(cellxgene_basepath, gateway_basepath)
else:
gateway_content = cellxgene_response.content
-
+
gateway_response = make_response(
- gateway_content,
+ gateway_content,
cellxgene_response.status_code,
- {"Content-Type": content_type }
+ {"Content-Type": content_type},
)
return gateway_response
diff --git a/cellxgene_exception.py b/cellxgene_gateway/cellxgene_exception.py
similarity index 99%
rename from cellxgene_exception.py
rename to cellxgene_gateway/cellxgene_exception.py
index ae645c3..d46d58c 100644
--- a/cellxgene_exception.py
+++ b/cellxgene_gateway/cellxgene_exception.py
@@ -7,6 +7,7 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
+
class CellxgeneException(Exception):
def __init__(self, message, http_status):
Exception.__init__(self)
diff --git a/dir_util.py b/cellxgene_gateway/dir_util.py
similarity index 90%
rename from dir_util.py
rename to cellxgene_gateway/dir_util.py
index 15d1370..d0a9af8 100644
--- a/dir_util.py
+++ b/cellxgene_gateway/dir_util.py
@@ -11,8 +11,8 @@ import os
from flask_api import status
-import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
def is_subdir(full_path, parent_path):
@@ -31,7 +31,8 @@ def create_dir(parent_path, dir_name):
)
elif not os.path.exists(parent_path):
raise CellxgeneException(
- "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST
+ "The selected User directory does not exist.",
+ status.HTTP_400_BAD_REQUEST,
)
elif os.path.exists(full_path):
raise CellxgeneException(
@@ -74,7 +75,8 @@ def recurse_dir(path):
}
else:
raise CellxgeneException(
- "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST
+ "Given path is neither file nor directory.",
+ status.HTTP_400_BAD_REQUEST,
)
return [make_entry(x) for x in os.listdir(path)]
diff --git a/env.py b/cellxgene_gateway/env.py
similarity index 54%
rename from env.py
rename to cellxgene_gateway/env.py
index 686bd4f..98bb912 100644
--- a/env.py
+++ b/cellxgene_gateway/env.py
@@ -15,3 +15,32 @@ cellxgene_data = os.environ.get("CELLXGENE_DATA")
gateway_host = os.environ.get("GATEWAY_HOST")
gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
ip = os.environ.get("GATEWAY_IP")
+
+env_vars = {
+ "DEPLOYMENT_ENV": deployment_env,
+ "CELLXGENE_LOCATION": cellxgene_location,
+ "CELLXGENE_DATA": cellxgene_data,
+ "GATEWAY_HOST": gateway_host,
+ "GATEWAY_PROTOCOL": gateway_protocol,
+ "GATEWAY_IP": ip,
+}
+
+if not all(env_vars.values()):
+ raise ValueError(
+ f"""
+Please ensure that environment variables are set correctly.
+The ones with None below are missing and need to be set.
+
+{env_vars}
+
+Set them at the terminal before running the gateway.
+An example is:
+
+ export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
+ export CELLXGENE_DATA=../cellxgene_data
+ export DEPLOYMENT_ENV=dev
+ export GATEWAY_HOST=localhost:5005
+ export GATEWAY_PROTOCOL=http
+ export GATEWAY_IP=127.0.0.1
+"""
+ )
diff --git a/extra_scripts.py b/cellxgene_gateway/extra_scripts.py
similarity index 99%
rename from extra_scripts.py
rename to cellxgene_gateway/extra_scripts.py
index 7c05b97..439c92e 100644
--- a/extra_scripts.py
+++ b/cellxgene_gateway/extra_scripts.py
@@ -7,6 +7,7 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
+
def get_extra_scripts():
# can be array of script tags to inject on every page, e.g. for google analytics could be
# ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
diff --git a/gateway.py b/cellxgene_gateway/gateway.py
similarity index 76%
rename from gateway.py
rename to cellxgene_gateway/gateway.py
index c92c35b..958a731 100644
--- a/gateway.py
+++ b/cellxgene_gateway/gateway.py
@@ -12,19 +12,25 @@ import datetime
import os
from threading import Thread
-from flask import Flask, redirect, render_template, request, send_from_directory
+from flask import (
+ Flask,
+ redirect,
+ render_template,
+ request,
+ send_from_directory,
+)
from flask_api import status
from werkzeug import secure_filename
-import env
-from backend_cache import BackendCache
-from cellxgene_exception import CellxgeneException
-from dir_util import create_dir, recurse_dir, render_entries
-from extra_scripts import get_extra_scripts
-from path_util import get_dataset, get_file_path
-from process_exception import ProcessException
-from prune_process_cache import PruneProcessCache
-from util import current_time_stamp
+from cellxgene_gateway import env
+from cellxgene_gateway.backend_cache import BackendCache
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
+from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries
+from cellxgene_gateway.extra_scripts import get_extra_scripts
+from cellxgene_gateway.path_util import get_dataset, get_file_path
+from cellxgene_gateway.process_exception import ProcessException
+from cellxgene_gateway.prune_process_cache import PruneProcessCache
+from cellxgene_gateway.util import current_time_stamp
app = Flask(__name__)
cache = BackendCache()
@@ -38,7 +44,9 @@ def handle_invalid_usage(error):
return (
render_template(
- "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message
+ "cellxgene_error.html",
+ extra_scripts=get_extra_scripts(),
+ message=message,
),
error.http_status,
)
@@ -56,7 +64,9 @@ def handle_invalid_process(error):
return (
render_template(
- "process_error.html", extra_scripts=get_extra_scripts(), message=message
+ "process_error.html",
+ extra_scripts=get_extra_scripts(),
+ message=message,
),
error.http_status,
)
@@ -116,7 +126,9 @@ def upload_file():
if "file" in request.files:
f = request.files["file"]
if f and f.filename.endswith(".h5ad"):
- f.save(full_upload_path + "/" + secure_filename(f.filename))
+ f.save(
+ full_upload_path + "/" + secure_filename(f.filename)
+ )
return redirect("/filecrawl.html", code=302)
else:
raise CellxgeneException(
@@ -125,10 +137,13 @@ def upload_file():
)
else:
raise CellxgeneException(
- "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST
+ "A file must be chosen to upload.",
+ status.HTTP_400_BAD_REQUEST,
)
else:
- raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST)
+ raise CellxgeneException(
+ "Invalid directory.", status.HTTP_400_BAD_REQUEST
+ )
return redirect(env.location, code=302)
@@ -139,7 +154,9 @@ def filecrawl():
entries = recurse_dir(env.cellxgene_data)
rendered_html = render_entries(entries)
return render_template(
- "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html
+ "filecrawl.html",
+ extra_scripts=get_extra_scripts(),
+ rendered_html=rendered_html,
)
@@ -166,8 +183,12 @@ def do_GET(path):
raise ProcessException.from_pid_object(match)
-if __name__ == "__main__":
+def main():
background_thread = Thread(target=PruneProcessCache(cache))
background_thread.start()
app.run(host="0.0.0.0", port=5005, debug=False)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/path_util.py b/cellxgene_gateway/path_util.py
similarity index 93%
rename from path_util.py
rename to cellxgene_gateway/path_util.py
index 0672184..6dc4fb6 100644
--- a/path_util.py
+++ b/cellxgene_gateway/path_util.py
@@ -11,8 +11,8 @@ import os
from flask_api import status
-import env
-from cellxgene_exception import CellxgeneException
+from cellxgene_gateway import env
+from cellxgene_gateway.cellxgene_exception import CellxgeneException
def get_dataset(path):
diff --git a/process_exception.py b/cellxgene_gateway/process_exception.py
similarity index 99%
rename from process_exception.py
rename to cellxgene_gateway/process_exception.py
index e54d842..5afdb00 100644
--- a/process_exception.py
+++ b/cellxgene_gateway/process_exception.py
@@ -7,6 +7,7 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
+
class ProcessException(Exception):
def __init__(self, message, stdout, stderr, http_status):
Exception.__init__(self)
diff --git a/prune_process_cache.py b/cellxgene_gateway/prune_process_cache.py
similarity index 96%
rename from prune_process_cache.py
rename to cellxgene_gateway/prune_process_cache.py
index 71bfcfa..f7240a0 100644
--- a/prune_process_cache.py
+++ b/cellxgene_gateway/prune_process_cache.py
@@ -11,7 +11,7 @@ import time
import psutil
-from util import current_time_stamp
+from cellxgene_gateway.util import current_time_stamp
class PruneProcessCache:
diff --git a/static/css/homepagestyle.css b/cellxgene_gateway/static/css/homepagestyle.css
similarity index 100%
rename from static/css/homepagestyle.css
rename to cellxgene_gateway/static/css/homepagestyle.css
diff --git a/static/nibr.ico b/cellxgene_gateway/static/nibr.ico
similarity index 100%
rename from static/nibr.ico
rename to cellxgene_gateway/static/nibr.ico
diff --git a/subprocess_backend.py b/cellxgene_gateway/subprocess_backend.py
similarity index 97%
rename from subprocess_backend.py
rename to cellxgene_gateway/subprocess_backend.py
index 10aabdc..a49f6e1 100644
--- a/subprocess_backend.py
+++ b/cellxgene_gateway/subprocess_backend.py
@@ -12,7 +12,7 @@ import subprocess
from flask_api import status
-from process_exception import ProcessException
+from cellxgene_gateway.process_exception import ProcessException
class SubprocessBackend:
diff --git a/templates/cellxgene_error.html b/cellxgene_gateway/templates/cellxgene_error.html
similarity index 100%
rename from templates/cellxgene_error.html
rename to cellxgene_gateway/templates/cellxgene_error.html
diff --git a/templates/filecrawl.html b/cellxgene_gateway/templates/filecrawl.html
similarity index 100%
rename from templates/filecrawl.html
rename to cellxgene_gateway/templates/filecrawl.html
diff --git a/templates/index.html b/cellxgene_gateway/templates/index.html
similarity index 99%
rename from templates/index.html
rename to cellxgene_gateway/templates/index.html
index 7aef319..2fcfd23 100644
--- a/templates/index.html
+++ b/cellxgene_gateway/templates/index.html
@@ -39,7 +39,6 @@
File Crawler: Allows you to view all uploaded data.
-