fork/cellxgene-gateway
1
0
Fork 0
mirror of https://github.com/Novartis/cellxgene-gateway.git synced 2026-09-29 04:18:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #41 from Novartis/grst_master

Browse Source 
Grst master
This commit is contained in:
Alokito
2021-02-22 08:59:05 -05:00
committed by GitHub
parent 942410bb44 893092f199
commit 25755d3f69
12 changed files with 74 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+9 -1
View File
@@ -60,6 +60,7 @@ Here's what the environment variables mean:
* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
* `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
Optional environment variables:
* `CELLXGENE_ARGS` - catch-all variable that can be used to pass additional command line args to cellxgene server
* `EXTERNAL_HOST` - the hostname and port from the perspective of the web browser, typically `localhost:5005` if running locally. Defaults to "localhost:{GATEWAY_PORT}"
@@ -69,7 +70,14 @@ Optional environment variables:
* `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
* `GATEWAY_ENABLE_UPLOAD` - Set to `true` or `1` to enable HTTP uploads. This is not recommended for a public server.
* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations.
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
* `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
* `PROXY_FIX_PROTO` - Number of upstream proxies setting X-Forwarded-Proto
* `PROXY_FIX_HOST` - Number of upstream proxies setting X-Forwarded-Host
* `PROXY_FIX_PORT` - Number of upstream proxies setting X-Forwarded-Port
* `PROXY_FIX_PREFIX` - Number of upstream proxies setting X-Forwarded-Prefix
The defaults should be fine if you set up a venv and cellxgene_data folder as above.
cellxgene_gateway/cache_entry.py
+4 -2
View File
@@ -8,6 +8,8 @@
# the specific language governing permissions and limitations under the License.
import datetime
import logging
from flask.helpers import url_for
from flask.wrappers import Response
import psutil
from enum import Enum
@@ -120,7 +122,7 @@ class CacheEntry:
return gateway_content
def gateway_basepath(self):
return f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/"
return url_for("do_view", path=self.key.pathpart) + "/"
def cellxgene_basepath(self):
return f"http://127.0.0.1:{self.port}"
@@ -130,7 +132,7 @@ class CacheEntry:
subpath = path[len(self.key.pathpart) :] # noqa: E203
if len(subpath) == 0:
r = make_response(f"Redirect to {gateway_basepath}\n", 301)
r = make_response(f"Redirect to {gateway_basepath}\n", 302)
r.headers["location"] = gateway_basepath + querystring()
return r
elif self.status == CacheEntryStatus.loading:
cellxgene_gateway/env.py
+11
View File
@@ -47,6 +47,12 @@ env_vars = {
"CELLXGENE_DATA": cellxgene_data,
}
proxy_fix_for = int(os.environ.get("PROXY_FIX_FOR", "0"))
proxy_fix_proto = int(os.environ.get("PROXY_FIX_PROTO", "0"))
proxy_fix_host = int(os.environ.get("PROXY_FIX_HOST", "0"))
proxy_fix_port = int(os.environ.get("PROXY_FIX_PORT", "0"))
proxy_fix_prefix = int(os.environ.get("PROXY_FIX_PREFIX", "0"))
optional_env_vars = {
"EXTERNAL_HOST": external_host,
"EXTERNAL_PROTOCOL": external_protocol,
@@ -58,6 +64,11 @@ optional_env_vars = {
"GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
"GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
"CELLXGENE_ARGS": cellxgene_args,
"PROXY_FIX_FOR": proxy_fix_for,
"PROXY_FIX_PROTO": proxy_fix_proto,
"PROXY_FIX_HOST": proxy_fix_host,
"PROXY_FIX_PORT": proxy_fix_port,
"PROXY_FIX_PREFIX": proxy_fix_prefix,
}
cellxgene_gateway/filecrawl.py
+2 -1
View File
@@ -14,6 +14,7 @@ from cellxgene_gateway.dir_util import (
make_annotations,
annotations_suffix,
)
from flask import url_for
def recurse_dir(path):
@@ -91,7 +92,7 @@ def render_entries(entries):
def get_url(entry):
return f"/view/{ entry['path'].lstrip('/') }/"
return url_for("do_view", path=entry["path"].lstrip("/") + "/")
def get_class(entry):
cellxgene_gateway/gateway.py
+20 -6
View File
@@ -25,7 +25,7 @@ from flask import (
)
from flask_api import status
from werkzeug.utils import secure_filename
from werkzeug.middleware.proxy_fix import ProxyFix
from cellxgene_gateway import env
from cellxgene_gateway.backend_cache import BackendCache
from cellxgene_gateway.cache_entry import CacheEntryStatus
@@ -50,9 +50,23 @@ def _force_https(app):
app.wsgi_app = _force_https(app.wsgi_app)
if (
env.proxy_fix_for > 0
or env.proxy_fix_proto > 0
or env.proxy_fix_host > 0
or env.proxy_fix_port > 0
or env.proxy_fix_prefix > 0
):
app.wsgi_app = ProxyFix(
app.wsgi_app,
x_for=env.proxy_fix_for,
x_proto=env.proxy_fix_proto,
x_host=env.proxy_fix_host,
x_port=env.proxy_fix_port,
x_prefix=env.proxy_fix_prefix,
)
cache = BackendCache()
location = f"{env.external_protocol}://{env.external_host}"
@app.errorhandler(CellxgeneException)
@@ -126,7 +140,7 @@ def make_user():
create_dir(env.cellxgene_data, dir_name)
return redirect(location, code=302)
return redirect(url_for("index"), code=302)
def make_subdir():
@@ -135,7 +149,7 @@ def make_subdir():
create_dir(parent_path, dir_name)
return redirect(location, code=302)
return redirect(url_for("index"), code=302)
def upload_file():
@@ -154,7 +168,7 @@ def upload_file():
full_upload_path, secure_filename(f.filename)
)
)
return redirect("/filecrawl.html", code=302)
return redirect(url_for("filecrawl"), code=302)
else:
raise CellxgeneException(
"Uploaded file must be in anndata (.h5ad) format.",
@@ -170,7 +184,7 @@ def upload_file():
"Invalid directory.", status.HTTP_400_BAD_REQUEST
)
return redirect(location, code=302)
return redirect(url_for("index"), code=302)
if env.enable_upload:
cellxgene_gateway/prune_process_cache.py
+8 -6
View File
@@ -10,14 +10,16 @@
import logging
import time
from cellxgene_gateway.env import ttl
from cellxgene_gateway.util import current_time_stamp
from cellxgene_gateway import env
from cellxgene_gateway import util
logger = logging.getLogger(__name__)
class PruneProcessCache:
def __init__(self, cache):
self.cache = cache
self.expire_seconds = 3600 if ttl is None else int(ttl)
self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
def __call__(self):
while True:
@@ -25,7 +27,7 @@ class PruneProcessCache:
self.prune()
def prune(self):
timestamp = current_time_stamp()
timestamp = util.current_time_stamp()
cutoff = timestamp - self.expire_seconds
processes_to_delete = [
p for p in self.cache.entry_list if p.timestamp < cutoff
@@ -33,9 +35,9 @@ class PruneProcessCache:
processes_to_keep = [
p for p in self.cache.entry_list if not p.timestamp < cutoff
]
logger = logging.getLogger("cellxgene_gateway")
logger.debug(
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}, pruning {processes_to_delete}"
)
for process in processes_to_delete:
cellxgene_gateway/templates/cellxgene_error.html
+2 -2
View File
@@ -28,11 +28,11 @@
<h4>{{ message }}</h4>
<a href="/filecrawl.html">
<a href="{{ url_for('filecrawl') }}">
Please click here to be redirected to the file directory.
</a>
<br>
<a href="/">
<a href="{{ url_for('index') }}">
Please click here to return to the homepage.
</a>
</div>
cellxgene_gateway/templates/filecrawl.html
+2 -2
View File
@@ -36,10 +36,10 @@
Navigation:
<ul>
{% if path %}
<li><a href="/filecrawl.html">top level</a></li>
<li><a href="{{ url_for('filecrawl') }}">top level</a></li>
{% else %}
{% endif %}
<li><a href="/">homepage</a></li>
<li><a href="{{ url_for('index') }}">homepage</a></li>
</ul>
</p>
<script>
cellxgene_gateway/templates/index.html
+2 -2
View File
@@ -35,12 +35,12 @@
Links:
</h1>
<div class="list-group" style="width:50%;padding-left:65px">
<a href="/filecrawl.html" class="list-group-item list-group-item-action">
<a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
<u>File Crawler: Allows you to view all uploaded data.</u></a>
</div>
<div class="list-group" style="width:50%;padding-left:65px">
<a href="/cache_status" class="list-group-item list-group-item-action">
<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
<u>Cache Status: view status of launched cellxgene servers.</u></a>
</div>
cellxgene_gateway/templates/loading.html
+2 -2
View File
@@ -35,11 +35,11 @@
The page will refresh shortly.
</p>
<a href="/filecrawl.html">
<a href="{{ url_for('filecrawl') }}">
Please click here to be redirected to the file directory.
</a>
<br>
<a href="/">
<a href="{{ url_for('index') }}">
Please click here to return to the homepage.
</a>
</div>
cellxgene_gateway/templates/process_error.html
+2 -2
View File
@@ -39,10 +39,10 @@
<li><a href="{{url_for('do_relaunch', path=dataset)}}">
Attempt to relaunch the cellxgene server.
</a></li>
<li><a href="/filecrawl.html">
<li><a href="{{ url_for('filecrawl') }}">
Return to the file directory.
</a></li>
<li><a href="/">
<li><a href="{{ url_for('index') }}">
Return to the homepage.
</a></li>
</ul>
tests/test_cache_entry.py
+10 -4
View File
@@ -1,11 +1,19 @@
import unittest
from flask import Flask
from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
from cellxgene_gateway.cache_key import CacheKey
from cellxgene_gateway.gateway import app
key = CacheKey("czi/pbmc3k.h5ad", "pbmc3k.h5ad", "tmp.csv")
class TestRenderEntry(unittest.TestCase):
def setUp(self):
self.app = app
self.app_context = self.app.test_request_context()
self.app_context.push()
self.client = self.app.test_client()
def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
entry = CacheEntry.for_key("some-key", 1)
self.assertEqual(entry.status, CacheEntryStatus.loading)
@@ -14,16 +22,14 @@ class TestRenderEntry(unittest.TestCase):
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
"src:url(/static/assets/"
)
expected = (
"src:url(http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/"
)
expected = "src:url(/view/czi/pbmc3k.h5ad/static/assets/"
self.assertEqual(actual, expected)
def test_GIVEN_absolute_src_THEN_include_path(self):
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
'<link rel="shortcut icon" href="/static/assets/favicon.ico">'
)
expected = '<link rel="shortcut icon" href="http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
expected = '<link rel="shortcut icon" href="/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
self.assertEqual(actual, expected)