mirror of
https://github.com/Novartis/cellxgene-gateway.git
synced 2026-09-16 05:17:55 +08:00
Compare commits
1 Commits
81_gene_se
...
patch-item
| Author | SHA1 | Date | |
|---|---|---|---|
|
a6c13a7606 |
13
.coveragerc
13
.coveragerc
@@ -1,13 +0,0 @@
|
||||
[run]
|
||||
branch = True
|
||||
source = cellxgene_gateway
|
||||
|
||||
[report]
|
||||
exclude_lines =
|
||||
if self.debug:
|
||||
pragma: no cover
|
||||
raise NotImplementedError
|
||||
if __name__ == .__main__.:
|
||||
ignore_errors = True
|
||||
omit =
|
||||
tests/*
|
||||
67
.github/workflows/codeql-analysis.yml
vendored
67
.github/workflows/codeql-analysis.yml
vendored
@@ -1,67 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master ]
|
||||
schedule:
|
||||
- cron: '18 6 * * 6'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'javascript', 'python' ]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
|
||||
# Learn more:
|
||||
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v1
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
68
.github/workflows/pr-checks.yaml
vendored
68
.github/workflows/pr-checks.yaml
vendored
@@ -1,68 +0,0 @@
|
||||
# Tests that run on every PR
|
||||
|
||||
name: Pull Request Checks
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
black:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
name: Checkout repository
|
||||
|
||||
- uses: actions/setup-python@v2
|
||||
name: Setup Python
|
||||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- name: Install black
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install black
|
||||
- name: Run black
|
||||
run: |
|
||||
black . --check
|
||||
# This job is copied over from `deploy.yaml`
|
||||
run-tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# See: https://github.com/marketplace/actions/setup-conda
|
||||
- uses: s-weigand/setup-conda@v1
|
||||
with:
|
||||
conda-channels: "conda-forge"
|
||||
|
||||
- name: Build environment
|
||||
run: |
|
||||
conda env create -f environment.yml
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate cellxgene-gateway
|
||||
pip install markupsafe==2.0.1 # temporary workaround for jinja2-2.11.3 calling soft_unicode in markupsafe
|
||||
python setup.py install
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate cellxgene-gateway
|
||||
coverage run -m unittest discover tests
|
||||
|
||||
- name: Check coverage
|
||||
run: |
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate cellxgene-gateway
|
||||
coverage report --fail-under 41
|
||||
coverage xml -i
|
||||
|
||||
- name: "Upload coverage to Codecov"
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
flags: unittests
|
||||
env_vars: OS,PYTHON
|
||||
name: codecov-umbrella
|
||||
fail_ci_if_error: true
|
||||
path_to_write_report: ./codecov_report.txt
|
||||
verbose: true
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -55,7 +55,6 @@ htmlcov/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
htmlcov
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
|
||||
@@ -7,7 +7,14 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
stages: [commit]
|
||||
- id: flake8
|
||||
name: flake8
|
||||
language: system
|
||||
entry: flake8
|
||||
types: [python]
|
||||
stages: [commit]
|
||||
- id: black
|
||||
language_version: python3.6+
|
||||
name: black
|
||||
language: system
|
||||
entry: black
|
||||
|
||||
60
Changelog.md
60
Changelog.md
@@ -1,63 +1,3 @@
|
||||
# 0.3.11
|
||||
|
||||
* #81 added support for gene sets
|
||||
|
||||
# 0.3.10
|
||||
|
||||
* #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.
|
||||
* Added GATEWAY_LOG_LEVEL to set the log level
|
||||
* #68 Close connections after reading response
|
||||
* #68 Background thread reads from output of cellxgene process until it exits
|
||||
|
||||
# 0.3.9
|
||||
|
||||
* Added S3_ENABLE_LISTINGS_CACHE variable (See README.md)
|
||||
|
||||
# 0.3.8
|
||||
|
||||
* Fixed bug #57 affecting deeply nested subdirectory listing
|
||||
|
||||
# 0.3.7
|
||||
|
||||
* added back /metadata/ip_address endpoint
|
||||
|
||||
# 0.3.6
|
||||
|
||||
* pinned version of werkzeug
|
||||
|
||||
# 0.3.5
|
||||
|
||||
* Pinned flask version to match cellxgene 0.17.0
|
||||
|
||||
# 0.3.4
|
||||
|
||||
* Fixed bug #50 affecting subdirectory listing
|
||||
|
||||
# 0.3.3
|
||||
|
||||
* Fixed bug #48 affecting cache pruning
|
||||
|
||||
# 0.3.2
|
||||
|
||||
* Fixed bug #45 affecting multi-level S3 folders
|
||||
* Added extra_scripts to cache_status page
|
||||
|
||||
# 0.3.1
|
||||
|
||||
* Added missing __init__.py
|
||||
|
||||
# 0.3.0
|
||||
|
||||
* Added support for itemsource interface, allowing s3 hosting
|
||||
* Removed support for http file uploads
|
||||
* Only set wsgi.url_scheme when EXTERNAL_PROTOCOL is set (see issue #43)
|
||||
* Dropped flake8 due to conflicts with black
|
||||
* Added code coverage metrics
|
||||
|
||||
# 0.2.3
|
||||
|
||||
* Added support for ProxyFix
|
||||
|
||||
# 0.2.2
|
||||
|
||||
* Fixed bug with annotations (missing annotation.js asset)
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
FROM python:3.9
|
||||
|
||||
RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
|
||||
|
||||
ENV CELLXGENE_DATA=/cellxgene-data
|
||||
ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
|
||||
|
||||
CMD ["cellxgene-gateway"]
|
||||
57
README.md
57
README.md
@@ -2,8 +2,6 @@
|
||||
|
||||
Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server.
|
||||
|
||||
[](https://codecov.io/gh/Novartis/cellxgene-gateway) [](https://pypi.org/project/cellxgene-gateway/) [](https://pypistats.org/packages/cellxgene-gateway)
|
||||
|
||||
# Running locally
|
||||
|
||||
## Prequisites
|
||||
@@ -73,49 +71,12 @@ Optional environment variables:
|
||||
* `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
|
||||
* `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
|
||||
* `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
|
||||
* `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
|
||||
* `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
|
||||
* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations and gene sets.
|
||||
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
|
||||
* `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
|
||||
* `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
|
||||
|
||||
If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
|
||||
* `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
|
||||
* `PROXY_FIX_PROTO` - Number of upstream proxies setting X-Forwarded-Proto
|
||||
* `PROXY_FIX_HOST` - Number of upstream proxies setting X-Forwarded-Host
|
||||
* `PROXY_FIX_PORT` - Number of upstream proxies setting X-Forwarded-Port
|
||||
* `PROXY_FIX_PREFIX` - Number of upstream proxies setting X-Forwarded-Prefix
|
||||
* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations.
|
||||
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
|
||||
|
||||
The defaults should be fine if you set up a venv and cellxgene_data folder as above.
|
||||
|
||||
## Running cellxgene-gateway with Docker
|
||||
|
||||
First, build Docker image:
|
||||
|
||||
```bash
|
||||
docker build -t cellxgene-gateway .
|
||||
```
|
||||
|
||||
Then, cellxgene-gateway can be launched as such:
|
||||
|
||||
```bash
|
||||
docker run -it --rm \
|
||||
-v <local_data_dir>:/cellxgene-data \
|
||||
-p 5005:5005 \
|
||||
cellxgene-gateway
|
||||
```
|
||||
|
||||
Additional environment variables can be provided with the `-e` parameter:
|
||||
|
||||
```bash
|
||||
docker run -it --rm \
|
||||
-v <local_data_dir>:/cellxgene-data \
|
||||
-e GATEWAY_PORT=8080 \
|
||||
-p 8080:8080 \
|
||||
cellxgene-gateway
|
||||
```
|
||||
|
||||
# Customization
|
||||
|
||||
The current paradigm for customization is to modify files during a build or deployment phase:
|
||||
@@ -156,14 +117,6 @@ python setup.py develop
|
||||
|
||||
For convenience, the code repo includes a `run.sh.example` shell script to run the gateway.
|
||||
|
||||
4. Install pre-commit hooks
|
||||
|
||||
```bash
|
||||
conda install -c conda-forge pre-commit
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
|
||||
## Running Tests
|
||||
|
||||
[](https://travis-ci.org/Novartis/cellxgene-gateway)
|
||||
@@ -172,12 +125,6 @@ pre-commit install
|
||||
python -m unittest discover tests
|
||||
```
|
||||
|
||||
## Code Coverage
|
||||
```bash
|
||||
coverage run -m unittest discover tests
|
||||
coverage html
|
||||
```
|
||||
|
||||
## Running Linters
|
||||
|
||||
pip install isort flake8 black
|
||||
|
||||
15
SECURITY.md
15
SECURITY.md
@@ -1,15 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Use this section to tell people about which versions of your project are
|
||||
currently being supported with security updates.
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 0.3.2 | :white_check_mark: |
|
||||
| <= 0.3.1 | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please file a bug report issue.
|
||||
2
cellxgene_gateway/__init__.py
Normal file → Executable file
2
cellxgene_gateway/__init__.py
Normal file → Executable file
@@ -7,4 +7,4 @@
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
__version__ = "0.3.10"
|
||||
__version__ = "0.2.2"
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
|
||||
import time
|
||||
from threading import Thread
|
||||
from typing import List
|
||||
|
||||
from flask_api import status
|
||||
|
||||
@@ -18,6 +17,7 @@ from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.subprocess_backend import SubprocessBackend
|
||||
from typing import List
|
||||
|
||||
process_backend = SubprocessBackend()
|
||||
|
||||
|
||||
@@ -8,21 +8,19 @@
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
import urllib.parse
|
||||
from enum import Enum
|
||||
|
||||
import psutil
|
||||
from flask import make_response, render_template, request
|
||||
from flask.wrappers import Response
|
||||
from requests import get, post, put
|
||||
import re
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.flask_util import querystring
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CacheEntryStatus(Enum):
|
||||
loaded = "loaded"
|
||||
@@ -58,6 +56,7 @@ class CacheEntry:
|
||||
|
||||
@classmethod
|
||||
def for_key(cls, key, port):
|
||||
|
||||
return cls(
|
||||
None,
|
||||
key,
|
||||
@@ -111,7 +110,9 @@ class CacheEntry:
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
logger.info(f"terminated {terminated}")
|
||||
logging.getLogger("cellxgene_gateway").info(
|
||||
f"terminated {terminated}"
|
||||
)
|
||||
self.status = CacheEntryStatus.terminated
|
||||
|
||||
def rewrite_text_content(self, cellxgene_content):
|
||||
@@ -119,22 +120,30 @@ class CacheEntry:
|
||||
gateway_content = (
|
||||
re.sub(
|
||||
'(="|\()/static/',
|
||||
f"\\1{self.key.gateway_basepath()}static/",
|
||||
f"\\1{self.gateway_basepath()}static/",
|
||||
cellxgene_content,
|
||||
)
|
||||
.replace("http://fonts.gstatic.com", "https://fonts.gstatic.com")
|
||||
.replace(self.cellxgene_basepath(), self.key.gateway_basepath())
|
||||
.replace(self.cellxgene_basepath(), self.gateway_basepath())
|
||||
)
|
||||
return gateway_content
|
||||
|
||||
def gateway_basepath(self):
|
||||
source_path = (
|
||||
f"/source/{urllib.parse.quote_plus(self.source_name)}"
|
||||
if self.source_name
|
||||
else ""
|
||||
)
|
||||
return f"{env.external_protocol}://{env.external_host}{source_path}/view/{self.key.descriptor}/"
|
||||
|
||||
def cellxgene_basepath(self):
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def serve_content(self, path):
|
||||
gateway_basepath = self.key.gateway_basepath()
|
||||
gateway_basepath = self.gateway_basepath()
|
||||
subpath = path[len(self.key.descriptor) :] # noqa: E203
|
||||
if len(subpath) == 0:
|
||||
r = make_response(f"Redirect to {gateway_basepath}\n", 302)
|
||||
r = make_response(f"Redirect to {gateway_basepath}\n", 301)
|
||||
r.headers["location"] = gateway_basepath + querystring()
|
||||
return r
|
||||
elif self.status == CacheEntryStatus.loading:
|
||||
@@ -169,43 +178,40 @@ class CacheEntry:
|
||||
|
||||
full_path = self.cellxgene_basepath() + subpath + querystring()
|
||||
|
||||
try:
|
||||
cellxgene_response = None
|
||||
if request.method in ["GET", "HEAD", "OPTIONS"]:
|
||||
cellxgene_response = get(full_path, headers=headers)
|
||||
elif request.method == "PUT":
|
||||
cellxgene_response = put(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
elif request.method == "POST":
|
||||
cellxgene_response = post(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
else:
|
||||
raise CellxgeneException(f"Unexpected method {request.method}", 400)
|
||||
content_type = cellxgene_response.headers["content-type"]
|
||||
if "text" in content_type:
|
||||
gateway_content = self.rewrite_text_content(
|
||||
cellxgene_response.content.decode()
|
||||
)
|
||||
else:
|
||||
gateway_content = cellxgene_response.content
|
||||
|
||||
resp_headers = {}
|
||||
for h in copy_headers:
|
||||
if h in cellxgene_response.headers:
|
||||
resp_headers[h] = cellxgene_response.headers[h]
|
||||
|
||||
gateway_response = make_response(
|
||||
gateway_content,
|
||||
cellxgene_response.status_code,
|
||||
resp_headers,
|
||||
if request.method in ["GET", "HEAD", "OPTIONS"]:
|
||||
cellxgene_response = get(full_path, headers=headers)
|
||||
elif request.method == "PUT":
|
||||
cellxgene_response = put(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
finally:
|
||||
if cellxgene_response is not None:
|
||||
cellxgene_response.close()
|
||||
elif request.method == "POST":
|
||||
cellxgene_response = post(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
f"Unexpected method {request.method}", 400
|
||||
)
|
||||
content_type = cellxgene_response.headers["content-type"]
|
||||
if "text" in content_type:
|
||||
gateway_content = self.rewrite_text_content(
|
||||
cellxgene_response.content.decode()
|
||||
)
|
||||
else:
|
||||
gateway_content = cellxgene_response.content
|
||||
|
||||
resp_headers = {}
|
||||
for h in copy_headers:
|
||||
if h in cellxgene_response.headers:
|
||||
resp_headers[h] = cellxgene_response.headers[h]
|
||||
|
||||
gateway_response = make_response(
|
||||
gateway_content,
|
||||
cellxgene_response.status_code,
|
||||
resp_headers,
|
||||
)
|
||||
return gateway_response
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
# 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
|
||||
# in this case, descriptor == 'somedir/dataset_annotations', dataset == 'somedir/dataset.h5ad'
|
||||
|
||||
from cellxgene_gateway import flask_util
|
||||
from cellxgene_gateway.items.item import Item
|
||||
from cellxgene_gateway.items.item_source import ItemSource, LookupResult
|
||||
|
||||
@@ -39,16 +38,6 @@ class CacheKey:
|
||||
else:
|
||||
return self.source.get_local_path(self.annotation_item)
|
||||
|
||||
def relaunch_url(self):
|
||||
return flask_util.relaunch_url(self.descriptor, self.source_name)
|
||||
|
||||
def gateway_basepath(self):
|
||||
return self.view_url + "/"
|
||||
|
||||
@property
|
||||
def view_url(self):
|
||||
return flask_util.view_url(self.descriptor, self.source_name)
|
||||
|
||||
@property
|
||||
def source_name(self):
|
||||
return self.source.name
|
||||
|
||||
@@ -14,6 +14,44 @@ from flask_api import status
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
|
||||
|
||||
def is_subdir(full_path, parent_path):
|
||||
subdir = os.path.realpath(full_path)
|
||||
parent = os.path.realpath(parent_path)
|
||||
return subdir.startswith(parent)
|
||||
|
||||
|
||||
def create_dir(parent_path, dir_name):
|
||||
full_path = os.path.join(parent_path, dir_name)
|
||||
|
||||
if "/" in dir_name:
|
||||
raise CellxgeneException(
|
||||
"Please have no slashes in the intended directory.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not os.path.exists(parent_path):
|
||||
raise CellxgeneException(
|
||||
"The selected User directory does not exist.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif os.path.exists(full_path):
|
||||
raise CellxgeneException(
|
||||
"The provided subdirectory already exists within Directory.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not is_subdir(full_path, parent_path):
|
||||
raise CellxgeneException(
|
||||
"The directory must be a subdirectory of the parent path.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not os.path.isdir(parent_path):
|
||||
raise CellxgeneException(
|
||||
"The parent is not a directory.", status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
else:
|
||||
os.mkdir(full_path)
|
||||
|
||||
|
||||
annotations_suffix = "_annotations"
|
||||
h5ad_suffix = ".h5ad"
|
||||
|
||||
@@ -24,8 +62,3 @@ def make_h5ad(el):
|
||||
|
||||
def make_annotations(el):
|
||||
return el[:-5] + annotations_suffix
|
||||
|
||||
|
||||
def ensure_dir_exists(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
os.makedirs(file_path)
|
||||
|
||||
@@ -9,9 +9,10 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
|
||||
cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA", "")
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA")
|
||||
cellxgene_args = os.environ.get("CELLXGENE_ARGS", None)
|
||||
gateway_port = int(os.environ.get("GATEWAY_PORT", "5005"))
|
||||
external_host = os.environ.get(
|
||||
@@ -19,50 +20,39 @@ external_host = os.environ.get(
|
||||
os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"),
|
||||
)
|
||||
external_protocol = os.environ.get(
|
||||
"EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", None)
|
||||
"EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http")
|
||||
)
|
||||
ip = os.environ.get("GATEWAY_IP")
|
||||
extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
|
||||
expire_seconds = int(
|
||||
os.environ.get("GATEWAY_EXPIRE_SECONDS", os.environ.get("GATEWAY_TTL", "3600"))
|
||||
)
|
||||
enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [
|
||||
ttl = os.environ.get("GATEWAY_TTL")
|
||||
enable_annotations = os.environ.get(
|
||||
"GATEWAY_ENABLE_ANNOTATIONS", ""
|
||||
).lower() in [
|
||||
"true",
|
||||
"1",
|
||||
]
|
||||
enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in [
|
||||
enable_backed_mode = os.environ.get(
|
||||
"GATEWAY_ENABLE_BACKED_MODE", ""
|
||||
).lower() in [
|
||||
"true",
|
||||
"1",
|
||||
]
|
||||
log_level = logging.getLevelName(os.environ.get("GATEWAY_LOG_LEVEL", "INFO"))
|
||||
|
||||
env_vars = {
|
||||
"CELLXGENE_LOCATION": cellxgene_location,
|
||||
}
|
||||
|
||||
proxy_fix_for = int(os.environ.get("PROXY_FIX_FOR", "0"))
|
||||
proxy_fix_proto = int(os.environ.get("PROXY_FIX_PROTO", "0"))
|
||||
proxy_fix_host = int(os.environ.get("PROXY_FIX_HOST", "0"))
|
||||
proxy_fix_port = int(os.environ.get("PROXY_FIX_PORT", "0"))
|
||||
proxy_fix_prefix = int(os.environ.get("PROXY_FIX_PREFIX", "0"))
|
||||
|
||||
optional_env_vars = {
|
||||
"EXTERNAL_HOST": external_host,
|
||||
"EXTERNAL_PROTOCOL": external_protocol,
|
||||
"GATEWAY_IP": ip,
|
||||
"GATEWAY_PORT": gateway_port,
|
||||
"GATEWAY_EXTRA_SCRIPTS": extra_scripts,
|
||||
"GATEWAY_EXPIRE_SECONDS": expire_seconds,
|
||||
"GATEWAY_TTL": ttl,
|
||||
"GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
|
||||
"GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
|
||||
"GATEWAY_LOG_LEVEL": log_level,
|
||||
"CELLXGENE_ARGS": cellxgene_args,
|
||||
"CELLXGENE_DATA": cellxgene_data,
|
||||
"PROXY_FIX_FOR": proxy_fix_for,
|
||||
"PROXY_FIX_PROTO": proxy_fix_proto,
|
||||
"PROXY_FIX_HOST": proxy_fix_host,
|
||||
"PROXY_FIX_PORT": proxy_fix_port,
|
||||
"PROXY_FIX_PREFIX": proxy_fix_prefix,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from json import loads
|
||||
from json.decoder import JSONDecodeError
|
||||
|
||||
from cellxgene_gateway import env
|
||||
|
||||
@@ -18,9 +17,4 @@ def get_extra_scripts():
|
||||
# ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
|
||||
# f"{env.external_protocol}://{env.external_host}/static/js/google_ua.js"]
|
||||
# where google_ua.js is a script you add to the static/js folder prior to deployment.
|
||||
try:
|
||||
return [] if env.extra_scripts is None else loads(env.extra_scripts)
|
||||
except JSONDecodeError as exc:
|
||||
raise Exception(
|
||||
f'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]'
|
||||
) from exc
|
||||
return [] if env.extra_scripts is None else loads(env.extra_scripts)
|
||||
|
||||
@@ -10,20 +10,14 @@
|
||||
import os
|
||||
import urllib.parse
|
||||
|
||||
from cellxgene_gateway import env, flask_util
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.dir_util import annotations_suffix, make_annotations, make_h5ad
|
||||
|
||||
|
||||
def render_annotations(item, item_source):
|
||||
url = flask_util.view_url(
|
||||
item_source.get_annotations_subpath(item), item_source.name
|
||||
)
|
||||
new_annotation = f"<a class='new' href='{url}'>new</a>"
|
||||
subpath = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/"
|
||||
new_annotation = f"<a class='new' href='{subpath}{item_source.get_annotations_subpath(item)}'>new</a>"
|
||||
annotations = (
|
||||
", ".join(
|
||||
[
|
||||
f"<a href='{CacheKey(item, item_source, a).view_url}/'>{a.name}</a>"
|
||||
f"<a href='{subpath}{a.descriptor}/'>{a.name}</a>"
|
||||
for a in item.annotations
|
||||
]
|
||||
)
|
||||
@@ -31,11 +25,13 @@ def render_annotations(item, item_source):
|
||||
if item.annotations
|
||||
else ""
|
||||
)
|
||||
|
||||
return " | annotations: " + annotations + new_annotation
|
||||
|
||||
|
||||
def render_item(item, item_source):
|
||||
item_string = f"<li> <a href='{ CacheKey(item, item_source).view_url }/'>{item.name}</a> {render_annotations(item, item_source)}</li>"
|
||||
url = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/{item.descriptor}/"
|
||||
item_string = f"<li> <a href='{ url }'>{item.name}</a> {render_annotations(item, item_source)}</li>"
|
||||
return item_string
|
||||
|
||||
|
||||
@@ -46,7 +42,9 @@ def render_item_tree(item_tree, item_source):
|
||||
else ""
|
||||
)
|
||||
branches = (
|
||||
"\n".join([render_item_tree(b, item_source) for b in item_tree.branches])
|
||||
"\n".join(
|
||||
[render_item_tree(b, item_source) for b in item_tree.branches]
|
||||
)
|
||||
if item_tree.branches
|
||||
else ""
|
||||
)
|
||||
@@ -54,7 +52,11 @@ def render_item_tree(item_tree, item_source):
|
||||
if item_tree.descriptor:
|
||||
descriptor = item_tree.descriptor.lstrip("/")
|
||||
url = f"/filecrawl/{descriptor}?source={item_source.name}"
|
||||
name = descriptor.rsplit("/", 1)[-1]
|
||||
name = (
|
||||
descriptor.rsplit("/")[1]
|
||||
if descriptor.find("/") >= 0
|
||||
else descriptor
|
||||
)
|
||||
return f"<li><a href='{url}'>{name}</a>{html}</li>"
|
||||
else:
|
||||
return html
|
||||
@@ -62,6 +64,5 @@ def render_item_tree(item_tree, item_source):
|
||||
|
||||
def render_item_source(item_source, filter=None):
|
||||
item_tree = item_source.list_items(filter)
|
||||
filterpart = "" if filter is None else ":" + filter
|
||||
heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a>{filterpart}</h6>"
|
||||
heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a></h6>"
|
||||
return heading + render_item_tree(item_tree, item_source)
|
||||
|
||||
@@ -7,27 +7,9 @@
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from flask import request, url_for
|
||||
from flask import request
|
||||
|
||||
|
||||
def querystring():
|
||||
qs = request.query_string.decode()
|
||||
return f"?{qs}" if len(qs) > 0 else ""
|
||||
|
||||
|
||||
include_source_in_url = False
|
||||
|
||||
|
||||
def url(endpoint, descriptor, source_name):
|
||||
if include_source_in_url:
|
||||
return url_for(endpoint, source_name=source_name, path=descriptor)
|
||||
else:
|
||||
return url_for(endpoint, path=descriptor)
|
||||
|
||||
|
||||
def view_url(descriptor, source_name):
|
||||
return url("do_view", descriptor, source_name)
|
||||
|
||||
|
||||
def relaunch_url(descriptor, source_name):
|
||||
return url("do_relaunch", descriptor, source_name)
|
||||
|
||||
@@ -22,18 +22,20 @@ from flask import (
|
||||
send_from_directory,
|
||||
url_for,
|
||||
)
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
from flask_api import status
|
||||
from werkzeug.utils import secure_filename
|
||||
|
||||
from cellxgene_gateway import env, flask_util
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.backend_cache import BackendCache
|
||||
from cellxgene_gateway.cache_entry import CacheEntryStatus
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.dir_util import create_dir, is_subdir
|
||||
from cellxgene_gateway.extra_scripts import get_extra_scripts
|
||||
from cellxgene_gateway.filecrawl import render_item_source
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
from cellxgene_gateway.prune_process_cache import PruneProcessCache
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@@ -43,43 +45,21 @@ default_item_source = None
|
||||
|
||||
def _force_https(app):
|
||||
def wrapper(environ, start_response):
|
||||
if env.external_protocol is not None:
|
||||
environ["wsgi.url_scheme"] = env.external_protocol
|
||||
environ["wsgi.url_scheme"] = env.external_protocol
|
||||
return app(environ, start_response)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def set_no_cache(resp):
|
||||
resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
|
||||
resp.headers["Pragma"] = "no-cache"
|
||||
resp.headers["Expires"] = "0"
|
||||
resp.headers["Cache-Control"] = "public, max-age=0"
|
||||
return resp
|
||||
|
||||
|
||||
app.wsgi_app = _force_https(app.wsgi_app)
|
||||
if (
|
||||
env.proxy_fix_for > 0
|
||||
or env.proxy_fix_proto > 0
|
||||
or env.proxy_fix_host > 0
|
||||
or env.proxy_fix_port > 0
|
||||
or env.proxy_fix_prefix > 0
|
||||
):
|
||||
app.wsgi_app = ProxyFix(
|
||||
app.wsgi_app,
|
||||
x_for=env.proxy_fix_for,
|
||||
x_proto=env.proxy_fix_proto,
|
||||
x_host=env.proxy_fix_host,
|
||||
x_port=env.proxy_fix_port,
|
||||
x_prefix=env.proxy_fix_prefix,
|
||||
)
|
||||
|
||||
cache = BackendCache()
|
||||
location = f"{env.external_protocol}://{env.external_host}"
|
||||
|
||||
|
||||
@app.errorhandler(CellxgeneException)
|
||||
def handle_invalid_usage(error):
|
||||
|
||||
message = f"{error.http_status} Error : {error.message}"
|
||||
|
||||
return (
|
||||
@@ -94,6 +74,7 @@ def handle_invalid_usage(error):
|
||||
|
||||
@app.errorhandler(ProcessException)
|
||||
def handle_invalid_process(error):
|
||||
|
||||
message = []
|
||||
|
||||
message.append(error.message)
|
||||
@@ -109,7 +90,7 @@ def handle_invalid_process(error):
|
||||
http_status=error.http_status,
|
||||
stdout=error.stdout,
|
||||
stderr=error.stderr,
|
||||
relaunch_url=error.key.relaunch_url(),
|
||||
dataset=error.key.h5ad_item.descriptor,
|
||||
annotation_file=error.key.annotation_descriptor,
|
||||
),
|
||||
error.http_status,
|
||||
@@ -161,7 +142,10 @@ def filecrawl(path=None):
|
||||
path=path,
|
||||
)
|
||||
)
|
||||
set_no_cache(resp)
|
||||
resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
|
||||
resp.headers["Pragma"] = "no-cache"
|
||||
resp.headers["Expires"] = "0"
|
||||
resp.headers["Cache-Control"] = "public, max-age=0"
|
||||
return resp
|
||||
|
||||
|
||||
@@ -210,21 +194,14 @@ def do_view(path, source_name=None):
|
||||
match.status == CacheEntryStatus.loaded
|
||||
or match.status == CacheEntryStatus.loading
|
||||
):
|
||||
if source.is_authorized(match.key.descriptor):
|
||||
return match.serve_content(path)
|
||||
else:
|
||||
raise CellxgeneException("User not authorized to access this data", 403)
|
||||
return match.serve_content(path)
|
||||
elif match.status == CacheEntryStatus.error:
|
||||
raise ProcessException.from_cache_entry(match)
|
||||
|
||||
|
||||
@app.route("/cache_status", methods=["GET"])
|
||||
def do_GET_status():
|
||||
return render_template(
|
||||
"cache_status.html",
|
||||
entry_list=cache.entry_list,
|
||||
extra_scripts=get_extra_scripts(),
|
||||
)
|
||||
return render_template("cache_status.html", entry_list=cache.entry_list)
|
||||
|
||||
|
||||
@app.route("/cache_status.json", methods=["GET"])
|
||||
@@ -248,14 +225,15 @@ def do_GET_status_json():
|
||||
|
||||
@app.route("/relaunch/<path:path>", methods=["GET"])
|
||||
def do_relaunch(path):
|
||||
source_name = request.args.get("source_name") or default_item_source.name
|
||||
source_name = request.args.get("source") or default_item_source.name
|
||||
source = matching_source(source_name)
|
||||
key = CacheKey.for_lookup(source, source.lookup(path))
|
||||
match = cache.check_entry(key)
|
||||
if not match is None:
|
||||
match.terminate()
|
||||
qs = request.query_string.decode()
|
||||
return redirect(
|
||||
key.view_url,
|
||||
url_for("do_view", path=path) + (f"?{qs}" if len(qs) > 0 else ""),
|
||||
code=302,
|
||||
)
|
||||
|
||||
@@ -271,12 +249,6 @@ def do_terminate(path):
|
||||
return redirect(url_for("do_GET_status"), code=302)
|
||||
|
||||
|
||||
@app.route("/metadata/ip_address", methods=["GET"])
|
||||
def ip_address():
|
||||
resp = make_response(env.ip)
|
||||
return set_no_cache(resp)
|
||||
|
||||
|
||||
def launch():
|
||||
env.validate()
|
||||
if not item_sources or not len(item_sources):
|
||||
@@ -297,7 +269,7 @@ def launch():
|
||||
|
||||
def main():
|
||||
logging.basicConfig(
|
||||
level=env.log_level,
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
|
||||
)
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA", None)
|
||||
@@ -315,7 +287,6 @@ def main():
|
||||
default_item_source = "local"
|
||||
if len(item_sources) == 0:
|
||||
raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
|
||||
flask_util.include_source_in_url = len(item_sources) > 1
|
||||
|
||||
launch()
|
||||
|
||||
|
||||
@@ -18,11 +18,10 @@ class FileItem(Item):
|
||||
The Item superclass expects a 'name' and 'type'.
|
||||
"""
|
||||
|
||||
def __init__(self, subpath: str, ext: str = "", *args, **kwargs):
|
||||
def __init__(self, subpath: str, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.subpath = subpath
|
||||
self.ext = ext
|
||||
|
||||
@property
|
||||
def descriptor(self) -> str:
|
||||
return os.path.join(self.subpath, self.name + self.ext).strip("/")
|
||||
return os.path.join(self.subpath, self.name).strip("/")
|
||||
|
||||
@@ -35,11 +35,6 @@ class FileItemSource(ItemSource):
|
||||
def name(self):
|
||||
return self._name or f"Files:{self.base_path}"
|
||||
|
||||
def is_gene_set(self, path: str) -> bool:
|
||||
return ("_gene_sets" in path or "-gene-sets" in path) and path.endswith(
|
||||
self.annotation_file_suffix
|
||||
)
|
||||
|
||||
def is_h5ad_file(self, path: str) -> bool:
|
||||
return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
|
||||
|
||||
@@ -56,7 +51,7 @@ class FileItemSource(ItemSource):
|
||||
return self.convert_h5ad_path_to_annotation(item.descriptor)
|
||||
|
||||
def list_items(self, filter: str = None) -> ItemTree:
|
||||
item_tree = self.scan_directory("" if filter is None else filter)
|
||||
item_tree = self.scan_directory()
|
||||
|
||||
"""def get_items(dir):
|
||||
if dir.branches:
|
||||
@@ -72,7 +67,9 @@ class FileItemSource(ItemSource):
|
||||
base_path = os.path.join(self.base_path, subpath)
|
||||
|
||||
if not os.path.exists(base_path):
|
||||
raise Exception(f"Path for local files '{base_path}' does not exist.")
|
||||
raise Exception(
|
||||
f"Path for local files '{base_path}' does not exist."
|
||||
)
|
||||
|
||||
filepath_map = dict(
|
||||
(filepath, os.path.join(base_path, filepath))
|
||||
@@ -98,12 +95,14 @@ class FileItemSource(ItemSource):
|
||||
]
|
||||
|
||||
items = [
|
||||
self.make_fileitem_from_path(filename, subpath) for filename in h5ad_paths
|
||||
self.make_fileitem_from_path(filename, subpath)
|
||||
for filename in h5ad_paths
|
||||
]
|
||||
branches = None
|
||||
if len(subdirs) > 0:
|
||||
branches = [
|
||||
self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
|
||||
self.scan_directory(os.path.join(subpath, subdir))
|
||||
for subdir in subdirs
|
||||
]
|
||||
|
||||
return ItemTree(subpath, items, branches)
|
||||
@@ -126,19 +125,17 @@ class FileItemSource(ItemSource):
|
||||
if self.is_h5ad_file(full_path):
|
||||
return self.shallowitem_from_descriptor(descriptor)
|
||||
|
||||
def is_authorized(self, descriptor):
|
||||
return True
|
||||
|
||||
def lookup(self, indescriptor: str) -> LookupResult:
|
||||
descriptor = indescriptor.strip("/")
|
||||
if descriptor.endswith(self.annotation_file_suffix):
|
||||
annotation_item = self.shallowitem_from_descriptor(descriptor, True)
|
||||
annotation_item = self.shallowitem_from_descriptor(
|
||||
descriptor, True
|
||||
)
|
||||
h5ad_descriptor = self.convert_annotation_path_to_h5ad(
|
||||
annotation_item.subpath
|
||||
)
|
||||
item = self.lookup_item(h5ad_descriptor)
|
||||
if item is not None:
|
||||
dir_util.ensure_dir_exists(self.full_path(annotation_item.subpath))
|
||||
return LookupResult(item, annotation_item)
|
||||
else:
|
||||
item = self.lookup_item(descriptor)
|
||||
@@ -158,16 +155,9 @@ class FileItemSource(ItemSource):
|
||||
def make_fileitem_from_path(
|
||||
self, filename, subpath, is_annotation=False, is_shallow=False
|
||||
) -> FileItem:
|
||||
if is_annotation and filename.endswith(self.annotation_file_suffix):
|
||||
name = filename[: -len(self.annotation_file_suffix)]
|
||||
ext = self.annotation_file_suffix
|
||||
else:
|
||||
name = filename
|
||||
ext = ""
|
||||
item = FileItem(
|
||||
subpath=subpath,
|
||||
name=name,
|
||||
ext=ext,
|
||||
name=filename,
|
||||
type=ItemType.annotation if is_annotation else ItemType.h5ad,
|
||||
)
|
||||
|
||||
@@ -182,11 +172,14 @@ class FileItemSource(ItemSource):
|
||||
annotations_fullpath = self.full_path(annotations_subpath)
|
||||
if os.path.isdir(annotations_fullpath):
|
||||
return [
|
||||
self.make_fileitem_from_path(annotation, annotations_subpath, True)
|
||||
self.make_fileitem_from_path(
|
||||
annotation, annotations_subpath, True
|
||||
)
|
||||
for annotation in sorted(os.listdir(annotations_fullpath))
|
||||
if annotation.endswith(self.annotation_file_suffix)
|
||||
and not self.is_gene_set(annotation)
|
||||
and os.path.isfile(os.path.join(annotations_fullpath, annotation))
|
||||
and os.path.isfile(
|
||||
os.path.join(annotations_fullpath, annotation)
|
||||
)
|
||||
]
|
||||
else:
|
||||
return None
|
||||
|
||||
@@ -18,7 +18,9 @@ class ItemType(Enum):
|
||||
|
||||
|
||||
class Item(ABC):
|
||||
def __init__(self, name: str, type: ItemType, annotations: List["Item"] = None):
|
||||
def __init__(
|
||||
self, name: str, type: ItemType, annotations: List["Item"] = None
|
||||
):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.annotations = annotations
|
||||
|
||||
@@ -40,10 +40,6 @@ class ItemSource(ABC):
|
||||
def update(self, item: Item) -> None:
|
||||
raise Exception('"update" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def is_authorized(self, descriptor: str) -> bool:
|
||||
raise Exception('"is_authorized" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, descriptor: str) -> LookupResult:
|
||||
raise Exception('"lookup" unimplemented')
|
||||
|
||||
@@ -7,21 +7,13 @@
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
from os.path import basename, dirname
|
||||
from typing import List
|
||||
|
||||
import flask
|
||||
import s3fs
|
||||
|
||||
from os.path import join, dirname, basename
|
||||
from cellxgene_gateway import dir_util
|
||||
import s3fs
|
||||
from cellxgene_gateway.items.s3.s3item import S3Item
|
||||
from cellxgene_gateway.items.item import ItemTree, ItemType
|
||||
from cellxgene_gateway.items.item_source import ItemSource, LookupResult
|
||||
from cellxgene_gateway.items.s3.s3item import S3Item
|
||||
|
||||
|
||||
def truthy(val: str):
|
||||
return val.lower() in ["true", "1"]
|
||||
|
||||
|
||||
class S3ItemSource(ItemSource):
|
||||
@@ -34,24 +26,14 @@ class S3ItemSource(ItemSource):
|
||||
annotation_file_suffix=".csv",
|
||||
):
|
||||
self._name = name
|
||||
enable_cache = os.environ.get("S3_ENABLE_LISTINGS_CACHE", "false").lower()
|
||||
assert enable_cache in ["0", "1", "false", "true"]
|
||||
self.use_listings_cache = truthy(enable_cache)
|
||||
self.s3 = s3fs.S3FileSystem(use_listings_cache=self.use_listings_cache)
|
||||
if bucket.startswith("s3://"):
|
||||
raise Exception(
|
||||
f"Bucket name should not include s3:// prefix, got {bucket}"
|
||||
)
|
||||
self.s3 = s3fs.S3FileSystem()
|
||||
self.bucket = bucket
|
||||
self.h5ad_suffix = h5ad_suffix
|
||||
self.annotation_dir_suffix = annotation_dir_suffix
|
||||
self.annotation_file_suffix = annotation_file_suffix
|
||||
|
||||
def url(self, key):
|
||||
return "s3://" + self.bucket + "/" + key
|
||||
|
||||
def remove_bucket(self, filepath):
|
||||
return filepath[len(self.bucket) :].lstrip("/")
|
||||
def url(self, path):
|
||||
return "s3://" + join(self.bucket, path)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
@@ -73,51 +55,51 @@ class S3ItemSource(ItemSource):
|
||||
return self.convert_h5ad_key_to_annotation(item.descriptor)
|
||||
|
||||
def list_items(self, filter: str = None) -> ItemTree:
|
||||
item_tree = self.scan_directory("" if filter is None else filter)
|
||||
item_tree = self.scan_directory()
|
||||
return item_tree
|
||||
|
||||
@property
|
||||
def refresh(self):
|
||||
return (
|
||||
truthy(flask.request.args.get("refresh", default="false"))
|
||||
or not self.use_listings_cache
|
||||
)
|
||||
|
||||
def scan_directory(self, directory_key="") -> dict:
|
||||
url = self.url(directory_key)
|
||||
def scan_directory(self, subpath="") -> dict:
|
||||
url = self.url(subpath)
|
||||
|
||||
if not self.s3.exists(url):
|
||||
raise Exception(f"S3 url '{url}' does not exist.")
|
||||
|
||||
s3key_map = dict(
|
||||
(self.remove_bucket(filepath), "s3://" + filepath)
|
||||
for filepath in sorted(self.s3.ls(url, refresh=self.refresh))
|
||||
(filepath[len(self.bucket) :], "s3://" + filepath)
|
||||
for filepath in sorted(self.s3.ls(url))
|
||||
)
|
||||
|
||||
def is_annotation_dir(dir_s3key):
|
||||
return (
|
||||
dir_s3key.endswith(self.annotation_dir_suffix)
|
||||
and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_keys
|
||||
and self.convert_annotation_key_to_h5ad(dir_s3key)
|
||||
in h5ad_paths
|
||||
)
|
||||
|
||||
h5ad_keys = [
|
||||
h5ad_paths = [
|
||||
filepath
|
||||
for filepath, item_url in s3key_map.items()
|
||||
if self.is_h5ad_url(item_url)
|
||||
]
|
||||
|
||||
subdir_keys = [
|
||||
subdirs = [
|
||||
filepath
|
||||
for filepath, item_url in s3key_map.items()
|
||||
if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
|
||||
]
|
||||
|
||||
items = [self.make_s3item_from_key(basename(key), key) for key in h5ad_keys]
|
||||
items = [
|
||||
self.make_s3item_from_key(filename, join(subpath, filename))
|
||||
for filename in h5ad_paths
|
||||
]
|
||||
branches = None
|
||||
if len(subdir_keys) > 0:
|
||||
branches = [self.scan_directory(key) for key in subdir_keys]
|
||||
if len(subdirs) > 0:
|
||||
branches = [
|
||||
self.scan_directory(join(subpath, subdir))
|
||||
for subdir in subdirs
|
||||
]
|
||||
|
||||
return ItemTree(directory_key, items, branches)
|
||||
return ItemTree(subpath, items, branches)
|
||||
|
||||
def create_annotation(self, item: S3Item, name: str) -> S3Item:
|
||||
annotation = self.make_s3item_from_key(
|
||||
@@ -129,9 +111,6 @@ class S3ItemSource(ItemSource):
|
||||
def update(self, item: S3Item) -> None:
|
||||
pass
|
||||
|
||||
def is_authorized(self, descriptor):
|
||||
return True
|
||||
|
||||
def lookup_item(self, descriptor):
|
||||
full_path = self.url(descriptor)
|
||||
if self.is_h5ad_url(full_path):
|
||||
@@ -140,7 +119,9 @@ class S3ItemSource(ItemSource):
|
||||
def lookup(self, indescriptor: str) -> LookupResult:
|
||||
descriptor = indescriptor.strip("/")
|
||||
if descriptor.endswith(self.annotation_file_suffix):
|
||||
annotation_item = self.shallowitem_from_descriptor(descriptor, True)
|
||||
annotation_item = self.shallowitem_from_descriptor(
|
||||
descriptor, True
|
||||
)
|
||||
if not self.s3.exists(self.url(annotation_item.s3key)):
|
||||
with self.s3.open(self.url(annotation_item.s3key), "w") as f:
|
||||
f.write("")
|
||||
@@ -180,13 +161,11 @@ class S3ItemSource(ItemSource):
|
||||
if self.s3.isdir(annotations_fullpath):
|
||||
return [
|
||||
self.make_s3item_from_key(
|
||||
basename(annotation), self.remove_bucket(annotation), True
|
||||
)
|
||||
for annotation in sorted(
|
||||
self.s3.ls(annotations_fullpath, refresh=self.refresh)
|
||||
annotation, join(annotations_subpath, annotation), True
|
||||
)
|
||||
for annotation in sorted(self.s3.ls(annotations_fullpath))
|
||||
if annotation.endswith(self.annotation_file_suffix)
|
||||
and self.s3.isfile("s3://" + annotation)
|
||||
and self.s3.isfile(join(annotations_fullpath, annotation))
|
||||
]
|
||||
else:
|
||||
return None
|
||||
|
||||
41
cellxgene_gateway/path_util.py
Normal file
41
cellxgene_gateway/path_util.py
Normal file
@@ -0,0 +1,41 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.dir_util import make_h5ad
|
||||
|
||||
|
||||
def validate_exists(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
raise CellxgeneException(
|
||||
"File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
|
||||
def validate_is_file(file_path):
|
||||
validate_exists(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise CellxgeneException(
|
||||
"Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
def validate_is_dir(file_path):
|
||||
validate_exists(file_path)
|
||||
if not os.path.isdir(file_path):
|
||||
raise CellxgeneException(
|
||||
"Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
return
|
||||
@@ -10,15 +10,14 @@
|
||||
import logging
|
||||
import time
|
||||
|
||||
from cellxgene_gateway import env, util
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from cellxgene_gateway.env import ttl
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
|
||||
|
||||
class PruneProcessCache:
|
||||
def __init__(self, cache):
|
||||
self.cache = cache
|
||||
self.expire_seconds = env.expire_seconds
|
||||
self.expire_seconds = 3600 if ttl is None else int(ttl)
|
||||
|
||||
def __call__(self):
|
||||
while True:
|
||||
@@ -26,22 +25,26 @@ class PruneProcessCache:
|
||||
self.prune()
|
||||
|
||||
def prune(self):
|
||||
timestamp = util.current_time_stamp()
|
||||
timestamp = current_time_stamp()
|
||||
cutoff = timestamp - self.expire_seconds
|
||||
processes_to_delete = [p for p in self.cache.entry_list if p.timestamp < cutoff]
|
||||
processes_to_delete = [
|
||||
p for p in self.cache.entry_list if p.timestamp < cutoff
|
||||
]
|
||||
processes_to_keep = [
|
||||
p for p in self.cache.entry_list if not p.timestamp < cutoff
|
||||
]
|
||||
|
||||
logger = logging.getLogger("cellxgene_gateway")
|
||||
logger.debug(
|
||||
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}, pruning {processes_to_delete}"
|
||||
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
|
||||
)
|
||||
|
||||
for process in processes_to_delete:
|
||||
try:
|
||||
logger.info(f"pruning process {process.pid} ({process.key.descriptor})")
|
||||
logger.info(
|
||||
f"pruning process {process.pid} ({process.key.dataset})"
|
||||
)
|
||||
self.cache.prune(process)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"failed to prune process {process.pid} ({process.key.descriptor})"
|
||||
"failed to prune process {process.pid} ({process.dataset})"
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// neandertal javascript
|
||||
// Annotations only work with file itemsources at the moment. If they work with others in the future we may need to revisit this.
|
||||
// TODO: rewrite this --
|
||||
const new_annotation_callback = (() =>{
|
||||
const suffix = `.csv`;
|
||||
return (e) => {
|
||||
|
||||
@@ -15,30 +15,29 @@ from flask_api import status
|
||||
from cellxgene_gateway.cache_entry import CacheEntryStatus
|
||||
from cellxgene_gateway.dir_util import make_annotations
|
||||
from cellxgene_gateway.env import (
|
||||
cellxgene_args,
|
||||
enable_annotations,
|
||||
enable_backed_mode,
|
||||
cellxgene_args,
|
||||
)
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SubprocessBackend:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path):
|
||||
def create_cmd(
|
||||
self, cellxgene_loc, file_path, port, scripts, annotation_file_path
|
||||
):
|
||||
if enable_annotations and not annotation_file_path is None:
|
||||
if annotation_file_path == "":
|
||||
extra_args = f" --annotations-dir {make_annotations(file_path)}"
|
||||
extra_args = (
|
||||
f" --annotations-dir {make_annotations(file_path)}"
|
||||
)
|
||||
else:
|
||||
extra_args = f" --annotations-file {annotation_file_path}"
|
||||
gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv"
|
||||
extra_args += f" --gene-sets-file {gene_sets_file_path}"
|
||||
else:
|
||||
extra_args = " --disable-annotations"
|
||||
extra_args += " --disable-gene-sets-save"
|
||||
if enable_backed_mode:
|
||||
extra_args += " --backed"
|
||||
if not cellxgene_args is None:
|
||||
@@ -64,7 +63,7 @@ class SubprocessBackend:
|
||||
scripts,
|
||||
cache_entry.key.annotation_file_path,
|
||||
)
|
||||
logger.info(f"launching {cmd}")
|
||||
logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
|
||||
process = subprocess.Popen(
|
||||
[cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
|
||||
)
|
||||
@@ -93,6 +92,5 @@ class SubprocessBackend:
|
||||
cache_entry.append_output(output)
|
||||
|
||||
cache_entry.set_loaded(process.pid)
|
||||
for output in process.communicate():
|
||||
logger.debug(f"cellxgene:{output}")
|
||||
logger.info(f"exiting {cmd}")
|
||||
|
||||
return
|
||||
|
||||
@@ -48,7 +48,7 @@
|
||||
<tr>
|
||||
<td>{{ entry.pid }}</td>
|
||||
<td><a
|
||||
href="{{ entry.key.view_url }}">{{ entry.key.h5ad_item.descriptor }}</a>
|
||||
href="{{ url_for('do_view', path=entry.key.descriptor, source_name=entry.key.source_name) }}">{{ entry.key.h5ad_item.descriptor }}</a>
|
||||
</td>
|
||||
<td>{{ entry.key.annotation_descriptor }}</td>
|
||||
<td>{{ entry.source_name }}</td>
|
||||
@@ -75,9 +75,7 @@
|
||||
const el = $(this);
|
||||
const ts = el.text();
|
||||
const dt = new Date(parseInt(ts * 1000));
|
||||
el.prepend(`${dt.toISOString()}<br>(`);
|
||||
el.append(')');
|
||||
|
||||
el.html(`${dt.toISOString()}<br>(${ts})`);
|
||||
});
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -28,11 +28,11 @@
|
||||
|
||||
<h4>{{ message }}</h4>
|
||||
|
||||
<a href="{{ url_for('filecrawl') }}">
|
||||
<a href="/filecrawl.html">
|
||||
Please click here to be redirected to the file directory.
|
||||
</a>
|
||||
<br>
|
||||
<a href="{{ url_for('index') }}">
|
||||
<a href="/">
|
||||
Please click here to return to the homepage.
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -36,10 +36,10 @@
|
||||
Navigation:
|
||||
<ul>
|
||||
{% if path %}
|
||||
<li><a href="{{ url_for('filecrawl') }}">top level</a></li>
|
||||
<li><a href="/filecrawl.html">top level</a></li>
|
||||
{% else %}
|
||||
{% endif %}
|
||||
<li><a href="{{ url_for('index') }}">homepage</a></li>
|
||||
<li><a href="/">homepage</a></li>
|
||||
</ul>
|
||||
</p>
|
||||
<script>
|
||||
|
||||
@@ -35,12 +35,12 @@
|
||||
Links:
|
||||
</h1>
|
||||
<div class="list-group" style="width:50%;padding-left:65px">
|
||||
<a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
|
||||
<a href="/filecrawl.html" class="list-group-item list-group-item-action">
|
||||
<u>File Crawler: Allows you to view all uploaded data.</u></a>
|
||||
|
||||
</div>
|
||||
<div class="list-group" style="width:50%;padding-left:65px">
|
||||
<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
|
||||
<a href="/cache_status" class="list-group-item list-group-item-action">
|
||||
<u>Cache Status: view status of launched cellxgene servers.</u></a>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -35,11 +35,11 @@
|
||||
The page will refresh shortly.
|
||||
</p>
|
||||
|
||||
<a href="{{ url_for('filecrawl') }}">
|
||||
<a href="/filecrawl.html">
|
||||
Please click here to be redirected to the file directory.
|
||||
</a>
|
||||
<br>
|
||||
<a href="{{ url_for('index') }}">
|
||||
<a href="/">
|
||||
Please click here to return to the homepage.
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -36,13 +36,13 @@
|
||||
<h4>Options</h4>
|
||||
Please choose one of the following, or use the back button:
|
||||
<ul>
|
||||
<li><a href="{{ relaunch_url }}">
|
||||
<li><a href="{{url_for('do_relaunch', path=dataset)}}">
|
||||
Attempt to relaunch the cellxgene server.
|
||||
</a></li>
|
||||
<li><a href="{{ url_for('filecrawl') }}">
|
||||
<li><a href="/filecrawl.html">
|
||||
Return to the file directory.
|
||||
</a></li>
|
||||
<li><a href="{{ url_for('index') }}">
|
||||
<li><a href="/">
|
||||
Return to the homepage.
|
||||
</a></li>
|
||||
</ul>
|
||||
|
||||
@@ -2,17 +2,12 @@ name: cellxgene-gateway
|
||||
channels:
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- python=3.9
|
||||
- python=3.7
|
||||
- requests
|
||||
- flask
|
||||
- psutil
|
||||
- black
|
||||
- twine
|
||||
- isort
|
||||
- coverage
|
||||
- pip
|
||||
- pip:
|
||||
- pre_commit
|
||||
- flask-api
|
||||
- werkzeug
|
||||
- cellxgene
|
||||
- cellxgene>=0.15
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
cellxgene
|
||||
cellxgene>=0.15
|
||||
flask
|
||||
flask-api
|
||||
werkzeug
|
||||
flask_api
|
||||
psutil
|
||||
requests
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
|
||||
|
||||
def stub_join(path):
|
||||
path.join = lambda x, y: x + "/" + y
|
||||
|
||||
|
||||
class TestFileItemSource(unittest.TestCase):
|
||||
@patch("os.path")
|
||||
@patch("os.listdir")
|
||||
def test_list_items_GIVEN_no_subpath_THEN_checks_dir(self, listdir, path):
|
||||
stub_join(path)
|
||||
source = FileItemSource("/tmp/unittest", "local")
|
||||
source.list_items()
|
||||
path.exists.assert_called_once_with("/tmp/unittest/")
|
||||
|
||||
@patch("os.path")
|
||||
@patch("os.listdir")
|
||||
def test_list_items_GIVEN_subpath_THEN_checks_subpath(self, listdir, path):
|
||||
stub_join(path)
|
||||
source = FileItemSource("/tmp/unittest", "local")
|
||||
source.list_items("foo")
|
||||
path.exists.assert_called_once_with("/tmp/unittest/foo")
|
||||
|
||||
def test_make_fileitem_from_path_GIVEN_annotation_file_THEN_name_lacks_csv(
|
||||
self,
|
||||
):
|
||||
source = FileItemSource(tempfile.gettempdir(), "local")
|
||||
item = source.make_fileitem_from_path(
|
||||
"customanno.csv", "someh5ad_annotations", True
|
||||
)
|
||||
self.assertEqual(item.name, "customanno")
|
||||
self.assertEqual(item.descriptor, "someh5ad_annotations/customanno.csv")
|
||||
|
||||
def test_make_fileitem_from_path_GIVEN_h5ad_file_THEN_returns_name(self):
|
||||
source = FileItemSource(tempfile.gettempdir(), "local")
|
||||
item = source.make_fileitem_from_path("someanalysis.h5ad", "studydir")
|
||||
self.assertEqual(item.name, "someanalysis.h5ad")
|
||||
self.assertEqual(item.descriptor, "studydir/someanalysis.h5ad")
|
||||
@@ -1,172 +0,0 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from cellxgene_gateway.gateway import app
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.items.s3.s3item import S3Item
|
||||
from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
|
||||
|
||||
|
||||
class TestScanDirectory(unittest.TestCase):
|
||||
@patch("s3fs.S3FileSystem")
|
||||
def test_GIVEN_invalid_bucket_THEN_throws_error(self, s3func):
|
||||
class S3Mock:
|
||||
def exists(path):
|
||||
if path in ["s3://my-bucket/"]:
|
||||
return False
|
||||
|
||||
s3func.return_value = S3Mock
|
||||
source = S3ItemSource("my-bucket")
|
||||
with self.assertRaises(Exception) as context:
|
||||
source.scan_directory()
|
||||
self.assertEqual(
|
||||
"S3 url 's3://my-bucket/' does not exist.",
|
||||
str(context.exception),
|
||||
)
|
||||
|
||||
@patch("s3fs.S3FileSystem")
|
||||
def test__GIVEN_multilevel_bucket_THEN_properly_recurses_suburls(self, s3func):
|
||||
class S3Mock:
|
||||
def exists(path):
|
||||
if path in [
|
||||
"s3://my-bucket/",
|
||||
"s3://my-bucket/pbmc3k.h5ad",
|
||||
"s3://my-bucket/lvl1",
|
||||
"s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
|
||||
"s3://my-bucket/lvl1/lvl2",
|
||||
"s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
|
||||
]:
|
||||
return True
|
||||
raise Exception("exists called with " + path)
|
||||
|
||||
def ls(path, refresh):
|
||||
assert refresh == True
|
||||
if path == "s3://my-bucket/":
|
||||
return [
|
||||
"my-bucket/lvl1",
|
||||
"my-bucket/pbmc3k.h5ad",
|
||||
"my-bucket/pbmc3k_annotations",
|
||||
]
|
||||
elif path == "s3://my-bucket/pbmc3k_annotations":
|
||||
return ["my-bucket/pbmc3k_annotations/annot.csv"]
|
||||
elif path == "s3://my-bucket/lvl1":
|
||||
return ["my-bucket/lvl1/lvl2", "my-bucket/lvl1/pbmc3k_l1.h5ad"]
|
||||
elif path == "s3://my-bucket/lvl1/lvl2":
|
||||
return ["my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad"]
|
||||
|
||||
raise Exception("ls called with " + path)
|
||||
|
||||
def isdir(path):
|
||||
if path in [
|
||||
"s3://my-bucket/lvl1",
|
||||
"s3://my-bucket/pbmc3k_annotations",
|
||||
"s3://my-bucket/lvl1/lvl2",
|
||||
]:
|
||||
return True
|
||||
if path in [
|
||||
"s3://my-bucket/pbmc3k.h5ad",
|
||||
"s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
|
||||
"s3://my-bucket/lvl1/pbmc3k_l1_annotations",
|
||||
"s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
|
||||
"s3://my-bucket/lvl1/lvl2/pbmc3k_l2_annotations",
|
||||
]:
|
||||
return False
|
||||
raise Exception("isdir called with " + path)
|
||||
|
||||
def isfile(path):
|
||||
if path in ["s3://my-bucket/pbmc3k_annotations/annot.csv"]:
|
||||
return True
|
||||
if path in ["s3://my-bucket/pbmc3k_annotations"]:
|
||||
return False
|
||||
raise Exception("isfile called with " + path)
|
||||
|
||||
s3func.return_value = S3Mock
|
||||
source = S3ItemSource("my-bucket")
|
||||
with app.test_request_context(query_string="refresh=true") as test_context:
|
||||
tree = source.scan_directory()
|
||||
|
||||
def s3item_compare(i1, i2, msg=""):
|
||||
self.assertEqual(i1.name, i2.name, "name equals")
|
||||
self.assertEqual(i1.type, i2.type, "type equals")
|
||||
self.assertEqual(i1.s3key, i2.s3key, "s3key equals")
|
||||
if i1.annotations is None:
|
||||
self.assertEqual(i1.annotations, i2.annotations, "annotations equals")
|
||||
else:
|
||||
self.assertEqual(
|
||||
len(i1.annotations),
|
||||
len(i2.annotations),
|
||||
"annotations length equals",
|
||||
)
|
||||
for a1, a2 in zip(i1.annotations, i2.annotations):
|
||||
self.assertEqual(a1, a2)
|
||||
return True
|
||||
|
||||
self.addTypeEqualityFunc(S3Item, s3item_compare)
|
||||
|
||||
def assertTree(t, descriptor, items):
|
||||
self.assertEqual(t.descriptor, descriptor)
|
||||
self.assertEqual(len(t.items), len(items))
|
||||
for i1, i2 in zip(t.items, items):
|
||||
self.assertEqual(i1, i2)
|
||||
|
||||
assertTree(
|
||||
tree,
|
||||
"",
|
||||
[
|
||||
S3Item(
|
||||
"pbmc3k.h5ad",
|
||||
name="pbmc3k.h5ad",
|
||||
type=ItemType.h5ad,
|
||||
annotations=[
|
||||
S3Item(
|
||||
"pbmc3k_annotations/annot.csv",
|
||||
name="annot.csv",
|
||||
type=ItemType.annotation,
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(len(tree.branches), 1)
|
||||
lvl1 = tree.branches[0]
|
||||
assertTree(
|
||||
lvl1,
|
||||
"lvl1",
|
||||
[
|
||||
S3Item(
|
||||
"lvl1/pbmc3k_l1.h5ad",
|
||||
name="pbmc3k_l1.h5ad",
|
||||
type=ItemType.h5ad,
|
||||
annotations=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(len(lvl1.branches), 1)
|
||||
lvl2 = lvl1.branches[0]
|
||||
assertTree(
|
||||
lvl2,
|
||||
"lvl1/lvl2",
|
||||
[
|
||||
S3Item(
|
||||
"lvl1/lvl2/pbmc3k_l2.h5ad",
|
||||
name="pbmc3k_l2.h5ad",
|
||||
type=ItemType.h5ad,
|
||||
annotations=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(lvl2.branches, None)
|
||||
|
||||
|
||||
class TestListItems(unittest.TestCase):
|
||||
def test_GIVEN_filter_THEN_pass_filter_into_scan_directory(self):
|
||||
source = S3ItemSource("my-bucket")
|
||||
source.scan_directory = MagicMock()
|
||||
tree = source.list_items("some-filter")
|
||||
source.scan_directory.assert_called_once_with("some-filter")
|
||||
|
||||
def test_GIVEN_no_filter_THEN_pass_empty_string_into_scan_directory(self):
|
||||
source = S3ItemSource("my-bucket")
|
||||
source.scan_directory = MagicMock()
|
||||
tree = source.list_items()
|
||||
source.scan_directory.assert_called_once_with("")
|
||||
@@ -1,28 +0,0 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.backend_cache import is_port_in_use
|
||||
|
||||
|
||||
class TestIsPortInUse(unittest.TestCase):
|
||||
@patch("socket.socket")
|
||||
def test_GIVEN_free_port_THEN_returns_true(self, socketMock):
|
||||
connectMock = socketMock()
|
||||
connectMock.connect_ex.return_value = 0
|
||||
connectMock.__enter__.return_value = connectMock
|
||||
self.assertEqual(is_port_in_use(123), True)
|
||||
self.assertTrue(connectMock.__enter__.calledOnce)
|
||||
self.assertTrue(connectMock.__exit__.calledOnce)
|
||||
self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
|
||||
self.assertTrue(socketMock.calledOnceWith("a"))
|
||||
|
||||
@patch("socket.socket")
|
||||
def test_GIVEN_used_port_THEN_returns_false(self, socketMock):
|
||||
connectMock = socketMock()
|
||||
connectMock.__enter__.return_value = connectMock
|
||||
connectMock.connect_ex.return_value = 1
|
||||
self.assertTrue(connectMock.__enter__.calledOnce)
|
||||
self.assertTrue(connectMock.__exit__.calledOnce)
|
||||
self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
|
||||
self.assertTrue(socketMock.calledOnceWith("a"))
|
||||
self.assertEqual(is_port_in_use(123), False)
|
||||
@@ -1,62 +1,33 @@
|
||||
import unittest
|
||||
|
||||
from flask import Flask
|
||||
|
||||
from cellxgene_gateway import flask_util
|
||||
from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.gateway import app
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItem("/czi/", "pbmc3k.h5ad", ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
)
|
||||
|
||||
|
||||
class TestRenderEntry(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = app
|
||||
self.app_context = self.app.test_request_context()
|
||||
self.app_context.push()
|
||||
self.client = self.app.test_client()
|
||||
|
||||
def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
|
||||
entry = CacheEntry.for_key("some-key", 1)
|
||||
self.assertEqual(entry.status, CacheEntryStatus.loading)
|
||||
|
||||
def test_GIVEN_absolute_static_url_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = False
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
"src:url(/static/assets/"
|
||||
)
|
||||
expected = "src:url(/view/czi/pbmc3k.h5ad/static/assets/"
|
||||
expected = "src:url(http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_GIVEN_absolute_src_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = False
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
'<link rel="shortcut icon" href="/static/assets/favicon.ico">'
|
||||
)
|
||||
expected = '<link rel="shortcut icon" href="/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_GIVEN_absolute_static_url_include_source_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = True
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
"src:url(/static/assets/"
|
||||
)
|
||||
expected = "src:url(/source/local/view/czi/pbmc3k.h5ad/static/assets/"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_GIVEN_absolute_src_include_source_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = True
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
'<link rel="shortcut icon" href="/static/assets/favicon.ico">'
|
||||
)
|
||||
expected = '<link rel="shortcut icon" href="/source/local/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
|
||||
expected = '<link rel="shortcut icon" href="http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.dir_util import ensure_dir_exists, make_annotations, make_h5ad
|
||||
|
||||
|
||||
class TestMakeH5ad(unittest.TestCase):
|
||||
def test_GIVEN_annotation_dir_THEN_returns_h5ad(self):
|
||||
self.assertEqual(make_h5ad("pbmc_annotations"), "pbmc.h5ad")
|
||||
|
||||
|
||||
class TestMakeAnnotations(unittest.TestCase):
|
||||
def test_GIVEN_h5ad_THEN_returns_annotations(self):
|
||||
self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
|
||||
|
||||
|
||||
class TestMakeAnnotations(unittest.TestCase):
|
||||
def test_GIVEN_h5ad_THEN_returns_annotations(self):
|
||||
self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
|
||||
|
||||
|
||||
class TestEnsureDirExists(unittest.TestCase):
|
||||
@patch("os.path.exists")
|
||||
@patch("os.makedirs")
|
||||
def test_GIVEN_existing_THEN_does_not_call_makedir(self, makedirsMock, existsMock):
|
||||
existsMock.return_value = True
|
||||
ensure_dir_exists("/foo")
|
||||
makedirsMock.assert_not_called()
|
||||
|
||||
@patch("os.path.exists")
|
||||
@patch("os.makedirs")
|
||||
def test_GIVEN_not_existing_THEN_calls_makedir(self, makedirsMock, existsMock):
|
||||
existsMock.return_value = False
|
||||
ensure_dir_exists("/foo")
|
||||
makedirsMock.assert_called_once_with("/foo")
|
||||
@@ -21,15 +21,6 @@ class TestExtraScripts(unittest.TestCase):
|
||||
def test_GIVEN_empty_string_THEN_returns_empty_array(self):
|
||||
self.assertEqual(get_extra_scripts(), [])
|
||||
|
||||
@patch("cellxgene_gateway.env.extra_scripts", new="'asdf'")
|
||||
def test_GIVEN_bare_string_THEN_throws_Exception(self):
|
||||
with self.assertRaises(Exception) as context:
|
||||
self.assertEqual(get_extra_scripts(), [])
|
||||
self.assertEqual(
|
||||
'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]',
|
||||
str(context.exception),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,21 +1,19 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.filecrawl import (
|
||||
render_item,
|
||||
render_item_source,
|
||||
render_item_tree,
|
||||
)
|
||||
from cellxgene_gateway.filecrawl import render_item
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.items.item import ItemTree, ItemType
|
||||
|
||||
source = FileItemSource("/tmp")
|
||||
|
||||
|
||||
class TestRenderEntry(unittest.TestCase):
|
||||
def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
|
||||
entry = FileItem(subpath="/somepath/", name="entry", type=ItemType.h5ad)
|
||||
entry = FileItem(
|
||||
subpath="/somepath/", name="entry", type=ItemType.h5ad
|
||||
)
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn("view/somepath/entry/'", rendered)
|
||||
|
||||
@@ -33,27 +31,3 @@ class TestRenderEntry(unittest.TestCase):
|
||||
entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn("view/somepath/entry/'", rendered)
|
||||
|
||||
|
||||
class TestRenderItemSource(unittest.TestCase):
|
||||
@patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
|
||||
def test_GIVEN_some_filter_THEN_includes_filterpart_in_heading(self, item_source):
|
||||
item_source.name = "FakeSource"
|
||||
item_source.list_items.return_value = ItemTree("rootdir", [], [])
|
||||
rendered = render_item_source(item_source, "some_filter")
|
||||
self.assertEqual(
|
||||
rendered,
|
||||
"<h6><a href='/filecrawl.html?source=FakeSource'>FakeSource</a>:some_filter</h6><li><a href='/filecrawl/rootdir?source=FakeSource'>rootdir</a><ul></ul></li>",
|
||||
)
|
||||
|
||||
|
||||
class TestRenderItemTree(unittest.TestCase):
|
||||
@patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
|
||||
def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
|
||||
item_source.name = "FakeSource"
|
||||
item_tree = ItemTree("foo/bar/baz", [], [])
|
||||
rendered = render_item_tree(item_tree, item_source)
|
||||
self.assertEqual(
|
||||
rendered,
|
||||
"<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>",
|
||||
)
|
||||
|
||||
@@ -1,21 +1,13 @@
|
||||
import unittest
|
||||
from unittest.mock import patch, seal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.backend_cache import BackendCache
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
)
|
||||
from cellxgene_gateway.cache_entry import CacheEntry
|
||||
|
||||
|
||||
class TestPruneProcessCache(unittest.TestCase):
|
||||
@patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
|
||||
@patch("cellxgene_gateway.env.expire_seconds", new=10)
|
||||
@patch("cellxgene_gateway.env.ttl", new="10")
|
||||
@patch("cellxgene_gateway.cache_entry.CacheEntry")
|
||||
@patch("cellxgene_gateway.cache_entry.CacheEntry")
|
||||
def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):
|
||||
@@ -23,23 +15,14 @@ class TestPruneProcessCache(unittest.TestCase):
|
||||
|
||||
cache = BackendCache()
|
||||
old.timestamp = -100
|
||||
old.foo = 12
|
||||
old.pid = 1
|
||||
old.key = key
|
||||
old.terminate.return_value = None
|
||||
seal(old)
|
||||
new.key = key
|
||||
cache.entry_list.append(old)
|
||||
new.timestamp = -5
|
||||
seal(new)
|
||||
cache.entry_list.append(new)
|
||||
self.assertEqual(len(cache.entry_list), 2)
|
||||
ppc = PruneProcessCache(cache)
|
||||
ppc.prune()
|
||||
self.assertEqual(len(cache.entry_list), 1)
|
||||
self.assertEqual(cache.entry_list[0], new)
|
||||
self.assertEqual(cache.entry_list[0], new)
|
||||
self.assertTrue(old.terminate.called)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.cache_entry import CacheEntry
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
|
||||
|
||||
class TestSubprocessBackend(unittest.TestCase):
|
||||
@patch("subprocess.Popen")
|
||||
def test_launch_GIVEN_no_stdout_THEN_throw_ProcessException(self, popen):
|
||||
subprocess = MagicMock()
|
||||
subprocess.stdout.readline().decode.return_value = ""
|
||||
subprocess.stderr.read().decode.return_value = "An unexpected error"
|
||||
popen.return_value = subprocess
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
)
|
||||
entry = CacheEntry.for_key(key, 8000)
|
||||
from cellxgene_gateway.subprocess_backend import SubprocessBackend
|
||||
|
||||
backend = SubprocessBackend()
|
||||
cellxgene_loc = "/some/cellxgene"
|
||||
scripts = ["http://example.com/script.js", "http://example.com/script2.js"]
|
||||
|
||||
with self.assertRaises(ProcessException) as context:
|
||||
backend.launch(cellxgene_loc, scripts, entry)
|
||||
popen.assert_called_once_with(
|
||||
[
|
||||
"yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
|
||||
],
|
||||
shell=True,
|
||||
stderr=-1,
|
||||
stdout=-1,
|
||||
)
|
||||
self.assertEqual("An unexpected error", context.exception.stderr)
|
||||
|
||||
@patch("subprocess.Popen")
|
||||
def test_launch_GIVEN_annotations_enabled_THEN_set_flags(self, popen):
|
||||
subprocess = MagicMock()
|
||||
subprocess.stdout.readline().decode.return_value = (
|
||||
"[cellxgene] Type CTRL-C at any time to exit.\n"
|
||||
)
|
||||
subprocess.stderr.read().decode.return_value = ""
|
||||
popen.return_value = subprocess
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
FileItem(
|
||||
"/czi/pbmc3k_annotations/", name="foo.csv", type=ItemType.annotation
|
||||
),
|
||||
)
|
||||
entry = CacheEntry.for_key(key, 8000)
|
||||
import cellxgene_gateway.subprocess_backend
|
||||
|
||||
cellxgene_gateway.subprocess_backend.enable_annotations = True
|
||||
try:
|
||||
backend = cellxgene_gateway.subprocess_backend.SubprocessBackend()
|
||||
cellxgene_loc = "/some/cellxgene"
|
||||
|
||||
backend.launch(cellxgene_loc, [], entry)
|
||||
finally:
|
||||
cellxgene_gateway.subprocess_backend.enable_annotations = False
|
||||
popen.assert_called_once_with(
|
||||
[
|
||||
"yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --annotations-file /tmp/czi/pbmc3k_annotations/foo.csv --gene-sets-file /tmp/czi/pbmc3k_annotations/foo_gene_sets.csv"
|
||||
],
|
||||
shell=True,
|
||||
stderr=-1,
|
||||
stdout=-1,
|
||||
)
|
||||
Reference in New Issue
Block a user