mirror of
https://github.com/Novartis/cellxgene-gateway.git
synced 2026-09-16 05:17:55 +08:00
Compare commits
1 Commits
reenable-c
...
patch-item
| Author | SHA1 | Date | |
|---|---|---|---|
|
a6c13a7606 |
13
.coveragerc
13
.coveragerc
@@ -1,13 +0,0 @@
|
||||
[run]
|
||||
branch = True
|
||||
source = cellxgene_gateway
|
||||
|
||||
[report]
|
||||
exclude_lines =
|
||||
if self.debug:
|
||||
pragma: no cover
|
||||
raise NotImplementedError
|
||||
if __name__ == .__main__.:
|
||||
ignore_errors = True
|
||||
omit =
|
||||
tests/*
|
||||
67
.github/workflows/codeql-analysis.yml
vendored
67
.github/workflows/codeql-analysis.yml
vendored
@@ -1,67 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master ]
|
||||
schedule:
|
||||
- cron: '18 6 * * 6'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'javascript', 'python' ]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
|
||||
# Learn more:
|
||||
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v1
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
91
.github/workflows/pr-checks.yaml
vendored
91
.github/workflows/pr-checks.yaml
vendored
@@ -1,91 +0,0 @@
|
||||
# Tests that run on every PR
|
||||
|
||||
name: Pull Request Checks
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
black:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
name: Checkout repository
|
||||
|
||||
- uses: actions/setup-python@v2
|
||||
name: Setup Python
|
||||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- name: Install black
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install black
|
||||
- name: Run black
|
||||
run: |
|
||||
black . --check
|
||||
# This job is copied over from `deploy.yaml`
|
||||
run-tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# See: https://github.com/marketplace/actions/setup-conda
|
||||
- uses: s-weigand/setup-conda@v1
|
||||
with:
|
||||
conda-channels: "conda-forge"
|
||||
|
||||
- name: Build environment
|
||||
run: |
|
||||
conda env create -f environment.yml
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate cellxgene-gateway
|
||||
python setup.py install
|
||||
pip install -r requirements-test.txt
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate cellxgene-gateway
|
||||
coverage run -m unittest discover tests
|
||||
|
||||
- name: Check coverage
|
||||
run: |
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate cellxgene-gateway
|
||||
coverage report --fail-under 41
|
||||
coverage report > coverage.txt
|
||||
coverage html -i
|
||||
coverage xml -i
|
||||
|
||||
- name: Upload coverage HTML report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-html
|
||||
path: htmlcov/
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload coverage xml
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-xml
|
||||
path: coverage.xml
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload coverage summary
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-summary
|
||||
path: coverage.txt
|
||||
retention-days: 30
|
||||
- name: "Upload coverage to Codecov"
|
||||
if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
flags: unittests
|
||||
env_vars: OS,PYTHON
|
||||
name: codecov-umbrella
|
||||
fail_ci_if_error: true
|
||||
path_to_write_report: ./codecov_report.txt
|
||||
verbose: true
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -55,7 +55,6 @@ htmlcov/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
htmlcov
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
|
||||
@@ -7,7 +7,14 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
stages: [commit]
|
||||
- id: flake8
|
||||
name: flake8
|
||||
language: system
|
||||
entry: flake8
|
||||
types: [python]
|
||||
stages: [commit]
|
||||
- id: black
|
||||
language_version: python3.6+
|
||||
name: black
|
||||
language: system
|
||||
entry: black
|
||||
|
||||
91
Changelog.md
91
Changelog.md
@@ -1,94 +1,3 @@
|
||||
# 0.4.2
|
||||
|
||||
* update package name
|
||||
|
||||
# 0.4.1
|
||||
|
||||
* Fix UnicodeDecodeError when viewing compressed datasets
|
||||
* Fix WSGI server initialization by extracting data source setup
|
||||
* Fix AttributeError by storing ItemSource objects in default_item_source
|
||||
* Delay itemsource initialization until first request is served
|
||||
* Set default_item_source and start pruner thread
|
||||
* Added start scripts for flask, gunicorn and uwsgi
|
||||
* Made pruner a daemon thread
|
||||
* Updated start scripts to run in subshells
|
||||
* Fixed bug in status.json
|
||||
|
||||
# 0.4.0
|
||||
|
||||
* Removed dependency on flask-api
|
||||
* Updated dependencies (python 3.11, numpy, unpinned flask, werkzeug)
|
||||
|
||||
# 0.3.12
|
||||
|
||||
* #81 List gene set annotations when cell annotations not present
|
||||
* #86 Upgrade pip within docker image
|
||||
* #73 Moved new link to front
|
||||
* #87 Temporarily pin versions of werkzeug and flask
|
||||
|
||||
|
||||
# 0.3.11
|
||||
|
||||
* #81 added support for gene sets
|
||||
* #79 added example for cellxgene-gateway customized docker image
|
||||
* #78 prune directories that do not contain h5ad files
|
||||
|
||||
# 0.3.10
|
||||
|
||||
* #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.
|
||||
* Added GATEWAY_LOG_LEVEL to set the log level
|
||||
* #68 Close connections after reading response
|
||||
* #68 Background thread reads from output of cellxgene process until it exits
|
||||
|
||||
# 0.3.9
|
||||
|
||||
* Added S3_ENABLE_LISTINGS_CACHE variable (See README.md)
|
||||
|
||||
# 0.3.8
|
||||
|
||||
* Fixed bug #57 affecting deeply nested subdirectory listing
|
||||
|
||||
# 0.3.7
|
||||
|
||||
* added back /metadata/ip_address endpoint
|
||||
|
||||
# 0.3.6
|
||||
|
||||
* pinned version of werkzeug
|
||||
|
||||
# 0.3.5
|
||||
|
||||
* Pinned flask version to match cellxgene 0.17.0
|
||||
|
||||
# 0.3.4
|
||||
|
||||
* Fixed bug #50 affecting subdirectory listing
|
||||
|
||||
# 0.3.3
|
||||
|
||||
* Fixed bug #48 affecting cache pruning
|
||||
|
||||
# 0.3.2
|
||||
|
||||
* Fixed bug #45 affecting multi-level S3 folders
|
||||
* Added extra_scripts to cache_status page
|
||||
|
||||
# 0.3.1
|
||||
|
||||
* Added missing __init__.py
|
||||
|
||||
# 0.3.0
|
||||
|
||||
* Added support for itemsource interface, allowing s3 hosting
|
||||
* Removed support for http file uploads
|
||||
* Only set wsgi.url_scheme when EXTERNAL_PROTOCOL is set (see issue #43)
|
||||
* Dropped flake8 due to conflicts with black
|
||||
* Added code coverage metrics
|
||||
|
||||
# 0.2.3
|
||||
|
||||
* Added support for ProxyFix
|
||||
|
||||
# 0.2.2
|
||||
|
||||
* Fixed bug with annotations (missing annotation.js asset)
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
FROM python:3.11
|
||||
|
||||
RUN pip install --upgrade pip
|
||||
RUN pip install "cellxgene-gateway>=0.4"
|
||||
|
||||
ENV CELLXGENE_DATA=/cellxgene-data
|
||||
ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
|
||||
|
||||
CMD ["cellxgene-gateway"]
|
||||
101
README.md
101
README.md
@@ -2,8 +2,6 @@
|
||||
|
||||
Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server.
|
||||
|
||||
[](https://codecov.io/gh/Novartis/cellxgene-gateway) [](https://pypi.org/project/cellxgene-gateway/) [](https://pypistats.org/packages/cellxgene-gateway)
|
||||
|
||||
# Running locally
|
||||
|
||||
## Prequisites
|
||||
@@ -73,64 +71,12 @@ Optional environment variables:
|
||||
* `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
|
||||
* `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
|
||||
* `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
|
||||
* `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
|
||||
* `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
|
||||
* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations and gene sets.
|
||||
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
|
||||
* `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
|
||||
* `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
|
||||
|
||||
If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
|
||||
* `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
|
||||
* `PROXY_FIX_PROTO` - Number of upstream proxies setting X-Forwarded-Proto
|
||||
* `PROXY_FIX_HOST` - Number of upstream proxies setting X-Forwarded-Host
|
||||
* `PROXY_FIX_PORT` - Number of upstream proxies setting X-Forwarded-Port
|
||||
* `PROXY_FIX_PREFIX` - Number of upstream proxies setting X-Forwarded-Prefix
|
||||
* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations.
|
||||
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
|
||||
|
||||
The defaults should be fine if you set up a venv and cellxgene_data folder as above.
|
||||
|
||||
## Running cellxgene-gateway with Docker
|
||||
|
||||
First, build Docker image:
|
||||
|
||||
```bash
|
||||
docker build -t cellxgene-gateway .
|
||||
```
|
||||
|
||||
Then, cellxgene-gateway can be launched as such:
|
||||
|
||||
```bash
|
||||
docker run -it --rm \
|
||||
-v <local_data_dir>:/cellxgene-data \
|
||||
-p 5005:5005 \
|
||||
cellxgene-gateway
|
||||
```
|
||||
|
||||
Additional environment variables can be provided with the `-e` parameter:
|
||||
|
||||
```bash
|
||||
docker run -it --rm \
|
||||
-v ../cellxgene_data:/cellxgene-data \
|
||||
-e GATEWAY_PORT=8080 \
|
||||
-p 8080:8080 \
|
||||
cellxgene-gateway
|
||||
```
|
||||
## Running cellxgene gateway with start scripts
|
||||
|
||||
For your convenience, we provide start scripts for flask, gunicorn and uwsgi.
|
||||
|
||||
First, set up a .env
|
||||
```bash
|
||||
cp env_example .env
|
||||
# edit .env
|
||||
open .env
|
||||
```
|
||||
|
||||
Then run the scripts in a subshell
|
||||
```bash
|
||||
( ./start_flask.sh )
|
||||
```
|
||||
|
||||
# Customization
|
||||
|
||||
The current paradigm for customization is to modify files during a build or deployment phase:
|
||||
@@ -171,14 +117,6 @@ python setup.py develop
|
||||
|
||||
For convenience, the code repo includes a `run.sh.example` shell script to run the gateway.
|
||||
|
||||
4. Install pre-commit hooks
|
||||
|
||||
```bash
|
||||
conda install -c conda-forge pre-commit
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
|
||||
## Running Tests
|
||||
|
||||
[](https://travis-ci.org/Novartis/cellxgene-gateway)
|
||||
@@ -187,12 +125,6 @@ pre-commit install
|
||||
python -m unittest discover tests
|
||||
```
|
||||
|
||||
## Code Coverage
|
||||
```bash
|
||||
coverage run -m unittest discover tests
|
||||
coverage html
|
||||
```
|
||||
|
||||
## Running Linters
|
||||
|
||||
pip install isort flake8 black
|
||||
@@ -206,35 +138,6 @@ black .
|
||||
|
||||
If you need help for any reason, please make a github ticket. One of the contributors should help you out.
|
||||
|
||||
# Releasing New Versions
|
||||
|
||||
## How to prepare for release
|
||||
|
||||
- Update Changelog.md and version number in __init__.py
|
||||
- Cut a release on github
|
||||
- Go to your project homepage on GitHub
|
||||
- On right side, you will see [Releases](https://github.com/Novartis/cellxgene-gateway/releases) link. Click on it.
|
||||
- Click on Draft a new release
|
||||
- Fill in all the details
|
||||
- Tag version should be the version number of your package release
|
||||
- Release Title can be anything you want, but we use v0.3.11 (the same as the tag to be created on publish)
|
||||
- Description should be changelog
|
||||
- Click Publish release at the bottom of the page
|
||||
- Now under Releases you can view all of your releases.
|
||||
- Copy the download link (tar.gz) and save it somewhere
|
||||
|
||||
## How to publish to PyPI
|
||||
|
||||
Make sure your `.pypirc` is set up for testpypi and pypi index servers.
|
||||
|
||||
|
||||
```bash
|
||||
rm -rf dist
|
||||
python setup.py sdist bdist_wheel
|
||||
python -m twine upload --repository testpypi dist/*
|
||||
python -m twine upload dist/*
|
||||
```
|
||||
|
||||
# Contributors
|
||||
|
||||
* Niket Patel - https://github.com/NiketPatel9
|
||||
|
||||
15
SECURITY.md
15
SECURITY.md
@@ -1,15 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Use this section to tell people about which versions of your project are
|
||||
currently being supported with security updates.
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 0.3.2 | :white_check_mark: |
|
||||
| <= 0.3.1 | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please file a bug report issue.
|
||||
2
cellxgene_gateway/__init__.py
Normal file → Executable file
2
cellxgene_gateway/__init__.py
Normal file → Executable file
@@ -7,4 +7,4 @@
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
__version__ = "0.4.2"
|
||||
__version__ = "0.2.2"
|
||||
|
||||
@@ -8,15 +8,16 @@
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from threading import Thread
|
||||
from typing import List
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.subprocess_backend import SubprocessBackend
|
||||
from typing import List
|
||||
|
||||
process_backend = SubprocessBackend()
|
||||
|
||||
@@ -52,7 +53,7 @@ class BackendCache:
|
||||
return matches[0]
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
"Found " + str(len(matches)) + " for " + path,
|
||||
)
|
||||
|
||||
@@ -70,7 +71,7 @@ class BackendCache:
|
||||
return matches[0]
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
"Found " + str(len(matches)) + " for " + key.dataset,
|
||||
)
|
||||
|
||||
|
||||
@@ -8,21 +8,19 @@
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
import urllib.parse
|
||||
from enum import Enum
|
||||
|
||||
import psutil
|
||||
from flask import make_response, render_template, request
|
||||
from flask.wrappers import Response
|
||||
from requests import get, post, put
|
||||
import re
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.flask_util import querystring
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CacheEntryStatus(Enum):
|
||||
loaded = "loaded"
|
||||
@@ -58,6 +56,7 @@ class CacheEntry:
|
||||
|
||||
@classmethod
|
||||
def for_key(cls, key, port):
|
||||
|
||||
return cls(
|
||||
None,
|
||||
key,
|
||||
@@ -111,7 +110,9 @@ class CacheEntry:
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
logger.info(f"terminated {terminated}")
|
||||
logging.getLogger("cellxgene_gateway").info(
|
||||
f"terminated {terminated}"
|
||||
)
|
||||
self.status = CacheEntryStatus.terminated
|
||||
|
||||
def rewrite_text_content(self, cellxgene_content):
|
||||
@@ -119,22 +120,30 @@ class CacheEntry:
|
||||
gateway_content = (
|
||||
re.sub(
|
||||
'(="|\()/static/',
|
||||
f"\\1{self.key.gateway_basepath()}static/",
|
||||
f"\\1{self.gateway_basepath()}static/",
|
||||
cellxgene_content,
|
||||
)
|
||||
.replace("http://fonts.gstatic.com", "https://fonts.gstatic.com")
|
||||
.replace(self.cellxgene_basepath(), self.key.gateway_basepath())
|
||||
.replace(self.cellxgene_basepath(), self.gateway_basepath())
|
||||
)
|
||||
return gateway_content
|
||||
|
||||
def gateway_basepath(self):
|
||||
source_path = (
|
||||
f"/source/{urllib.parse.quote_plus(self.source_name)}"
|
||||
if self.source_name
|
||||
else ""
|
||||
)
|
||||
return f"{env.external_protocol}://{env.external_host}{source_path}/view/{self.key.descriptor}/"
|
||||
|
||||
def cellxgene_basepath(self):
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def serve_content(self, path):
|
||||
gateway_basepath = self.key.gateway_basepath()
|
||||
gateway_basepath = self.gateway_basepath()
|
||||
subpath = path[len(self.key.descriptor) :] # noqa: E203
|
||||
if len(subpath) == 0:
|
||||
r = make_response(f"Redirect to {gateway_basepath}\n", 302)
|
||||
r = make_response(f"Redirect to {gateway_basepath}\n", 301)
|
||||
r.headers["location"] = gateway_basepath + querystring()
|
||||
return r
|
||||
elif self.status == CacheEntryStatus.loading:
|
||||
@@ -148,7 +157,7 @@ class CacheEntry:
|
||||
headers = {}
|
||||
copy_headers = [
|
||||
"accept",
|
||||
# "accept-encoding" - removed: let requests library handle compression/decompression
|
||||
"accept-encoding",
|
||||
"accept-language",
|
||||
"cache-control",
|
||||
"connection",
|
||||
@@ -168,43 +177,41 @@ class CacheEntry:
|
||||
headers[h] = request.headers[h]
|
||||
|
||||
full_path = self.cellxgene_basepath() + subpath + querystring()
|
||||
cellxgene_response = None
|
||||
try:
|
||||
if request.method in ["GET", "HEAD", "OPTIONS"]:
|
||||
cellxgene_response = get(full_path, headers=headers)
|
||||
elif request.method == "PUT":
|
||||
cellxgene_response = put(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
elif request.method == "POST":
|
||||
cellxgene_response = post(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
else:
|
||||
raise CellxgeneException(f"Unexpected method {request.method}", 400)
|
||||
content_type = cellxgene_response.headers["content-type"]
|
||||
if "text" in content_type:
|
||||
gateway_content = self.rewrite_text_content(
|
||||
cellxgene_response.content.decode()
|
||||
)
|
||||
else:
|
||||
gateway_content = cellxgene_response.content
|
||||
|
||||
resp_headers = {}
|
||||
for h in copy_headers:
|
||||
if h in cellxgene_response.headers:
|
||||
resp_headers[h] = cellxgene_response.headers[h]
|
||||
|
||||
gateway_response = make_response(
|
||||
gateway_content,
|
||||
cellxgene_response.status_code,
|
||||
resp_headers,
|
||||
if request.method in ["GET", "HEAD", "OPTIONS"]:
|
||||
cellxgene_response = get(full_path, headers=headers)
|
||||
elif request.method == "PUT":
|
||||
cellxgene_response = put(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
finally:
|
||||
if cellxgene_response is not None:
|
||||
cellxgene_response.close()
|
||||
elif request.method == "POST":
|
||||
cellxgene_response = post(
|
||||
full_path,
|
||||
headers=headers,
|
||||
data=request.data,
|
||||
)
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
f"Unexpected method {request.method}", 400
|
||||
)
|
||||
content_type = cellxgene_response.headers["content-type"]
|
||||
if "text" in content_type:
|
||||
gateway_content = self.rewrite_text_content(
|
||||
cellxgene_response.content.decode()
|
||||
)
|
||||
else:
|
||||
gateway_content = cellxgene_response.content
|
||||
|
||||
resp_headers = {}
|
||||
for h in copy_headers:
|
||||
if h in cellxgene_response.headers:
|
||||
resp_headers[h] = cellxgene_response.headers[h]
|
||||
|
||||
gateway_response = make_response(
|
||||
gateway_content,
|
||||
cellxgene_response.status_code,
|
||||
resp_headers,
|
||||
)
|
||||
return gateway_response
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
# 3) somedir/dataset_annotations: an annotation directory. The corresponding h5ad must exist, but the directory may not.
|
||||
# in this case, descriptor == 'somedir/dataset_annotations', dataset == 'somedir/dataset.h5ad'
|
||||
|
||||
from cellxgene_gateway import flask_util
|
||||
from cellxgene_gateway.items.item import Item
|
||||
from cellxgene_gateway.items.item_source import ItemSource, LookupResult
|
||||
|
||||
@@ -39,16 +38,6 @@ class CacheKey:
|
||||
else:
|
||||
return self.source.get_local_path(self.annotation_item)
|
||||
|
||||
def relaunch_url(self):
|
||||
return flask_util.relaunch_url(self.descriptor, self.source_name)
|
||||
|
||||
def gateway_basepath(self):
|
||||
return self.view_url + "/"
|
||||
|
||||
@property
|
||||
def view_url(self):
|
||||
return flask_util.view_url(self.descriptor, self.source_name)
|
||||
|
||||
@property
|
||||
def source_name(self):
|
||||
return self.source.name
|
||||
|
||||
@@ -9,9 +9,49 @@
|
||||
|
||||
import os
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
|
||||
|
||||
def is_subdir(full_path, parent_path):
|
||||
subdir = os.path.realpath(full_path)
|
||||
parent = os.path.realpath(parent_path)
|
||||
return subdir.startswith(parent)
|
||||
|
||||
|
||||
def create_dir(parent_path, dir_name):
|
||||
full_path = os.path.join(parent_path, dir_name)
|
||||
|
||||
if "/" in dir_name:
|
||||
raise CellxgeneException(
|
||||
"Please have no slashes in the intended directory.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not os.path.exists(parent_path):
|
||||
raise CellxgeneException(
|
||||
"The selected User directory does not exist.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif os.path.exists(full_path):
|
||||
raise CellxgeneException(
|
||||
"The provided subdirectory already exists within Directory.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not is_subdir(full_path, parent_path):
|
||||
raise CellxgeneException(
|
||||
"The directory must be a subdirectory of the parent path.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not os.path.isdir(parent_path):
|
||||
raise CellxgeneException(
|
||||
"The parent is not a directory.", status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
else:
|
||||
os.mkdir(full_path)
|
||||
|
||||
|
||||
annotations_suffix = "_annotations"
|
||||
h5ad_suffix = ".h5ad"
|
||||
|
||||
@@ -22,8 +62,3 @@ def make_h5ad(el):
|
||||
|
||||
def make_annotations(el):
|
||||
return el[:-5] + annotations_suffix
|
||||
|
||||
|
||||
def ensure_dir_exists(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
os.makedirs(file_path)
|
||||
|
||||
@@ -9,9 +9,10 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
|
||||
cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA", "")
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA")
|
||||
cellxgene_args = os.environ.get("CELLXGENE_ARGS", None)
|
||||
gateway_port = int(os.environ.get("GATEWAY_PORT", "5005"))
|
||||
external_host = os.environ.get(
|
||||
@@ -19,50 +20,39 @@ external_host = os.environ.get(
|
||||
os.environ.get("GATEWAY_HOST", f"localhost:{gateway_port}"),
|
||||
)
|
||||
external_protocol = os.environ.get(
|
||||
"EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", None)
|
||||
"EXTERNAL_PROTOCOL", os.environ.get("GATEWAY_PROTOCOL", "http")
|
||||
)
|
||||
ip = os.environ.get("GATEWAY_IP")
|
||||
extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
|
||||
expire_seconds = int(
|
||||
os.environ.get("GATEWAY_EXPIRE_SECONDS", os.environ.get("GATEWAY_TTL", "3600"))
|
||||
)
|
||||
enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [
|
||||
ttl = os.environ.get("GATEWAY_TTL")
|
||||
enable_annotations = os.environ.get(
|
||||
"GATEWAY_ENABLE_ANNOTATIONS", ""
|
||||
).lower() in [
|
||||
"true",
|
||||
"1",
|
||||
]
|
||||
enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in [
|
||||
enable_backed_mode = os.environ.get(
|
||||
"GATEWAY_ENABLE_BACKED_MODE", ""
|
||||
).lower() in [
|
||||
"true",
|
||||
"1",
|
||||
]
|
||||
log_level = logging.getLevelName(os.environ.get("GATEWAY_LOG_LEVEL", "INFO"))
|
||||
|
||||
env_vars = {
|
||||
"CELLXGENE_LOCATION": cellxgene_location,
|
||||
}
|
||||
|
||||
proxy_fix_for = int(os.environ.get("PROXY_FIX_FOR", "0"))
|
||||
proxy_fix_proto = int(os.environ.get("PROXY_FIX_PROTO", "0"))
|
||||
proxy_fix_host = int(os.environ.get("PROXY_FIX_HOST", "0"))
|
||||
proxy_fix_port = int(os.environ.get("PROXY_FIX_PORT", "0"))
|
||||
proxy_fix_prefix = int(os.environ.get("PROXY_FIX_PREFIX", "0"))
|
||||
|
||||
optional_env_vars = {
|
||||
"EXTERNAL_HOST": external_host,
|
||||
"EXTERNAL_PROTOCOL": external_protocol,
|
||||
"GATEWAY_IP": ip,
|
||||
"GATEWAY_PORT": gateway_port,
|
||||
"GATEWAY_EXTRA_SCRIPTS": extra_scripts,
|
||||
"GATEWAY_EXPIRE_SECONDS": expire_seconds,
|
||||
"GATEWAY_TTL": ttl,
|
||||
"GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
|
||||
"GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
|
||||
"GATEWAY_LOG_LEVEL": log_level,
|
||||
"CELLXGENE_ARGS": cellxgene_args,
|
||||
"CELLXGENE_DATA": cellxgene_data,
|
||||
"PROXY_FIX_FOR": proxy_fix_for,
|
||||
"PROXY_FIX_PROTO": proxy_fix_proto,
|
||||
"PROXY_FIX_HOST": proxy_fix_host,
|
||||
"PROXY_FIX_PORT": proxy_fix_port,
|
||||
"PROXY_FIX_PREFIX": proxy_fix_prefix,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from json import loads
|
||||
from json.decoder import JSONDecodeError
|
||||
|
||||
from cellxgene_gateway import env
|
||||
|
||||
@@ -18,9 +17,4 @@ def get_extra_scripts():
|
||||
# ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
|
||||
# f"{env.external_protocol}://{env.external_host}/static/js/google_ua.js"]
|
||||
# where google_ua.js is a script you add to the static/js folder prior to deployment.
|
||||
try:
|
||||
return [] if env.extra_scripts is None else loads(env.extra_scripts)
|
||||
except JSONDecodeError as exc:
|
||||
raise Exception(
|
||||
f'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]'
|
||||
) from exc
|
||||
return [] if env.extra_scripts is None else loads(env.extra_scripts)
|
||||
|
||||
@@ -7,36 +7,31 @@
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import html
|
||||
import os
|
||||
import urllib.parse
|
||||
|
||||
from cellxgene_gateway import flask_util
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.dir_util import annotations_suffix, make_annotations, make_h5ad
|
||||
from cellxgene_gateway.env import enable_annotations
|
||||
|
||||
|
||||
def render_annotations(item, item_source):
|
||||
if not enable_annotations:
|
||||
return ""
|
||||
url = flask_util.view_url(
|
||||
item_source.get_annotations_subpath(item), item_source.name
|
||||
)
|
||||
new_annotation = [f"<a class='new' href='{url}'>new</a>"]
|
||||
|
||||
subpath = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/"
|
||||
new_annotation = f"<a class='new' href='{subpath}{item_source.get_annotations_subpath(item)}'>new</a>"
|
||||
annotations = (
|
||||
[
|
||||
f"<a href='{CacheKey(item, item_source, a).view_url}/'>{html.escape(a.name)}</a>"
|
||||
for a in item.annotations
|
||||
]
|
||||
", ".join(
|
||||
[
|
||||
f"<a href='{subpath}{a.descriptor}/'>{a.name}</a>"
|
||||
for a in item.annotations
|
||||
]
|
||||
)
|
||||
+ ", "
|
||||
if item.annotations
|
||||
else []
|
||||
else ""
|
||||
)
|
||||
return "| annotations: " + ", ".join(new_annotation + annotations)
|
||||
|
||||
return " | annotations: " + annotations + new_annotation
|
||||
|
||||
|
||||
def render_item(item, item_source):
|
||||
item_string = f"<li> <a href='{ CacheKey(item, item_source).view_url }/'>{item.name}</a> {render_annotations(item, item_source)}</li>"
|
||||
url = f"/source/{urllib.parse.quote_plus(item_source.name)}/view/{item.descriptor}/"
|
||||
item_string = f"<li> <a href='{ url }'>{item.name}</a> {render_annotations(item, item_source)}</li>"
|
||||
return item_string
|
||||
|
||||
|
||||
@@ -47,7 +42,9 @@ def render_item_tree(item_tree, item_source):
|
||||
else ""
|
||||
)
|
||||
branches = (
|
||||
"\n".join([render_item_tree(b, item_source) for b in item_tree.branches])
|
||||
"\n".join(
|
||||
[render_item_tree(b, item_source) for b in item_tree.branches]
|
||||
)
|
||||
if item_tree.branches
|
||||
else ""
|
||||
)
|
||||
@@ -55,7 +52,11 @@ def render_item_tree(item_tree, item_source):
|
||||
if item_tree.descriptor:
|
||||
descriptor = item_tree.descriptor.lstrip("/")
|
||||
url = f"/filecrawl/{descriptor}?source={item_source.name}"
|
||||
name = descriptor.rsplit("/", 1)[-1]
|
||||
name = (
|
||||
descriptor.rsplit("/")[1]
|
||||
if descriptor.find("/") >= 0
|
||||
else descriptor
|
||||
)
|
||||
return f"<li><a href='{url}'>{name}</a>{html}</li>"
|
||||
else:
|
||||
return html
|
||||
@@ -63,6 +64,5 @@ def render_item_tree(item_tree, item_source):
|
||||
|
||||
def render_item_source(item_source, filter=None):
|
||||
item_tree = item_source.list_items(filter)
|
||||
filterpart = "" if filter is None else ":" + filter
|
||||
heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a>{filterpart}</h6>"
|
||||
heading = f"<h6><a href='/filecrawl.html?source={urllib.parse.quote_plus(item_source.name)}'>{item_source.name}</a></h6>"
|
||||
return heading + render_item_tree(item_tree, item_source)
|
||||
|
||||
@@ -7,27 +7,9 @@
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from flask import request, url_for
|
||||
from flask import request
|
||||
|
||||
|
||||
def querystring():
|
||||
qs = request.query_string.decode()
|
||||
return f"?{qs}" if len(qs) > 0 else ""
|
||||
|
||||
|
||||
include_source_in_url = False
|
||||
|
||||
|
||||
def url(endpoint, descriptor, source_name):
|
||||
if include_source_in_url:
|
||||
return url_for(endpoint, source_name=source_name, path=descriptor)
|
||||
else:
|
||||
return url_for(endpoint, path=descriptor)
|
||||
|
||||
|
||||
def view_url(descriptor, source_name):
|
||||
return url("do_view", descriptor, source_name)
|
||||
|
||||
|
||||
def relaunch_url(descriptor, source_name):
|
||||
return url("do_relaunch", descriptor, source_name)
|
||||
|
||||
@@ -22,148 +22,44 @@ from flask import (
|
||||
send_from_directory,
|
||||
url_for,
|
||||
)
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
from flask_api import status
|
||||
from werkzeug.utils import secure_filename
|
||||
|
||||
from cellxgene_gateway import env, flask_util
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.backend_cache import BackendCache
|
||||
from cellxgene_gateway.cache_entry import CacheEntryStatus
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.dir_util import create_dir, is_subdir
|
||||
from cellxgene_gateway.extra_scripts import get_extra_scripts
|
||||
from cellxgene_gateway.filecrawl import render_item_source
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
from cellxgene_gateway.prune_process_cache import PruneProcessCache
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
item_sources = []
|
||||
default_item_source = None
|
||||
|
||||
# Guard for lazy initialization so tests can import this module without
|
||||
# triggering environment-dependent side effects. initialize_data_sources()
|
||||
# will set this to True when it has run.
|
||||
data_sources_initialized = False
|
||||
data_sources_init_lock = Lock()
|
||||
|
||||
|
||||
def _force_https(app):
|
||||
def wrapper(environ, start_response):
|
||||
if env.external_protocol is not None:
|
||||
environ["wsgi.url_scheme"] = env.external_protocol
|
||||
environ["wsgi.url_scheme"] = env.external_protocol
|
||||
return app(environ, start_response)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def set_no_cache(resp):
|
||||
resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
|
||||
resp.headers["Pragma"] = "no-cache"
|
||||
resp.headers["Expires"] = "0"
|
||||
resp.headers["Cache-Control"] = "public, max-age=0"
|
||||
return resp
|
||||
|
||||
|
||||
app.wsgi_app = _force_https(app.wsgi_app)
|
||||
if (
|
||||
env.proxy_fix_for > 0
|
||||
or env.proxy_fix_proto > 0
|
||||
or env.proxy_fix_host > 0
|
||||
or env.proxy_fix_port > 0
|
||||
or env.proxy_fix_prefix > 0
|
||||
):
|
||||
app.wsgi_app = ProxyFix(
|
||||
app.wsgi_app,
|
||||
x_for=env.proxy_fix_for,
|
||||
x_proto=env.proxy_fix_proto,
|
||||
x_host=env.proxy_fix_host,
|
||||
x_port=env.proxy_fix_port,
|
||||
x_prefix=env.proxy_fix_prefix,
|
||||
)
|
||||
|
||||
|
||||
# WSGI middleware to ensure data sources are initialized before the first
|
||||
# WSGI request is handled. This guarantees initialization works under
|
||||
# Gunicorn/uWSGI (which import the module but don't call main()). The
|
||||
# initialize_data_sources() function is idempotent-protected by
|
||||
# data_sources_initialized and data_sources_init_lock.
|
||||
def _init_on_first_wsgi_request(wsgi_app):
|
||||
def middleware(environ, start_response):
|
||||
global data_sources_initialized
|
||||
if not data_sources_initialized:
|
||||
with data_sources_init_lock:
|
||||
if not app.extensions.get("cellxgene_gateway", {}).get("launchtime"):
|
||||
app.extensions.setdefault("cellxgene_gateway", {})[
|
||||
"launchtime"
|
||||
] = current_time_stamp()
|
||||
|
||||
if not data_sources_initialized:
|
||||
initialize_data_sources()
|
||||
|
||||
env.validate()
|
||||
if not item_sources or not len(item_sources):
|
||||
raise Exception(
|
||||
"No data sources specified for Cellxgene Gateway"
|
||||
)
|
||||
|
||||
global default_item_source
|
||||
if default_item_source is None:
|
||||
default_item_source = item_sources[0]
|
||||
|
||||
data_sources_initialized = True
|
||||
return wsgi_app(environ, start_response)
|
||||
|
||||
return middleware
|
||||
|
||||
|
||||
# Wrap the WSGI app so Gunicorn/uWSGI will trigger initialization when the
|
||||
# first request comes in. Tests that need initialization can call
|
||||
# initialize_data_sources() directly.
|
||||
app.wsgi_app = _init_on_first_wsgi_request(app.wsgi_app)
|
||||
|
||||
cache = BackendCache()
|
||||
|
||||
|
||||
# Initialize data sources - this is defined later in the file but called here
|
||||
# to ensure initialization happens when WSGI servers (Gunicorn) import the module
|
||||
def initialize_data_sources():
|
||||
"""Initialize data sources from environment variables.
|
||||
Called at module import time for WSGI server compatibility (Gunicorn).
|
||||
Uses a guard flag to prevent double initialization within a process."""
|
||||
global default_item_source
|
||||
|
||||
logging.basicConfig(
|
||||
level=env.log_level,
|
||||
format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA", None)
|
||||
cellxgene_bucket = os.environ.get("CELLXGENE_BUCKET", None)
|
||||
|
||||
if cellxgene_bucket is not None:
|
||||
from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
|
||||
|
||||
s3_source = S3ItemSource(cellxgene_bucket, name="s3")
|
||||
item_sources.append(s3_source)
|
||||
default_item_source = s3_source
|
||||
logger.info("Initialized S3 data source")
|
||||
logger.debug(f"S3 bucket: {cellxgene_bucket}")
|
||||
if cellxgene_data is not None:
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
|
||||
file_source = FileItemSource(cellxgene_data, name="local")
|
||||
item_sources.append(file_source)
|
||||
default_item_source = file_source
|
||||
logger.info("Initialized local file data source")
|
||||
logger.debug(f"Data directory: {cellxgene_data}")
|
||||
if len(item_sources) == 0:
|
||||
raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
|
||||
flask_util.include_source_in_url = len(item_sources) > 1
|
||||
location = f"{env.external_protocol}://{env.external_host}"
|
||||
|
||||
|
||||
@app.errorhandler(CellxgeneException)
|
||||
def handle_invalid_usage(error):
|
||||
|
||||
message = f"{error.http_status} Error : {error.message}"
|
||||
|
||||
return (
|
||||
@@ -178,6 +74,7 @@ def handle_invalid_usage(error):
|
||||
|
||||
@app.errorhandler(ProcessException)
|
||||
def handle_invalid_process(error):
|
||||
|
||||
message = []
|
||||
|
||||
message.append(error.message)
|
||||
@@ -193,7 +90,7 @@ def handle_invalid_process(error):
|
||||
http_status=error.http_status,
|
||||
stdout=error.stdout,
|
||||
stderr=error.stderr,
|
||||
relaunch_url=error.key.relaunch_url(),
|
||||
dataset=error.key.h5ad_item.descriptor,
|
||||
annotation_file=error.key.annotation_descriptor,
|
||||
),
|
||||
error.http_status,
|
||||
@@ -245,7 +142,10 @@ def filecrawl(path=None):
|
||||
path=path,
|
||||
)
|
||||
)
|
||||
set_no_cache(resp)
|
||||
resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
|
||||
resp.headers["Pragma"] = "no-cache"
|
||||
resp.headers["Expires"] = "0"
|
||||
resp.headers["Cache-Control"] = "public, max-age=0"
|
||||
return resp
|
||||
|
||||
|
||||
@@ -253,7 +153,7 @@ entry_lock = Lock()
|
||||
|
||||
|
||||
def matching_source(source_name):
|
||||
if source_name is None and default_item_source is not None:
|
||||
if source_name is None:
|
||||
source_name = default_item_source.name
|
||||
matching = [i for i in item_sources if i.name == source_name]
|
||||
if len(matching) != 1:
|
||||
@@ -294,111 +194,100 @@ def do_view(path, source_name=None):
|
||||
match.status == CacheEntryStatus.loaded
|
||||
or match.status == CacheEntryStatus.loading
|
||||
):
|
||||
if source.is_authorized(match.key.descriptor):
|
||||
return match.serve_content(path)
|
||||
else:
|
||||
raise CellxgeneException("User not authorized to access this data", 403)
|
||||
return match.serve_content(path)
|
||||
elif match.status == CacheEntryStatus.error:
|
||||
raise ProcessException.from_cache_entry(match)
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
f"Unexpected cache entry status {match.status} for key {match.key.descriptor}",
|
||||
500,
|
||||
)
|
||||
|
||||
|
||||
@app.route("/cache_status", methods=["GET"])
|
||||
def do_GET_status():
|
||||
return render_template(
|
||||
"cache_status.html",
|
||||
entry_list=cache.entry_list,
|
||||
extra_scripts=get_extra_scripts(),
|
||||
)
|
||||
return render_template("cache_status.html", entry_list=cache.entry_list)
|
||||
|
||||
|
||||
@app.route("/cache_status.json", methods=["GET"])
|
||||
def do_GET_status_json():
|
||||
def map_entry(entry):
|
||||
dataset = entry.key.h5ad_item.descriptor
|
||||
annotation_file = entry.key.annotation_descriptor
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"annotation_file": annotation_file,
|
||||
"launchtime": entry.launchtime,
|
||||
"last_access": entry.timestamp,
|
||||
"status": entry.status.name,
|
||||
}
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"launchtime": app.extensions.get("cellxgene_gateway", {}).get("launchtime"),
|
||||
"entry_list": [map_entry(entry) for entry in cache.entry_list],
|
||||
"launchtime": app.launchtime,
|
||||
"entry_list": [
|
||||
{
|
||||
"dataset": entry.key.dataset,
|
||||
"annotation_file": entry.key.annotation_file,
|
||||
"launchtime": entry.launchtime,
|
||||
"last_access": entry.timestamp,
|
||||
"status": entry.status,
|
||||
}
|
||||
for entry in cache.entry_list
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_cache_key(path):
|
||||
if request.args.get("source_name"):
|
||||
source_name = request.args.get("source_name")
|
||||
elif default_item_source:
|
||||
source_name = default_item_source.name
|
||||
else:
|
||||
source_name = None
|
||||
source = matching_source(source_name)
|
||||
key = CacheKey.for_lookup(source, source.lookup(path))
|
||||
return key
|
||||
|
||||
|
||||
@app.route("/relaunch/<path:path>", methods=["GET"])
|
||||
def do_relaunch(path):
|
||||
key = get_cache_key(path)
|
||||
source_name = request.args.get("source") or default_item_source.name
|
||||
source = matching_source(source_name)
|
||||
key = CacheKey.for_lookup(source, source.lookup(path))
|
||||
match = cache.check_entry(key)
|
||||
if not match is None:
|
||||
match.terminate()
|
||||
qs = request.query_string.decode()
|
||||
return redirect(
|
||||
key.view_url,
|
||||
url_for("do_view", path=path) + (f"?{qs}" if len(qs) > 0 else ""),
|
||||
code=302,
|
||||
)
|
||||
|
||||
|
||||
@app.route("/terminate/<path:path>", methods=["GET"])
|
||||
def do_terminate(path):
|
||||
key = get_cache_key(path)
|
||||
source_name = request.args.get("source_name") or default_item_source.name
|
||||
source = matching_source(source_name)
|
||||
key = CacheKey.for_lookup(source, source.lookup(path))
|
||||
match = cache.check_entry(key)
|
||||
if not match is None:
|
||||
match.terminate()
|
||||
return redirect(url_for("do_GET_status"), code=302)
|
||||
|
||||
|
||||
@app.route("/metadata/ip_address", methods=["GET"])
|
||||
def ip_address():
|
||||
resp = make_response(env.ip)
|
||||
return set_no_cache(resp)
|
||||
def launch():
|
||||
env.validate()
|
||||
if not item_sources or not len(item_sources):
|
||||
raise Exception("No data sources specified for Cellxgene Gateway")
|
||||
|
||||
global default_item_source
|
||||
if default_item_source is None:
|
||||
default_item_source = item_sources[0]
|
||||
|
||||
def start_pruner_thread():
|
||||
pruner = PruneProcessCache(cache)
|
||||
# Run the pruner as a daemon thread so it won't block interpreter
|
||||
# shutdown (for example when Ctrl-C is used in the main thread).
|
||||
# This avoids "Exception ignored in: <module 'threading'...>" at exit.
|
||||
background_thread = Thread(target=pruner, daemon=True)
|
||||
|
||||
background_thread = Thread(target=pruner)
|
||||
background_thread.start()
|
||||
|
||||
|
||||
def launch():
|
||||
start_pruner_thread()
|
||||
|
||||
app.extensions.setdefault("cellxgene_gateway", {})[
|
||||
"launchtime"
|
||||
] = current_time_stamp()
|
||||
app.launchtime = current_time_stamp()
|
||||
app.run(host="0.0.0.0", port=env.gateway_port, debug=False)
|
||||
|
||||
|
||||
app.extensions.setdefault("cellxgene_gateway", {})["launchtime"] = None
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point for Flask development server."""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
|
||||
)
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA", None)
|
||||
cellxgene_bucket = os.environ.get("CELLXGENE_BUCKET", None)
|
||||
|
||||
if cellxgene_bucket is not None:
|
||||
from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
|
||||
|
||||
item_sources.append(S3ItemSource(cellxgene_bucket, name="s3"))
|
||||
default_item_source = "s3"
|
||||
if cellxgene_data is not None:
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
|
||||
item_sources.append(FileItemSource(cellxgene_data, name="local"))
|
||||
default_item_source = "local"
|
||||
if len(item_sources) == 0:
|
||||
raise Exception("Please specify CELLXGENE_DATA or CELLXGENE_BUCKET")
|
||||
|
||||
launch()
|
||||
|
||||
|
||||
|
||||
@@ -18,11 +18,10 @@ class FileItem(Item):
|
||||
The Item superclass expects a 'name' and 'type'.
|
||||
"""
|
||||
|
||||
def __init__(self, subpath: str, ext: str = "", *args, **kwargs):
|
||||
def __init__(self, subpath: str, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.subpath = subpath
|
||||
self.ext = ext
|
||||
|
||||
@property
|
||||
def descriptor(self) -> str:
|
||||
return os.path.join(self.subpath, self.name + self.ext).strip("/")
|
||||
return os.path.join(self.subpath, self.name).strip("/")
|
||||
|
||||
@@ -24,22 +24,17 @@ class FileItemSource(ItemSource):
|
||||
h5ad_suffix=dir_util.h5ad_suffix,
|
||||
annotation_dir_suffix=dir_util.annotations_suffix,
|
||||
annotation_file_suffix=".csv",
|
||||
gene_set_file_suffix="_gene_sets.csv",
|
||||
):
|
||||
self._name = name
|
||||
self.base_path = base_path
|
||||
self.h5ad_suffix = h5ad_suffix
|
||||
self.annotation_dir_suffix = annotation_dir_suffix
|
||||
self.annotation_file_suffix = annotation_file_suffix
|
||||
self.gene_set_file_suffix = gene_set_file_suffix
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._name or f"Files:{self.base_path}"
|
||||
|
||||
def is_gene_set(self, path: str) -> bool:
|
||||
return path.endswith(self.gene_set_file_suffix)
|
||||
|
||||
def is_h5ad_file(self, path: str) -> bool:
|
||||
return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
|
||||
|
||||
@@ -56,7 +51,7 @@ class FileItemSource(ItemSource):
|
||||
return self.convert_h5ad_path_to_annotation(item.descriptor)
|
||||
|
||||
def list_items(self, filter: str = None) -> ItemTree:
|
||||
item_tree = self.scan_directory("" if filter is None else filter)
|
||||
item_tree = self.scan_directory()
|
||||
|
||||
"""def get_items(dir):
|
||||
if dir.branches:
|
||||
@@ -68,11 +63,13 @@ class FileItemSource(ItemSource):
|
||||
|
||||
return item_tree
|
||||
|
||||
def scan_directory(self, subpath: str = "") -> ItemTree:
|
||||
def scan_directory(self, subpath="") -> dict:
|
||||
base_path = os.path.join(self.base_path, subpath)
|
||||
|
||||
if not os.path.exists(base_path):
|
||||
raise Exception(f"Path for local files '{base_path}' does not exist.")
|
||||
raise Exception(
|
||||
f"Path for local files '{base_path}' does not exist."
|
||||
)
|
||||
|
||||
filepath_map = dict(
|
||||
(filepath, os.path.join(base_path, filepath))
|
||||
@@ -98,17 +95,14 @@ class FileItemSource(ItemSource):
|
||||
]
|
||||
|
||||
items = [
|
||||
self.make_fileitem_from_path(filename, subpath) for filename in h5ad_paths
|
||||
self.make_fileitem_from_path(filename, subpath)
|
||||
for filename in h5ad_paths
|
||||
]
|
||||
branches = None
|
||||
if len(subdirs) > 0:
|
||||
branches = [
|
||||
self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
|
||||
]
|
||||
# Exclude branches without files as leaves. Since traversal is applied pre-order,
|
||||
# branch.branches has already been processed and we don't need to check deeper nesting.
|
||||
branches = [
|
||||
branch for branch in branches if branch.items or branch.branches
|
||||
self.scan_directory(os.path.join(subpath, subdir))
|
||||
for subdir in subdirs
|
||||
]
|
||||
|
||||
return ItemTree(subpath, items, branches)
|
||||
@@ -131,19 +125,17 @@ class FileItemSource(ItemSource):
|
||||
if self.is_h5ad_file(full_path):
|
||||
return self.shallowitem_from_descriptor(descriptor)
|
||||
|
||||
def is_authorized(self, descriptor):
|
||||
return True
|
||||
|
||||
def lookup(self, indescriptor: str) -> LookupResult:
|
||||
descriptor = indescriptor.strip("/")
|
||||
if descriptor.endswith(self.annotation_file_suffix):
|
||||
annotation_item = self.shallowitem_from_descriptor(descriptor, True)
|
||||
annotation_item = self.shallowitem_from_descriptor(
|
||||
descriptor, True
|
||||
)
|
||||
h5ad_descriptor = self.convert_annotation_path_to_h5ad(
|
||||
annotation_item.subpath
|
||||
)
|
||||
item = self.lookup_item(h5ad_descriptor)
|
||||
if item is not None:
|
||||
dir_util.ensure_dir_exists(self.full_path(annotation_item.subpath))
|
||||
return LookupResult(item, annotation_item)
|
||||
else:
|
||||
item = self.lookup_item(descriptor)
|
||||
@@ -163,16 +155,9 @@ class FileItemSource(ItemSource):
|
||||
def make_fileitem_from_path(
|
||||
self, filename, subpath, is_annotation=False, is_shallow=False
|
||||
) -> FileItem:
|
||||
if is_annotation and filename.endswith(self.annotation_file_suffix):
|
||||
name = filename[: -len(self.annotation_file_suffix)]
|
||||
ext = self.annotation_file_suffix
|
||||
else:
|
||||
name = filename
|
||||
ext = ""
|
||||
item = FileItem(
|
||||
subpath=subpath,
|
||||
name=name,
|
||||
ext=ext,
|
||||
name=filename,
|
||||
type=ItemType.annotation if is_annotation else ItemType.h5ad,
|
||||
)
|
||||
|
||||
@@ -186,29 +171,15 @@ class FileItemSource(ItemSource):
|
||||
annotations_subpath = self.get_annotations_subpath(item)
|
||||
annotations_fullpath = self.full_path(annotations_subpath)
|
||||
if os.path.isdir(annotations_fullpath):
|
||||
sorted_files = sorted(os.listdir(annotations_fullpath))
|
||||
annotation_files = [
|
||||
self.make_fileitem_from_path(annotation, annotations_subpath, True)
|
||||
for annotation in sorted_files
|
||||
if annotation.endswith(self.annotation_file_suffix)
|
||||
and not self.is_gene_set(annotation)
|
||||
and os.path.isfile(os.path.join(annotations_fullpath, annotation))
|
||||
]
|
||||
|
||||
# Catch gene sets without accompanying [annotations].csv
|
||||
gene_sets_files = [
|
||||
return [
|
||||
self.make_fileitem_from_path(
|
||||
annotation[: -len(self.gene_set_file_suffix)] + ".csv",
|
||||
annotations_subpath,
|
||||
True,
|
||||
annotation, annotations_subpath, True
|
||||
)
|
||||
for annotation in sorted(os.listdir(annotations_fullpath))
|
||||
if annotation.endswith(self.annotation_file_suffix)
|
||||
and os.path.isfile(
|
||||
os.path.join(annotations_fullpath, annotation)
|
||||
)
|
||||
for annotation in sorted_files
|
||||
if self.is_gene_set(annotation)
|
||||
and annotation[: -len(self.gene_set_file_suffix)]
|
||||
not in [a.name for a in annotation_files]
|
||||
and os.path.isfile(os.path.join(annotations_fullpath, annotation))
|
||||
]
|
||||
|
||||
return sorted(annotation_files + gene_sets_files, key=lambda x: x.name)
|
||||
else:
|
||||
return None
|
||||
|
||||
@@ -18,7 +18,9 @@ class ItemType(Enum):
|
||||
|
||||
|
||||
class Item(ABC):
|
||||
def __init__(self, name: str, type: ItemType, annotations: List["Item"] = None):
|
||||
def __init__(
|
||||
self, name: str, type: ItemType, annotations: List["Item"] = None
|
||||
):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.annotations = annotations
|
||||
|
||||
@@ -40,10 +40,6 @@ class ItemSource(ABC):
|
||||
def update(self, item: Item) -> None:
|
||||
raise Exception('"update" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def is_authorized(self, descriptor: str) -> bool:
|
||||
raise Exception('"is_authorized" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, descriptor: str) -> LookupResult:
|
||||
raise Exception('"lookup" unimplemented')
|
||||
|
||||
@@ -7,21 +7,13 @@
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
from os.path import basename, dirname
|
||||
from typing import List
|
||||
|
||||
import flask
|
||||
import s3fs
|
||||
|
||||
from os.path import join, dirname, basename
|
||||
from cellxgene_gateway import dir_util
|
||||
import s3fs
|
||||
from cellxgene_gateway.items.s3.s3item import S3Item
|
||||
from cellxgene_gateway.items.item import ItemTree, ItemType
|
||||
from cellxgene_gateway.items.item_source import ItemSource, LookupResult
|
||||
from cellxgene_gateway.items.s3.s3item import S3Item
|
||||
|
||||
|
||||
def truthy(val: str):
|
||||
return val.lower() in ["true", "1"]
|
||||
|
||||
|
||||
class S3ItemSource(ItemSource):
|
||||
@@ -34,24 +26,14 @@ class S3ItemSource(ItemSource):
|
||||
annotation_file_suffix=".csv",
|
||||
):
|
||||
self._name = name
|
||||
enable_cache = os.environ.get("S3_ENABLE_LISTINGS_CACHE", "false").lower()
|
||||
assert enable_cache in ["0", "1", "false", "true"]
|
||||
self.use_listings_cache = truthy(enable_cache)
|
||||
self.s3 = s3fs.S3FileSystem(use_listings_cache=self.use_listings_cache)
|
||||
if bucket.startswith("s3://"):
|
||||
raise Exception(
|
||||
f"Bucket name should not include s3:// prefix, got {bucket}"
|
||||
)
|
||||
self.s3 = s3fs.S3FileSystem()
|
||||
self.bucket = bucket
|
||||
self.h5ad_suffix = h5ad_suffix
|
||||
self.annotation_dir_suffix = annotation_dir_suffix
|
||||
self.annotation_file_suffix = annotation_file_suffix
|
||||
|
||||
def url(self, key):
|
||||
return "s3://" + self.bucket + "/" + key
|
||||
|
||||
def remove_bucket(self, filepath):
|
||||
return filepath[len(self.bucket) :].lstrip("/")
|
||||
def url(self, path):
|
||||
return "s3://" + join(self.bucket, path)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
@@ -73,54 +55,51 @@ class S3ItemSource(ItemSource):
|
||||
return self.convert_h5ad_key_to_annotation(item.descriptor)
|
||||
|
||||
def list_items(self, filter: str = None) -> ItemTree:
|
||||
item_tree = self.scan_directory("" if filter is None else filter)
|
||||
item_tree = self.scan_directory()
|
||||
return item_tree
|
||||
|
||||
@property
|
||||
def refresh(self):
|
||||
return (
|
||||
truthy(flask.request.args.get("refresh", default="false"))
|
||||
or not self.use_listings_cache
|
||||
)
|
||||
|
||||
def scan_directory(self, directory_key="") -> dict:
|
||||
url = self.url(directory_key)
|
||||
def scan_directory(self, subpath="") -> dict:
|
||||
url = self.url(subpath)
|
||||
|
||||
if not self.s3.exists(url):
|
||||
raise Exception(f"S3 url '{url}' does not exist.")
|
||||
|
||||
s3key_map = dict(
|
||||
(self.remove_bucket(filepath), "s3://" + filepath)
|
||||
for filepath in sorted(self.s3.ls(url, refresh=self.refresh))
|
||||
(filepath[len(self.bucket) :], "s3://" + filepath)
|
||||
for filepath in sorted(self.s3.ls(url))
|
||||
)
|
||||
|
||||
def is_annotation_dir(dir_s3key):
|
||||
return (
|
||||
dir_s3key.endswith(self.annotation_dir_suffix)
|
||||
and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_keys
|
||||
and self.convert_annotation_key_to_h5ad(dir_s3key)
|
||||
in h5ad_paths
|
||||
)
|
||||
|
||||
h5ad_keys = [
|
||||
h5ad_paths = [
|
||||
filepath
|
||||
for filepath, item_url in s3key_map.items()
|
||||
if self.is_h5ad_url(item_url)
|
||||
]
|
||||
|
||||
subdir_keys = [
|
||||
subdirs = [
|
||||
filepath
|
||||
for filepath, item_url in s3key_map.items()
|
||||
if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
|
||||
]
|
||||
|
||||
items = [self.make_s3item_from_key(basename(key), key) for key in h5ad_keys]
|
||||
items = [
|
||||
self.make_s3item_from_key(filename, join(subpath, filename))
|
||||
for filename in h5ad_paths
|
||||
]
|
||||
branches = None
|
||||
if len(subdir_keys) > 0:
|
||||
branches = [self.scan_directory(key) for key in subdir_keys]
|
||||
if len(subdirs) > 0:
|
||||
branches = [
|
||||
branch for branch in branches if branch.items or branch.branches
|
||||
self.scan_directory(join(subpath, subdir))
|
||||
for subdir in subdirs
|
||||
]
|
||||
|
||||
return ItemTree(directory_key, items, branches)
|
||||
return ItemTree(subpath, items, branches)
|
||||
|
||||
def create_annotation(self, item: S3Item, name: str) -> S3Item:
|
||||
annotation = self.make_s3item_from_key(
|
||||
@@ -132,9 +111,6 @@ class S3ItemSource(ItemSource):
|
||||
def update(self, item: S3Item) -> None:
|
||||
pass
|
||||
|
||||
def is_authorized(self, descriptor):
|
||||
return True
|
||||
|
||||
def lookup_item(self, descriptor):
|
||||
full_path = self.url(descriptor)
|
||||
if self.is_h5ad_url(full_path):
|
||||
@@ -143,7 +119,9 @@ class S3ItemSource(ItemSource):
|
||||
def lookup(self, indescriptor: str) -> LookupResult:
|
||||
descriptor = indescriptor.strip("/")
|
||||
if descriptor.endswith(self.annotation_file_suffix):
|
||||
annotation_item = self.shallowitem_from_descriptor(descriptor, True)
|
||||
annotation_item = self.shallowitem_from_descriptor(
|
||||
descriptor, True
|
||||
)
|
||||
if not self.s3.exists(self.url(annotation_item.s3key)):
|
||||
with self.s3.open(self.url(annotation_item.s3key), "w") as f:
|
||||
f.write("")
|
||||
@@ -183,13 +161,11 @@ class S3ItemSource(ItemSource):
|
||||
if self.s3.isdir(annotations_fullpath):
|
||||
return [
|
||||
self.make_s3item_from_key(
|
||||
basename(annotation), self.remove_bucket(annotation), True
|
||||
)
|
||||
for annotation in sorted(
|
||||
self.s3.ls(annotations_fullpath, refresh=self.refresh)
|
||||
annotation, join(annotations_subpath, annotation), True
|
||||
)
|
||||
for annotation in sorted(self.s3.ls(annotations_fullpath))
|
||||
if annotation.endswith(self.annotation_file_suffix)
|
||||
and self.s3.isfile("s3://" + annotation)
|
||||
and self.s3.isfile(join(annotations_fullpath, annotation))
|
||||
]
|
||||
else:
|
||||
return None
|
||||
|
||||
41
cellxgene_gateway/path_util.py
Normal file
41
cellxgene_gateway/path_util.py
Normal file
@@ -0,0 +1,41 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.dir_util import make_h5ad
|
||||
|
||||
|
||||
def validate_exists(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
raise CellxgeneException(
|
||||
"File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
|
||||
def validate_is_file(file_path):
|
||||
validate_exists(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise CellxgeneException(
|
||||
"Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
def validate_is_dir(file_path):
|
||||
validate_exists(file_path)
|
||||
if not os.path.isdir(file_path):
|
||||
raise CellxgeneException(
|
||||
"Path is not dir: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
return
|
||||
@@ -10,15 +10,14 @@
|
||||
import logging
|
||||
import time
|
||||
|
||||
from cellxgene_gateway import env, util
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from cellxgene_gateway.env import ttl
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
|
||||
|
||||
class PruneProcessCache:
|
||||
def __init__(self, cache):
|
||||
self.cache = cache
|
||||
self.expire_seconds = env.expire_seconds
|
||||
self.expire_seconds = 3600 if ttl is None else int(ttl)
|
||||
|
||||
def __call__(self):
|
||||
while True:
|
||||
@@ -26,22 +25,26 @@ class PruneProcessCache:
|
||||
self.prune()
|
||||
|
||||
def prune(self):
|
||||
timestamp = util.current_time_stamp()
|
||||
timestamp = current_time_stamp()
|
||||
cutoff = timestamp - self.expire_seconds
|
||||
processes_to_delete = [p for p in self.cache.entry_list if p.timestamp < cutoff]
|
||||
processes_to_delete = [
|
||||
p for p in self.cache.entry_list if p.timestamp < cutoff
|
||||
]
|
||||
processes_to_keep = [
|
||||
p for p in self.cache.entry_list if not p.timestamp < cutoff
|
||||
]
|
||||
|
||||
logger = logging.getLogger("cellxgene_gateway")
|
||||
logger.debug(
|
||||
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}, pruning {processes_to_delete}"
|
||||
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
|
||||
)
|
||||
|
||||
for process in processes_to_delete:
|
||||
try:
|
||||
logger.info(f"pruning process {process.pid} ({process.key.descriptor})")
|
||||
logger.info(
|
||||
f"pruning process {process.pid} ({process.key.dataset})"
|
||||
)
|
||||
self.cache.prune(process)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"failed to prune process {process.pid} ({process.key.descriptor})"
|
||||
"failed to prune process {process.pid} ({process.dataset})"
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// neandertal javascript
|
||||
// Annotations only work with file itemsources at the moment. If they work with others in the future we may need to revisit this.
|
||||
// TODO: rewrite this --
|
||||
const new_annotation_callback = (() =>{
|
||||
const suffix = `.csv`;
|
||||
return (e) => {
|
||||
|
||||
@@ -9,31 +9,35 @@
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from http import HTTPStatus
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway.cache_entry import CacheEntryStatus
|
||||
from cellxgene_gateway.dir_util import make_annotations
|
||||
from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
|
||||
from cellxgene_gateway.env import (
|
||||
enable_annotations,
|
||||
enable_backed_mode,
|
||||
cellxgene_args,
|
||||
)
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SubprocessBackend:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def create_cmd(self, cellxgene_loc, file_path, port, scripts, annotation_file_path):
|
||||
def create_cmd(
|
||||
self, cellxgene_loc, file_path, port, scripts, annotation_file_path
|
||||
):
|
||||
if enable_annotations and not annotation_file_path is None:
|
||||
if annotation_file_path == "":
|
||||
extra_args = f" --annotations-dir {make_annotations(file_path)}"
|
||||
extra_args = (
|
||||
f" --annotations-dir {make_annotations(file_path)}"
|
||||
)
|
||||
else:
|
||||
extra_args = f" --annotations-file {annotation_file_path}"
|
||||
gene_sets_file_path = annotation_file_path[:-4] + "_gene_sets.csv"
|
||||
extra_args += f" --gene-sets-file {gene_sets_file_path}"
|
||||
else:
|
||||
extra_args = " --disable-annotations"
|
||||
extra_args += " --disable-gene-sets-save"
|
||||
if enable_backed_mode:
|
||||
extra_args += " --backed"
|
||||
if not cellxgene_args is None:
|
||||
@@ -59,7 +63,7 @@ class SubprocessBackend:
|
||||
scripts,
|
||||
cache_entry.key.annotation_file_path,
|
||||
)
|
||||
logger.info(f"launching {cmd}")
|
||||
logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
|
||||
process = subprocess.Popen(
|
||||
[cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
|
||||
)
|
||||
@@ -75,10 +79,10 @@ class SubprocessBackend:
|
||||
or "Could not open file" in stderr
|
||||
):
|
||||
message = "File was invalid."
|
||||
http_status = HTTPStatus.BAD_REQUEST
|
||||
http_status = status.HTTP_400_BAD_REQUEST
|
||||
else:
|
||||
message = "Cellxgene failed to launch dataset."
|
||||
http_status = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
|
||||
cache_entry.status = CacheEntryStatus.error
|
||||
cache_entry.set_error(message, stderr, http_status)
|
||||
@@ -88,6 +92,5 @@ class SubprocessBackend:
|
||||
cache_entry.append_output(output)
|
||||
|
||||
cache_entry.set_loaded(process.pid)
|
||||
for output in process.communicate():
|
||||
logger.debug(f"cellxgene:{output}")
|
||||
logger.info(f"exiting {cmd}")
|
||||
|
||||
return
|
||||
|
||||
@@ -48,7 +48,7 @@
|
||||
<tr>
|
||||
<td>{{ entry.pid }}</td>
|
||||
<td><a
|
||||
href="{{ entry.key.view_url }}">{{ entry.key.h5ad_item.descriptor }}</a>
|
||||
href="{{ url_for('do_view', path=entry.key.descriptor, source_name=entry.key.source_name) }}">{{ entry.key.h5ad_item.descriptor }}</a>
|
||||
</td>
|
||||
<td>{{ entry.key.annotation_descriptor }}</td>
|
||||
<td>{{ entry.source_name }}</td>
|
||||
@@ -75,9 +75,7 @@
|
||||
const el = $(this);
|
||||
const ts = el.text();
|
||||
const dt = new Date(parseInt(ts * 1000));
|
||||
el.prepend(`${dt.toISOString()}<br>(`);
|
||||
el.append(')');
|
||||
|
||||
el.html(`${dt.toISOString()}<br>(${ts})`);
|
||||
});
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -28,11 +28,11 @@
|
||||
|
||||
<h4>{{ message }}</h4>
|
||||
|
||||
<a href="{{ url_for('filecrawl') }}">
|
||||
<a href="/filecrawl.html">
|
||||
Please click here to be redirected to the file directory.
|
||||
</a>
|
||||
<br>
|
||||
<a href="{{ url_for('index') }}">
|
||||
<a href="/">
|
||||
Please click here to return to the homepage.
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -36,10 +36,10 @@
|
||||
Navigation:
|
||||
<ul>
|
||||
{% if path %}
|
||||
<li><a href="{{ url_for('filecrawl') }}">top level</a></li>
|
||||
<li><a href="/filecrawl.html">top level</a></li>
|
||||
{% else %}
|
||||
{% endif %}
|
||||
<li><a href="{{ url_for('index') }}">homepage</a></li>
|
||||
<li><a href="/">homepage</a></li>
|
||||
</ul>
|
||||
</p>
|
||||
<script>
|
||||
|
||||
@@ -35,12 +35,12 @@
|
||||
Links:
|
||||
</h1>
|
||||
<div class="list-group" style="width:50%;padding-left:65px">
|
||||
<a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
|
||||
<a href="/filecrawl.html" class="list-group-item list-group-item-action">
|
||||
<u>File Crawler: Allows you to view all uploaded data.</u></a>
|
||||
|
||||
</div>
|
||||
<div class="list-group" style="width:50%;padding-left:65px">
|
||||
<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
|
||||
<a href="/cache_status" class="list-group-item list-group-item-action">
|
||||
<u>Cache Status: view status of launched cellxgene servers.</u></a>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -35,11 +35,11 @@
|
||||
The page will refresh shortly.
|
||||
</p>
|
||||
|
||||
<a href="{{ url_for('filecrawl') }}">
|
||||
<a href="/filecrawl.html">
|
||||
Please click here to be redirected to the file directory.
|
||||
</a>
|
||||
<br>
|
||||
<a href="{{ url_for('index') }}">
|
||||
<a href="/">
|
||||
Please click here to return to the homepage.
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -36,13 +36,13 @@
|
||||
<h4>Options</h4>
|
||||
Please choose one of the following, or use the back button:
|
||||
<ul>
|
||||
<li><a href="{{ relaunch_url }}">
|
||||
<li><a href="{{url_for('do_relaunch', path=dataset)}}">
|
||||
Attempt to relaunch the cellxgene server.
|
||||
</a></li>
|
||||
<li><a href="{{ url_for('filecrawl') }}">
|
||||
<li><a href="/filecrawl.html">
|
||||
Return to the file directory.
|
||||
</a></li>
|
||||
<li><a href="{{ url_for('index') }}">
|
||||
<li><a href="/">
|
||||
Return to the homepage.
|
||||
</a></li>
|
||||
</ul>
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
export CELLXGENE_LOCATION=$(pwd)/.venv/bin/cellxgene
|
||||
export CELLXGENE_DATA=../cellxgene_data
|
||||
export GATEWAY_IP=127.0.0.1
|
||||
@@ -2,16 +2,12 @@ name: cellxgene-gateway
|
||||
channels:
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- python=3.11
|
||||
- python=3.7
|
||||
- requests
|
||||
- flask
|
||||
- psutil
|
||||
- black
|
||||
- twine
|
||||
- isort
|
||||
- coverage
|
||||
- pip
|
||||
- pip:
|
||||
- pre_commit
|
||||
- werkzeug
|
||||
- cellxgene
|
||||
- flask-api
|
||||
- cellxgene>=0.15
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
FROM python:3.9
|
||||
|
||||
RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
|
||||
|
||||
COPY customize_ui.sh customize_ui.sh
|
||||
RUN CELLXGENE_GATEWAY_DIR=/usr/local/lib/python3.9/site-packages/cellxgene_gateway . ./customize_ui.sh
|
||||
|
||||
ENV CELLXGENE_DATA=/cellxgene-data
|
||||
ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
|
||||
EXPOSE 5005
|
||||
|
||||
RUN mkdir /cellxgene-data
|
||||
|
||||
CMD ["cellxgene-gateway"]
|
||||
@@ -1,14 +0,0 @@
|
||||
# Purpose
|
||||
|
||||
This is a simple example of how to make a small script to customize the UI of cellxgene-gateway. The script that does the customization is `customize_ui.sh`, it simply makes the main header green using CSS but you could do anything you want there (including adding more script tags, etc).
|
||||
|
||||
# Usage
|
||||
|
||||
```
|
||||
docker build -t cellxgene_custom .
|
||||
CELLXGENE_DATA=`pwd`/../../../cellxgene_data
|
||||
docker run -p 5005:5005 --mount src=$CELLXGENE_DATA,target=/cellxgene-data,type=bind cellxgene_custom
|
||||
```
|
||||
|
||||
If you now open http://localhost:5005 you should see a green cellxgene gateway header.
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
# make the header bright green
|
||||
find "${CELLXGENE_GATEWAY_DIR}/templates" -name index.html -exec sed -i -e 's/<head>/<head>\
|
||||
> <style> header h3 {color: #0F0;} <\/style>/g' {} \;
|
||||
@@ -1 +0,0 @@
|
||||
freezegun~=1.5.5
|
||||
@@ -1,5 +1,5 @@
|
||||
cellxgene
|
||||
cellxgene>=0.15
|
||||
flask
|
||||
werkzeug
|
||||
flask_api
|
||||
psutil
|
||||
requests
|
||||
|
||||
6
run.sh.example
Normal file
6
run.sh.example
Normal file
@@ -0,0 +1,6 @@
|
||||
export CELLXGENE_LOCATION=$(pwd)/.cellxgene-gateway/bin/cellxgene
|
||||
export CELLXGENE_DATA=../cellxgene_data
|
||||
export GATEWAY_IP=127.0.0.1
|
||||
|
||||
#Once these are set, you run like a normal Flask app
|
||||
cellxgene-gateway
|
||||
2
setup.py
2
setup.py
@@ -38,7 +38,7 @@ install_reqs = parse_requirements()
|
||||
|
||||
setup(
|
||||
# mandatory
|
||||
name="cellxgene_gateway",
|
||||
name="cellxgene-gateway",
|
||||
# mandatory
|
||||
version=get_version("cellxgene_gateway/__init__.py"),
|
||||
# mandatory
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# start_gunicorn.sh - Start Cellxgene Gateway with Gunicorn
|
||||
#
|
||||
# PREREQUISITES:
|
||||
# - Gunicorn installed (included with cellxgene 1.3.0, or: pip install gunicorn)
|
||||
# - Virtual environment activated
|
||||
# - .env file with CELLXGENE_LOCATION and CELLXGENE_DATA (or CELLXGENE_BUCKET)
|
||||
#
|
||||
# USAGE:
|
||||
# ./start_gunicorn.sh
|
||||
|
||||
# Exit on error
|
||||
set -e
|
||||
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
|
||||
# Source environment variables
|
||||
echo "Loading environment variables..."
|
||||
if [ -f "$SCRIPT_DIR/.env" ]; then
|
||||
source "$SCRIPT_DIR/.env"
|
||||
else
|
||||
echo "Error: .env file not found at $SCRIPT_DIR/.env"
|
||||
echo "Please create it with required environment variables"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify required environment variables
|
||||
if [ -z "$CELLXGENE_LOCATION" ]; then
|
||||
echo "Error: CELLXGENE_LOCATION not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$CELLXGENE_DATA" ] && [ -z "$CELLXGENE_BUCKET" ]; then
|
||||
echo "Error: Either CELLXGENE_DATA or CELLXGENE_BUCKET must be set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec cellxgene-gateway
|
||||
@@ -1,91 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# start_gunicorn.sh - Start Cellxgene Gateway with Gunicorn
|
||||
#
|
||||
# PREREQUISITES:
|
||||
# - Gunicorn installed (included with cellxgene 1.3.0, or: pip install gunicorn)
|
||||
# - Virtual environment activated
|
||||
# - .env file with CELLXGENE_LOCATION and CELLXGENE_DATA (or CELLXGENE_BUCKET)
|
||||
#
|
||||
# USAGE:
|
||||
# ./start_gunicorn.sh
|
||||
|
||||
# Exit on error
|
||||
set -e
|
||||
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
# Source environment variables
|
||||
echo "Loading environment variables..."
|
||||
if [ -f "$SCRIPT_DIR/.env" ]; then
|
||||
source "$SCRIPT_DIR/.env"
|
||||
else
|
||||
echo "Error: .env file not found at $SCRIPT_DIR/.env"
|
||||
echo "Please create it with required environment variables"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify required environment variables
|
||||
if [ -z "$CELLXGENE_LOCATION" ]; then
|
||||
echo "Error: CELLXGENE_LOCATION not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$CELLXGENE_DATA" ] && [ -z "$CELLXGENE_BUCKET" ]; then
|
||||
echo "Error: Either CELLXGENE_DATA or CELLXGENE_BUCKET must be set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Gunicorn configuration
|
||||
# WARNING: Multi-worker mode has cache synchronization issues (see plans/002-shared-cache-implementation.md)
|
||||
# Each worker maintains its own in-memory cache, causing 404s for static assets when different
|
||||
# workers handle requests for the same dataset. Use GUNICORN_WORKERS=1 until shared cache is implemented.
|
||||
WORKERS=${GUNICORN_WORKERS:-1}
|
||||
BIND=${GATEWAY_IP:-0.0.0.0}:${GATEWAY_PORT:-5005}
|
||||
TIMEOUT=${GUNICORN_TIMEOUT:-120}
|
||||
WORKER_CLASS=${GUNICORN_WORKER_CLASS:-sync}
|
||||
KEEPALIVE=${GUNICORN_KEEPALIVE:-5}
|
||||
LOG_LEVEL=${GUNICORN_LOG_LEVEL:-info}
|
||||
|
||||
# Production optimization: enable backed mode to reduce memory usage
|
||||
export GATEWAY_ENABLE_BACKED_MODE=${GATEWAY_ENABLE_BACKED_MODE:-true}
|
||||
|
||||
# Check if gunicorn is installed
|
||||
if ! command -v gunicorn &> /dev/null; then
|
||||
echo "Error: gunicorn not found. Install with: pip install gunicorn"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Display configuration
|
||||
echo "Starting Cellxgene Gateway with Gunicorn..."
|
||||
echo "Configuration:"
|
||||
echo " Data source: ${CELLXGENE_DATA:-$CELLXGENE_BUCKET}"
|
||||
echo " Binding to: $BIND"
|
||||
echo " Workers: $WORKERS"
|
||||
echo " Worker class: $WORKER_CLASS"
|
||||
echo " Timeout: ${TIMEOUT}s"
|
||||
echo " Keepalive: ${KEEPALIVE}s"
|
||||
echo " Log level: $LOG_LEVEL"
|
||||
echo " Backed mode: ${GATEWAY_ENABLE_BACKED_MODE}"
|
||||
echo ""
|
||||
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Start Gunicorn with optimized settings
|
||||
# Additional options you can add via environment variables:
|
||||
# - GUNICORN_MAX_REQUESTS: Restart worker after N requests (prevents memory leaks)
|
||||
# - GUNICORN_MAX_REQUESTS_JITTER: Add randomness to max-requests
|
||||
exec gunicorn cellxgene_gateway.gateway:app \
|
||||
--workers "$WORKERS" \
|
||||
--worker-class "$WORKER_CLASS" \
|
||||
--bind "$BIND" \
|
||||
--timeout "$TIMEOUT" \
|
||||
--keep-alive "$KEEPALIVE" \
|
||||
--access-logfile - \
|
||||
--error-logfile - \
|
||||
--log-level "$LOG_LEVEL" \
|
||||
--preload \
|
||||
${GUNICORN_MAX_REQUESTS:+--max-requests "$GUNICORN_MAX_REQUESTS"} \
|
||||
${GUNICORN_MAX_REQUESTS_JITTER:+--max-requests-jitter "$GUNICORN_MAX_REQUESTS_JITTER"} \
|
||||
"$@"
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# start_uwsgi.sh - Start Cellxgene Gateway with uWSGI
|
||||
#
|
||||
# PREREQUISITES:
|
||||
# - uWSGI installed (pip install uwsgi)
|
||||
# - Virtual environment activated
|
||||
# - .env file with CELLXGENE_LOCATION and CELLXGENE_DATA (or CELLXGENE_BUCKET)
|
||||
#
|
||||
# USAGE:
|
||||
# ./start_uwsgi.sh
|
||||
|
||||
# Exit on error
|
||||
set -e
|
||||
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
# Source environment variables
|
||||
echo "Loading environment variables..."
|
||||
if [ -f "$SCRIPT_DIR/.env" ]; then
|
||||
source "$SCRIPT_DIR/.env"
|
||||
else
|
||||
echo "Error: .env file not found at $SCRIPT_DIR/.env"
|
||||
echo "Please create it with required environment variables"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify required environment variables
|
||||
if [ -z "$CELLXGENE_LOCATION" ]; then
|
||||
echo "Error: CELLXGENE_LOCATION not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$CELLXGENE_DATA" ] && [ -z "$CELLXGENE_BUCKET" ]; then
|
||||
echo "Error: Either CELLXGENE_DATA or CELLXGENE_BUCKET must be set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# uWSGI configuration
|
||||
# WARNING: Multi-worker mode has cache synchronization issues (see plans/002-shared-cache-implementation.md)
|
||||
# Each worker maintains its own in-memory cache, causing 404s for static assets when different
|
||||
# workers handle requests for the same dataset. Use UWSGI_WORKERS=1 until shared cache is implemented.
|
||||
WORKERS=${UWSGI_WORKERS:-1}
|
||||
HOST=${GATEWAY_IP:-0.0.0.0}
|
||||
PORT=${GATEWAY_PORT:-5005}
|
||||
TIMEOUT=${UWSGI_TIMEOUT:-120}
|
||||
THREADS=${UWSGI_THREADS:-1}
|
||||
|
||||
# Production optimization: enable backed mode to reduce memory usage
|
||||
export GATEWAY_ENABLE_BACKED_MODE=${GATEWAY_ENABLE_BACKED_MODE:-true}
|
||||
|
||||
# Check if uwsgi is installed
|
||||
if ! command -v uwsgi &> /dev/null; then
|
||||
echo "Error: uwsgi not found. Install with: pip install uwsgi"
|
||||
exit 1
|
||||
else
|
||||
# Display configuration
|
||||
echo "Starting Cellxgene Gateway with uWSGI..."
|
||||
echo "Configuration:"
|
||||
echo " Data source: ${CELLXGENE_DATA:-$CELLXGENE_BUCKET}"
|
||||
echo " Binding to: $HOST:$PORT"
|
||||
echo " Workers: $WORKERS"
|
||||
echo " Threads: $THREADS"
|
||||
echo " Timeout: ${TIMEOUT}s"
|
||||
echo " Backed mode: ${GATEWAY_ENABLE_BACKED_MODE}"
|
||||
echo ""
|
||||
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Start uWSGI with optimized settings
|
||||
# Additional options you can add via environment variables:
|
||||
# - UWSGI_MAX_REQUESTS: Restart worker after N requests (prevents memory leaks)
|
||||
exec uwsgi \
|
||||
--http "$HOST:$PORT" \
|
||||
--module cellxgene_gateway.gateway:app \
|
||||
--workers "$WORKERS" \
|
||||
--threads "$THREADS" \
|
||||
--harakiri "$TIMEOUT" \
|
||||
--master \
|
||||
--enable-threads \
|
||||
--single-interpreter \
|
||||
--need-app \
|
||||
--die-on-term \
|
||||
--log-x-forwarded-for \
|
||||
${UWSGI_MAX_REQUESTS:+--max-requests "$UWSGI_MAX_REQUESTS"} \
|
||||
"$@"
|
||||
fi
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
|
||||
|
||||
def stub_join(path):
|
||||
path.join = lambda x, y: x + "/" + y
|
||||
|
||||
|
||||
class TestFileItemSource(unittest.TestCase):
|
||||
@patch("os.path")
|
||||
@patch("os.listdir")
|
||||
def test_list_items_GIVEN_no_subpath_THEN_checks_dir(self, listdir, path):
|
||||
stub_join(path)
|
||||
source = FileItemSource("/tmp/unittest", "local")
|
||||
source.list_items()
|
||||
path.exists.assert_called_once_with("/tmp/unittest/")
|
||||
|
||||
@patch("os.path")
|
||||
@patch("os.listdir")
|
||||
def test_list_items_GIVEN_subpath_THEN_checks_subpath(self, listdir, path):
|
||||
stub_join(path)
|
||||
source = FileItemSource("/tmp/unittest", "local")
|
||||
source.list_items("foo")
|
||||
path.exists.assert_called_once_with("/tmp/unittest/foo")
|
||||
|
||||
def test_make_fileitem_from_path_GIVEN_annotation_file_THEN_name_lacks_csv(
|
||||
self,
|
||||
):
|
||||
source = FileItemSource(tempfile.gettempdir(), "local")
|
||||
item = source.make_fileitem_from_path(
|
||||
"customanno.csv", "someh5ad_annotations", True
|
||||
)
|
||||
self.assertEqual(item.name, "customanno")
|
||||
self.assertEqual(item.descriptor, "someh5ad_annotations/customanno.csv")
|
||||
|
||||
def test_make_fileitem_from_path_GIVEN_h5ad_file_THEN_returns_name(self):
|
||||
source = FileItemSource(tempfile.gettempdir(), "local")
|
||||
item = source.make_fileitem_from_path("someanalysis.h5ad", "studydir")
|
||||
self.assertEqual(item.name, "someanalysis.h5ad")
|
||||
self.assertEqual(item.descriptor, "studydir/someanalysis.h5ad")
|
||||
@@ -1,184 +0,0 @@
|
||||
import unittest
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.items.s3.s3item import S3Item
|
||||
from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
|
||||
from cellxgene_gateway.gateway import app
|
||||
|
||||
|
||||
class TestScanDirectory(unittest.TestCase):
|
||||
def setUp(self):
|
||||
|
||||
self.app = app
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
@patch("s3fs.S3FileSystem")
|
||||
def test_GIVEN_invalid_bucket_THEN_throws_error(self, s3func):
|
||||
class S3Mock:
|
||||
def exists(path):
|
||||
if path in ["s3://my-bucket/"]:
|
||||
return False
|
||||
|
||||
s3func.return_value = S3Mock
|
||||
source = S3ItemSource("my-bucket")
|
||||
with self.assertRaises(Exception) as context:
|
||||
source.scan_directory()
|
||||
self.assertEqual(
|
||||
"S3 url 's3://my-bucket/' does not exist.",
|
||||
str(context.exception),
|
||||
)
|
||||
|
||||
@patch("s3fs.S3FileSystem")
|
||||
def test__GIVEN_multilevel_bucket_THEN_properly_recurses_suburls(self, s3func):
|
||||
|
||||
class S3Mock:
|
||||
def exists(path):
|
||||
if path in [
|
||||
"s3://my-bucket/",
|
||||
"s3://my-bucket/pbmc3k.h5ad",
|
||||
"s3://my-bucket/lvl1",
|
||||
"s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
|
||||
"s3://my-bucket/lvl1/lvl2",
|
||||
"s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
|
||||
]:
|
||||
return True
|
||||
raise Exception("exists called with " + path)
|
||||
|
||||
def ls(path, refresh):
|
||||
assert refresh == True
|
||||
if path == "s3://my-bucket/":
|
||||
return [
|
||||
"my-bucket/lvl1",
|
||||
"my-bucket/pbmc3k.h5ad",
|
||||
"my-bucket/pbmc3k_annotations",
|
||||
]
|
||||
elif path == "s3://my-bucket/pbmc3k_annotations":
|
||||
return ["my-bucket/pbmc3k_annotations/annot.csv"]
|
||||
elif path == "s3://my-bucket/lvl1":
|
||||
return ["my-bucket/lvl1/lvl2", "my-bucket/lvl1/pbmc3k_l1.h5ad"]
|
||||
elif path == "s3://my-bucket/lvl1/lvl2":
|
||||
return ["my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad"]
|
||||
|
||||
raise Exception("ls called with " + path)
|
||||
|
||||
def isdir(path):
|
||||
if path in [
|
||||
"s3://my-bucket/lvl1",
|
||||
"s3://my-bucket/pbmc3k_annotations",
|
||||
"s3://my-bucket/lvl1/lvl2",
|
||||
]:
|
||||
return True
|
||||
if path in [
|
||||
"s3://my-bucket/pbmc3k.h5ad",
|
||||
"s3://my-bucket/lvl1/pbmc3k_l1.h5ad",
|
||||
"s3://my-bucket/lvl1/pbmc3k_l1_annotations",
|
||||
"s3://my-bucket/lvl1/lvl2/pbmc3k_l2.h5ad",
|
||||
"s3://my-bucket/lvl1/lvl2/pbmc3k_l2_annotations",
|
||||
]:
|
||||
return False
|
||||
raise Exception("isdir called with " + path)
|
||||
|
||||
def isfile(path):
|
||||
if path in ["s3://my-bucket/pbmc3k_annotations/annot.csv"]:
|
||||
return True
|
||||
if path in ["s3://my-bucket/pbmc3k_annotations"]:
|
||||
return False
|
||||
raise Exception("isfile called with " + path)
|
||||
|
||||
s3func.return_value = S3Mock
|
||||
source = S3ItemSource("my-bucket")
|
||||
with self.app.test_request_context(query_string="refresh=true") as test_context:
|
||||
tree = source.scan_directory()
|
||||
|
||||
def s3item_compare(i1, i2, msg=""):
|
||||
self.assertEqual(i1.name, i2.name, "name equals")
|
||||
self.assertEqual(i1.type, i2.type, "type equals")
|
||||
self.assertEqual(i1.s3key, i2.s3key, "s3key equals")
|
||||
if i1.annotations is None:
|
||||
self.assertEqual(i1.annotations, i2.annotations, "annotations equals")
|
||||
else:
|
||||
self.assertEqual(
|
||||
len(i1.annotations),
|
||||
len(i2.annotations),
|
||||
"annotations length equals",
|
||||
)
|
||||
for a1, a2 in zip(i1.annotations, i2.annotations):
|
||||
self.assertEqual(a1, a2)
|
||||
return True
|
||||
|
||||
self.addTypeEqualityFunc(S3Item, s3item_compare)
|
||||
|
||||
def assertTree(t, descriptor, items):
|
||||
self.assertEqual(t.descriptor, descriptor)
|
||||
self.assertEqual(len(t.items), len(items))
|
||||
for i1, i2 in zip(t.items, items):
|
||||
self.assertEqual(i1, i2)
|
||||
|
||||
assertTree(
|
||||
tree,
|
||||
"",
|
||||
[
|
||||
S3Item(
|
||||
"pbmc3k.h5ad",
|
||||
name="pbmc3k.h5ad",
|
||||
type=ItemType.h5ad,
|
||||
annotations=[
|
||||
S3Item(
|
||||
"pbmc3k_annotations/annot.csv",
|
||||
name="annot.csv",
|
||||
type=ItemType.annotation,
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(len(tree.branches), 1)
|
||||
lvl1 = tree.branches[0]
|
||||
assertTree(
|
||||
lvl1,
|
||||
"lvl1",
|
||||
[
|
||||
S3Item(
|
||||
"lvl1/pbmc3k_l1.h5ad",
|
||||
name="pbmc3k_l1.h5ad",
|
||||
type=ItemType.h5ad,
|
||||
annotations=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(len(lvl1.branches), 1)
|
||||
lvl2 = lvl1.branches[0]
|
||||
assertTree(
|
||||
lvl2,
|
||||
"lvl1/lvl2",
|
||||
[
|
||||
S3Item(
|
||||
"lvl1/lvl2/pbmc3k_l2.h5ad",
|
||||
name="pbmc3k_l2.h5ad",
|
||||
type=ItemType.h5ad,
|
||||
annotations=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(lvl2.branches, None)
|
||||
|
||||
|
||||
class TestListItems(unittest.TestCase):
|
||||
def test_GIVEN_filter_THEN_pass_filter_into_scan_directory(self):
|
||||
source = S3ItemSource("my-bucket")
|
||||
source.scan_directory = MagicMock()
|
||||
tree = source.list_items("some-filter")
|
||||
source.scan_directory.assert_called_once_with("some-filter")
|
||||
|
||||
def test_GIVEN_no_filter_THEN_pass_empty_string_into_scan_directory(self):
|
||||
source = S3ItemSource("my-bucket")
|
||||
source.scan_directory = MagicMock()
|
||||
tree = source.list_items()
|
||||
source.scan_directory.assert_called_once_with("")
|
||||
@@ -1,492 +0,0 @@
|
||||
import unittest
|
||||
import pytest
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import MagicMock, Mock, patch, call
|
||||
from freezegun import freeze_time
|
||||
|
||||
from cellxgene_gateway.backend_cache import BackendCache, is_port_in_use
|
||||
from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
|
||||
|
||||
class TestIsPortInUse(unittest.TestCase):
|
||||
@patch("socket.socket")
|
||||
def test_GIVEN_free_port_THEN_returns_true(self, socketMock):
|
||||
connectMock = socketMock()
|
||||
connectMock.connect_ex.return_value = 0
|
||||
connectMock.__enter__.return_value = connectMock
|
||||
self.assertEqual(is_port_in_use(123), True)
|
||||
self.assertTrue(connectMock.__enter__.calledOnce)
|
||||
self.assertTrue(connectMock.__exit__.calledOnce)
|
||||
self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
|
||||
self.assertTrue(socketMock.calledOnceWith("a"))
|
||||
|
||||
@patch("socket.socket")
|
||||
def test_GIVEN_used_port_THEN_returns_false(self, socketMock):
|
||||
connectMock = socketMock()
|
||||
connectMock.__enter__.return_value = connectMock
|
||||
connectMock.connect_ex.return_value = 1
|
||||
self.assertTrue(connectMock.__enter__.calledOnce)
|
||||
self.assertTrue(connectMock.__exit__.calledOnce)
|
||||
self.assertTrue(connectMock.connect_ex.calledOnceWith("a"))
|
||||
self.assertTrue(socketMock.calledOnceWith("a"))
|
||||
self.assertEqual(is_port_in_use(123), False)
|
||||
|
||||
|
||||
class TestBackendCacheInit(unittest.TestCase):
|
||||
def test_GIVEN_new_backend_cache_THEN_entry_list_is_empty(self):
|
||||
"""BackendCache should initialize with an empty entry list."""
|
||||
cache = BackendCache()
|
||||
self.assertEqual(cache.entry_list, [])
|
||||
|
||||
|
||||
class TestBackendCacheGetPorts(unittest.TestCase):
|
||||
def test_GIVEN_empty_cache_THEN_get_ports_returns_empty_list(self):
|
||||
"""get_ports should return empty list when no entries exist."""
|
||||
cache = BackendCache()
|
||||
ports = cache.get_ports()
|
||||
self.assertEqual(ports, [])
|
||||
|
||||
def test_GIVEN_cache_with_entries_THEN_get_ports_returns_all_ports(self):
|
||||
"""get_ports should return list of all ports from entries."""
|
||||
cache = BackendCache()
|
||||
|
||||
# Create mock entries with different ports
|
||||
entry1 = Mock()
|
||||
entry1.port = 8000
|
||||
entry2 = Mock()
|
||||
entry2.port = 8001
|
||||
entry3 = Mock()
|
||||
entry3.port = 8002
|
||||
|
||||
cache.entry_list = [entry1, entry2, entry3]
|
||||
|
||||
ports = cache.get_ports()
|
||||
self.assertEqual(ports, [8000, 8001, 8002])
|
||||
|
||||
def test_GIVEN_single_entry_THEN_get_ports_returns_single_port(self):
|
||||
"""get_ports should correctly handle a single entry."""
|
||||
cache = BackendCache()
|
||||
entry = Mock()
|
||||
entry.port = 9000
|
||||
|
||||
cache.entry_list = [entry]
|
||||
|
||||
ports = cache.get_ports()
|
||||
self.assertEqual(ports, [9000])
|
||||
|
||||
|
||||
class TestBackendCacheCheckPath(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.cache = BackendCache()
|
||||
|
||||
def test_GIVEN_no_matching_path_THEN_check_path_returns_none(self):
|
||||
"""check_path should return None when no entries match the path."""
|
||||
source = Mock()
|
||||
source.name = "test_source"
|
||||
|
||||
cache_entry = Mock()
|
||||
cache_entry.status = CacheEntryStatus.loaded
|
||||
cache_entry.key = Mock()
|
||||
cache_entry.key.source.name = "other_source"
|
||||
cache_entry.key.descriptor = "/some/path"
|
||||
|
||||
self.cache.entry_list = [cache_entry]
|
||||
|
||||
result = self.cache.check_path(source, "/test/path")
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_GIVEN_terminated_entry_THEN_check_path_ignores_it(self):
|
||||
"""check_path should ignore entries with terminated status."""
|
||||
source = Mock()
|
||||
source.name = "test_source"
|
||||
|
||||
cache_entry = Mock()
|
||||
cache_entry.status = CacheEntryStatus.terminated
|
||||
cache_entry.key = Mock()
|
||||
cache_entry.key.source.name = "test_source"
|
||||
cache_entry.key.descriptor = "/data"
|
||||
|
||||
self.cache.entry_list = [cache_entry]
|
||||
|
||||
result = self.cache.check_path(source, "/data/file.txt")
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_GIVEN_single_matching_entry_THEN_check_path_returns_it(self):
|
||||
"""check_path should return the matching entry when exactly one matches."""
|
||||
source = Mock()
|
||||
source.name = "test_source"
|
||||
|
||||
cache_entry = Mock()
|
||||
cache_entry.status = CacheEntryStatus.loaded
|
||||
cache_entry.key = Mock()
|
||||
cache_entry.key.source.name = "test_source"
|
||||
cache_entry.key.descriptor = "/data"
|
||||
|
||||
self.cache.entry_list = [cache_entry]
|
||||
|
||||
result = self.cache.check_path(source, "/data/file.txt")
|
||||
self.assertIs(result, cache_entry)
|
||||
|
||||
def test_GIVEN_path_that_does_not_start_with_descriptor_THEN_check_path_returns_none(
|
||||
self,
|
||||
):
|
||||
"""check_path should return None if path doesn't start with descriptor."""
|
||||
source = Mock()
|
||||
source.name = "test_source"
|
||||
|
||||
cache_entry = Mock()
|
||||
cache_entry.status = CacheEntryStatus.loaded
|
||||
cache_entry.key = Mock()
|
||||
cache_entry.key.source.name = "test_source"
|
||||
cache_entry.key.descriptor = "/data"
|
||||
|
||||
self.cache.entry_list = [cache_entry]
|
||||
|
||||
result = self.cache.check_path(source, "/other/file.txt")
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_GIVEN_multiple_matching_entries_THEN_check_path_raises_exception(self):
|
||||
"""check_path should raise exception when multiple entries match."""
|
||||
source = Mock()
|
||||
source.name = "test_source"
|
||||
|
||||
cache_entry1 = Mock()
|
||||
cache_entry1.status = CacheEntryStatus.loaded
|
||||
cache_entry1.key = Mock()
|
||||
cache_entry1.key.source.name = "test_source"
|
||||
cache_entry1.key.descriptor = "/data"
|
||||
|
||||
cache_entry2 = Mock()
|
||||
cache_entry2.status = CacheEntryStatus.loaded
|
||||
cache_entry2.key = Mock()
|
||||
cache_entry2.key.source.name = "test_source"
|
||||
cache_entry2.key.descriptor = "/data"
|
||||
|
||||
self.cache.entry_list = [cache_entry1, cache_entry2]
|
||||
|
||||
with self.assertRaises(CellxgeneException) as context:
|
||||
self.cache.check_path(source, "/data/file.txt")
|
||||
|
||||
# The CellxgeneException is raised with HTTPStatus as first arg and message as second
|
||||
self.assertEqual(context.exception.message, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
self.assertIn("Found 2", context.exception.http_status)
|
||||
|
||||
def test_GIVEN_mixed_entries_THEN_check_path_returns_only_matching_active_entry(
|
||||
self,
|
||||
):
|
||||
"""check_path should correctly filter by source, path, and status."""
|
||||
source = Mock()
|
||||
source.name = "target_source"
|
||||
|
||||
# Terminated entry - should be ignored
|
||||
terminated_entry = Mock()
|
||||
terminated_entry.status = CacheEntryStatus.terminated
|
||||
terminated_entry.key = Mock()
|
||||
terminated_entry.key.source.name = "target_source"
|
||||
terminated_entry.key.descriptor = "/data"
|
||||
|
||||
# Different source - should be ignored
|
||||
other_source_entry = Mock()
|
||||
other_source_entry.status = CacheEntryStatus.loaded
|
||||
other_source_entry.key = Mock()
|
||||
other_source_entry.key.source.name = "other_source"
|
||||
other_source_entry.key.descriptor = "/data"
|
||||
|
||||
# Matching entry - should be returned
|
||||
matching_entry = Mock()
|
||||
matching_entry.status = CacheEntryStatus.loaded
|
||||
matching_entry.key = Mock()
|
||||
matching_entry.key.source.name = "target_source"
|
||||
matching_entry.key.descriptor = "/data"
|
||||
|
||||
self.cache.entry_list = [terminated_entry, other_source_entry, matching_entry]
|
||||
|
||||
result = self.cache.check_path(source, "/data/file.txt")
|
||||
self.assertIs(result, matching_entry)
|
||||
|
||||
|
||||
class TestBackendCacheCheckEntry(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.cache = BackendCache()
|
||||
|
||||
def test_GIVEN_no_matching_entry_THEN_check_entry_returns_none(self):
|
||||
"""check_entry should return None when no entries match the key."""
|
||||
key = Mock()
|
||||
|
||||
cache_entry = Mock()
|
||||
cache_entry.status = CacheEntryStatus.loaded
|
||||
cache_entry.key = Mock()
|
||||
cache_entry.key.equals.return_value = False
|
||||
|
||||
self.cache.entry_list = [cache_entry]
|
||||
|
||||
result = self.cache.check_entry(key)
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_GIVEN_terminated_entry_THEN_check_entry_ignores_it(self):
|
||||
"""check_entry should ignore entries with terminated status."""
|
||||
key = Mock()
|
||||
|
||||
cache_entry = Mock()
|
||||
cache_entry.status = CacheEntryStatus.terminated
|
||||
cache_entry.key = Mock()
|
||||
cache_entry.key.equals.return_value = True
|
||||
|
||||
self.cache.entry_list = [cache_entry]
|
||||
|
||||
result = self.cache.check_entry(key)
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_GIVEN_single_matching_entry_THEN_check_entry_returns_it(self):
|
||||
"""check_entry should return the matching entry when exactly one matches."""
|
||||
key = Mock()
|
||||
|
||||
cache_entry = Mock()
|
||||
cache_entry.status = CacheEntryStatus.loaded
|
||||
cache_entry.key = Mock()
|
||||
cache_entry.key.equals.return_value = True
|
||||
|
||||
self.cache.entry_list = [cache_entry]
|
||||
|
||||
result = self.cache.check_entry(key)
|
||||
self.assertIs(result, cache_entry)
|
||||
|
||||
def test_GIVEN_multiple_matching_entries_THEN_check_entry_raises_exception(self):
|
||||
"""check_entry should raise exception when multiple entries match."""
|
||||
key = Mock()
|
||||
key.dataset = "test_dataset"
|
||||
|
||||
cache_entry1 = Mock()
|
||||
cache_entry1.status = CacheEntryStatus.loaded
|
||||
cache_entry1.key = Mock()
|
||||
cache_entry1.key.equals.return_value = True
|
||||
|
||||
cache_entry2 = Mock()
|
||||
cache_entry2.status = CacheEntryStatus.loaded
|
||||
cache_entry2.key = Mock()
|
||||
cache_entry2.key.equals.return_value = True
|
||||
|
||||
self.cache.entry_list = [cache_entry1, cache_entry2]
|
||||
|
||||
with self.assertRaises(CellxgeneException) as context:
|
||||
self.cache.check_entry(key)
|
||||
|
||||
# The CellxgeneException is raised with HTTPStatus as first arg and message as second
|
||||
self.assertEqual(context.exception.message, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
self.assertIn("Found 2", context.exception.http_status)
|
||||
|
||||
def test_GIVEN_mixed_entries_THEN_check_entry_returns_only_matching_active_entry(
|
||||
self,
|
||||
):
|
||||
"""check_entry should correctly filter by key equality and status."""
|
||||
key = Mock()
|
||||
|
||||
# Terminated entry - should be ignored
|
||||
terminated_entry = Mock()
|
||||
terminated_entry.status = CacheEntryStatus.terminated
|
||||
terminated_entry.key = Mock()
|
||||
terminated_entry.key.equals.return_value = True
|
||||
|
||||
# Non-matching entry - should be ignored
|
||||
non_matching_entry = Mock()
|
||||
non_matching_entry.status = CacheEntryStatus.loaded
|
||||
non_matching_entry.key = Mock()
|
||||
non_matching_entry.key.equals.return_value = False
|
||||
|
||||
# Matching entry - should be returned
|
||||
matching_entry = Mock()
|
||||
matching_entry.status = CacheEntryStatus.loaded
|
||||
matching_entry.key = Mock()
|
||||
matching_entry.key.equals.return_value = True
|
||||
|
||||
self.cache.entry_list = [terminated_entry, non_matching_entry, matching_entry]
|
||||
|
||||
result = self.cache.check_entry(key)
|
||||
self.assertIs(result, matching_entry)
|
||||
|
||||
|
||||
class TestBackendCachePrune(unittest.TestCase):
|
||||
def test_GIVEN_entry_in_cache_THEN_prune_removes_it(self):
|
||||
"""prune should remove the entry from the cache."""
|
||||
cache = BackendCache()
|
||||
|
||||
entry_mock = Mock()
|
||||
cache.entry_list = [entry_mock]
|
||||
|
||||
cache.prune(entry_mock)
|
||||
|
||||
self.assertEqual(len(cache.entry_list), 0)
|
||||
self.assertNotIn(entry_mock, cache.entry_list)
|
||||
|
||||
def test_GIVEN_entry_in_cache_THEN_prune_terminates_it(self):
|
||||
"""prune should call terminate on the entry."""
|
||||
cache = BackendCache()
|
||||
|
||||
entry_mock = Mock()
|
||||
cache.entry_list = [entry_mock]
|
||||
|
||||
cache.prune(entry_mock)
|
||||
|
||||
entry_mock.terminate.assert_called_once()
|
||||
|
||||
def test_GIVEN_multiple_entries_THEN_prune_removes_only_target_entry(self):
|
||||
"""prune should only remove the specified entry, not others."""
|
||||
cache = BackendCache()
|
||||
|
||||
entry1 = Mock()
|
||||
entry2 = Mock()
|
||||
entry3 = Mock()
|
||||
|
||||
cache.entry_list = [entry1, entry2, entry3]
|
||||
|
||||
cache.prune(entry2)
|
||||
|
||||
self.assertEqual(len(cache.entry_list), 2)
|
||||
self.assertIn(entry1, cache.entry_list)
|
||||
self.assertNotIn(entry2, cache.entry_list)
|
||||
self.assertIn(entry3, cache.entry_list)
|
||||
|
||||
# Verify only entry2 was terminated
|
||||
entry1.terminate.assert_not_called()
|
||||
entry2.terminate.assert_called_once()
|
||||
entry3.terminate.assert_not_called()
|
||||
|
||||
def test_GIVEN_empty_cache_THEN_prune_raises_value_error(self):
|
||||
"""prune should raise ValueError if entry is not in cache."""
|
||||
cache = BackendCache()
|
||||
|
||||
entry_mock = Mock()
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
cache.prune(entry_mock)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def connect_mock():
|
||||
with patch("socket.socket") as socketMock:
|
||||
connectMock = socketMock()
|
||||
connectMock.__enter__.return_value = connectMock
|
||||
connectMock.connect_ex.return_value = 1 # Port not in use by default
|
||||
yield connectMock
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def process_backend_mock():
|
||||
with patch("cellxgene_gateway.backend_cache.process_backend") as mock:
|
||||
yield mock
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def sleep_mock():
|
||||
from time import sleep
|
||||
|
||||
with patch("cellxgene_gateway.backend_cache.time.sleep") as mock:
|
||||
mock.side_effect = lambda x: sleep(0.01) # short sleep
|
||||
yield mock
|
||||
|
||||
|
||||
@freeze_time("2025-12-01 10:00:00")
|
||||
def test_GIVEN_empty_cache_THEN_create_entry_uses_starting_port(process_backend_mock):
|
||||
"""create_entry should use port 8000 when cache is empty and port is free."""
|
||||
key = Mock()
|
||||
scripts = ["script1", "script2"]
|
||||
|
||||
cache = BackendCache()
|
||||
result = cache.create_entry(key, scripts)
|
||||
|
||||
# Verify port 8000 was used
|
||||
assert result.port == 8000
|
||||
|
||||
# Verify entry was added to list
|
||||
assert result in cache.entry_list
|
||||
# Verify thread was started
|
||||
process_backend_mock.launch.assert_called_once()
|
||||
|
||||
# Verify CacheEntry was created with correct key
|
||||
assert result.key is key
|
||||
assert result.status == CacheEntryStatus.loading
|
||||
|
||||
|
||||
@freeze_time("2025-12-01 10:00:00")
|
||||
def test_GIVEN_port_in_use_THEN_create_entry_finds_next_available_port(connect_mock):
|
||||
"""create_entry should increment port until finding one that's free."""
|
||||
# Ports 8000 and 8001 are in use, 8002 is free
|
||||
connect_mock.connect_ex.side_effect = [0, 0, 1]
|
||||
|
||||
key = Mock()
|
||||
scripts = []
|
||||
|
||||
cache = BackendCache()
|
||||
result = cache.create_entry(key, scripts)
|
||||
|
||||
# Verify port 8002 was used (after skipping 8000 and 8001)
|
||||
assert result.port == 8002
|
||||
|
||||
|
||||
@freeze_time("2025-12-01 10:00:00")
|
||||
def test_GIVEN_port_exists_in_cache_THEN_create_entry_skips_it():
|
||||
"""create_entry should skip ports already in the cache."""
|
||||
key = Mock()
|
||||
scripts = []
|
||||
|
||||
# Pre-populate cache with port 8000
|
||||
existing_entry = Mock()
|
||||
existing_entry.port = 8000
|
||||
|
||||
cache = BackendCache()
|
||||
cache.entry_list = [existing_entry]
|
||||
|
||||
result = cache.create_entry(key, scripts)
|
||||
|
||||
# Verify port 8001 was used (skipping the existing 8000)
|
||||
assert result.port == 8001
|
||||
|
||||
|
||||
@freeze_time("2025-12-01 10:00:00")
|
||||
def test_GIVEN_create_entry_THEN_background_thread_is_started(process_backend_mock):
|
||||
"""create_entry should start a background thread with correct args."""
|
||||
key = Mock()
|
||||
scripts = ["script1"]
|
||||
|
||||
with patch("cellxgene_gateway.backend_cache.env") as env_mock:
|
||||
env_mock.cellxgene_location = "/path/to/cellxgene"
|
||||
|
||||
cache = BackendCache()
|
||||
result = cache.create_entry(key, scripts)
|
||||
|
||||
# Verify Thread was created with correct target and args
|
||||
process_backend_mock.launch.assert_called_once()
|
||||
call_kwargs = process_backend_mock.launch.call_args[0]
|
||||
assert call_kwargs == ("/path/to/cellxgene", ["script1"], result)
|
||||
|
||||
|
||||
@freeze_time("2025-12-01 10:00:00")
|
||||
def test_GIVEN_create_entry_THEN_entry_is_added_to_cache():
|
||||
"""create_entry should add the created entry to the cache list."""
|
||||
key = Mock()
|
||||
scripts = []
|
||||
|
||||
cache = BackendCache()
|
||||
initial_count = len(cache.entry_list)
|
||||
|
||||
result = cache.create_entry(key, scripts)
|
||||
|
||||
# Verify entry was added
|
||||
assert len(cache.entry_list) == initial_count + 1
|
||||
assert result in cache.entry_list
|
||||
|
||||
|
||||
@freeze_time("2025-12-01 10:00:00")
|
||||
def test_GIVEN_create_entry_THEN_returns_created_entry():
|
||||
"""create_entry should return the created entry."""
|
||||
key = Mock()
|
||||
scripts = []
|
||||
|
||||
cache = BackendCache()
|
||||
result = cache.create_entry(key, scripts)
|
||||
|
||||
# Verify the result is a CacheEntry instance
|
||||
assert isinstance(result, CacheEntry)
|
||||
assert result.status == CacheEntryStatus.loading
|
||||
@@ -1,67 +1,33 @@
|
||||
import unittest
|
||||
import tempfile
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from flask import Flask
|
||||
from cellxgene_gateway import flask_util
|
||||
from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.gateway import app
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItem("/czi/", "pbmc3k.h5ad", ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
)
|
||||
|
||||
|
||||
class TestRenderEntry(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = app
|
||||
self.app_context = self.app.test_request_context()
|
||||
self.app_context.push()
|
||||
self.client = self.app.test_client()
|
||||
|
||||
def tearDown(self):
|
||||
self.app_context.pop()
|
||||
|
||||
def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
|
||||
entry = CacheEntry.for_key("some-key", 1)
|
||||
self.assertEqual(entry.status, CacheEntryStatus.loading)
|
||||
|
||||
def test_GIVEN_absolute_static_url_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = False
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
"src:url(/static/assets/"
|
||||
)
|
||||
expected = "src:url(/view/czi/pbmc3k.h5ad/static/assets/"
|
||||
expected = "src:url(http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_GIVEN_absolute_src_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = False
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
'<link rel="shortcut icon" href="/static/assets/favicon.ico">'
|
||||
)
|
||||
expected = '<link rel="shortcut icon" href="/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_GIVEN_absolute_static_url_include_source_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = True
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
"src:url(/static/assets/"
|
||||
)
|
||||
expected = "src:url(/source/local/view/czi/pbmc3k.h5ad/static/assets/"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_GIVEN_absolute_src_include_source_THEN_include_path(self):
|
||||
flask_util.include_source_in_url = True
|
||||
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
|
||||
'<link rel="shortcut icon" href="/static/assets/favicon.ico">'
|
||||
)
|
||||
expected = '<link rel="shortcut icon" href="/source/local/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
|
||||
expected = '<link rel="shortcut icon" href="http://localhost:5005/source/local/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.dir_util import ensure_dir_exists, make_annotations, make_h5ad
|
||||
|
||||
|
||||
class TestMakeH5ad(unittest.TestCase):
|
||||
def test_GIVEN_annotation_dir_THEN_returns_h5ad(self):
|
||||
self.assertEqual(make_h5ad("pbmc_annotations"), "pbmc.h5ad")
|
||||
|
||||
|
||||
class TestMakeAnnotations(unittest.TestCase):
|
||||
def test_GIVEN_h5ad_THEN_returns_annotations(self):
|
||||
self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
|
||||
|
||||
|
||||
class TestMakeAnnotations(unittest.TestCase):
|
||||
def test_GIVEN_h5ad_THEN_returns_annotations(self):
|
||||
self.assertEqual(make_annotations("pbmc.h5ad"), "pbmc_annotations")
|
||||
|
||||
|
||||
class TestEnsureDirExists(unittest.TestCase):
|
||||
@patch("os.path.exists")
|
||||
@patch("os.makedirs")
|
||||
def test_GIVEN_existing_THEN_does_not_call_makedir(self, makedirsMock, existsMock):
|
||||
existsMock.return_value = True
|
||||
ensure_dir_exists("/foo")
|
||||
makedirsMock.assert_not_called()
|
||||
|
||||
@patch("os.path.exists")
|
||||
@patch("os.makedirs")
|
||||
def test_GIVEN_not_existing_THEN_calls_makedir(self, makedirsMock, existsMock):
|
||||
existsMock.return_value = False
|
||||
ensure_dir_exists("/foo")
|
||||
makedirsMock.assert_called_once_with("/foo")
|
||||
@@ -21,15 +21,6 @@ class TestExtraScripts(unittest.TestCase):
|
||||
def test_GIVEN_empty_string_THEN_returns_empty_array(self):
|
||||
self.assertEqual(get_extra_scripts(), [])
|
||||
|
||||
@patch("cellxgene_gateway.env.extra_scripts", new="'asdf'")
|
||||
def test_GIVEN_bare_string_THEN_throws_Exception(self):
|
||||
with self.assertRaises(Exception) as context:
|
||||
self.assertEqual(get_extra_scripts(), [])
|
||||
self.assertEqual(
|
||||
'Error parsing GATEWAY_EXTRA_SCRIPTS, expected JSON array e.g. ["https://example.com/path/to/script.js"]',
|
||||
str(context.exception),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,170 +1,33 @@
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections import defaultdict
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.filecrawl import (
|
||||
render_item,
|
||||
render_item_source,
|
||||
render_item_tree,
|
||||
)
|
||||
from cellxgene_gateway.filecrawl import render_item
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.items.item import ItemTree, ItemType
|
||||
from cellxgene_gateway.gateway import app
|
||||
|
||||
source = FileItemSource("/tmp")
|
||||
|
||||
|
||||
def make_entry(subpath="somepath", annotations=None):
|
||||
return FileItem(
|
||||
subpath=subpath,
|
||||
name="entry",
|
||||
ext=".h5ad",
|
||||
type=ItemType.h5ad,
|
||||
annotations=annotations,
|
||||
)
|
||||
|
||||
|
||||
class TestRenderEntry(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = app
|
||||
self.app_context = self.app.test_request_context()
|
||||
self.app_context.push()
|
||||
|
||||
def tearDown(self):
|
||||
self.app_context.pop()
|
||||
|
||||
def test_GIVEN_path_both_slash_THEN_view_has_single_slash(self):
|
||||
entry = make_entry(subpath="/somepath/")
|
||||
entry = FileItem(
|
||||
subpath="/somepath/", name="entry", type=ItemType.h5ad
|
||||
)
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn("view/somepath/entry.h5ad/'", rendered)
|
||||
self.assertIn("view/somepath/entry/'", rendered)
|
||||
|
||||
def test_GIVEN_path_starts_slash_THEN_view_has_single_slash(self):
|
||||
entry = make_entry(subpath="/somepath")
|
||||
entry = FileItem(subpath="/somepath", name="entry", type=ItemType.h5ad)
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn("view/somepath/entry.h5ad/'", rendered)
|
||||
self.assertIn("view/somepath/entry/'", rendered)
|
||||
|
||||
def test_GIVEN_path_ends_slash_THEN_view_has_single_slash(self):
|
||||
entry = make_entry(subpath="somepath/")
|
||||
entry = FileItem(subpath="somepath/", name="entry", type=ItemType.h5ad)
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn("view/somepath/entry.h5ad/'", rendered)
|
||||
self.assertIn("view/somepath/entry/'", rendered)
|
||||
|
||||
def test_GIVEN_path_no_slash_THEN_view_has_single_slash(self):
|
||||
entry = make_entry(subpath="somepath")
|
||||
entry = FileItem(subpath="somepath", name="entry", type=ItemType.h5ad)
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn("view/somepath/entry.h5ad/'", rendered)
|
||||
|
||||
|
||||
class TestRenderAnnotation(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.app = app
|
||||
self.app_context = self.app.test_request_context()
|
||||
self.app_context.push()
|
||||
|
||||
def tearDown(self):
|
||||
self.app_context.pop()
|
||||
|
||||
@patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
|
||||
def test_GIVEN_no_annotation_THEN_new_alone(self):
|
||||
entry = make_entry(annotations=None)
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn(
|
||||
"> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a></li>",
|
||||
rendered,
|
||||
)
|
||||
|
||||
@patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
|
||||
def test_GIVEN_annotation_THEN_new_before(self):
|
||||
annotation = FileItem(
|
||||
subpath="somepath/entry_annotations",
|
||||
name="annot",
|
||||
ext=".csv",
|
||||
type=ItemType.annotation,
|
||||
)
|
||||
entry = make_entry(annotations=[annotation])
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn(
|
||||
"> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
|
||||
" <a href='/source/Files:/tmp/view/somepath/entry_annotations/annot.csv/'>annot</a></li>",
|
||||
rendered,
|
||||
)
|
||||
|
||||
@patch("cellxgene_gateway.filecrawl.enable_annotations", new=True)
|
||||
def test_GIVEN_annotation_THEN_escaped(self):
|
||||
annotation = FileItem(
|
||||
subpath="somepath/entry_annotations",
|
||||
name="hot&cold",
|
||||
ext=".csv",
|
||||
type=ItemType.annotation,
|
||||
)
|
||||
entry = make_entry(annotations=[annotation])
|
||||
rendered = render_item(entry, source)
|
||||
self.assertIn(
|
||||
"> | annotations: <a class='new' href='/source/Files:/tmp/view/somepath/entry_annotations'>new</a>,"
|
||||
" <a href='/source/Files:/tmp/view/somepath/entry_annotations/hot&cold.csv/'>hot&cold</a></li>",
|
||||
rendered,
|
||||
)
|
||||
|
||||
|
||||
class TestRenderItemSource(unittest.TestCase):
|
||||
@patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
|
||||
def test_GIVEN_some_filter_THEN_includes_filterpart_in_heading(self, item_source):
|
||||
item_source.name = "FakeSource"
|
||||
item_source.list_items.return_value = ItemTree("rootdir", [], [])
|
||||
rendered = render_item_source(item_source, "some_filter")
|
||||
self.assertEqual(
|
||||
rendered,
|
||||
"<h6><a href='/filecrawl.html?source=FakeSource'>FakeSource</a>:some_filter</h6><li><a href='/filecrawl/rootdir?source=FakeSource'>rootdir</a><ul></ul></li>",
|
||||
)
|
||||
|
||||
|
||||
class TestRenderItemTree(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.app = app
|
||||
self.app_context = self.app.test_request_context()
|
||||
self.app_context.push()
|
||||
|
||||
def tearDown(self):
|
||||
self.app_context.pop()
|
||||
|
||||
@patch("cellxgene_gateway.items.file.fileitem_source.FileItemSource")
|
||||
def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
|
||||
item_source.name = "FakeSource"
|
||||
item_source.get_annotations_subpath = lambda _: "FakeAnnotations"
|
||||
file_item = FileItem(
|
||||
subpath="foo/bar/baz", name="file.h5ad", type=ItemType.h5ad
|
||||
)
|
||||
item_tree = ItemTree("foo/bar/baz", [file_item], [])
|
||||
rendered = render_item_tree(item_tree, item_source)
|
||||
self.assertEqual(
|
||||
rendered,
|
||||
"<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul>"
|
||||
"<li> <a href='/source/FakeSource/view/foo/bar/baz/file.h5ad/'>file.h5ad</a>"
|
||||
" </li></ul></li>",
|
||||
)
|
||||
|
||||
@patch(
|
||||
"os.listdir",
|
||||
side_effect=lambda parent: defaultdict(
|
||||
list, {"tmp": ["foo"], "tmp/foo": ["bar"]}
|
||||
)[parent],
|
||||
)
|
||||
@patch("os.path.exists", return_value=True)
|
||||
def test_GIVEN_dirs_without_h5ad_THEN_excludes_dirs_in_output(
|
||||
self, listdir, exists
|
||||
):
|
||||
# Directories:
|
||||
# - tmp
|
||||
# - foo
|
||||
# - bar (no h5ad files)
|
||||
item_source = FileItemSource("tmp", name="local")
|
||||
item_tree = item_source.list_items("foo")
|
||||
rendered = render_item_tree(item_tree, item_source)
|
||||
self.assertEqual(
|
||||
rendered,
|
||||
"<li><a href='/filecrawl/foo?source=local'>foo</a><ul></ul></li>",
|
||||
)
|
||||
self.assertIn("view/somepath/entry/'", rendered)
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
import json
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from cellxgene_gateway.gateway import do_GET_status_json, app, cache
|
||||
from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
|
||||
|
||||
|
||||
class TestGatewayStatusJson(unittest.TestCase):
|
||||
def test_do_GET_status_json_returns_expected_structure(self):
|
||||
# Create a minimal fake key with required attributes
|
||||
h5ad_item = SimpleNamespace(descriptor="somedir/dataset.h5ad")
|
||||
key = SimpleNamespace(
|
||||
h5ad_item=h5ad_item,
|
||||
annotation_descriptor="somedir/dataset_annotations/foo.csv",
|
||||
)
|
||||
|
||||
# Create a CacheEntry with known launchtime/timestamp/status
|
||||
entry = CacheEntry(
|
||||
None,
|
||||
key,
|
||||
8000,
|
||||
111,
|
||||
222,
|
||||
CacheEntryStatus.loaded,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
# Install into the gateway cache and set app launchtime
|
||||
cache.entry_list = [entry]
|
||||
app.extensions.setdefault("cellxgene_gateway", {})["launchtime"] = "LAUNCH_TIME"
|
||||
|
||||
rv = do_GET_status_json()
|
||||
|
||||
data = json.loads(rv)
|
||||
# top-level launchtime comes from app.extensions
|
||||
self.assertEqual("LAUNCH_TIME", data["launchtime"])
|
||||
|
||||
self.assertIn("entry_list", data)
|
||||
self.assertEqual(1, len(data["entry_list"]))
|
||||
|
||||
e = data["entry_list"][0]
|
||||
self.assertEqual("somedir/dataset.h5ad", e["dataset"])
|
||||
self.assertEqual("somedir/dataset_annotations/foo.csv", e["annotation_file"])
|
||||
self.assertEqual("loaded", e["status"])
|
||||
self.assertEqual(111, e["launchtime"])
|
||||
self.assertEqual(222, e["last_access"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,21 +1,13 @@
|
||||
import unittest
|
||||
from unittest.mock import patch, seal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.backend_cache import BackendCache
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
)
|
||||
from cellxgene_gateway.cache_entry import CacheEntry
|
||||
|
||||
|
||||
class TestPruneProcessCache(unittest.TestCase):
|
||||
@patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
|
||||
@patch("cellxgene_gateway.env.expire_seconds", new=10)
|
||||
@patch("cellxgene_gateway.env.ttl", new="10")
|
||||
@patch("cellxgene_gateway.cache_entry.CacheEntry")
|
||||
@patch("cellxgene_gateway.cache_entry.CacheEntry")
|
||||
def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):
|
||||
@@ -23,23 +15,14 @@ class TestPruneProcessCache(unittest.TestCase):
|
||||
|
||||
cache = BackendCache()
|
||||
old.timestamp = -100
|
||||
old.foo = 12
|
||||
old.pid = 1
|
||||
old.key = key
|
||||
old.terminate.return_value = None
|
||||
seal(old)
|
||||
new.key = key
|
||||
cache.entry_list.append(old)
|
||||
new.timestamp = -5
|
||||
seal(new)
|
||||
cache.entry_list.append(new)
|
||||
self.assertEqual(len(cache.entry_list), 2)
|
||||
ppc = PruneProcessCache(cache)
|
||||
ppc.prune()
|
||||
self.assertEqual(len(cache.entry_list), 1)
|
||||
self.assertEqual(cache.entry_list[0], new)
|
||||
self.assertEqual(cache.entry_list[0], new)
|
||||
self.assertTrue(old.terminate.called)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from cellxgene_gateway.cache_entry import CacheEntry
|
||||
from cellxgene_gateway.cache_key import CacheKey
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.file.fileitem_source import FileItemSource
|
||||
from cellxgene_gateway.items.item import ItemType
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
|
||||
|
||||
class TestSubprocessBackend(unittest.TestCase):
|
||||
@patch("subprocess.Popen")
|
||||
def test_launch_GIVEN_no_stdout_THEN_throw_ProcessException(self, popen):
|
||||
subprocess = MagicMock()
|
||||
subprocess.stdout.readline().decode.return_value = ""
|
||||
subprocess.stderr.read().decode.return_value = "An unexpected error"
|
||||
popen.return_value = subprocess
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
)
|
||||
entry = CacheEntry.for_key(key, 8000)
|
||||
from cellxgene_gateway.subprocess_backend import SubprocessBackend
|
||||
|
||||
backend = SubprocessBackend()
|
||||
cellxgene_loc = "/some/cellxgene"
|
||||
scripts = ["http://example.com/script.js", "http://example.com/script2.js"]
|
||||
|
||||
with self.assertRaises(ProcessException) as context:
|
||||
backend.launch(cellxgene_loc, scripts, entry)
|
||||
popen.assert_called_once_with(
|
||||
[
|
||||
"yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --disable-annotations --disable-gene-sets-save --scripts http://example.com/script.js --scripts http://example.com/script2.js"
|
||||
],
|
||||
shell=True,
|
||||
stderr=-1,
|
||||
stdout=-1,
|
||||
)
|
||||
self.assertEqual("An unexpected error", context.exception.stderr)
|
||||
|
||||
@patch("subprocess.Popen")
|
||||
def test_launch_GIVEN_annotations_enabled_THEN_set_flags(self, popen):
|
||||
subprocess = MagicMock()
|
||||
subprocess.stdout.readline().decode.return_value = (
|
||||
"[cellxgene] Type CTRL-C at any time to exit.\n"
|
||||
)
|
||||
subprocess.stderr.read().decode.return_value = ""
|
||||
popen.return_value = subprocess
|
||||
|
||||
key = CacheKey(
|
||||
FileItem("/czi/", name="pbmc3k.h5ad", type=ItemType.h5ad),
|
||||
FileItemSource("/tmp", "local"),
|
||||
FileItem(
|
||||
"/czi/pbmc3k_annotations/", name="foo.csv", type=ItemType.annotation
|
||||
),
|
||||
)
|
||||
entry = CacheEntry.for_key(key, 8000)
|
||||
import cellxgene_gateway.subprocess_backend
|
||||
|
||||
cellxgene_gateway.subprocess_backend.enable_annotations = True
|
||||
try:
|
||||
backend = cellxgene_gateway.subprocess_backend.SubprocessBackend()
|
||||
cellxgene_loc = "/some/cellxgene"
|
||||
|
||||
backend.launch(cellxgene_loc, [], entry)
|
||||
finally:
|
||||
cellxgene_gateway.subprocess_backend.enable_annotations = False
|
||||
popen.assert_called_once_with(
|
||||
[
|
||||
"yes | /some/cellxgene launch /tmp/czi/pbmc3k.h5ad --port 8000 --host 127.0.0.1 --annotations-file /tmp/czi/pbmc3k_annotations/foo.csv --gene-sets-file /tmp/czi/pbmc3k_annotations/foo_gene_sets.csv"
|
||||
],
|
||||
shell=True,
|
||||
stderr=-1,
|
||||
stdout=-1,
|
||||
)
|
||||
Reference in New Issue
Block a user