#79 added cellxgene-data directory so it actually works

68 read from subprocess

Changelog.md

@@ -1,3 +1,10 @@
+# 0.3.10
+
+* #65 Added GATEWAY_EXPIRE_SECONDS to set how long cellxgene servers can remain idle before being terminated.
+* Added GATEWAY_LOG_LEVEL to set the log level
+* #68 Close connections after reading response
+* #68 Background thread reads from output of cellxgene process until it exits
+
 # 0.3.9
 
 * Added S3_ENABLE_LISTINGS_CACHE variable (See README.md)

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9
+
+RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
+
+ENV CELLXGENE_DATA=/cellxgene-data
+ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
+
+CMD ["cellxgene-gateway"]

README.md

@@ -73,10 +73,12 @@ Optional environment variables:
 * `EXTERNAL_PROTOCOL` - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy that performs https termination. Default value "http"
 * `GATEWAY_IP` - ip addess of instance gateway is running on, mostly used to display SSH instructions. Defaults to `socket.gethostbyname(socket.gethostname())`
 * `GATEWAY_PORT` - local port that the gateway should bind to, defaults to 5005
+* `GATEWAY_EXPIRE_SECONDS` - time in seconds that a cellxgene process will remain idle before being terminated. Defaults to 3600 (one hour)
 * `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
 * `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations. 
 * `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
-* `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. Can be overridden by setting `filecrawl.html?refresh=true` query parameter.
+* `GATEWAY_LOG_LEVEL` - default is `INFO`. set to `DEBUG` to increase logging and to `WARNING` to decrease logging.
+* `S3_ENABLE_LISTINGS_CACHE` - Set to `true` or to `1` to cache listings of S3 folders for performance. If the cache becomes stale, set `filecrawl.html?refresh=true` query parameter to refresh the cache.
 
 If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
 * `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
@@ -87,6 +89,33 @@ If any of the following optional variables are set, [ProxyFix](https://werkzeug.
 
 The defaults should be fine if you set up a venv and cellxgene_data folder as above.
 
+## Running cellxgene-gateway with Docker
+
+First, build Docker image:
+
+```bash
+docker build -t cellxgene-gateway .
+```
+
+Then, cellxgene-gateway can be launched as such:
+
+```bash
+docker run -it --rm \
+-v <local_data_dir>:/cellxgene-data \
+-p 5005:5005 \
+cellxgene-gateway
+```
+
+Additional environment variables can be provided with the `-e` parameter:
+
+```bash
+docker run -it --rm \
+-v <local_data_dir>:/cellxgene-data \
+-e GATEWAY_PORT=8080 \
+-p 8080:8080 \
+cellxgene-gateway
+```
+
 # Customization
 
 The current paradigm for customization is to modify files during a build or deployment phase:

cellxgene_gateway/__init__.py

@@ -7,4 +7,4 @@
 # OR CONDITIONS OF ANY KIND, either express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 
-__version__ = "0.3.9"
+__version__ = "0.3.10"

cellxgene_gateway/cache_entry.py

@@ -9,7 +9,6 @@
 import datetime
 import logging
 import re
-import urllib.parse
 from enum import Enum
 
 import psutil
@@ -22,6 +21,8 @@ from cellxgene_gateway.cellxgene_exception import CellxgeneException
 from cellxgene_gateway.flask_util import querystring
 from cellxgene_gateway.util import current_time_stamp
 
+logger = logging.getLogger(__name__)
+
 
 class CacheEntryStatus(Enum):
     loaded = "loaded"
@@ -111,7 +112,7 @@ class CacheEntry:
             except psutil.NoSuchProcess:
                 pass
 
-            logging.getLogger("cellxgene_gateway").info(f"terminated {terminated}")
+            logger.info(f"terminated {terminated}")
         self.status = CacheEntryStatus.terminated
 
     def rewrite_text_content(self, cellxgene_content):
@@ -169,38 +170,43 @@ class CacheEntry:
 
         full_path = self.cellxgene_basepath() + subpath + querystring()
 
-        if request.method in ["GET", "HEAD", "OPTIONS"]:
+        try:
-            cellxgene_response = get(full_path, headers=headers)
+            cellxgene_response = None
-        elif request.method == "PUT":
+            if request.method in ["GET", "HEAD", "OPTIONS"]:
-            cellxgene_response = put(
+                cellxgene_response = get(full_path, headers=headers)
-                full_path,
+            elif request.method == "PUT":
-                headers=headers,
+                cellxgene_response = put(
-                data=request.data,
+                    full_path,
-            )
+                    headers=headers,
-        elif request.method == "POST":
+                    data=request.data,
-            cellxgene_response = post(
+                )
-                full_path,
+            elif request.method == "POST":
-                headers=headers,
+                cellxgene_response = post(
-                data=request.data,
+                    full_path,
-            )
+                    headers=headers,
-        else:
+                    data=request.data,
-            raise CellxgeneException(f"Unexpected method {request.method}", 400)
+                )
-        content_type = cellxgene_response.headers["content-type"]
+            else:
-        if "text" in content_type:
+                raise CellxgeneException(f"Unexpected method {request.method}", 400)
-            gateway_content = self.rewrite_text_content(
+            content_type = cellxgene_response.headers["content-type"]
-                cellxgene_response.content.decode()
+            if "text" in content_type:
-            )
+                gateway_content = self.rewrite_text_content(
-        else:
+                    cellxgene_response.content.decode()
-            gateway_content = cellxgene_response.content
+                )
+            else:
+                gateway_content = cellxgene_response.content
 
-        resp_headers = {}
+            resp_headers = {}
-        for h in copy_headers:
+            for h in copy_headers:
-            if h in cellxgene_response.headers:
+                if h in cellxgene_response.headers:
-                resp_headers[h] = cellxgene_response.headers[h]
+                    resp_headers[h] = cellxgene_response.headers[h]
 
-        gateway_response = make_response(
+            gateway_response = make_response(
-            gateway_content,
+                gateway_content,
-            cellxgene_response.status_code,
+                cellxgene_response.status_code,
-            resp_headers,
+                resp_headers,
-        )
+            )
+        finally:
+            if cellxgene_response is not None:
+                cellxgene_response.close()
         return gateway_response

cellxgene_gateway/env.py

@@ -9,7 +9,6 @@
 
 import logging
 import os
-import socket
 
 cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
 cellxgene_data = os.environ.get("CELLXGENE_DATA", "")
@@ -24,7 +23,9 @@ external_protocol = os.environ.get(
 )
 ip = os.environ.get("GATEWAY_IP")
 extra_scripts = os.environ.get("GATEWAY_EXTRA_SCRIPTS")
-ttl = os.environ.get("GATEWAY_TTL")
+expire_seconds = int(
+    os.environ.get("GATEWAY_EXPIRE_SECONDS", os.environ.get("GATEWAY_TTL", "3600"))
+)
 enable_annotations = os.environ.get("GATEWAY_ENABLE_ANNOTATIONS", "").lower() in [
     "true",
     "1",
@@ -33,6 +34,7 @@ enable_backed_mode = os.environ.get("GATEWAY_ENABLE_BACKED_MODE", "").lower() in
     "true",
     "1",
 ]
+log_level = logging.getLevelName(os.environ.get("GATEWAY_LOG_LEVEL", "INFO"))
 
 env_vars = {
     "CELLXGENE_LOCATION": cellxgene_location,
@@ -50,9 +52,10 @@ optional_env_vars = {
     "GATEWAY_IP": ip,
     "GATEWAY_PORT": gateway_port,
     "GATEWAY_EXTRA_SCRIPTS": extra_scripts,
-    "GATEWAY_TTL": ttl,
+    "GATEWAY_EXPIRE_SECONDS": expire_seconds,
     "GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
     "GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
+    "GATEWAY_LOG_LEVEL": log_level,
     "CELLXGENE_ARGS": cellxgene_args,
     "CELLXGENE_DATA": cellxgene_data,
     "PROXY_FIX_FOR": proxy_fix_for,

cellxgene_gateway/gateway.py

@@ -299,7 +299,7 @@ def launch():
 
 def main():
     logging.basicConfig(
-        level=logging.INFO,
+        level=env.log_level,
         format="%(asctime)s:%(name)s:%(levelname)s:%(message)s",
     )
     cellxgene_data = os.environ.get("CELLXGENE_DATA", None)

cellxgene_gateway/prune_process_cache.py

@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 class PruneProcessCache:
     def __init__(self, cache):
         self.cache = cache
-        self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
+        self.expire_seconds = env.expire_seconds
 
     def __call__(self):
         while True:

cellxgene_gateway/subprocess_backend.py

@@ -17,6 +17,8 @@ from cellxgene_gateway.dir_util import make_annotations
 from cellxgene_gateway.env import cellxgene_args, enable_annotations, enable_backed_mode
 from cellxgene_gateway.process_exception import ProcessException
 
+logger = logging.getLogger(__name__)
+
 
 class SubprocessBackend:
     def __init__(self):
@@ -55,7 +57,7 @@ class SubprocessBackend:
             scripts,
             cache_entry.key.annotation_file_path,
         )
-        logging.getLogger("cellxgene_gateway").info(f"launching {cmd}")
+        logger.info(f"launching {cmd}")
         process = subprocess.Popen(
             [cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
         )
@@ -84,5 +86,6 @@ class SubprocessBackend:
                 cache_entry.append_output(output)
 
         cache_entry.set_loaded(process.pid)
-
+        for output in process.communicate():
-        return
+            logger.debug(f"cellxgene:{output}")
+        logger.info(f"exiting {cmd}")

environment.yml

@@ -2,9 +2,9 @@ name: cellxgene-gateway
 channels:
 - conda-forge
 dependencies:
-- python=3.7
+- python=3.9
 - requests
-- flask<2.0.0,>=1.0.2
+- flask
 - psutil
 - black
 - twine
@@ -13,6 +13,6 @@ dependencies:
 - pip
 - pip:
   - pre_commit
-  - flask-api==2.0
+  - flask-api
-  - werkzeug==1.0.1
+  - werkzeug
-  - cellxgene>=0.15
+  - cellxgene

examples/customized_docker_image/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.9
+
+RUN pip install cellxgene-gateway 'MarkupSafe<2.1'
+
+COPY customize_ui.sh customize_ui.sh
+RUN CELLXGENE_GATEWAY_DIR=/usr/local/lib/python3.9/site-packages/cellxgene_gateway . ./customize_ui.sh
+
+ENV CELLXGENE_DATA=/cellxgene-data
+ENV CELLXGENE_LOCATION=/usr/local/bin/cellxgene
+EXPOSE 5005
+
+RUN mkdir /cellxgene-data
+
+CMD ["cellxgene-gateway"]

examples/customized_docker_image/README.md

@@ -0,0 +1,14 @@
+# Purpose
+
+This is a simple example of how to make a small script to customize the UI of cellxgene-gateway. The script that does the customization is `customize_ui.sh`, it simply makes the main header green using CSS but you could do anything you want there (including adding more script tags, etc).
+
+# Usage
+
+```
+docker build -t cellxgene_custom .
+CELLXGENE_DATA=`pwd`/../../../cellxgene_data
+docker run -p 5005:5005 --mount src=$CELLXGENE_DATA,target=/cellxgene-data,type=bind cellxgene_custom
+```
+
+If you now open http://localhost:5005 you should see a green cellxgene gateway header.
+

examples/customized_docker_image/customize_ui.sh

@@ -0,0 +1,3 @@
+# make the header bright green
+find "${CELLXGENE_GATEWAY_DIR}/templates" -name index.html -exec sed -i -e 's/<head>/<head>\
+> <style> header h3 {color: #0F0;} <\/style>/g' {} \;

requirements.txt

@@ -1,6 +1,6 @@
-cellxgene>=0.15
+cellxgene
-flask<2.0.0,>=1.0.2
+flask
-flask-api==2.0
+flask-api
-werkzeug==1.0.1
+werkzeug
 psutil
 requests

tests/items/s3/test_s3item_source.py

@@ -1,6 +1,7 @@
 import unittest
 from unittest.mock import MagicMock, Mock, patch
 
+from cellxgene_gateway.gateway import app
 from cellxgene_gateway.items.item import ItemType
 from cellxgene_gateway.items.s3.s3item import S3Item
 from cellxgene_gateway.items.s3.s3item_source import S3ItemSource
@@ -24,10 +25,7 @@ class TestScanDirectory(unittest.TestCase):
         )
 
     @patch("s3fs.S3FileSystem")
-    @patch("flask.request")
+    def test__GIVEN_multilevel_bucket_THEN_properly_recurses_suburls(self, s3func):
-    def test__GIVEN_multilevel_bucket_THEN_properly_recurses_suburls(
-        self, requestMock, s3func
-    ):
         class S3Mock:
             def exists(path):
                 if path in [
@@ -83,9 +81,9 @@ class TestScanDirectory(unittest.TestCase):
                 raise Exception("isfile called with " + path)
 
         s3func.return_value = S3Mock
-        requestMock.args.get.return_value = "true"
         source = S3ItemSource("my-bucket")
-        tree = source.scan_directory()
+        with app.test_request_context(query_string="refresh=true") as test_context:
+            tree = source.scan_directory()
 
         def s3item_compare(i1, i2, msg=""):
             self.assertEqual(i1.name, i2.name, "name equals")

tests/test_prune_process_cache.py

@@ -15,7 +15,7 @@ key = CacheKey(
 
 class TestPruneProcessCache(unittest.TestCase):
     @patch("cellxgene_gateway.util.current_time_stamp", new=lambda: 0)
-    @patch("cellxgene_gateway.env.ttl", new="10")
+    @patch("cellxgene_gateway.env.expire_seconds", new=10)
     @patch("cellxgene_gateway.cache_entry.CacheEntry")
     @patch("cellxgene_gateway.cache_entry.CacheEntry")
     def test_GIVEN_one_old_one_new_THEN_prune_old(self, old, new):