From 8a918d9d5f02bf6a7f6728ce0e5d2ae2f2365833 Mon Sep 17 00:00:00 2001 From: Alok Saldanha Date: Tue, 27 Aug 2019 16:14:10 -0400 Subject: [PATCH] the first commit --- .flake8 | 2 + .gitignore | 5 + .isort.cfg | 6 + .pre-commit-config.yaml | 30 +++++ LICENSE.txt | 201 +++++++++++++++++++++++++++++++++ Readme.md | 72 ++++++++++++ __init__.py | 9 ++ backend_cache.py | 62 ++++++++++ cache_entry.py | 108 ++++++++++++++++++ cellxgene_exception.py | 14 +++ dir_util.py | 91 +++++++++++++++ env.py | 17 +++ extra_scripts.py | 15 +++ gateway.py | 173 ++++++++++++++++++++++++++++ path_util.py | 49 ++++++++ process_exception.py | 25 ++++ prune_process_cache.py | 39 +++++++ requirements.txt | 6 + run.sh.example | 9 ++ static/css/homepagestyle.css | 25 ++++ static/nibr.ico | Bin 0 -> 65106 bytes subprocess_backend.py | 71 ++++++++++++ templates/cellxgene_error.html | 40 +++++++ templates/filecrawl.html | 34 ++++++ templates/index.html | 123 ++++++++++++++++++++ templates/loading.html | 53 +++++++++ templates/process_error.html | 43 +++++++ util.py | 16 +++ 28 files changed, 1338 insertions(+) create mode 100644 .flake8 create mode 100644 .gitignore create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml create mode 100644 LICENSE.txt create mode 100644 Readme.md create mode 100755 __init__.py create mode 100644 backend_cache.py create mode 100644 cache_entry.py create mode 100644 cellxgene_exception.py create mode 100644 dir_util.py create mode 100644 env.py create mode 100644 extra_scripts.py create mode 100644 gateway.py create mode 100644 path_util.py create mode 100644 process_exception.py create mode 100644 prune_process_cache.py create mode 100644 requirements.txt create mode 100644 run.sh.example create mode 100644 static/css/homepagestyle.css create mode 100644 static/nibr.ico create mode 100644 subprocess_backend.py create mode 100644 templates/cellxgene_error.html create mode 100644 templates/filecrawl.html create mode 100644 templates/index.html create mode 100644 templates/loading.html create mode 100644 templates/process_error.html create mode 100644 util.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..6444710 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 95 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24dd510 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +__pycache__ +*.pyc +run.sh +.cellxgene-gateway diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..2ca6a6e --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +line_length=88 +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e7e8bae --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: +- repo: local + hooks: + - id: isort + name: isort + entry: isort + language: system + types: [python] + stages: [commit] + - id: flake8 + name: flake8 + language: system + entry: flake8 + types: [python] + stages: [commit] + - id: black + language_version: python3.6+ + name: black + language: system + entry: black + types: [python] + stages: [commit] + - id: pytest + name: pytest + language: system + entry: pytest + types: [python] + pass_filenames: false + always_run: true + stages: [push] diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..7d1f8d2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Novartis Institues for Biomedical Research Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..ce41bf8 --- /dev/null +++ b/Readme.md @@ -0,0 +1,72 @@ +# Overview # + +Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server. + +# Running locally # + +Before running the gateway, make sure you can run the Cellxgene Server from CZI, and that you have a folder with h5ad files ready to view. + +The first step is to set up an environment and install requirements: +``` +# create an environment for packages +python -m venv .cellxgene-gateway +source .cellxgene-gateway/bin/activate +# install requirements +pip install -r requirements.txt +``` +Then copy run.sh.example to run.sh and edit +``` +cp run.sh.example run.sh +``` + +`run.sh` defines various environment variables, you probably only need to edit CELLXGENE_LOCATION and CELLXGENE_DATA: + +* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd' +* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene +* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data +* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally +* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. + +Finally, execute run.sh: +``` +source run.sh +``` + +# Customization # + +The current paradigm for customization is to modify files during a build or deployment phase: + +* To modify CSS or JS on particular gateway pages, overwrite or append to the templates +* To add script tags such as for user analytics to all pages, overwrite the extra_scripts.py file. + * these scripts will also be run on the pages served by cellxgene server via the --scripts parameter + * See https://github.com/chanzuckerberg/cellxgene/pull/680 for details on --scripts parameter + +Currently we use a build.sh that copies the gateway to a "build" directory before modifying with sed and the like. + +# Development # + +## Running Linters ## + +pip install isort flake8 black + +``` +isort -rc . +``` + +``` +flake8 . +``` + +``` +black . +``` + +# Getting Help # + +If you need help for any reason, please make a github ticket. One of the contributors should help you out. + +# Contributors # + +* Niket Patel - https://github.com/NiketPatel9 +* Alok Saldanha - https://github.com/alokito +* Yohann Potier - https://github.com/ypotier diff --git a/__init__.py b/__init__.py new file mode 100755 index 0000000..8605fcc --- /dev/null +++ b/__init__.py @@ -0,0 +1,9 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + diff --git a/backend_cache.py b/backend_cache.py new file mode 100644 index 0000000..8293c80 --- /dev/null +++ b/backend_cache.py @@ -0,0 +1,62 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import time +from threading import Thread + +from flask_api import status + +import env +from cache_entry import CacheEntry +from cellxgene_exception import CellxgeneException +from subprocess_backend import SubprocessBackend + +process_backend = SubprocessBackend() + + +class BackendCache: + def __init__(self): + self.entry_list = [] + + def get_ports(self): + contents = self.entry_list + return [c.port for c in contents] + + def check_entry(self, dataset): + contents = self.entry_list + matches = [c for c in contents if c.dataset == dataset] + + if len(matches) == 0: + return None + elif len(matches) == 1: + return matches[0] + else: + raise CellxgeneException( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "Found " + str(len(matches)) + " for " + dataset, + ) + + def create_entry(self, dataset, file_path, scripts): + port = 8000 + existing_ports = self.get_ports() + while port in existing_ports: + port += 1 + + entry = CacheEntry.for_dataset(dataset, file_path, port) + + background_thread = Thread( + target=process_backend.launch, args=(env.cellxgene_location, scripts, entry) + ) + background_thread.start() + + self.entry_list.append(entry) + + time.sleep(1) # Automatic refresh is too fast, needs a second to pause + + return entry diff --git a/cache_entry.py b/cache_entry.py new file mode 100644 index 0000000..0a63a63 --- /dev/null +++ b/cache_entry.py @@ -0,0 +1,108 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +from flask import Response, request +from requests import get + +import env +from util import current_time_stamp + + +class CacheEntry: + def __init__( + self, + pid, + dataset, + file_path, + port, + launchtime, + timestamp, + status, + message, + all_output, + stderr, + http_status, + ): + self.pid = pid + self.dataset = dataset + self.file_path = file_path + self.port = port + self.launchtime = launchtime + self.timestamp = timestamp + self.status = status + self.message = message + self.all_output = all_output + self.stderr = stderr + self.http_status = http_status + + @classmethod + def for_dataset(cls, dataset, file_path, port): + return cls( + "", + dataset, + file_path, + port, + current_time_stamp(), + current_time_stamp(), + "loading", + "", + "", + "", + "", + ) + + def set_loaded(self, pid): + self.pid = pid + self.status = "loaded" + + def set_error(self, message, stderr, http_status): + self.message = message + self.stderr = stderr + self.http_status = http_status + self.status = "error" + + def serve_content(self, path): + dataset = self.dataset + + gateway_basepath = ( + f"{env.gateway_protocol}://{env.gateway_host}/view/{dataset}/" + ) + subpath = path[len(dataset) :] # noqa: E203 + + if len(subpath) == 0: + r = Response(f"Redirect to {gateway_basepath}\n", status=301) + r.headers["location"] = gateway_basepath + return r + + port = self.port + cellxgene_basepath = f"http://127.0.0.1:{port}" + + headers = ( + {"accept": request.headers["accept"]} if "accept" in request.headers else {} + ) + + cellxgene_response = get(cellxgene_basepath + subpath, headers=headers) + + content_type = cellxgene_response.headers["content-type"] + + if "text" in content_type: + cellxgene_content = cellxgene_response.content.decode() + gateway_content = cellxgene_content.replace( + "http://fonts.gstatic.com", "https://fonts.gstatic.com" + ).replace(cellxgene_basepath, gateway_basepath) + else: + gateway_content = cellxgene_response.content + + gateway_response = Response( + gateway_content, status=cellxgene_response.status_code + ) + + gateway_response.headers["content-type"] = content_type + + return gateway_response diff --git a/cellxgene_exception.py b/cellxgene_exception.py new file mode 100644 index 0000000..ae645c3 --- /dev/null +++ b/cellxgene_exception.py @@ -0,0 +1,14 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +class CellxgeneException(Exception): + def __init__(self, message, http_status): + Exception.__init__(self) + self.message = message + self.http_status = http_status diff --git a/dir_util.py b/dir_util.py new file mode 100644 index 0000000..1a23369 --- /dev/null +++ b/dir_util.py @@ -0,0 +1,91 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import os + +from flask_api import status + +import env +from cellxgene_exception import CellxgeneException + + +def is_subdir(full_path, parent_path): + subdir = os.path.realpath(full_path) + parent = os.path.realpath(parent_path) + return subdir.startswith(parent) + + +def create_dir(parent_path, dir_name): + full_path = os.path.join(parent_path, dir_name) + + if "/" in dir_name: + raise CellxgeneException( + "Please have no slashes in the intended directory.", + status.HTTP_400_BAD_REQUEST, + ) + elif not os.path.exists(parent_path): + raise CellxgeneException( + "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST + ) + elif os.path.exists(full_path): + raise CellxgeneException( + "The provided subdirectory already exists within Directory.", + status.HTTP_400_BAD_REQUEST, + ) + elif not is_subdir(full_path, parent_path): + raise CellxgeneException( + "The directory must be a subdirectory of the parent path.", + status.HTTP_400_BAD_REQUEST, + ) + elif not os.path.isdir(parent_path): + raise CellxgeneException( + "The parent is not a directory.", status.HTTP_400_BAD_REQUEST + ) + else: + os.mkdir(full_path) + + +def recurse_dir(path): + if not os.path.exists(path): + raise CellxgeneException( + "The given path does not exist.", status.HTTP_400_BAD_REQUEST + ) + + def make_entry(el): + full_path = os.path.join(path, el) + if os.path.isfile(full_path): + return { + "path": full_path.replace(env.cellxgene_data, ""), + "name": el, + "type": "file", + } + elif os.path.isdir(full_path): + return { + "path": full_path, + "name": el, + "type": "directory", + "children": recurse_dir(full_path), + } + else: + raise CellxgeneException( + "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST + ) + + return [make_entry(x) for x in os.listdir(path)] + + +def render_entries(entries): + return "" + + +def render_entry(entry): + if entry["type"] == "file": + return f"
  • {entry['name']}
    • " + elif entry["type"] == "directory": + return f"
  • {entry['name']}{render_entries(entry['children'])}
    • " diff --git a/env.py b/env.py new file mode 100644 index 0000000..686bd4f --- /dev/null +++ b/env.py @@ -0,0 +1,17 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import os + +deployment_env = os.environ.get("DEPLOYMENT_ENV") +cellxgene_location = os.environ.get("CELLXGENE_LOCATION") +cellxgene_data = os.environ.get("CELLXGENE_DATA") +gateway_host = os.environ.get("GATEWAY_HOST") +gateway_protocol = os.environ.get("GATEWAY_PROTOCOL") +ip = os.environ.get("GATEWAY_IP") diff --git a/extra_scripts.py b/extra_scripts.py new file mode 100644 index 0000000..7c05b97 --- /dev/null +++ b/extra_scripts.py @@ -0,0 +1,15 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +def get_extra_scripts(): + # can be array of script tags to inject on every page, e.g. for google analytics could be + # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2', + # f"{env.gateway_protocol}://{env.gateway_host}/static/js/google_ua.js"] + # where google_ua.js is a script you add to the static/js folder prior to deployment. + return [] diff --git a/gateway.py b/gateway.py new file mode 100644 index 0000000..61e7f2d --- /dev/null +++ b/gateway.py @@ -0,0 +1,173 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +# import BaseHTTPServer +import datetime +import os +from threading import Thread + +from flask import Flask, redirect, render_template, request, send_from_directory +from flask_api import status +from werkzeug import secure_filename + +import env +from backend_cache import BackendCache +from cellxgene_exception import CellxgeneException +from dir_util import create_dir, recurse_dir, render_entries +from extra_scripts import get_extra_scripts +from path_util import get_dataset, get_file_path +from process_exception import ProcessException +from prune_process_cache import PruneProcessCache +from util import current_time_stamp + +app = Flask(__name__) +cache = BackendCache() +location = f"{env.gateway_protocol}://{env.gateway_host}" + + +@app.errorhandler(CellxgeneException) +def handle_invalid_usage(error): + + message = f"{error.http_status} Error : {error.message}" + + return ( + render_template( + "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message + ), + error.http_status, + ) + + +@app.errorhandler(ProcessException) +def handle_invalid_process(error): + + message = [] + + message.append(error.message) + message.append(f"{error.http_status} Error.") + message.append(f"Stdout: {error.stdout}") + message.append(f"Stderr: {error.stderr}") + + return ( + render_template( + "process_error.html", extra_scripts=get_extra_scripts(), message=message + ), + error.http_status, + ) + + +@app.route("/favicon.ico") +def favicon(): + return send_from_directory( + os.path.join(app.root_path, "static"), + "nibr.ico", + mimetype="image/vnd.microsof.icon", + ) + + +@app.route("/") +def index(): + users = [ + name + for name in os.listdir(env.cellxgene_data) + if os.path.isdir(os.path.join(env.cellxgene_data, name)) + ] + return render_template( + "index.html", + ip=env.ip, + cellxgene_data=env.cellxgene_data, + extra_scripts=get_extra_scripts(), + users=users, + ) + + +@app.route("/make_user", methods=["POST"]) +def make_user(): + dir_name = request.form["directory"] + + create_dir(env.cellxgene_data, dir_name) + + return redirect(location, code=302) + + +@app.route("/make_subdir", methods=["POST"]) +def make_subdir(): + parent_path = os.path.join(env.cellxgene_data, request.form["usernames"]) + dir_name = request.form["directory"] + + create_dir(parent_path, dir_name) + + return redirect(location, code=302) + + +@app.route("/upload_file", methods=["POST"]) +def upload_file(): + upload_dir = request.form["path"] + + full_upload_path = env.cellxgene_data + "/" + upload_dir + if os.path.isdir(full_upload_path): + if request.method == "POST": + if "file" in request.files: + f = request.files["file"] + if f and f.filename.endswith(".h5ad"): + f.save(full_upload_path + "/" + secure_filename(f.filename)) + return redirect("/filecrawl.html", code=302) + else: + raise CellxgeneException( + "Uploaded file must be in anndata (.h5ad) format.", + status.HTTP_400_BAD_REQUEST, + ) + else: + raise CellxgeneException( + "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST + ) + else: + raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST) + + return redirect(env.location, code=302) + + +@app.route("/filecrawl.html") +def filecrawl(): + + entries = recurse_dir(env.cellxgene_data) + rendered_html = render_entries(entries) + return render_template( + "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html + ) + + +@app.route("/view/") +def do_GET(path): + + dataset = get_dataset(path) + file_path = get_file_path(dataset) + match = cache.check_entry(dataset) + if match is None: + uascripts = get_extra_scripts() + match = cache.create_entry(dataset, file_path, uascripts) + + match.timestamp = current_time_stamp() + + if match.status == "loaded": + return match.serve_content(path) + elif match.status == "loading": + launch_time = datetime.datetime.fromtimestamp(match.launchtime) + return render_template( + "loading.html", launchtime=launch_time, all_output=match.all_output + ) + elif match.status == "error": + raise ProcessException.from_pid_object(match) + + +if __name__ == "__main__": + background_thread = Thread(target=PruneProcessCache(cache)) + background_thread.start() + + app.run(host="0.0.0.0", port=5005, debug=False) diff --git a/path_util.py b/path_util.py new file mode 100644 index 0000000..0672184 --- /dev/null +++ b/path_util.py @@ -0,0 +1,49 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import os + +from flask_api import status + +import env +from cellxgene_exception import CellxgeneException + + +def get_dataset(path): + if path == "/" or path == "": + raise CellxgeneException( + "No matching dataset found.", status.HTTP_404_NOT_FOUND + ) + + trimmed = path[:-1] if path[-1] == "/" else path + + try: + get_file_path(trimmed) + return trimmed + except CellxgeneException: + split = os.path.split(trimmed) + return get_dataset(split[0]) + + +def validate_path(file_path): + if not os.path.exists(file_path): + raise CellxgeneException( + "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST + ) + if not os.path.isfile(file_path): + raise CellxgeneException( + "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST + ) + return + + +def get_file_path(dataset): + file_path = os.path.join(env.cellxgene_data, dataset) + validate_path(file_path) + return file_path diff --git a/process_exception.py b/process_exception.py new file mode 100644 index 0000000..e54d842 --- /dev/null +++ b/process_exception.py @@ -0,0 +1,25 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +class ProcessException(Exception): + def __init__(self, message, stdout, stderr, http_status): + Exception.__init__(self) + self.message = message + self.stdout = stdout + self.stderr = stderr + self.http_status = http_status + + @classmethod + def from_pid_object(cls, pid_object): + return cls( + pid_object.message, + pid_object.all_output, + pid_object.stderr, + pid_object.http_status, + ) diff --git a/prune_process_cache.py b/prune_process_cache.py new file mode 100644 index 0000000..71bfcfa --- /dev/null +++ b/prune_process_cache.py @@ -0,0 +1,39 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import time + +import psutil + +from util import current_time_stamp + + +class PruneProcessCache: + def __init__(self, cache): + self.cache = cache + + def __call__(self): + while True: + time.sleep(60) + + timestamp = current_time_stamp() + + processes_to_delete = [] + for p in self.cache.entry_list: + if timestamp - p.timestamp > 3600: + processes_to_delete.append(p) + processes_to_delete + + for process in processes_to_delete: + self.cache.entry_list.remove(process) + pid = process.pid + p = psutil.Process(pid) + p.terminate() + p = psutil.Process(pid + 2) + p.terminate() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c89a503 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +cellxgene +flask +flask_api +psutil +requests + diff --git a/run.sh.example b/run.sh.example new file mode 100644 index 0000000..baa9e83 --- /dev/null +++ b/run.sh.example @@ -0,0 +1,9 @@ +export CELLXGENE_LOCATION=/path/to/cellxgene +export CELLXGENE_DATA=/path/to/datasets +export DEPLOYMENT_ENV=dev +export GATEWAY_HOST=localhost:5005 +export GATEWAY_PROTOCOL=http +export GATEWAY_IP=127.0.0.1 + +#Once these are set, you run like a normal Flask app +python gateway.py diff --git a/static/css/homepagestyle.css b/static/css/homepagestyle.css new file mode 100644 index 0000000..a4ae50e --- /dev/null +++ b/static/css/homepagestyle.css @@ -0,0 +1,25 @@ +/* + Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed + under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy + of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless + required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. +*/ + +pre { + display: block; + padding: 8.5px; + margin: 0 0 9px; + font-size: 6px; + line-height: 1.385; + color: #333333; + word-break: break-all; + word-wrap: break-word; + background-color: #f5f5f5; + border: 1px solid #cccccc; + border-radius: 3px; + width: 80%; +} diff --git a/static/nibr.ico b/static/nibr.ico new file mode 100644 index 0000000000000000000000000000000000000000..81749029d05d7c2ffbf66706333bd65862720b40 GIT binary patch literal 65106 zcmeHQ2Y6If_K#)%Zqwawiy}={0nt@qb=S3bT^s6JKmln2qBL0nK@boXM5Onw2q7VK z2qpC1TPUIT^vp~qy>kA)-<#az<>gIzlgWsX`+aX-nfGox_nv#&JtrjOnvh?G{PB+= zIM)hke|<6cW;)LD7AkkPv&#kdP0%xb9c4 z3<(+cTu4Yg5?WAfxoVu75q708~8c`{;Gro1vgRl2TA*YeSQ zDt2d~eE5}J?t3RemTk_~|5fTb>TIsmpJ9_H!yU3GA;0*)*~>HKwin~2)3**OC@6j` z1$kMLpPg3x0L_zccVs+4qWlbqW&Tit5sRcJ%4$tm*cf>g0( zyE=2^taPdVLY%zRAz9duDxKdEA%_a=+QDqeM%z`&!*!40sZ#CvIBDM7E}7Y`|MrLX zl#Rn)lS>78l9_i&4kYDEd`3zA)b~8^Jd`Wt zfT}`(jx_wGeGi;vOon1h-R59yolhu`ZHIFua%Z;8*_JKgI4_FMk&PI$C+!82fp@H? zi@-lKmSjrXFYWSXcbj}P%pu=|rOVirnKEHrrVLq~AsrT`$*Y*ljh8rO`0gxijOUvV z@#5lhQJ09QEP4LJWV!S81nE53A-hobjI6SC%)-07NNQq=6!=1c5Mz{OgAqi zA46zDM26h;MuPn9<#-u9J*{{?7FWRJA`|0(*zQaXH&vUMBJZF*Cmki@QGK#;Pma`Y zlqfY`h?jRivth27*F(LpKNDYeS_@o-{8>i(RcWQ*!a3L(IJm}t;-%J0@iJvWhW_*7 z>%2^dWZKRb-|H`S#O297KiIS~?wn?m4Htk5ic)-Xfjr$NNyGJZUyGO3+jG=krOwXS zBNDrBX{l%G#tz)}(mde0qBe}*UxF`>pUanj0)O3xy4PtKFDtf~uSJP5>*UzV5h}}4 zXO2_iSe5@+UH)g|3gnUIs6X;&{}9e*>D=~k**o(S{8e^5C1n=Mvk?w0|JqY+vih70 zUojuzP_DlVF^b{k8)o<*U57Zmb&cXc@fEzHh$VDYqhT(gFvQOV)7p?U7UY@=!Au4A_ny zBT^+__Zo59_DL-zvU|Pab%8aE!@AUabB4Aq_rn@_CbjI?XBngq`i*yLV~FkeH_D1R zT~a_nUXE<|{$*L%@m6uh#%R?lKmmIa^M&-sIcI^k4r_n7`8jwpPabUQY6Iasd0q*e zb2IGPx-_@#O_FqMvzr_|KPHagdd>dfTqFKbbWcX|Ic?1hZ*_xQ*frmCMjrl++?1`& z0p{PTpIwOan7FPj=ukyl*G{Y+?;!`ze?_0jlLke2i11K!Q^KJ&+L{~oeuzZR?dv%c z5C8554MBS3AB_`b$H9^@dUi{w%xU$1G8_5toYEFk%+)RrS(W*VblA8#85+GhZDB@n z-}A+oWy2(_#kIovZyfrX6k$-;x%8EKu2N zv`)4oUNVx;Xh&wsIVsGy-v0rskS-%5JrwAB(qXl5JcWaNq?X748j`L`UN6bpII*V0 zfrmRDmoJAd4H(Z<~}NT|g@)VT`y-XV9mwbV}CN zF_!&At0Xy$wbS3kI*_ks`87~Jk6xQHG(O*$C&%`{DQQ}pNoSK5`2NRq&u{}!i6cEF z^4O%GRQbGKhG)h^en-)=q!I3UD?zrJ&{F#EvQrbperAvUQJHHe@H%OKj)R(LKk2xS zz1h|*bLAqCqFH_G)%jpf%itpWBe6u~XxO~Fm8O)1B(5?FP z23F-i4%nZI_uRJ$hu27SL~pNa_7uoB*#B%l%YSsXU5xzJWZ)OVoaM9sDYrP} zW$aOwvu%7+tSM#Tsc+ka16jr86Amwe-%zq1_1?VIS(?0t_?9vjoe0DjqXBv0x?-p01GeA1FVN2ZpdWsI300A0VV=RL)A zxAA}8bH2X}+MeaJ@9Y0)ldZ93=QR1wHzDs~xh$XapJU(@))%uxx_`%{*0Qj3wPL#5 z?7f*svNc@HvhM&+CLT27=h;tVAPZ3KS8x~`o?*I%6y#=sFL*`fw)=x5AKzkDwCsjV zl4Fhh&3$M$%O-wymrk6|mnT}g_(25&$^gmd8dGk%4fN7Opj|=N?_OByZ_aJ<$?W$B zX4<9GnsnKMJ~zU=)JM3X?DJd5Y}Gtq9k>RCt;jUK3VQ0S##e+jzfMl9nON%oQ;;)E zhdgrb@falhv zJ??K}9}JpWvJP-OkS9C}d7wVUA6x~#M;pw z37gkA6|%^8ARlG?q`Hs3D@{bGGW=X$sPHEJ`CJ%p*BzJ+9x}#}=)N7L{ z|M(A0G!j&~bdHU)RgqCi1J@ zAtR%Vn{*55C!U{0J4V9>gF0<9Ij_5NDa$(n8P{m&L7#+9lsps<@^X~t(C%XRo=l0w z_$p7LE@3YC->0C<(`Y{QHEFN9uO&#gp{Ygo1OAu5(Q*{+psh*msgMJq{z?|W^BW6O zW$pP~ul$i&4cMO(=VoX!LdCnX?xb12pOG%kjKGc6G?X<0{atT5_7kQY}n83hZ@jj*FRKEq)b zY)l^2@roM1h<3aIdFcfmy>6v`zPwABS{IBZ&IQ(89lZZHWUb`2y(RW-C)f$8F{Apu zPQwHZCf>d;@Bg&jqRo=A4V>F8mpjFw)AO2@wFx+q`Z~pnK>dY}!*Nu!-WytmPo5?QzQuVtwcEInn|Jos0UfQnmQRno;Q;>sB zwM>8SO30Ic!2Ux&m)Q==XnSl(HrY*E30_tZb6xRLdrgiMdJJtNNHSQ-J_H>)W=dZla`-AHIhVkJ>7 z)d8|GWBoZNls%tWe&h`sV2#y}eQo@s{?r**P2^)}ACYB`we%_RcTdQ2t=3=J<{9fx zTtvFZ9pCG&a+E_a?N%FdSnqaq`aCFy{d9fV`m;ThT_(F3KL?JNmXE19!1_1uZId)N zHi`POORizZ82*{Y&!?Q&Uh>$7E81AKgpSMD|D3;_F|ORnJcX{7x?JUF7R$+Its3w><~jA>$fAyR@*Q?sX4oplRMmKhqXO^}RY% zZ!rh9aK0wm<8B<*5HghO<>b1_zJ8{?NBbL%7SgT3gavVrCz_PJ-jE+kWW1{GJX7Y? z{_C>xW_J~%4vsdhvmuMr+vd7^Up*&3#<8#Od#(e2hppTE6FH_j2lXw_wA-V7v~dpe zp0Z==%za5*yCQqsrP+Aw{;>nrFyF@BdgvGG>((x|%aSK6p-9@W?<85H%xw8>L_Px|>T(A2CuX>4E9PfIhQ zrvnUhbhE1eV_4seIs>zO4b_4H@P^o`6tK%f2+wrm-V_2_Ce=N-V%+L*$bOF z+7OaXxVMN8o^mj&j4swR1CxiG$JE>20UedT?!4w&@Z!hGashgEPwB{#!J7P%Hd>26 zxm{8(94hlVX-$rI(q!ZzCqkbaC^-b{_ew{X&4a!T98=_f!&iF1rRactv$|vKT&u~V z=YRAUQ#}A>43wFWrZ>}lmT9ns2ZRCT_0$pR>#tzo3769TYwNgX8t$Yldrs@$%lYkd zEtDRZSIH~B3_Yun7b70xdhFW2P`0eROvnwWhXL-_^bjp?kmIYym3^^-pAZJrT^swK zbC`DH%iZju;XaC$Up!c>$DuCwz??o_?I|jvIkfXpw$R3QOw!rQ>dQ{Oco=%1+QoVx zuE9H|eDt&E0mw6wcT_r4)dr3!<31wQhUDX0u_vTTi5^MQ<-Q^LIo5dym1lX*v|&~C zR%g{>}X)o&kwyypdJAEPhSGMBnJ)TKt?*5^J4tG+IdmU(@KSP(dTK{U$u4#is{)Jf4_mM_uaQOj7*O~6jS^%o_WrrkFObe$ z^UbrxxL>s)ti`Wj+c7NI)z|FL9)3Rw`yQcZX_lnf-m*>Z68AMM>s7yO{jrAa59<|B zdmMLlP|xF_^Jxj&;5yiY)Vz;PIFBh~^pqUu4{G!#=d9{`o=Hp4Cf=J(p{Km9{KtB7 z?;7iUx7=S-7am{xF5FE zfoI%Llyb84K5O@R1^*w~d`HjihdnEGWhwTZl{&iJso6#)9o-Q8iuqb1CJp6Qw4+Kl zI}%U+Y&58^O{jeD>goZ5IZ)+okb8i3S|8##jYHXK&0Jiz5?9x#tZZ#%4_v)HK>c$b z)J3`A)9AA!8%GxOCz(YDuj%iC&vWn&^`ZR6Jy&=2_N6~9C+}m2EyzjOh3ddjAUA)QD1ZGZLH^nS=gx_8 zOV>oH(hbLFiBk3RB&pFiNooy%P3*vAsrw!5Vn<>h;dFg(}R?X7sJ4mnx% zcN3%rY?5m<#{t;X_y7lBR--*&^)Zf)IJ)2fOlp3L=iLF1&v5j@(Hk-=z@~bC90QW1 z+Sf@^6)?JWP?A&`oFq2`R#k^5%iYj?q>r8mi zkHu?UVB-y&Q_5LsLsA{`%9?;lEz0iS#2J1%Zfk^RfLC=MO%kL|>qNP?Lz3LzHCZ0+ znJh1TZIic$+NH^8hqRoSD($AF$_F@_PIkzfz;VwEx5+;TL(e=gMQVQy7-+Cbmg=x8 zuL+pk2HW!6hC}y_c~pHA;57#CjC0v?-Z{;sQ||@as>LUBsXQ42 z-46HN(w2n!2Ev2qV!IORb)kp84X~+>xl}58=r^NTzSJH5%L+%`%BDVlO)i7^ra$f`Bp=>|ZnYK)X6%X2`bN>eY9_al?S*gd* z`7V86eg}Io`k|tJ*Qn#w!-R9^R``yr_um9*+21Zd!oHrq9jxrT@TaY$tCKLd7J+ta zgZ(9Uj!%)B0iWtPE0~y_IgdD>Xfxdo{@dt3%pc!+{?xfV-;mx~ygp0l8-;ytwB69d zgx6K!%aUjMGl|^jyPl&wbu|0d0h{S4%!4M2ol+hBev23Q5N4c9Rq!rt18IYF)aILw zo>}=7{VW!?G_)b6KR)_Ub%(nM3)&WNt~>+UjD8ud_#UjtcF?CD{i)J-!!7U)YqZZd zn@5~Ow*VJDg*{QTu@C2Rn?v+}#<9aj1 zgiBq}$F%dJZTv4G5kHN@K0MOUqyda`#q6GVknnjj+%B8o!#miCa9IW$FZ$Bt+Tso# z>N)8G+6E8BK1`$km|)l4YI(Hfo`5~VR^UUJ-3<6VhINbkS%aDAQ}gMzkbM-K33Y_`#OIJpIbriJ%N3fn|;_LYPC*g@1KRA#>PE> zgW*r!3>L;`q{kX{Pm%NR>G(@d^t07_bGqhx%h&aX@ZlQ7{X>ULd=Uq-KQh6;^cj=t zlC^kPTNHc_2h4DG*1L8x{6%`@Zj)QFRtxrL#KnCLu9{T6i zU`xt<##;P{D+MrU2W8Pi;oHCJMDV7V4@Nk+yC-gYIMgmX0oy<)`ZT}em9qO0_3%*F zlxtRdK3=}$9=~7qGGp3})E97H{LMbdcJ#2KpGad21=9j7JmmheHxUDccGar?J)J54 zzaM+sz4@NCP=(9x)gKOJ$!)+>R_FrC87O~S7eKmt)-vuNb;-dz!9@K>7&1nO)6)L- z%dY!9R*3P0UY+mI=r;Y_@ODi(-6z<7Tw%iHc)CY@r5-N4CLQ-2VkRAdzA~68CUP)h zku~?pS3scZ<;*RVHY?LC<2A~2=#Qy9c#ZgsahS;48)4w?p16-bK9UjNAed=?SP%Hs zeo1oRV>XzHe%0vH&f7Yp|DU=$>U+*WKkjd$uNV3k_I0fxyeJFk0^2gHd!GEQqxx>= z)HV{;?_Y?6cuGAA%9%Q8(q6vk;S_wx>)Z{W9_69ew(if7yP(JSb*&+6DA(*aJ~b%) zq6(ou=q;n((tI#E&h1l=gL0+~8-3ncp`}SXkT?1XF+}}M3*e8OvVCuQe?5MqzM1~* zgPFLGaNW1hVNcsFl6=g2+{i%IjsDjt%d`R;&YwSzO_uTSapF&+?=bpsBU}_+tcQWw zHRS<(&RBB6Oxz<$d=>$}BXI|o1+!?)&k*SWe_%tsgFlJ#NyhY`?8OWY?w%1oL;Ub- z=T9BgH}pvu*}bmjpW)orQ1v({XZm$B^Y5wF7I){rVBQRYPfC9h{SGs3uoW7O@FETU z)g<@vU;L?yw{MbS)@%5Tz5jO!KDRa_b=7p4Uwq6|Mu!5(Mmsf zdzU_r-&|ykhNdi@MOrF{i3tfSk4|BEVo1JNI&;)h|+#}~tB7Z|UKl|EO{)Y30?=qGX;~WMfM) zA^1-ESfO9M#s9lq>}U4|qiC%R+HW?P;fmXC1_$Fa%62HTVa)pS?6uP3C_*kk_!!rG zPxr)YjCDmiyrL6*Z>{*^U)r1@oko8*@dsB{bg{+x(>}esF2v+w zGyKERKc5wO8evA+=nIgMCSosgMJL9Jj{NNJ+MJ;;rMYcy)M8Clbh*r*`?bE%#p*Q9 z5wm-xA27xfwOZdJ*t>Ju8{Xpo z5ch4kl`Q;zHx~N^=rfwSOfxtbpOKdH=11Mv%Frhp`A=VLp7pSy+=lkJUrkD@aJ_Wg z{zzHy@y(hqFZv=2ZE_v-pM8C8V!7U11Dg!uCtvAj!jLqbr&!d!R)~HP7)!(|YzQaX z4ZhMb#d5q9U+bpcI}N@wqRsD${p5x=xmNbh>{{V+n>?rv`)$e4;NpwD0O3bJ>5AnT+VmRPHlbOC^Ph}8MjnN0 zyAgMp-IG4+j(7!D{NsD;16LdjtT{f~+30Z_=i8@49gn^El!KKe$f9VQY?*s6Ad8{Ie)bfnRo&d(YtKx%QC+8JyZCAn1M@1z1wB{aL>NNH9 z!rL|ZP_8=@A`sUcV=d5$F_^ZFeNQ-7)V!fgZE@GyVn4Gd(8X!_QMTE0V}?eXd0S_= z|K=Xh3XDx&5jN_q5aU}hZo5@Jhp?mm=XLl;iG{6upc7+cul@Gf(y#$7J@3P+l5}ii zpo`P;V?Q(IBIT84>xz1IGh}ArN_f4hQ#@ys=ddNbO0EP-#SiiC@5;MV=Q zW#S5|{`YpKo}t|Ud%)vXlJ8v9`EQymtpqlRSJLW6p-v}|i8A9?u%3I`7eyRKzQ1Xt zQ}zIk!Ai6{Z`IfC^Zpw>yod`ao8~^zK&GUa4Tu>~1+XdcqaMb%?2ru{Su)6SOh$Kg zW89?=V6VY`H_j8@bG;|b7|-!L*Z|OuJ=lr<{uy`Hid@eO%dMNCpHa4GYF>CdbFPst2oycdxG0Afe5uID_A_Pw^xr@phP)$TLj1-38I9mKYV_VLIRMy|$KKo@7%9`bvlupzsV(byA7ULGx70+&%$o?iq~4p-W$eBzSqNW{oE!9QL;nK5ghU^Yj74cx zufzF6yha}d<)sPovz@Yb;ERAwiM&R^W_Gh{Wl@)!l5%3JrS2ro4hBDBsZthSrtj=Q z@DuEdk0FK6)R}Yyb)qNFmyiDEKAWh1kCu|9DcF$zoZIdX61#V?#g<(eud6uIR_6}! z8f9~HqRMCM!Fbw{J^l`PP1$+FSZQJJ|5i?K{PD`@Y>TZRPWl{v2CG=K$<^>$k*x<~ zHu{?w--ogG^n9n9H`L|KdiMtCrbqgl{V)Ez?a_>72%o_gbvxA4Jq>@5-s1Lp>Y$5z z7C>%xV`;v#6dnC5IW&KOr!21M|L8llGjuyv?Q)6ND8pH?#eZ9mlk2BSXtUpF_M9d2 zMbG10tHp(6Y~ZUk(Z?Ov4}WCTL(8#;>gclJ8s0L43!hOouzA$mLAAfV+{OUo);I29 zlUsf83#n)|o+-oO{vN;991y$F5q&{Fm&_9*jHqv0{ng__9naO8$RAIHUXysrie8ko zG-Y%CtU0uS-Z||<34O0b{%i)DrQI4-sBZd~0CW1bf3!uSHW&1B!_zh83ZCNddTKZ0 zP;VUmnufQGFj4okTVK%eP6_tDX;1b3IsVgpM1Ur+vj32@T7yRL6$PE0)s&fXPFD4Q zq?9gN!9<-&ThD7>1G?$p?whvSmSDX91NIGY|A4P}jqlwFTkEwut;^P&aWS%_>wO+! zqjb|3cFhY$^DpZ%=)dOS=7&a8JsK+i!pU0{7p zyIlG;Q1u8~2_Gw1b9{o0MhhfZrv<><99ua8w&^$ED6NZDFwxGfu9umOuL1uFzg>BO zx4#Nhp9L!)`hoVbP>a6Le`#plax#xq`1($6( zM`4pry&3sR1p{|y!jC?RscW>73Lq2MG_t9NuM`a2ok_&YZ zXUY7z4l=jj$(hZwuDHIo+92+M^0wY+^ciC396#%yEk`2!am{OIu(LJJ7u|hm(_ce3 zJ<4iRuDJJi!WR9X?|G)vYQmrA&>8;kpT4krG3LyTZuOldbH*hf2yJ@390>0P{xi5f zkuNvkG6st$-}v1B$vD#f=xkJ5Pj8%|;VHHE)(8vZJ?(IIOll?hIl=W)sp@+Z0P^?KFuG5bs3GmeKCl=KqRn8l;)}NCbE zPV6x%PvX9SZR4ByB(G60qAYr8w+CPweWE-?|6;#o{K58L!w&-16TKXO@&O*b;J=gp zSIV1m(o^6E`UPvSf#0r0UGA)~ebeP_Q$>HzJ??j5Kbu~TM%;Ecc>gaaI)%14fk=#_ zvix({Kvk8~I3$X}j{(`-BG-!=mKbyEVcwwU!3 z{c+82U&XBrv^)5OV*gZh#FsD|2n=pDtJ~%DjJ{I;^=_iP&@Nfq|GW%(|B8-DOrHS} zA7JmTCw-?*UZY?&3wsQA|Je1akqsmw#{KTYd7A9GynE1m9dj|uhPv6);hEo|Y+%dS zW{?eJ_*zuudlq`&67bRfaN0n7*LSMIWj9-#f_4T#0^U4{E5u7IXr_E}!owE%0) z@32=b1hksP1K-d#ntoR+ZKDGZ>ck`K;5W6YY#Gzozhm?}a(va8t41~ucnG-9u@HOF z^I$hf+d;pM411ByK;Y!!?rU55Ph}5O_CRG1RQ5n+4^;L*We-&LKxGe9_CRG1RQ7=C Yfso2n*#nh5P}u{OJy6*LS3wW_KedO>l>h($ literal 0 HcmV?d00001 diff --git a/subprocess_backend.py b/subprocess_backend.py new file mode 100644 index 0000000..10aabdc --- /dev/null +++ b/subprocess_backend.py @@ -0,0 +1,71 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import logging +import subprocess + +from flask_api import status + +from process_exception import ProcessException + + +class SubprocessBackend: + def __init__(self): + pass + + def create_cmd(self, cellxgene_loc, file_path, port, scripts): + + cmd = ( + f"yes | {cellxgene_loc} launch {file_path}" + + " --port " + + str(port) + + " --host 127.0.0.1" + ) + + for s in scripts: + cmd += f" --scripts {s}" + + return cmd + + def launch(self, cellxgene_loc, scripts, cache_entry): + + cmd = self.create_cmd( + cellxgene_loc, cache_entry.file_path, cache_entry.port, scripts + ) + logging.getLogger("werkzeug").info(f"launching {cmd}") + process = subprocess.Popen( + [cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) + + while True: + output = process.stdout.readline().decode() + if output == "[cellxgene] Type CTRL-C at any time to exit.\n": + break + elif output == "": + stderr = process.stderr.read().decode() + if ( + "Error while loading file" in stderr + or "Could not open file" in stderr + ): + message = "File was invalid." + http_status = status.HTTP_400_BAD_REQUEST + else: + message = "Cellxgene failed to launch dataset." + http_status = status.HTTP_500_INTERNAL_SERVER_ERROR + + cache_entry.status = "error" + cache_entry.set_error(message, stderr, http_status) + + raise ProcessException.from_pid_object(cache_entry) + else: + cache_entry.all_output += output + + cache_entry.set_loaded(process.pid) + + return diff --git a/templates/cellxgene_error.html b/templates/cellxgene_error.html new file mode 100644 index 0000000..908b5bf --- /dev/null +++ b/templates/cellxgene_error.html @@ -0,0 +1,40 @@ + + + + + Cellxgene Gateway - Error + + + {% for script in extra_scripts %} + + {% endfor %} + + + +
    +

    Cellxgene Gateway - Error

    +
    +
    + + + diff --git a/templates/filecrawl.html b/templates/filecrawl.html new file mode 100644 index 0000000..acdf372 --- /dev/null +++ b/templates/filecrawl.html @@ -0,0 +1,34 @@ + + + + + Cellxgene Gateway - FILE CRAWLER + + + {% for script in extra_scripts %} + + {% endfor %} + + + +
    +

    Cellxgene Gateway - FILE CRAWLER

    +
    +
    + +

    Please wait until a dataset is done loading before trying to launch a different one

    + +
    + {{ rendered_html|safe }} + + + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..62921c2 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,123 @@ + + + + + Cellxgene Gateway + + + + {% for script in extra_scripts %} + + {% endfor %} + + + + +
    +

    Cellxgene Gateway - HOME

    +
    +
    + +

    + Links: +

    +
    + + File Crawler: Allows you to view all uploaded data. + +
    + +
    + +

    + How To Upload Data: +

    +
      +
    1. + Create a folder for your Username: +
      2. +
      +
      + Username + +
      +
    2. + Create a subdirectory under the selected Folder: +
      3. +
      +
      + +
      + Subdirectory Name + +
      +
    3. Choose a folder to copy your data to, then upload your data file (must be in .h5ad format).
      4. +
      +
      + Type in the name of the directory and subdirectory you wish to upload to, i.e. "USER/cells". +
      + File:
      + +
      +
      +
    4. Take a look at your data using the file crawler link above
      5. +
    + + +
    + +

    + How To Upload Data via SSH and SCP (Linux): +

    +
      +
    1. + Confirm that your ssh key has been properly added. Running the following command should connect you to the server: +
      2. + 
      ssh ec2-user@{{ ip }}
      +
    2. + Prepare your data on your own machine. They should be in a common folder called 'USER' and should look
      similar to the following structure: +
      3. + 
      - USER
+    - hpc.h5ad
+    - pbmc3k.h5ad
      +
    3. Copy your data to the server (replace USER with your username):
      4. + 
      ssh ec2-user@{{ ip }} 'mkdir -p {{ cellxgene_data }}/USER/'
+ssh ec2-user@{{ ip }} 'mkdir -p {{ cellxgene_data }}/USER/DATA/'
+scp -r pbmc3k.h5ad ec2-user@{{ ip }}:{{ cellxgene_data }}/USER/DATA/
      +
    4. Take a look at your data using the file crawler link above
      5. +
    + +
    +
    + + + + diff --git a/templates/loading.html b/templates/loading.html new file mode 100644 index 0000000..f2183f3 --- /dev/null +++ b/templates/loading.html @@ -0,0 +1,53 @@ + + + + + Cellxgene Gateway - Loading + + + + + + +
    +

    Cellxgene Gateway - Loading

    +
    + +
    +
    + +
    +

    Output:

    + 
    {{ all_output }}
    +

    + Launch Time: {{ launchtime.isoformat(' ') }} +

    +

    + The page will refresh shortly. +

    + + + Please click here to be redirected to the file directory. + +
    + + Please click here to return to the homepage. + +
    + + + diff --git a/templates/process_error.html b/templates/process_error.html new file mode 100644 index 0000000..1c5d13e --- /dev/null +++ b/templates/process_error.html @@ -0,0 +1,43 @@ + + + + + Cellxgene Gateway - Process Error + + + {% for script in extra_scripts %} + + {% endfor %} + + + +
    +

    Cellxgene Gateway - Process Error

    +
    +
    +
    + +

    {{ message[0] }}

    +

    {{ message[1] }}

    +

    {{ message[2] }}

    +

    {{ message[3] }}

    + + + Please click here to be redirected to the file directory. + +
    + + Please click here to return to the homepage. + +
    + + diff --git a/util.py b/util.py new file mode 100644 index 0000000..7dab25a --- /dev/null +++ b/util.py @@ -0,0 +1,16 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +from datetime import datetime + + +def current_time_stamp(): + now = datetime.now() + timestamp = datetime.timestamp(now) + return timestamp