commit 8a918d9d5f02bf6a7f6728ce0e5d2ae2f2365833 Author: Alok Saldanha Date: Tue Aug 27 16:14:10 2019 -0400 the first commit diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..6444710 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 95 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24dd510 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +__pycache__ +*.pyc +run.sh +.cellxgene-gateway diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..2ca6a6e --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +line_length=88 +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e7e8bae --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: +- repo: local + hooks: + - id: isort + name: isort + entry: isort + language: system + types: [python] + stages: [commit] + - id: flake8 + name: flake8 + language: system + entry: flake8 + types: [python] + stages: [commit] + - id: black + language_version: python3.6+ + name: black + language: system + entry: black + types: [python] + stages: [commit] + - id: pytest + name: pytest + language: system + entry: pytest + types: [python] + pass_filenames: false + always_run: true + stages: [push] diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..7d1f8d2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Novartis Institues for Biomedical Research Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..ce41bf8 --- /dev/null +++ b/Readme.md @@ -0,0 +1,72 @@ +# Overview # + +Cellxgene Gateway allows you to use the Cellxgene Server provided by the Chan Zuckerberg Institute (https://github.com/chanzuckerberg/cellxgene) with multiple datasets. It displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server instance that loads that particular data file and once it is available proxies requests to that server. + +# Running locally # + +Before running the gateway, make sure you can run the Cellxgene Server from CZI, and that you have a folder with h5ad files ready to view. + +The first step is to set up an environment and install requirements: +``` +# create an environment for packages +python -m venv .cellxgene-gateway +source .cellxgene-gateway/bin/activate +# install requirements +pip install -r requirements.txt +``` +Then copy run.sh.example to run.sh and edit +``` +cp run.sh.example run.sh +``` + +`run.sh` defines various environment variables, you probably only need to edit CELLXGENE_LOCATION and CELLXGENE_DATA: + +* DEPLOYMENT_ENV - expects 'dev', 'tst' or 'prd' +* CELLXGENE_LOCATION - the location of the cellxgene executable, e.g. ~/anaconda2/envs/cellxgene/bin/cellxgene +* CELLXGENE_DATA - a directory that can contain subdirectories with .h5ad data files, *without* trailing slash, e.g. /mnt/cellxgene_data +* GATEWAY_HOST - the hostname and port that the gateway will run on, typically localhost:5005 if running locally +* GATEWAY_PROTOCOL - typically http when running locally, can be https when deployed if the gateway is behind a load balancer or reverse proxy. + +Finally, execute run.sh: +``` +source run.sh +``` + +# Customization # + +The current paradigm for customization is to modify files during a build or deployment phase: + +* To modify CSS or JS on particular gateway pages, overwrite or append to the templates +* To add script tags such as for user analytics to all pages, overwrite the extra_scripts.py file. + * these scripts will also be run on the pages served by cellxgene server via the --scripts parameter + * See https://github.com/chanzuckerberg/cellxgene/pull/680 for details on --scripts parameter + +Currently we use a build.sh that copies the gateway to a "build" directory before modifying with sed and the like. + +# Development # + +## Running Linters ## + +pip install isort flake8 black + +``` +isort -rc . +``` + +``` +flake8 . +``` + +``` +black . +``` + +# Getting Help # + +If you need help for any reason, please make a github ticket. One of the contributors should help you out. + +# Contributors # + +* Niket Patel - https://github.com/NiketPatel9 +* Alok Saldanha - https://github.com/alokito +* Yohann Potier - https://github.com/ypotier diff --git a/__init__.py b/__init__.py new file mode 100755 index 0000000..8605fcc --- /dev/null +++ b/__init__.py @@ -0,0 +1,9 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + diff --git a/backend_cache.py b/backend_cache.py new file mode 100644 index 0000000..8293c80 --- /dev/null +++ b/backend_cache.py @@ -0,0 +1,62 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import time +from threading import Thread + +from flask_api import status + +import env +from cache_entry import CacheEntry +from cellxgene_exception import CellxgeneException +from subprocess_backend import SubprocessBackend + +process_backend = SubprocessBackend() + + +class BackendCache: + def __init__(self): + self.entry_list = [] + + def get_ports(self): + contents = self.entry_list + return [c.port for c in contents] + + def check_entry(self, dataset): + contents = self.entry_list + matches = [c for c in contents if c.dataset == dataset] + + if len(matches) == 0: + return None + elif len(matches) == 1: + return matches[0] + else: + raise CellxgeneException( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "Found " + str(len(matches)) + " for " + dataset, + ) + + def create_entry(self, dataset, file_path, scripts): + port = 8000 + existing_ports = self.get_ports() + while port in existing_ports: + port += 1 + + entry = CacheEntry.for_dataset(dataset, file_path, port) + + background_thread = Thread( + target=process_backend.launch, args=(env.cellxgene_location, scripts, entry) + ) + background_thread.start() + + self.entry_list.append(entry) + + time.sleep(1) # Automatic refresh is too fast, needs a second to pause + + return entry diff --git a/cache_entry.py b/cache_entry.py new file mode 100644 index 0000000..0a63a63 --- /dev/null +++ b/cache_entry.py @@ -0,0 +1,108 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +from flask import Response, request +from requests import get + +import env +from util import current_time_stamp + + +class CacheEntry: + def __init__( + self, + pid, + dataset, + file_path, + port, + launchtime, + timestamp, + status, + message, + all_output, + stderr, + http_status, + ): + self.pid = pid + self.dataset = dataset + self.file_path = file_path + self.port = port + self.launchtime = launchtime + self.timestamp = timestamp + self.status = status + self.message = message + self.all_output = all_output + self.stderr = stderr + self.http_status = http_status + + @classmethod + def for_dataset(cls, dataset, file_path, port): + return cls( + "", + dataset, + file_path, + port, + current_time_stamp(), + current_time_stamp(), + "loading", + "", + "", + "", + "", + ) + + def set_loaded(self, pid): + self.pid = pid + self.status = "loaded" + + def set_error(self, message, stderr, http_status): + self.message = message + self.stderr = stderr + self.http_status = http_status + self.status = "error" + + def serve_content(self, path): + dataset = self.dataset + + gateway_basepath = ( + f"{env.gateway_protocol}://{env.gateway_host}/view/{dataset}/" + ) + subpath = path[len(dataset) :] # noqa: E203 + + if len(subpath) == 0: + r = Response(f"Redirect to {gateway_basepath}\n", status=301) + r.headers["location"] = gateway_basepath + return r + + port = self.port + cellxgene_basepath = f"http://127.0.0.1:{port}" + + headers = ( + {"accept": request.headers["accept"]} if "accept" in request.headers else {} + ) + + cellxgene_response = get(cellxgene_basepath + subpath, headers=headers) + + content_type = cellxgene_response.headers["content-type"] + + if "text" in content_type: + cellxgene_content = cellxgene_response.content.decode() + gateway_content = cellxgene_content.replace( + "http://fonts.gstatic.com", "https://fonts.gstatic.com" + ).replace(cellxgene_basepath, gateway_basepath) + else: + gateway_content = cellxgene_response.content + + gateway_response = Response( + gateway_content, status=cellxgene_response.status_code + ) + + gateway_response.headers["content-type"] = content_type + + return gateway_response diff --git a/cellxgene_exception.py b/cellxgene_exception.py new file mode 100644 index 0000000..ae645c3 --- /dev/null +++ b/cellxgene_exception.py @@ -0,0 +1,14 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +class CellxgeneException(Exception): + def __init__(self, message, http_status): + Exception.__init__(self) + self.message = message + self.http_status = http_status diff --git a/dir_util.py b/dir_util.py new file mode 100644 index 0000000..1a23369 --- /dev/null +++ b/dir_util.py @@ -0,0 +1,91 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import os + +from flask_api import status + +import env +from cellxgene_exception import CellxgeneException + + +def is_subdir(full_path, parent_path): + subdir = os.path.realpath(full_path) + parent = os.path.realpath(parent_path) + return subdir.startswith(parent) + + +def create_dir(parent_path, dir_name): + full_path = os.path.join(parent_path, dir_name) + + if "/" in dir_name: + raise CellxgeneException( + "Please have no slashes in the intended directory.", + status.HTTP_400_BAD_REQUEST, + ) + elif not os.path.exists(parent_path): + raise CellxgeneException( + "The selected User directory does not exist.", status.HTTP_400_BAD_REQUEST + ) + elif os.path.exists(full_path): + raise CellxgeneException( + "The provided subdirectory already exists within Directory.", + status.HTTP_400_BAD_REQUEST, + ) + elif not is_subdir(full_path, parent_path): + raise CellxgeneException( + "The directory must be a subdirectory of the parent path.", + status.HTTP_400_BAD_REQUEST, + ) + elif not os.path.isdir(parent_path): + raise CellxgeneException( + "The parent is not a directory.", status.HTTP_400_BAD_REQUEST + ) + else: + os.mkdir(full_path) + + +def recurse_dir(path): + if not os.path.exists(path): + raise CellxgeneException( + "The given path does not exist.", status.HTTP_400_BAD_REQUEST + ) + + def make_entry(el): + full_path = os.path.join(path, el) + if os.path.isfile(full_path): + return { + "path": full_path.replace(env.cellxgene_data, ""), + "name": el, + "type": "file", + } + elif os.path.isdir(full_path): + return { + "path": full_path, + "name": el, + "type": "directory", + "children": recurse_dir(full_path), + } + else: + raise CellxgeneException( + "Given path is neither file nor directory.", status.HTTP_400_BAD_REQUEST + ) + + return [make_entry(x) for x in os.listdir(path)] + + +def render_entries(entries): + return "" + + +def render_entry(entry): + if entry["type"] == "file": + return f"
  • {entry['name']}
    • " + elif entry["type"] == "directory": + return f"
  • {entry['name']}{render_entries(entry['children'])}
    • " diff --git a/env.py b/env.py new file mode 100644 index 0000000..686bd4f --- /dev/null +++ b/env.py @@ -0,0 +1,17 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import os + +deployment_env = os.environ.get("DEPLOYMENT_ENV") +cellxgene_location = os.environ.get("CELLXGENE_LOCATION") +cellxgene_data = os.environ.get("CELLXGENE_DATA") +gateway_host = os.environ.get("GATEWAY_HOST") +gateway_protocol = os.environ.get("GATEWAY_PROTOCOL") +ip = os.environ.get("GATEWAY_IP") diff --git a/extra_scripts.py b/extra_scripts.py new file mode 100644 index 0000000..7c05b97 --- /dev/null +++ b/extra_scripts.py @@ -0,0 +1,15 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +def get_extra_scripts(): + # can be array of script tags to inject on every page, e.g. for google analytics could be + # ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2', + # f"{env.gateway_protocol}://{env.gateway_host}/static/js/google_ua.js"] + # where google_ua.js is a script you add to the static/js folder prior to deployment. + return [] diff --git a/gateway.py b/gateway.py new file mode 100644 index 0000000..61e7f2d --- /dev/null +++ b/gateway.py @@ -0,0 +1,173 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +# import BaseHTTPServer +import datetime +import os +from threading import Thread + +from flask import Flask, redirect, render_template, request, send_from_directory +from flask_api import status +from werkzeug import secure_filename + +import env +from backend_cache import BackendCache +from cellxgene_exception import CellxgeneException +from dir_util import create_dir, recurse_dir, render_entries +from extra_scripts import get_extra_scripts +from path_util import get_dataset, get_file_path +from process_exception import ProcessException +from prune_process_cache import PruneProcessCache +from util import current_time_stamp + +app = Flask(__name__) +cache = BackendCache() +location = f"{env.gateway_protocol}://{env.gateway_host}" + + +@app.errorhandler(CellxgeneException) +def handle_invalid_usage(error): + + message = f"{error.http_status} Error : {error.message}" + + return ( + render_template( + "cellxgene_error.html", extra_scripts=get_extra_scripts(), message=message + ), + error.http_status, + ) + + +@app.errorhandler(ProcessException) +def handle_invalid_process(error): + + message = [] + + message.append(error.message) + message.append(f"{error.http_status} Error.") + message.append(f"Stdout: {error.stdout}") + message.append(f"Stderr: {error.stderr}") + + return ( + render_template( + "process_error.html", extra_scripts=get_extra_scripts(), message=message + ), + error.http_status, + ) + + +@app.route("/favicon.ico") +def favicon(): + return send_from_directory( + os.path.join(app.root_path, "static"), + "nibr.ico", + mimetype="image/vnd.microsof.icon", + ) + + +@app.route("/") +def index(): + users = [ + name + for name in os.listdir(env.cellxgene_data) + if os.path.isdir(os.path.join(env.cellxgene_data, name)) + ] + return render_template( + "index.html", + ip=env.ip, + cellxgene_data=env.cellxgene_data, + extra_scripts=get_extra_scripts(), + users=users, + ) + + +@app.route("/make_user", methods=["POST"]) +def make_user(): + dir_name = request.form["directory"] + + create_dir(env.cellxgene_data, dir_name) + + return redirect(location, code=302) + + +@app.route("/make_subdir", methods=["POST"]) +def make_subdir(): + parent_path = os.path.join(env.cellxgene_data, request.form["usernames"]) + dir_name = request.form["directory"] + + create_dir(parent_path, dir_name) + + return redirect(location, code=302) + + +@app.route("/upload_file", methods=["POST"]) +def upload_file(): + upload_dir = request.form["path"] + + full_upload_path = env.cellxgene_data + "/" + upload_dir + if os.path.isdir(full_upload_path): + if request.method == "POST": + if "file" in request.files: + f = request.files["file"] + if f and f.filename.endswith(".h5ad"): + f.save(full_upload_path + "/" + secure_filename(f.filename)) + return redirect("/filecrawl.html", code=302) + else: + raise CellxgeneException( + "Uploaded file must be in anndata (.h5ad) format.", + status.HTTP_400_BAD_REQUEST, + ) + else: + raise CellxgeneException( + "A file must be chosen to upload.", status.HTTP_400_BAD_REQUEST + ) + else: + raise CellxgeneException("Invalid directory.", status.HTTP_400_BAD_REQUEST) + + return redirect(env.location, code=302) + + +@app.route("/filecrawl.html") +def filecrawl(): + + entries = recurse_dir(env.cellxgene_data) + rendered_html = render_entries(entries) + return render_template( + "filecrawl.html", extra_scripts=get_extra_scripts(), rendered_html=rendered_html + ) + + +@app.route("/view/") +def do_GET(path): + + dataset = get_dataset(path) + file_path = get_file_path(dataset) + match = cache.check_entry(dataset) + if match is None: + uascripts = get_extra_scripts() + match = cache.create_entry(dataset, file_path, uascripts) + + match.timestamp = current_time_stamp() + + if match.status == "loaded": + return match.serve_content(path) + elif match.status == "loading": + launch_time = datetime.datetime.fromtimestamp(match.launchtime) + return render_template( + "loading.html", launchtime=launch_time, all_output=match.all_output + ) + elif match.status == "error": + raise ProcessException.from_pid_object(match) + + +if __name__ == "__main__": + background_thread = Thread(target=PruneProcessCache(cache)) + background_thread.start() + + app.run(host="0.0.0.0", port=5005, debug=False) diff --git a/path_util.py b/path_util.py new file mode 100644 index 0000000..0672184 --- /dev/null +++ b/path_util.py @@ -0,0 +1,49 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import os + +from flask_api import status + +import env +from cellxgene_exception import CellxgeneException + + +def get_dataset(path): + if path == "/" or path == "": + raise CellxgeneException( + "No matching dataset found.", status.HTTP_404_NOT_FOUND + ) + + trimmed = path[:-1] if path[-1] == "/" else path + + try: + get_file_path(trimmed) + return trimmed + except CellxgeneException: + split = os.path.split(trimmed) + return get_dataset(split[0]) + + +def validate_path(file_path): + if not os.path.exists(file_path): + raise CellxgeneException( + "File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST + ) + if not os.path.isfile(file_path): + raise CellxgeneException( + "Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST + ) + return + + +def get_file_path(dataset): + file_path = os.path.join(env.cellxgene_data, dataset) + validate_path(file_path) + return file_path diff --git a/process_exception.py b/process_exception.py new file mode 100644 index 0000000..e54d842 --- /dev/null +++ b/process_exception.py @@ -0,0 +1,25 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +class ProcessException(Exception): + def __init__(self, message, stdout, stderr, http_status): + Exception.__init__(self) + self.message = message + self.stdout = stdout + self.stderr = stderr + self.http_status = http_status + + @classmethod + def from_pid_object(cls, pid_object): + return cls( + pid_object.message, + pid_object.all_output, + pid_object.stderr, + pid_object.http_status, + ) diff --git a/prune_process_cache.py b/prune_process_cache.py new file mode 100644 index 0000000..71bfcfa --- /dev/null +++ b/prune_process_cache.py @@ -0,0 +1,39 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import time + +import psutil + +from util import current_time_stamp + + +class PruneProcessCache: + def __init__(self, cache): + self.cache = cache + + def __call__(self): + while True: + time.sleep(60) + + timestamp = current_time_stamp() + + processes_to_delete = [] + for p in self.cache.entry_list: + if timestamp - p.timestamp > 3600: + processes_to_delete.append(p) + processes_to_delete + + for process in processes_to_delete: + self.cache.entry_list.remove(process) + pid = process.pid + p = psutil.Process(pid) + p.terminate() + p = psutil.Process(pid + 2) + p.terminate() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c89a503 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +cellxgene +flask +flask_api +psutil +requests + diff --git a/run.sh.example b/run.sh.example new file mode 100644 index 0000000..baa9e83 --- /dev/null +++ b/run.sh.example @@ -0,0 +1,9 @@ +export CELLXGENE_LOCATION=/path/to/cellxgene +export CELLXGENE_DATA=/path/to/datasets +export DEPLOYMENT_ENV=dev +export GATEWAY_HOST=localhost:5005 +export GATEWAY_PROTOCOL=http +export GATEWAY_IP=127.0.0.1 + +#Once these are set, you run like a normal Flask app +python gateway.py diff --git a/static/css/homepagestyle.css b/static/css/homepagestyle.css new file mode 100644 index 0000000..a4ae50e --- /dev/null +++ b/static/css/homepagestyle.css @@ -0,0 +1,25 @@ +/* + Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed + under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy + of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless + required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. +*/ + +pre { + display: block; + padding: 8.5px; + margin: 0 0 9px; + font-size: 6px; + line-height: 1.385; + color: #333333; + word-break: break-all; + word-wrap: break-word; + background-color: #f5f5f5; + border: 1px solid #cccccc; + border-radius: 3px; + width: 80%; +} diff --git a/static/nibr.ico b/static/nibr.ico new file mode 100644 index 0000000..8174902 Binary files /dev/null and b/static/nibr.ico differ diff --git a/subprocess_backend.py b/subprocess_backend.py new file mode 100644 index 0000000..10aabdc --- /dev/null +++ b/subprocess_backend.py @@ -0,0 +1,71 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +import logging +import subprocess + +from flask_api import status + +from process_exception import ProcessException + + +class SubprocessBackend: + def __init__(self): + pass + + def create_cmd(self, cellxgene_loc, file_path, port, scripts): + + cmd = ( + f"yes | {cellxgene_loc} launch {file_path}" + + " --port " + + str(port) + + " --host 127.0.0.1" + ) + + for s in scripts: + cmd += f" --scripts {s}" + + return cmd + + def launch(self, cellxgene_loc, scripts, cache_entry): + + cmd = self.create_cmd( + cellxgene_loc, cache_entry.file_path, cache_entry.port, scripts + ) + logging.getLogger("werkzeug").info(f"launching {cmd}") + process = subprocess.Popen( + [cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) + + while True: + output = process.stdout.readline().decode() + if output == "[cellxgene] Type CTRL-C at any time to exit.\n": + break + elif output == "": + stderr = process.stderr.read().decode() + if ( + "Error while loading file" in stderr + or "Could not open file" in stderr + ): + message = "File was invalid." + http_status = status.HTTP_400_BAD_REQUEST + else: + message = "Cellxgene failed to launch dataset." + http_status = status.HTTP_500_INTERNAL_SERVER_ERROR + + cache_entry.status = "error" + cache_entry.set_error(message, stderr, http_status) + + raise ProcessException.from_pid_object(cache_entry) + else: + cache_entry.all_output += output + + cache_entry.set_loaded(process.pid) + + return diff --git a/templates/cellxgene_error.html b/templates/cellxgene_error.html new file mode 100644 index 0000000..908b5bf --- /dev/null +++ b/templates/cellxgene_error.html @@ -0,0 +1,40 @@ + + + + + Cellxgene Gateway - Error + + + {% for script in extra_scripts %} + + {% endfor %} + + + +
    +

    Cellxgene Gateway - Error

    +
    +
    +
    + +

    {{ message }}

    + + + Please click here to be redirected to the file directory. + +
    + + Please click here to return to the homepage. + +
    + + diff --git a/templates/filecrawl.html b/templates/filecrawl.html new file mode 100644 index 0000000..acdf372 --- /dev/null +++ b/templates/filecrawl.html @@ -0,0 +1,34 @@ + + + + + Cellxgene Gateway - FILE CRAWLER + + + {% for script in extra_scripts %} + + {% endfor %} + + + +
    +

    Cellxgene Gateway - FILE CRAWLER

    +
    +
    + +

    Please wait until a dataset is done loading before trying to launch a different one

    + +
    + {{ rendered_html|safe }} + + + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..62921c2 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,123 @@ + + + + + Cellxgene Gateway + + + + {% for script in extra_scripts %} + + {% endfor %} + + + + +
    +

    Cellxgene Gateway - HOME

    +
    +
    + +

    + Links: +

    +
    + + File Crawler: Allows you to view all uploaded data. + +
    + +
    + +

    + How To Upload Data: +

    +
      +
    1. + Create a folder for your Username: +
      2. +
      +
      + Username + +
      +
    2. + Create a subdirectory under the selected Folder: +
      3. +
      +
      + +
      + Subdirectory Name + +
      +
    3. Choose a folder to copy your data to, then upload your data file (must be in .h5ad format).
      4. +
      +
      + Type in the name of the directory and subdirectory you wish to upload to, i.e. "USER/cells". +
      + File:
      + +
      +
      +
    4. Take a look at your data using the file crawler link above
      5. +
    + + +
    + +

    + How To Upload Data via SSH and SCP (Linux): +

    +
      +
    1. + Confirm that your ssh key has been properly added. Running the following command should connect you to the server: +
      2. + 
      ssh ec2-user@{{ ip }}
      +
    2. + Prepare your data on your own machine. They should be in a common folder called 'USER' and should look
      similar to the following structure: +
      3. + 
      - USER
+    - hpc.h5ad
+    - pbmc3k.h5ad
      +
    3. Copy your data to the server (replace USER with your username):
      4. + 
      ssh ec2-user@{{ ip }} 'mkdir -p {{ cellxgene_data }}/USER/'
+ssh ec2-user@{{ ip }} 'mkdir -p {{ cellxgene_data }}/USER/DATA/'
+scp -r pbmc3k.h5ad ec2-user@{{ ip }}:{{ cellxgene_data }}/USER/DATA/
      +
    4. Take a look at your data using the file crawler link above
      5. +
    + +
    +
    + + + + diff --git a/templates/loading.html b/templates/loading.html new file mode 100644 index 0000000..f2183f3 --- /dev/null +++ b/templates/loading.html @@ -0,0 +1,53 @@ + + + + + Cellxgene Gateway - Loading + + + + + + +
    +

    Cellxgene Gateway - Loading

    +
    + +
    +
    + +
    +

    Output:

    + 
    {{ all_output }}
    +

    + Launch Time: {{ launchtime.isoformat(' ') }} +

    +

    + The page will refresh shortly. +

    + + + Please click here to be redirected to the file directory. + +
    + + Please click here to return to the homepage. + +
    + + + diff --git a/templates/process_error.html b/templates/process_error.html new file mode 100644 index 0000000..1c5d13e --- /dev/null +++ b/templates/process_error.html @@ -0,0 +1,43 @@ + + + + + Cellxgene Gateway - Process Error + + + {% for script in extra_scripts %} + + {% endfor %} + + + +
    +

    Cellxgene Gateway - Process Error

    +
    +
    +
    + +

    {{ message[0] }}

    +

    {{ message[1] }}

    +

    {{ message[2] }}

    +

    {{ message[3] }}

    + + + Please click here to be redirected to the file directory. + +
    + + Please click here to return to the homepage. + +
    + + diff --git a/util.py b/util.py new file mode 100644 index 0000000..7dab25a --- /dev/null +++ b/util.py @@ -0,0 +1,16 @@ +# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed +# under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless +# required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the License for +# the specific language governing permissions and limitations under the License. + +from datetime import datetime + + +def current_time_stamp(): + now = datetime.now() + timestamp = datetime.timestamp(now) + return timestamp