mirror of
https://github.com/Novartis/cellxgene-gateway.git
synced 2026-10-09 08:18:11 +08:00
refactor to allow use with console entry points
This commit is contained in:
Eric Ma
Executable
+8
@@ -0,0 +1,8 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import time
|
||||
from threading import Thread
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cache_entry import CacheEntry
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.subprocess_backend import SubprocessBackend
|
||||
|
||||
process_backend = SubprocessBackend()
|
||||
|
||||
|
||||
class BackendCache:
|
||||
def __init__(self):
|
||||
self.entry_list = []
|
||||
|
||||
def get_ports(self):
|
||||
contents = self.entry_list
|
||||
return [c.port for c in contents]
|
||||
|
||||
def check_entry(self, dataset):
|
||||
contents = self.entry_list
|
||||
matches = [c for c in contents if c.dataset == dataset]
|
||||
|
||||
if len(matches) == 0:
|
||||
return None
|
||||
elif len(matches) == 1:
|
||||
return matches[0]
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
"Found " + str(len(matches)) + " for " + dataset,
|
||||
)
|
||||
|
||||
def create_entry(self, dataset, file_path, scripts):
|
||||
port = 8000
|
||||
existing_ports = self.get_ports()
|
||||
while port in existing_ports:
|
||||
port += 1
|
||||
|
||||
entry = CacheEntry.for_dataset(dataset, file_path, port)
|
||||
|
||||
background_thread = Thread(
|
||||
target=process_backend.launch,
|
||||
args=(env.cellxgene_location, scripts, entry),
|
||||
)
|
||||
background_thread.start()
|
||||
|
||||
self.entry_list.append(entry)
|
||||
|
||||
time.sleep(1) # Automatic refresh is too fast, needs a second to pause
|
||||
|
||||
return entry
|
||||
@@ -0,0 +1,130 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
from flask import make_response, request
|
||||
from requests import get, post, put
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
|
||||
|
||||
class CacheEntry:
|
||||
def __init__(
|
||||
self,
|
||||
pid,
|
||||
dataset,
|
||||
file_path,
|
||||
port,
|
||||
launchtime,
|
||||
timestamp,
|
||||
status,
|
||||
message,
|
||||
all_output,
|
||||
stderr,
|
||||
http_status,
|
||||
):
|
||||
self.pid = pid
|
||||
self.dataset = dataset
|
||||
self.file_path = file_path
|
||||
self.port = port
|
||||
self.launchtime = launchtime
|
||||
self.timestamp = timestamp
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.all_output = all_output
|
||||
self.stderr = stderr
|
||||
self.http_status = http_status
|
||||
|
||||
@classmethod
|
||||
def for_dataset(cls, dataset, file_path, port):
|
||||
return cls(
|
||||
"",
|
||||
dataset,
|
||||
file_path,
|
||||
port,
|
||||
current_time_stamp(),
|
||||
current_time_stamp(),
|
||||
"loading",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
)
|
||||
|
||||
def set_loaded(self, pid):
|
||||
self.pid = pid
|
||||
self.status = "loaded"
|
||||
|
||||
def set_error(self, message, stderr, http_status):
|
||||
self.message = message
|
||||
self.stderr = stderr
|
||||
self.http_status = http_status
|
||||
self.status = "error"
|
||||
|
||||
def serve_content(self, path):
|
||||
dataset = self.dataset
|
||||
|
||||
gateway_basepath = (
|
||||
f"{env.gateway_protocol}://{env.gateway_host}/view/{dataset}/"
|
||||
)
|
||||
subpath = path[len(dataset) :] # noqa: E203
|
||||
|
||||
if len(subpath) == 0:
|
||||
r = make_response(f"Redirect to {gateway_basepath}\n", 301)
|
||||
r.headers["location"] = gateway_basepath
|
||||
return r
|
||||
|
||||
port = self.port
|
||||
cellxgene_basepath = f"http://127.0.0.1:{port}"
|
||||
|
||||
headers = {}
|
||||
|
||||
if "accept" in request.headers:
|
||||
headers["accept"] = request.headers["accept"]
|
||||
if "user-agent" in request.headers:
|
||||
headers["user-agent"] = request.headers["user-agent"]
|
||||
if "content-type" in request.headers:
|
||||
headers["content-type"] = request.headers["content-type"]
|
||||
|
||||
if request.method in ["GET", "HEAD", "OPTIONS"]:
|
||||
cellxgene_response = get(
|
||||
cellxgene_basepath + subpath, headers=headers
|
||||
)
|
||||
elif request.method == "PUT":
|
||||
cellxgene_response = put(
|
||||
cellxgene_basepath + subpath,
|
||||
headers=headers,
|
||||
data=request.data.decode(),
|
||||
)
|
||||
elif request.method == "POST":
|
||||
cellxgene_response = post(
|
||||
cellxgene_basepath + subpath,
|
||||
headers=headers,
|
||||
data=request.data.decode(),
|
||||
)
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
f"Unexpected method {request.method}", 400
|
||||
)
|
||||
content_type = cellxgene_response.headers["content-type"]
|
||||
if "text" in content_type:
|
||||
cellxgene_content = cellxgene_response.content.decode()
|
||||
gateway_content = cellxgene_content.replace(
|
||||
"http://fonts.gstatic.com", "https://fonts.gstatic.com"
|
||||
).replace(cellxgene_basepath, gateway_basepath)
|
||||
else:
|
||||
gateway_content = cellxgene_response.content
|
||||
|
||||
gateway_response = make_response(
|
||||
gateway_content,
|
||||
cellxgene_response.status_code,
|
||||
{"Content-Type": content_type},
|
||||
)
|
||||
|
||||
return gateway_response
|
||||
@@ -0,0 +1,15 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
class CellxgeneException(Exception):
|
||||
def __init__(self, message, http_status):
|
||||
Exception.__init__(self)
|
||||
self.message = message
|
||||
self.http_status = http_status
|
||||
@@ -0,0 +1,93 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
|
||||
|
||||
def is_subdir(full_path, parent_path):
|
||||
subdir = os.path.realpath(full_path)
|
||||
parent = os.path.realpath(parent_path)
|
||||
return subdir.startswith(parent)
|
||||
|
||||
|
||||
def create_dir(parent_path, dir_name):
|
||||
full_path = os.path.join(parent_path, dir_name)
|
||||
|
||||
if "/" in dir_name:
|
||||
raise CellxgeneException(
|
||||
"Please have no slashes in the intended directory.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not os.path.exists(parent_path):
|
||||
raise CellxgeneException(
|
||||
"The selected User directory does not exist.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif os.path.exists(full_path):
|
||||
raise CellxgeneException(
|
||||
"The provided subdirectory already exists within Directory.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not is_subdir(full_path, parent_path):
|
||||
raise CellxgeneException(
|
||||
"The directory must be a subdirectory of the parent path.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
elif not os.path.isdir(parent_path):
|
||||
raise CellxgeneException(
|
||||
"The parent is not a directory.", status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
else:
|
||||
os.mkdir(full_path)
|
||||
|
||||
|
||||
def recurse_dir(path):
|
||||
if not os.path.exists(path):
|
||||
raise CellxgeneException(
|
||||
"The given path does not exist.", status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
def make_entry(el):
|
||||
full_path = os.path.join(path, el)
|
||||
if os.path.isfile(full_path):
|
||||
return {
|
||||
"path": full_path.replace(env.cellxgene_data, ""),
|
||||
"name": el,
|
||||
"type": "file",
|
||||
}
|
||||
elif os.path.isdir(full_path):
|
||||
return {
|
||||
"path": full_path,
|
||||
"name": el,
|
||||
"type": "directory",
|
||||
"children": recurse_dir(full_path),
|
||||
}
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
"Given path is neither file nor directory.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
return [make_entry(x) for x in os.listdir(path)]
|
||||
|
||||
|
||||
def render_entries(entries):
|
||||
return "<ul>" + "\n".join([render_entry(e) for e in entries]) + "</ul>"
|
||||
|
||||
|
||||
def render_entry(entry):
|
||||
if entry["type"] == "file":
|
||||
return f"<li> <a href='view{entry['path']}'>{entry['name']}</a></li>"
|
||||
elif entry["type"] == "directory":
|
||||
return f"<li>{entry['name']}{render_entries(entry['children'])}</li>"
|
||||
@@ -0,0 +1,46 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
deployment_env = os.environ.get("DEPLOYMENT_ENV")
|
||||
cellxgene_location = os.environ.get("CELLXGENE_LOCATION")
|
||||
cellxgene_data = os.environ.get("CELLXGENE_DATA")
|
||||
gateway_host = os.environ.get("GATEWAY_HOST")
|
||||
gateway_protocol = os.environ.get("GATEWAY_PROTOCOL")
|
||||
ip = os.environ.get("GATEWAY_IP")
|
||||
|
||||
env_vars = {
|
||||
"DEPLOYMENT_ENV": deployment_env,
|
||||
"CELLXGENE_LOCATION": cellxgene_location,
|
||||
"CELLXGENE_DATA": cellxgene_data,
|
||||
"GATEWAY_HOST": gateway_host,
|
||||
"GATEWAY_PROTOCOL": gateway_protocol,
|
||||
"GATEWAY_IP": ip,
|
||||
}
|
||||
|
||||
if not all(env_vars.values()):
|
||||
raise ValueError(
|
||||
f"""
|
||||
Please ensure that environment variables are set correctly.
|
||||
The ones with None below are missing and need to be set.
|
||||
|
||||
{env_vars}
|
||||
|
||||
Set them at the terminal before running the gateway.
|
||||
An example is:
|
||||
|
||||
export CELLXGENE_LOCATION=~/anaconda/envs/cellxgene-dev/bin/cellxgene
|
||||
export CELLXGENE_DATA=../cellxgene_data
|
||||
export DEPLOYMENT_ENV=dev
|
||||
export GATEWAY_HOST=localhost:5005
|
||||
export GATEWAY_PROTOCOL=http
|
||||
export GATEWAY_IP=127.0.0.1
|
||||
"""
|
||||
)
|
||||
@@ -0,0 +1,16 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
def get_extra_scripts():
|
||||
# can be array of script tags to inject on every page, e.g. for google analytics could be
|
||||
# ['https://www.googletagmanager.com/gtag/js?id=UA-123456-2',
|
||||
# f"{env.gateway_protocol}://{env.gateway_host}/static/js/google_ua.js"]
|
||||
# where google_ua.js is a script you add to the static/js folder prior to deployment.
|
||||
return []
|
||||
@@ -0,0 +1,194 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
# import BaseHTTPServer
|
||||
import datetime
|
||||
import os
|
||||
from threading import Thread
|
||||
|
||||
from flask import (
|
||||
Flask,
|
||||
redirect,
|
||||
render_template,
|
||||
request,
|
||||
send_from_directory,
|
||||
)
|
||||
from flask_api import status
|
||||
from werkzeug import secure_filename
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.backend_cache import BackendCache
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
from cellxgene_gateway.dir_util import create_dir, recurse_dir, render_entries
|
||||
from cellxgene_gateway.extra_scripts import get_extra_scripts
|
||||
from cellxgene_gateway.path_util import get_dataset, get_file_path
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
from cellxgene_gateway.prune_process_cache import PruneProcessCache
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
|
||||
app = Flask(__name__)
|
||||
cache = BackendCache()
|
||||
location = f"{env.gateway_protocol}://{env.gateway_host}"
|
||||
|
||||
|
||||
@app.errorhandler(CellxgeneException)
|
||||
def handle_invalid_usage(error):
|
||||
|
||||
message = f"{error.http_status} Error : {error.message}"
|
||||
|
||||
return (
|
||||
render_template(
|
||||
"cellxgene_error.html",
|
||||
extra_scripts=get_extra_scripts(),
|
||||
message=message,
|
||||
),
|
||||
error.http_status,
|
||||
)
|
||||
|
||||
|
||||
@app.errorhandler(ProcessException)
|
||||
def handle_invalid_process(error):
|
||||
|
||||
message = []
|
||||
|
||||
message.append(error.message)
|
||||
message.append(f"{error.http_status} Error.")
|
||||
message.append(f"Stdout: {error.stdout}")
|
||||
message.append(f"Stderr: {error.stderr}")
|
||||
|
||||
return (
|
||||
render_template(
|
||||
"process_error.html",
|
||||
extra_scripts=get_extra_scripts(),
|
||||
message=message,
|
||||
),
|
||||
error.http_status,
|
||||
)
|
||||
|
||||
|
||||
@app.route("/favicon.ico")
|
||||
def favicon():
|
||||
return send_from_directory(
|
||||
os.path.join(app.root_path, "static"),
|
||||
"nibr.ico",
|
||||
mimetype="image/vnd.microsof.icon",
|
||||
)
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
users = [
|
||||
name
|
||||
for name in os.listdir(env.cellxgene_data)
|
||||
if os.path.isdir(os.path.join(env.cellxgene_data, name))
|
||||
]
|
||||
return render_template(
|
||||
"index.html",
|
||||
ip=env.ip,
|
||||
cellxgene_data=env.cellxgene_data,
|
||||
extra_scripts=get_extra_scripts(),
|
||||
users=users,
|
||||
)
|
||||
|
||||
|
||||
@app.route("/make_user", methods=["POST"])
|
||||
def make_user():
|
||||
dir_name = request.form["directory"]
|
||||
|
||||
create_dir(env.cellxgene_data, dir_name)
|
||||
|
||||
return redirect(location, code=302)
|
||||
|
||||
|
||||
@app.route("/make_subdir", methods=["POST"])
|
||||
def make_subdir():
|
||||
parent_path = os.path.join(env.cellxgene_data, request.form["usernames"])
|
||||
dir_name = request.form["directory"]
|
||||
|
||||
create_dir(parent_path, dir_name)
|
||||
|
||||
return redirect(location, code=302)
|
||||
|
||||
|
||||
@app.route("/upload_file", methods=["POST"])
|
||||
def upload_file():
|
||||
upload_dir = request.form["path"]
|
||||
|
||||
full_upload_path = env.cellxgene_data + "/" + upload_dir
|
||||
if os.path.isdir(full_upload_path):
|
||||
if request.method == "POST":
|
||||
if "file" in request.files:
|
||||
f = request.files["file"]
|
||||
if f and f.filename.endswith(".h5ad"):
|
||||
f.save(
|
||||
full_upload_path + "/" + secure_filename(f.filename)
|
||||
)
|
||||
return redirect("/filecrawl.html", code=302)
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
"Uploaded file must be in anndata (.h5ad) format.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
"A file must be chosen to upload.",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
else:
|
||||
raise CellxgeneException(
|
||||
"Invalid directory.", status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
return redirect(env.location, code=302)
|
||||
|
||||
|
||||
@app.route("/filecrawl.html")
|
||||
def filecrawl():
|
||||
|
||||
entries = recurse_dir(env.cellxgene_data)
|
||||
rendered_html = render_entries(entries)
|
||||
return render_template(
|
||||
"filecrawl.html",
|
||||
extra_scripts=get_extra_scripts(),
|
||||
rendered_html=rendered_html,
|
||||
)
|
||||
|
||||
|
||||
@app.route("/view/<path:path>", methods=["GET", "PUT", "POST"])
|
||||
def do_GET(path):
|
||||
|
||||
dataset = get_dataset(path)
|
||||
file_path = get_file_path(dataset)
|
||||
match = cache.check_entry(dataset)
|
||||
if match is None:
|
||||
uascripts = get_extra_scripts()
|
||||
match = cache.create_entry(dataset, file_path, uascripts)
|
||||
|
||||
match.timestamp = current_time_stamp()
|
||||
|
||||
if match.status == "loaded":
|
||||
return match.serve_content(path)
|
||||
elif match.status == "loading":
|
||||
launch_time = datetime.datetime.fromtimestamp(match.launchtime)
|
||||
return render_template(
|
||||
"loading.html", launchtime=launch_time, all_output=match.all_output
|
||||
)
|
||||
elif match.status == "error":
|
||||
raise ProcessException.from_pid_object(match)
|
||||
|
||||
|
||||
def main():
|
||||
background_thread = Thread(target=PruneProcessCache(cache))
|
||||
background_thread.start()
|
||||
|
||||
app.run(host="0.0.0.0", port=5005, debug=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,49 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway import env
|
||||
from cellxgene_gateway.cellxgene_exception import CellxgeneException
|
||||
|
||||
|
||||
def get_dataset(path):
|
||||
if path == "/" or path == "":
|
||||
raise CellxgeneException(
|
||||
"No matching dataset found.", status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
trimmed = path[:-1] if path[-1] == "/" else path
|
||||
|
||||
try:
|
||||
get_file_path(trimmed)
|
||||
return trimmed
|
||||
except CellxgeneException:
|
||||
split = os.path.split(trimmed)
|
||||
return get_dataset(split[0])
|
||||
|
||||
|
||||
def validate_path(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
raise CellxgeneException(
|
||||
"File does not exist: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
if not os.path.isfile(file_path):
|
||||
raise CellxgeneException(
|
||||
"Path is not file: " + file_path, status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
def get_file_path(dataset):
|
||||
file_path = os.path.join(env.cellxgene_data, dataset)
|
||||
validate_path(file_path)
|
||||
return file_path
|
||||
@@ -0,0 +1,26 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
class ProcessException(Exception):
|
||||
def __init__(self, message, stdout, stderr, http_status):
|
||||
Exception.__init__(self)
|
||||
self.message = message
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
self.http_status = http_status
|
||||
|
||||
@classmethod
|
||||
def from_pid_object(cls, pid_object):
|
||||
return cls(
|
||||
pid_object.message,
|
||||
pid_object.all_output,
|
||||
pid_object.stderr,
|
||||
pid_object.http_status,
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import time
|
||||
|
||||
import psutil
|
||||
|
||||
from cellxgene_gateway.util import current_time_stamp
|
||||
|
||||
|
||||
class PruneProcessCache:
|
||||
def __init__(self, cache):
|
||||
self.cache = cache
|
||||
|
||||
def __call__(self):
|
||||
while True:
|
||||
time.sleep(60)
|
||||
|
||||
timestamp = current_time_stamp()
|
||||
|
||||
processes_to_delete = []
|
||||
for p in self.cache.entry_list:
|
||||
if timestamp - p.timestamp > 3600:
|
||||
processes_to_delete.append(p)
|
||||
processes_to_delete
|
||||
|
||||
for process in processes_to_delete:
|
||||
self.cache.entry_list.remove(process)
|
||||
pid = process.pid
|
||||
p = psutil.Process(pid)
|
||||
p.terminate()
|
||||
p = psutil.Process(pid + 2)
|
||||
p.terminate()
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy
|
||||
of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
required by applicable law or agreed to in writing, software distributed
|
||||
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
the specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
pre {
|
||||
display: block;
|
||||
padding: 8.5px;
|
||||
margin: 0 0 9px;
|
||||
font-size: 6px;
|
||||
line-height: 1.385;
|
||||
color: #333333;
|
||||
word-break: break-all;
|
||||
word-wrap: break-word;
|
||||
background-color: #f5f5f5;
|
||||
border: 1px solid #cccccc;
|
||||
border-radius: 3px;
|
||||
width: 80%;
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
@@ -0,0 +1,71 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
from flask_api import status
|
||||
|
||||
from cellxgene_gateway.process_exception import ProcessException
|
||||
|
||||
|
||||
class SubprocessBackend:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def create_cmd(self, cellxgene_loc, file_path, port, scripts):
|
||||
|
||||
cmd = (
|
||||
f"yes | {cellxgene_loc} launch {file_path}"
|
||||
+ " --port "
|
||||
+ str(port)
|
||||
+ " --host 127.0.0.1"
|
||||
)
|
||||
|
||||
for s in scripts:
|
||||
cmd += f" --scripts {s}"
|
||||
|
||||
return cmd
|
||||
|
||||
def launch(self, cellxgene_loc, scripts, cache_entry):
|
||||
|
||||
cmd = self.create_cmd(
|
||||
cellxgene_loc, cache_entry.file_path, cache_entry.port, scripts
|
||||
)
|
||||
logging.getLogger("werkzeug").info(f"launching {cmd}")
|
||||
process = subprocess.Popen(
|
||||
[cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
|
||||
)
|
||||
|
||||
while True:
|
||||
output = process.stdout.readline().decode()
|
||||
if output == "[cellxgene] Type CTRL-C at any time to exit.\n":
|
||||
break
|
||||
elif output == "":
|
||||
stderr = process.stderr.read().decode()
|
||||
if (
|
||||
"Error while loading file" in stderr
|
||||
or "Could not open file" in stderr
|
||||
):
|
||||
message = "File was invalid."
|
||||
http_status = status.HTTP_400_BAD_REQUEST
|
||||
else:
|
||||
message = "Cellxgene failed to launch dataset."
|
||||
http_status = status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
|
||||
cache_entry.status = "error"
|
||||
cache_entry.set_error(message, stderr, http_status)
|
||||
|
||||
raise ProcessException.from_pid_object(cache_entry)
|
||||
else:
|
||||
cache_entry.all_output += output
|
||||
|
||||
cache_entry.set_loaded(process.pid)
|
||||
|
||||
return
|
||||
@@ -0,0 +1,40 @@
|
||||
<!--
|
||||
Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy
|
||||
of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
required by applicable law or agreed to in writing, software distributed
|
||||
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
the specific language governing permissions and limitations under the License.
|
||||
-->
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>Cellxgene Gateway - Error</title>
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
|
||||
<link rel="icon" type="image/png" href="{{ url_for('static', filename='nibr.ico') }}">
|
||||
{% for script in extra_scripts %}
|
||||
<script src="{{ script }}"></script>
|
||||
{% endfor %}
|
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
|
||||
</head>
|
||||
<body>
|
||||
<header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
|
||||
<h3>Cellxgene Gateway - Error</h3>
|
||||
</header>
|
||||
<br>
|
||||
<div style="margin-left:20px">
|
||||
|
||||
<h4>{{ message }}</h4>
|
||||
|
||||
<a href="/filecrawl.html">
|
||||
Please click here to be redirected to the file directory.
|
||||
</a>
|
||||
<br>
|
||||
<a href="/">
|
||||
Please click here to return to the homepage.
|
||||
</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,34 @@
|
||||
<!--
|
||||
Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy
|
||||
of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
required by applicable law or agreed to in writing, software distributed
|
||||
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
the specific language governing permissions and limitations under the License.
|
||||
-->
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>Cellxgene Gateway - FILE CRAWLER</title>
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
|
||||
<link rel="icon" type="image/png" href="{{ url_for('static', filename='nibr.ico') }}">
|
||||
{% for script in extra_scripts %}
|
||||
<script src="{{ script }}"></script>
|
||||
{% endfor %}
|
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
|
||||
</head>
|
||||
<body>
|
||||
<header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
|
||||
<h3>Cellxgene Gateway - FILE CRAWLER</h3>
|
||||
</header>
|
||||
<br>
|
||||
|
||||
<h4>Please wait until a dataset is done loading before trying to launch a different one</h4>
|
||||
|
||||
<br>
|
||||
{{ rendered_html|safe }}
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,122 @@
|
||||
<!--
|
||||
Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy
|
||||
of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
required by applicable law or agreed to in writing, software distributed
|
||||
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
the specific language governing permissions and limitations under the License.
|
||||
-->
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>Cellxgene Gateway</title>
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/homepagestyle.css') }}">
|
||||
<link rel="icon" type="image/png" href="{{ url_for('static', filename='nibr.ico') }}">
|
||||
{% for script in extra_scripts %}
|
||||
<script src="{{ script }}"></script>
|
||||
{% endfor %}
|
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
|
||||
<style>
|
||||
li {
|
||||
font-size: 20px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
|
||||
<h3>Cellxgene Gateway - HOME</h3>
|
||||
</header>
|
||||
<br>
|
||||
|
||||
<h1 style="padding-left:35px">
|
||||
Links:
|
||||
</h1>
|
||||
<div class="list-group" style="width:50%;padding-left:65px">
|
||||
<a href="/filecrawl.html" class="list-group-item list-group-item-action">
|
||||
<u>File Crawler: Allows you to view all uploaded data.</u></a>
|
||||
|
||||
</div>
|
||||
<br>
|
||||
|
||||
<h1 style="padding-left:35px">
|
||||
How To Upload Data:
|
||||
</h1>
|
||||
<ol style="padding-left:85px;">
|
||||
<li>
|
||||
Create a folder for your Username:
|
||||
</li>
|
||||
<br>
|
||||
<form action="{{ url_for('make_user') }}" method="post">
|
||||
Username <input type="text" name="directory">
|
||||
<input type="submit" value="Create">
|
||||
</form>
|
||||
<li>
|
||||
Create a subdirectory under the selected Folder:
|
||||
</li>
|
||||
<br>
|
||||
<form action="{{ url_for('make_subdir') }}" method="post">
|
||||
<select name="usernames" id="usernames">
|
||||
{% for user in users %}
|
||||
<option value="{{ user }}">{{ user }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<br>
|
||||
Subdirectory Name <input type="text" name="directory">
|
||||
<input type="submit" value="Create">
|
||||
</form>
|
||||
<li>Choose a folder to copy your data to, then upload your data file (must be in .h5ad format).</li>
|
||||
<br>
|
||||
<form action="{{ url_for('upload_file') }}" method="post" enctype="multipart/form-data">
|
||||
Type in the name of the directory and subdirectory you wish to upload to, i.e. "USER/cells". <input type="text" name="path">
|
||||
<br>
|
||||
File: <input type="file" name="file"><br>
|
||||
<input style="position:relative; top:10px;" type="submit" value="Upload">
|
||||
</form>
|
||||
<br>
|
||||
<li>Take a look at your data using the file crawler link above</li>
|
||||
</ol>
|
||||
|
||||
|
||||
<br>
|
||||
|
||||
<h1 style="padding-left:35px">
|
||||
How To Upload Data via SSH and SCP (Linux):
|
||||
</h1>
|
||||
<ol style="padding-left:85px;">
|
||||
<li>
|
||||
Confirm that your ssh key has been properly added. Running the following command should connect you to the server:
|
||||
</li>
|
||||
<pre>ssh ec2-user@{{ ip }}</pre>
|
||||
<li>
|
||||
Prepare your data on your own machine. They should be in a common folder called 'USER' and should look <br>similar to the following structure:
|
||||
</li>
|
||||
<pre>- USER
|
||||
- hpc.h5ad
|
||||
- pbmc3k.h5ad</pre>
|
||||
<li>Copy your data to the server (replace USER with your username):</li>
|
||||
<pre>ssh ec2-user@{{ ip }} 'mkdir -p {{ cellxgene_data }}/USER/'
|
||||
ssh ec2-user@{{ ip }} 'mkdir -p {{ cellxgene_data }}/USER/DATA/'
|
||||
scp -r pbmc3k.h5ad ec2-user@{{ ip }}:{{ cellxgene_data }}/USER/DATA/</pre>
|
||||
<li>Take a look at your data using the file crawler link above</li>
|
||||
</ol>
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
<script>
|
||||
if (NIBRIam && NIBRIam.NIBR521 !== undefined) {
|
||||
var usernames = document.getElementById("usernames")
|
||||
var options = usernames.options
|
||||
|
||||
for (i = 0; i < options.length; i++) {
|
||||
if (options[i].value.toLowerCase() == currentUser.toLowerCase()) {
|
||||
options[i].setAttribute("selected", "selected")
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,53 @@
|
||||
<!--
|
||||
Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy
|
||||
of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
required by applicable law or agreed to in writing, software distributed
|
||||
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
the specific language governing permissions and limitations under the License.
|
||||
-->
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>Cellxgene Gateway - Loading</title>
|
||||
<link rel="icon" type="image/png" href="nibr.png">
|
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
|
||||
|
||||
<meta http-equiv="refresh" content="5">
|
||||
</head>
|
||||
<body>
|
||||
<header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
|
||||
<h3>Cellxgene Gateway - Loading<span id='dots'></span></h3>
|
||||
</header>
|
||||
|
||||
<br>
|
||||
<div style="margin-left:20px">
|
||||
|
||||
<br>
|
||||
<h4>Output:</h4>
|
||||
<pre>{{ all_output }}</pre>
|
||||
<p>
|
||||
Launch Time: <span id="launch_time"> {{ launchtime.isoformat(' ') }} </span>
|
||||
</p>
|
||||
<p>
|
||||
The page will refresh shortly.
|
||||
</p>
|
||||
|
||||
<a href="/filecrawl.html">
|
||||
Please click here to be redirected to the file directory.
|
||||
</a>
|
||||
<br>
|
||||
<a href="/">
|
||||
Please click here to return to the homepage.
|
||||
</a>
|
||||
</div>
|
||||
<script>
|
||||
window.setInterval(function(){
|
||||
var dots = document.getElementById('dots');
|
||||
dots.textContent = dots.textContent + '.';
|
||||
}, 1000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,43 @@
|
||||
<!--
|
||||
Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy
|
||||
of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
required by applicable law or agreed to in writing, software distributed
|
||||
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
the specific language governing permissions and limitations under the License.
|
||||
-->
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>Cellxgene Gateway - Process Error</title>
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
|
||||
<link rel="icon" type="image/png" href="nibr.png">
|
||||
{% for script in extra_scripts %}
|
||||
<script src="{{ script }}"></script>
|
||||
{% endfor %}
|
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
|
||||
</head>
|
||||
<body>
|
||||
<header class="navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar">
|
||||
<h3>Cellxgene Gateway - Process Error</h3>
|
||||
</header>
|
||||
<br>
|
||||
<div style="margin-left:20px">
|
||||
|
||||
<h4>{{ message[0] }}</h4>
|
||||
<h4>{{ message[1] }}</h4>
|
||||
<h4>{{ message[2] }}</h4>
|
||||
<h4>{{ message[3] }}</h4>
|
||||
|
||||
<a href="/filecrawl.html">
|
||||
Please click here to be redirected to the file directory.
|
||||
</a>
|
||||
<br>
|
||||
<a href="/">
|
||||
Please click here to return to the homepage.
|
||||
</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,16 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def current_time_stamp():
|
||||
now = datetime.now()
|
||||
timestamp = datetime.timestamp(now)
|
||||
return timestamp
|
||||
Reference in New Issue
Block a user