diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d2058a44..6a3692ae 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -13,3 +13,7 @@ replace = version="{new_version}" search = "version": "{current_version}" replace = "version": "{new_version}" +[bumpversion:file:server/__init__.py] +search = "__version__ = "{current_version}" +replace = "__version__ = "{new_version}" + diff --git a/cellxgene-osx.spec b/cellxgene-osx.spec new file mode 100644 index 00000000..7bac23c4 --- /dev/null +++ b/cellxgene-osx.spec @@ -0,0 +1,41 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['server/gui/main.py'], + pathex=['/Users/charlotteweaver/Documents/Git/cellxgene'], + binaries=[('/System/Library/Frameworks/Tk.framework/Tk', 'tk'), ('/System/Library/Frameworks/Tcl.framework/Tcl', 'tcl')], + datas=[('server/app/web/templates/', 'server/app/web/templates/'), ('server/app/web/static/', 'server/app/web/static/')], + hiddenimports=['sklearn', 'sklearn.utils._cython_blas', 'sklearn.neighbors.typedefs', 'sklearn.neighbors.quad_tree', 'sklearn.tree', 'sklearn.tree._utils'], + hookspath=['server/gui/'], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + [], + exclude_binaries=True, + name='cellxgene', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False , icon='server/gui/images/cxg_icons.icns') +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='cellxgene') +app = BUNDLE(coll, + name='cellxgene.app', + icon='server/gui/images/cxg_icons.icns', + bundle_identifier=None) diff --git a/cellxgene-windows.spec b/cellxgene-windows.spec new file mode 100644 index 00000000..2bd7054a --- /dev/null +++ b/cellxgene-windows.spec @@ -0,0 +1,36 @@ +# -*- mode: python -*- + +block_cipher = None + + +a = Analysis(['server\\gui\\main.py'], + pathex=['C:\\Users\\Charlotte\\Documents\\git\\cellxgene'], + binaries=[], + datas=[('server/app/web/templates/', 'server/app/web/templates'), ('server/app/web/static/', 'server/app/web/static')], + hiddenimports=[], + hookspath=['server/gui/'], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + [], + exclude_binaries=True, + name='cellxgene', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False , icon='server\\gui\\images\\icon.ico') +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + name='cellxgene') diff --git a/makefile b/makefile index 388d308e..5e61f72f 100644 --- a/makefile +++ b/makefile @@ -1,7 +1,7 @@ BUILDDIR := build CLIENTBUILD := $(BUILDDIR)/client SERVERBUILD := $(BUILDDIR)/server -CLEANFILES := $(BUILDDIR)/ client/build dist cellxgene.egg-info +CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info PART ?= patch @@ -84,6 +84,9 @@ release-directly-to-prod : dev-env pydist twine-prod dev-env : pip install -r server/requirements-dev.txt +gui-env : dev-env + pip install -r server/requirements-gui.txt + # give PART=[major, minor, part] as param to make bump bump : bumpversion --config-file .bumpversion.cfg $(PART) @@ -136,4 +139,21 @@ uninstall : build-assets : pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py -.PHONY : build-assets +gui-spec-osx : clean-lite gui-env + pip install -e .[gui] + pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl' --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py + mv cellxgene.spec cellxgene-osx.spec + +gui-spec-windows : clean-lite dev-env + pip install -e .[gui] + pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py + mv cellxgene.spec cellxgene-windows.spec + +gui-build-osx : clean-lite + pyinstaller --clean cellxgene-osx.spec + +gui-build-windows : clean-lite + pyinstaller --clean cellxgene-windows.spec + +.PHONY : build-assets gui-build-osx gui-build-windows gui-build-osx gui-build-windows + diff --git a/server/__init__.py b/server/__init__.py index e69de29b..1f4c4d43 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -0,0 +1 @@ +__version__ = "0.10.1" diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index aafe175a..b94c5439 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -1,9 +1,10 @@ from http import HTTPStatus -import pkg_resources import warnings from flask import Blueprint, current_app, jsonify, make_response, request from flask_restful import Api, Resource +from server import __version__ as cellxgene_version +from anndata import __version__ as anndata_version from server.app.util.constants import ( Axis, @@ -59,16 +60,15 @@ class ConfigAPI(Resource): }, ], "displayNames": { - "engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}", + "engine": f"cellxgene Scanpy engine version ", "dataset": current_app.config["DATASET_TITLE"], }, "parameters": { "max-category-items": current_app.data.config["max_category_items"] }, "library_versions": { - "scanpy": pkg_resources.get_distribution("scanpy").version, - "cellxgene": pkg_resources.get_distribution("cellxgene").version, - "anndata": pkg_resources.get_distribution("cellxgene").version + "cellxgene": cellxgene_version, + "anndata": anndata_version } } } diff --git a/server/gui/cellxgene.spec b/server/gui/cellxgene.spec new file mode 100644 index 00000000..0dc65d3b --- /dev/null +++ b/server/gui/cellxgene.spec @@ -0,0 +1,29 @@ +# -*- mode: python -*- + +block_cipher = None + + +a = Analysis(['main.py'], + pathex=['/Users/charlotteweaver/Documents/Git/cellxgene/server/gui'], + hookspath=["/Users/charlotteweaver/Documents/Git/cellxgene/server/gui/"], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) + +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='cellxgene', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + runtime_tmpdir=None, + console=True ) diff --git a/server/gui/hook-cefpython3.py b/server/gui/hook-cefpython3.py new file mode 100644 index 00000000..4c6e0a9d --- /dev/null +++ b/server/gui/hook-cefpython3.py @@ -0,0 +1,237 @@ +""" +This is PyInstaller hook file for CEF Python. This file +helps PyInstaller find CEF Python dependencies that are +required to run final executable. + +See PyInstaller docs for hooks: +https://pyinstaller.readthedocs.io/en/stable/hooks.html +""" + +import glob +import os +import platform +import re +import sys +import PyInstaller +from PyInstaller.utils.hooks import is_module_satisfies, get_package_paths +from PyInstaller.compat import is_win, is_darwin, is_linux, is_py2 +from PyInstaller import log as logging + +# Constants +CEFPYTHON_MIN_VERSION = "57.0" +PYINSTALLER_MIN_VERSION = "3.2.1" + +# Makes assumption that using "python.exe" and not "pyinstaller.exe" +# TODO: use this code to work cross-platform: +# from PyInstaller.utils.hooks import get_package_paths +# get_package_paths("cefpython3") + +CEFPYTHON3_DIR = get_package_paths("cefpython3")[1] + +CYTHON_MODULE_EXT = ".pyd" if is_win else ".so" + +# Globals +logger = logging.getLogger(__name__) + + +# Functions +def check_platforms(): + if not is_win and not is_darwin and not is_linux: + raise SystemExit("Error: Currently only Windows, Linux and Darwin " + "platforms are supported, see Issue #135.") + + +def check_pyinstaller_version(): + """Using is_module_satisfies() for pyinstaller fails when + installed using 'pip install develop.zip' command + (PyInstaller Issue #2802).""" + # Example version string for dev version of pyinstaller: + # > 3.3.dev0+g5dc9557c + version = PyInstaller.__version__ + match = re.search(r"^\d+\.\d+(\.\d+)?", version) + if not (match.group(0) >= PYINSTALLER_MIN_VERSION): + raise SystemExit("Error: pyinstaller %s or higher is required" + % PYINSTALLER_MIN_VERSION) + + +def check_cefpython3_version(): + if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION): + raise SystemExit("Error: cefpython3 %s or higher is required" + % CEFPYTHON_MIN_VERSION) + + +def get_cefpython_modules(): + """Get all cefpython Cython modules in the cefpython3 package. + It returns a list of names without file extension. Eg. + 'cefpython_py27'. """ + pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, + "cefpython_py*" + CYTHON_MODULE_EXT)) + assert len(pyds) > 1, "Missing cefpython3 Cython modules" + modules = [] + for path in pyds: + filename = os.path.basename(path) + mod = filename.replace(CYTHON_MODULE_EXT, "") + modules.append(mod) + return modules + + +def get_excluded_cefpython_modules(): + """CEF Python package includes Cython modules for various Python + versions. When using Python 2.7 pyinstaller should not + bundle modules for eg. Python 3.6, otherwise it will + cause to include Python 3 dll dependencies. Returns a list + of fully qualified names eg. 'cefpython3.cefpython_py27'.""" + pyver = "".join(map(str, sys.version_info[:2])) + pyver_string = "py%s" % pyver + modules = get_cefpython_modules() + excluded = [] + for mod in modules: + if pyver_string in mod: + continue + excluded.append("cefpython3.%s" % mod) + logger.info("Exclude cefpython3 module: %s" % excluded[-1]) + return excluded + + +def get_cefpython3_datas(): + """Returning almost all of cefpython binaries as DATAS (see exception + below), because pyinstaller does strange things and fails if these are + returned as BINARIES. It first updates manifest in .dll files: + >> Updating manifest in chrome_elf.dll + + And then because of that it fails to load the library: + >> hsrc = win32api.LoadLibraryEx(filename, 0, LOAD_LIBRARY_AS_DATAFILE) + >> pywintypes.error: (5, 'LoadLibraryEx', 'Access is denied.') + + It is not required for pyinstaller to modify in any way + CEF binaries or to look for its dependencies. CEF binaries + does not have any external dependencies like MSVCR or similar. + + The .pak .dat and .bin files cannot be marked as BINARIES + as pyinstaller would fail to find binary depdendencies on + these files. + + One exception is subprocess (subprocess.exe on Windows) executable + file, which is passed to pyinstaller as BINARIES in order to collect + its dependecies. + + DATAS are in format: tuple(full_path, dest_subdir). + """ + ret = list() + + if is_win: + cefdatadir = "." + elif is_darwin or is_linux: + cefdatadir = "." + else: + assert False, "Unsupported system {}".format(platform.system()) + + # Binaries, licenses and readmes in the cefpython3/ directory + for filename in os.listdir(CEFPYTHON3_DIR): + # Ignore Cython modules which are already handled by + # pyinstaller automatically. + if filename[:-len(CYTHON_MODULE_EXT)] in get_cefpython_modules(): + continue + + # CEF binaries and datas + extension = os.path.splitext(filename)[1] + if extension in \ + [".exe", ".dll", ".pak", ".dat", ".bin", ".txt", ".so", ".plist"] \ + or filename.lower().startswith("license"): + logger.info("Include cefpython3 data: {}".format(filename)) + ret.append((os.path.join(CEFPYTHON3_DIR, filename), cefdatadir)) + + if is_darwin: + # "Chromium Embedded Framework.framework/Resources" with subdirectories + # is required. Contain .pak files and locales (each locale in separate + # subdirectory). + resources_subdir = \ + os.path.join("Chromium Embedded Framework.framework", "Resources") + base_path = os.path.join(CEFPYTHON3_DIR, resources_subdir) + assert os.path.exists(base_path), \ + "{} dir not found in cefpython3".format(resources_subdir) + for path, dirs, files in os.walk(base_path): + for file in files: + absolute_file_path = os.path.join(path, file) + dest_path = os.path.relpath(path, CEFPYTHON3_DIR) + ret.append((absolute_file_path, dest_path)) + logger.info("Include cefpython3 data: {}/{}".format(dest_path, file)) + elif is_win or is_linux: + # The .pak files in cefpython3/locales/ directory + locales_dir = os.path.join(CEFPYTHON3_DIR, "locales") + assert os.path.exists(locales_dir), \ + "locales/ dir not found in cefpython3" + for filename in os.listdir(locales_dir): + logger.info("Include cefpython3 data: {}/{}".format( + os.path.basename(locales_dir), filename)) + ret.append((os.path.join(locales_dir, filename), + os.path.join(cefdatadir, "locales"))) + + # Optional .so/.dll files in cefpython3/swiftshader/ directory + swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader") + if os.path.isdir(swiftshader_dir): + for filename in os.listdir(swiftshader_dir): + logger.info("Include cefpython3 data: {}/{}".format( + os.path.basename(swiftshader_dir), filename)) + ret.append((os.path.join(swiftshader_dir, filename), + os.path.join(cefdatadir, "swiftshader"))) + return ret + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- + +# Checks +check_platforms() +check_pyinstaller_version() +check_cefpython3_version() + +# Info +logger.info("CEF Python package directory: %s" % CEFPYTHON3_DIR) + +# Hidden imports. +# PyInstaller has no way on detecting imports made by Cython +# modules, so all pure Python imports made in cefpython .pyx +# files need to be manually entered here. +# TODO: Write a tool script that would find such imports in +# .pyx files automatically. +hiddenimports = [ + "codecs", + "copy", + "datetime", + "inspect", + "json", + "os", + "platform", + "random", + "re", + "sys", + "time", + "traceback", + "types", + "urllib", + "weakref", +] +if is_py2: + hiddenimports += [ + "urlparse", + ] + +# Excluded modules +excludedimports = get_excluded_cefpython_modules() + +# Include binaries requiring to collect its dependencies +if is_darwin or is_linux: + binaries = [(os.path.join(CEFPYTHON3_DIR, "subprocess"), ".")] +elif is_win: + binaries = [(os.path.join(CEFPYTHON3_DIR, "subprocess.exe"), ".")] +else: + binaries = [] + +# Include datas +datas = get_cefpython3_datas() + +# Notify pyinstaller.spec code that this hook was executed +# and that it succeeded. +os.environ["PYINSTALLER_CEFPYTHON3_HOOK_SUCCEEDED"] = "1" diff --git a/server/gui/images/cxg_icons.icns b/server/gui/images/cxg_icons.icns new file mode 100644 index 00000000..d57d20cf Binary files /dev/null and b/server/gui/images/cxg_icons.icns differ diff --git a/server/gui/images/icon.ico b/server/gui/images/icon.ico new file mode 100644 index 00000000..cea1f139 Binary files /dev/null and b/server/gui/images/icon.ico differ diff --git a/server/gui/main.py b/server/gui/main.py index 4aedf8af..72f3e2a7 100644 --- a/server/gui/main.py +++ b/server/gui/main.py @@ -1,6 +1,6 @@ # flake8: noqa F403, F405 from functools import partialmethod -from multiprocessing import Pipe, Process +from multiprocessing import Pipe, Process, freeze_support from os import environ from os.path import splitext, basename, dirname, join import sys @@ -172,7 +172,7 @@ class LoadWidget(QFrame): # UI section # TODO add cancel button to send back to browser (if available) - self.file_area = FileArea() + self.file_area = FileArea(self) self.file_name.signals.changed.connect(self.updatePath) self.launch_widget = QLabel("Select a file to launch cellxgene") @@ -301,7 +301,7 @@ class FilePath(QObject): class FileArea(QFrame): - def __init__(self): + def __init__(self, parent): super(FileArea, self).__init__() self.setFrameShape(QFrame.Box) self.setMinimumHeight(100) @@ -357,6 +357,7 @@ class FileArea(QFrame): def main(): + freeze_support() # This generates an error.log file on error sys.excepthook = cef.ExceptHook # To shutdown all CEF processes on error settings = {} diff --git a/server/requirements-gui.txt b/server/requirements-gui.txt new file mode 100644 index 00000000..352eb4da --- /dev/null +++ b/server/requirements-gui.txt @@ -0,0 +1,4 @@ +cefpython3>=66 +requests +PyInstaller>=3.4 +PySide2>=5.12.3