Hooks for sentry integration (#1460)

* add sentry webpack plugin

* allow override of webpack config

* work around cheerio inability to parse jinga templates

* webpack can not minify jinja templates

* allow script injection to specify other attributes

* allow script injection to specify other attributes

* Adjustments to make plugin systems work

* Add sourcemaps for javascript in prod webpack

* Update .gitignore

* Fix spelling errors

Co-authored-by: Matt Weiden <538456+mweiden@users.noreply.github.com>

.gitignore

@@ -24,9 +24,14 @@ server/common/web/static/img/
 server/common/web/static/media/
 server/common/web/static/fonts/
 server/common/web/static/js/
-server/common/web/templates/index\.html
+server/common/web/templates/
 server/common/web/csp-hashes.json
 
+# eb build
+artifact.dir
+artifact.zip
+customize
+
 # Jupyter Notebook
 .ipynb_checkpoints
 *.ipynb

client/Makefile

@@ -1,8 +1,4 @@
 include ../common.mk
-# https://stackoverflow.com/a/14777895/9587410
-ifeq ($(shell uname),Darwin)     # is Windows_NT on XP, 2000, 7, Vista, 10...
-    IS_DARWIN := "true"
-endif
 
 ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
@@ -21,8 +17,9 @@ install:
 	npm install
 
 .PHONY: build
+WEBPACK_CONFIG ?= configuration/webpack/webpack.config.prod.js
 build:
-	npm run build
+	npm run build $(WEBPACK_CONFIG)
 
 # Formatting code
 

client/configuration/webpack/cspHashPlugin.js

@@ -32,17 +32,15 @@ class CspHashPlugin {
             };
           }
 
-          // remove no-csp-hash attributes
+          // Remove no-csp-hash attributes. Cheerio does not parse Jinja templates
-          let foundOne = false;
+          // correctly, so we brute force this with a regular expression.
-          $("script[no-csp-hash]").each((i, elmt) => {
+          data.html = data.html
-            $(elmt).removeAttr("no-csp-hash");
+            .replace(/(<script .*)no-csp-hash(.*>)/, (match, p1, p2) =>
-            foundOne = true;
+              [p1, p2].join("")
-          });
+            )
-          $("style[no-csp-hash]").each((i, elmt) => {
+            .replace(/(<style .*)no-csp-hash(.*>)/, (match, p1, p2) =>
-            $(elmt).removeAttr("no-csp-hash");
+              [p1, p2].join("")
-            foundOne = true;
+            );
-          });
-          if (foundOne) data.html = $.html();
 
           // Tell webpack to move on
           cb(null, data);

client/configuration/webpack/webpack.config.prod.js

@@ -2,7 +2,6 @@
 const path = require("path");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const HtmlWebpackInlineSourcePlugin = require("html-webpack-inline-source-plugin");
 const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const TerserJSPlugin = require("terser-webpack-plugin");
@@ -38,6 +37,7 @@ module.exports = {
       }),
     ],
   },
+  devtool: "source-map",
   module: {
     rules: [
       {
@@ -87,18 +87,7 @@ module.exports = {
       filename: "index.html",
       template: path.resolve("index_template.html"),
       decodeEntities: false,
-      minify: {
+      minify: false,
-        removeComments: true,
-        collapseWhitespace: true,
-        removeRedundantAttributes: true,
-        useShortDoctype: true,
-        removeEmptyAttributes: true,
-        removeStyleLinkTypeAttributes: true,
-        keepClosingSlash: true,
-        minifyJS: true,
-        minifyCSS: true,
-        minifyURLs: true,
-      },
     }),
     new CleanWebpackPlugin({
       verbose: true,

client/index_template.html

@@ -50,12 +50,15 @@
     </noscript>
 
     <div id="root"></div>
-    {% for script in SCRIPTS %}
+    {% for script in SCRIPTS -%}
-    <script type="text/javascript" src="{{script | safe}}"></script>
+    <script type="text/javascript"
-    {% endfor %}
+      {{ ('integrity="%s"' % script.integrity) | safe if script.integrity }}
+      {{ ('crossorigin="%s"' % script.crossorigin) | safe if script.crossorigin }}
+      src="{{script.src | safe}}"
+    ></script>
+    {%- endfor %}
 
     {% for ils in INLINE_SCRIPTS %}
-    <!-- caution: do not change white space in this script element -->
     <script type="text/javascript" no-csp-hash>{% include ils %}</script>
     {% endfor %}
   </body>

client/package-lock.json

@@ -2547,6 +2547,49 @@
         }
       }
     },
+    "@sentry/cli": {
+      "version": "1.52.3",
+      "resolved": "https://registry.npmjs.org/@sentry/cli/-/cli-1.52.3.tgz",
+      "integrity": "sha512-QOSIg5hxAEa6v6H7oEeF6A/Rpa0wloMhbu0Qed6zHv3lyoqf0Z34Kq2jCXdqGsOE3IzkO+3CNy81F6361j5TKg==",
+      "dev": true,
+      "requires": {
+        "https-proxy-agent": "^5.0.0",
+        "mkdirp": "^0.5.5",
+        "node-fetch": "^2.6.0",
+        "progress": "^2.0.3",
+        "proxy-from-env": "^1.1.0"
+      },
+      "dependencies": {
+        "https-proxy-agent": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
+          "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
+          "dev": true,
+          "requires": {
+            "agent-base": "6",
+            "debug": "4"
+          }
+        },
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        }
+      }
+    },
+    "@sentry/webpack-plugin": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@sentry/webpack-plugin/-/webpack-plugin-1.11.1.tgz",
+      "integrity": "sha512-7uQiqHXK84GYy2mL/c+cW/QBtr/D8fNmf8U5nUDN4+44aNrQxSJ4vJmYJbxYg1zO6qN/xZyqJt3uKBlgA3FkZg==",
+      "dev": true,
+      "requires": {
+        "@sentry/cli": "^1.52.3"
+      }
+    },
     "@sinonjs/commons": {
       "version": "1.7.1",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz",

client/package.json

@@ -6,8 +6,9 @@
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
     "clean": "rimraf build",
-    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
+    "build": "npm run clean && webpack --config",
-    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
+    "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
+    "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
     "fmt": "eslint --fix src",
     "lint": "eslint src"
   },
@@ -62,6 +63,7 @@
     "@babel/preset-react": "^7.9.4",
     "@babel/register": "^7.9.0",
     "@babel/runtime": "^7.9.2",
+    "@sentry/webpack-plugin": "^1.11.1",
     "babel-eslint": "^10.1.0",
     "babel-jest": "^25.2.6",
     "babel-loader": "^8.1.0",

common.mk

@@ -10,6 +10,12 @@ define get_or_else_dev_env_default
 $(if $($(1)),$($(1)),$(shell VAR=$$(sed -n 's/$(1)=\(.*\)/\1/p' $(PROJECT_ROOT)/environment.default); eval "echo \"$$VAR\""))
 endef
 
+# https://stackoverflow.com/a/14777895/9587410
+ifeq ($(shell uname),Darwin)     # is Windows_NT on XP, 2000, 7, Vista, 10...
+    IS_DARWIN := "true"
+endif
+
+export CELLXGENE_COMMIT := $(shell git rev-parse --short HEAD)
 export CXG_SERVER_PORT := $(call get_or_else_dev_env_default,CXG_SERVER_PORT)
 export CXG_CLIENT_PORT := $(call get_or_else_dev_env_default,CXG_CLIENT_PORT)
 export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV)

server/common/app_config.py

@@ -287,6 +287,17 @@ class AppConfig(object):
                 elif not isinstance(v, str):
                     raise ConfigurationError(f"CSP directive value must be a string or list of strings.")
 
+        # scripts can be string (filename) or dict (attributes).   Convert string to dict.
+        scripts = []
+        for s in self.server__scripts:
+            if isinstance(s, str):
+                scripts.append({"src": s})
+            elif isinstance(s, dict) and isinstance(s["src"], str):
+                scripts.append(s)
+            else:
+                raise ConfigurationError("Scripts must be string or dict")
+        self.server__scripts = scripts
+
     def handle_data_locator(self, context):
         self.__check_attr("data_locator__s3__region_name", (type(None), bool, str))
         if self.data_locator__s3__region_name is True:

server/common/default_config.py

@@ -8,8 +8,13 @@ server:
   debug: false
   host: "127.0.0.1"
   port : null
-  scripts : []
+
+  # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin.
+  # these will be injected into the index template as script tags with these attributes set.
+  scripts: []
+  # Inline scripts are a list of file names, where the contents of the file will be injected into the index.
   inline_scripts: []
+
   open_browser: false
   about_legal_tos: null
   about_legal_privacy: null

server/eb/Makefile

@@ -28,7 +28,7 @@ build: clean
 	  if [ -f customize/requirements.txt ] ; then \
 	     pip install requirements-parser ; \
 	     pip install packaging ; \
-	     python check_requirements.py ../requirements.txt customize/requirements.txt; \
+	     python3 check_requirements.py ../requirements.txt customize/requirements.txt; \
 	     cp customize/requirements.txt artifact.dir;  \
 	  fi ; \
 	  if [ -d customize/deploy ] ; then \
@@ -36,6 +36,7 @@ build: clean
 	  fi; \
 	  if [ -d customize/inline_scripts ] ; then \
 	      cp -r customize/inline_scripts/* artifact.dir/server/common/web/templates;  \
+	      find artifact.dir/server/common/web/templates/ -type f -exec sed -i$(if $(IS_DARWIN), '',) 's/__CELLXGENE_COMMIT__/$(CELLXGENE_COMMIT)/g' {} \;; \
 	  fi; \
 	  if [ -d customize/ebextensions ] ; then \
 	      cp -r customize/ebextensions/* artifact.dir/.ebextensions;  \
@@ -43,6 +44,7 @@ build: clean
 	fi; \
 	if [ -d customize/plugins ] ; then \
 	  cp -r customize/plugins artifact.dir/server/;  \
+	  find artifact.dir/server/plugins/ -type f -exec sed -i$(if $(IS_DARWIN), '',) 's/__CELLXGENE_COMMIT__/$(CELLXGENE_COMMIT)/g' {} \;; \
 	fi; \
 	(cd artifact.dir; \
 	  cp -r server/common/web/static static; \

server/eb/app.py

@@ -60,6 +60,7 @@ class WSGIServer(Server):
         script_hashes, style_hashes = WSGIServer.get_csp_hashes(app, app_config)
         csp = {
             "default-src": ["'self'"],
+            "connect-src": ["'self'"],
             "script-src": ["'self'", "'unsafe-eval'", "'unsafe-inline'"] + script_hashes,
             "style-src": ["'self'", "'unsafe-inline'"] + style_hashes,
             "img-src": ["'self'", "data:"],