Integrate Stable Diffusion with the Imaginate layer (#784)

* Add AI Artist layer

* WIP add a button to download the rendered folder under an AI Artist layer

* Successfully download the correct image

* Break out image downloading JS into helper function

* Change file download from using data URLs to blob URLs

* WIP rasterize to blob

* Remove dimensions from AI Artist layer

* Successfully draw rasterized image on layer after calculation

* Working txt2img generation based on user prompt

* Add img2img and the main parameters

* Fix ability to rasterize multi-depth documents with blob URL images by switching them to base64

* Fix test

* Rasterize with artboard background color

* Allow aspect ratio stretch of AI Artist images

* Add automatic resolution choosing

* Add a terminate button, and make the lifecycle more robust

* Add negative prompt

* Add range bounds for parameter inputs

* Add seed

* Add tiling and restore faces

* Add server status check, server hostname customization, and resizing layer to fit AI Artist resolution

* Fix background color of infinite canvas rasterization

* Escape prompt text sent in the JSON

* Revoke blob URLs when cleared/replaced to reduce memory leak

* Fix welcome screen logo color

* Add PreferencesMessageHandler

* Add persistent storage of preferences

* Fix crash introduced in previous commit when moving mouse on page load

* Add tooltips to the AI Artist layer properties

* Integrate AI Artist tool into the raster section of the tool shelf

* Add a refresh button to the connection status

* Fix crash when generating and switching to a different document tab

* Add persistent image storage to AI Artist layers and fix duplication bugs

* Add a generate with random seed button

* Simplify and standardize message names

* Majorly improve robustness of networking code

* Fix race condition causing default server hostname to show disconnected when app loads with AI Artist layer selected (probably, not confirmed fixed)

* Clean up messages and function calls by changing arguments into structs

* Update API to more recent server commit

* Add support for picking the sampling method

* Add machinery for filtering selected layers with type

* Replace placeholder button icons

* Improve the random icon by tilting the dice

* Use selected_layers() instead of repeating that code

* Fix borrow error

* Change message flow in progress towards fixing #797

* Allow loading image on non-active document (fixes #797)

* Reduce code duplication with rasterization

* Add AI Artist tool and layer icons, and remove ugly node layer icon style

* Rename "AI Artist" codename to "Imaginate" feature name

Co-authored-by: otdavies <oliver@psyfer.io>
Co-authored-by: 0hypercube <0hypercube@gmail.com>

frontend/src/utility-functions/escape.ts

@@ -0,0 +1,14 @@
+/* eslint-disable no-useless-escape */
+/* eslint-disable quotes */
+
+export function escapeJSON(str: string): string {
+	return str
+		.replace(/[\\]/g, "\\\\")
+		.replace(/[\"]/g, '\\"')
+		.replace(/[\/]/g, "\\/")
+		.replace(/[\b]/g, "\\b")
+		.replace(/[\f]/g, "\\f")
+		.replace(/[\n]/g, "\\n")
+		.replace(/[\r]/g, "\\r")
+		.replace(/[\t]/g, "\\t");
+}

frontend/src/utility-functions/files.ts

@@ -52,3 +52,31 @@ export async function upload<T extends "text" | "data">(acceptedExtensions: stri
 }
 export type UploadResult<T> = { filename: string; type: string; content: UploadResultType<T> };
 type UploadResultType<T> = T extends "text" ? string : T extends "data" ? Uint8Array : never;
+
+export function blobToBase64(blob: Blob): Promise<string> {
+	return new Promise((resolve) => {
+		const reader = new FileReader();
+		reader.onloadend = (): void => resolve(typeof reader.result === "string" ? reader.result : "");
+		reader.readAsDataURL(blob);
+	});
+}
+
+export async function replaceBlobURLsWithBase64(svg: string): Promise<string> {
+	const splitByBlobs = svg.split(/(?<=")(blob:.*?)(?=")/);
+	const onlyBlobs = splitByBlobs.filter((_, i) => i % 2 === 1);
+
+	const onlyBlobsConverted = onlyBlobs.map(async (blobURL) => {
+		const data = await fetch(blobURL);
+		const dataBlob = await data.blob();
+		return blobToBase64(dataBlob);
+	});
+	const base64Images = await Promise.all(onlyBlobsConverted);
+
+	const substituted = splitByBlobs.map((segment, i) => {
+		if (i % 2 === 0) return segment;
+
+		const blobsIndex = Math.floor(i / 2);
+		return base64Images[blobsIndex];
+	});
+	return substituted.join("");
+}

frontend/src/utility-functions/icons.ts

@@ -101,6 +101,7 @@ import NodeColorCorrection from "@/../assets/icon-16px-solid/node-color-correcti
 import NodeFolder from "@/../assets/icon-16px-solid/node-folder.svg";
 import NodeGradient from "@/../assets/icon-16px-solid/node-gradient.svg";
 import NodeImage from "@/../assets/icon-16px-solid/node-image.svg";
+import NodeImaginate from "@/../assets/icon-16px-solid/node-imaginate.svg";
 import NodeMagicWand from "@/../assets/icon-16px-solid/node-magic-wand.svg";
 import NodeMask from "@/../assets/icon-16px-solid/node-mask.svg";
 import NodeMotionBlur from "@/../assets/icon-16px-solid/node-motion-blur.svg";
@@ -109,6 +110,12 @@ import NodeShape from "@/../assets/icon-16px-solid/node-shape.svg";
 import NodeText from "@/../assets/icon-16px-solid/node-text.svg";
 import NodeTransform from "@/../assets/icon-16px-solid/node-transform.svg";
 import Paste from "@/../assets/icon-16px-solid/paste.svg";
+import Random from "@/../assets/icon-16px-solid/random.svg";
+import Regenerate from "@/../assets/icon-16px-solid/regenerate.svg";
+import Reload from "@/../assets/icon-16px-solid/reload.svg";
+import Rescale from "@/../assets/icon-16px-solid/rescale.svg";
+import Reset from "@/../assets/icon-16px-solid/reset.svg";
+import Settings from "@/../assets/icon-16px-solid/settings.svg";
 import Trash from "@/../assets/icon-16px-solid/trash.svg";
 import ViewModeNormal from "@/../assets/icon-16px-solid/view-mode-normal.svg";
 import ViewModeOutline from "@/../assets/icon-16px-solid/view-mode-outline.svg";
@@ -142,6 +149,7 @@ const SOLID_16PX = {
 	FlipVertical: { component: FlipVertical, size: 16 },
 	Folder: { component: Folder, size: 16 },
 	GraphiteLogo: { component: GraphiteLogo, size: 16 },
+	NodeImaginate: { component: NodeImaginate, size: 16 },
 	NodeArtboard: { component: NodeArtboard, size: 16 },
 	NodeBlur: { component: NodeBlur, size: 16 },
 	NodeBrushwork: { component: NodeBrushwork, size: 16 },
@@ -157,6 +165,12 @@ const SOLID_16PX = {
 	NodeText: { component: NodeText, size: 16 },
 	NodeTransform: { component: NodeTransform, size: 16 },
 	Paste: { component: Paste, size: 16 },
+	Random: { component: Random, size: 16 },
+	Regenerate: { component: Regenerate, size: 16 },
+	Reload: { component: Reload, size: 16 },
+	Rescale: { component: Rescale, size: 16 },
+	Reset: { component: Reset, size: 16 },
+	Settings: { component: Settings, size: 16 },
 	Trash: { component: Trash, size: 16 },
 	ViewModeNormal: { component: ViewModeNormal, size: 16 },
 	ViewModeOutline: { component: ViewModeOutline, size: 16 },
@@ -205,6 +219,7 @@ import RasterBrushTool from "@/../assets/icon-24px-two-tone/raster-brush-tool.sv
 import RasterCloneTool from "@/../assets/icon-24px-two-tone/raster-clone-tool.svg";
 import RasterDetailTool from "@/../assets/icon-24px-two-tone/raster-detail-tool.svg";
 import RasterHealTool from "@/../assets/icon-24px-two-tone/raster-heal-tool.svg";
+import RasterImaginateTool from "@/../assets/icon-24px-two-tone/raster-imaginate-tool.svg";
 import RasterPatchTool from "@/../assets/icon-24px-two-tone/raster-patch-tool.svg";
 import RasterRelightTool from "@/../assets/icon-24px-two-tone/raster-relight-tool.svg";
 import VectorEllipseTool from "@/../assets/icon-24px-two-tone/vector-ellipse-tool.svg";
@@ -220,10 +235,11 @@ import VectorTextTool from "@/../assets/icon-24px-two-tone/vector-text-tool.svg"
 const TWO_TONE_24PX = {
 	GeneralArtboardTool: { component: GeneralArtboardTool, size: 24 },
 	GeneralEyedropperTool: { component: GeneralEyedropperTool, size: 24 },
-	GeneralNavigateTool: { component: GeneralNavigateTool, size: 24 },
-	GeneralSelectTool: { component: GeneralSelectTool, size: 24 },
 	GeneralFillTool: { component: GeneralFillTool, size: 24 },
 	GeneralGradientTool: { component: GeneralGradientTool, size: 24 },
+	GeneralNavigateTool: { component: GeneralNavigateTool, size: 24 },
+	GeneralSelectTool: { component: GeneralSelectTool, size: 24 },
+	RasterImaginateTool: { component: RasterImaginateTool, size: 24 },
 	RasterBrushTool: { component: RasterBrushTool, size: 24 },
 	RasterCloneTool: { component: RasterCloneTool, size: 24 },
 	RasterDetailTool: { component: RasterDetailTool, size: 24 },
@@ -256,7 +272,6 @@ export const ICON_COMPONENTS = Object.fromEntries(Object.entries(ICONS).map(([na
 
 export type IconName = keyof typeof ICONS;
 export type IconSize = undefined | 12 | 16 | 24 | 32;
-export type IconStyle = "Normal" | "Node";
 
 // The following helper type declarations allow us to avoid manually maintaining the `IconName` type declaration as a string union paralleling the keys of the
 // icon definitions. It lets TypeScript do that for us. Our goal is to define the big key-value pair of icons by constraining its values, but inferring its keys.

frontend/src/utility-functions/imaginate.ts

@@ -0,0 +1,439 @@
+import { escapeJSON } from "@/utility-functions/escape";
+import { blobToBase64 } from "@/utility-functions/files";
+import { type RequestResult, requestWithUploadDownloadProgress } from "@/utility-functions/network";
+import { stripIndents } from "@/utility-functions/strip-indents";
+import { type Editor } from "@/wasm-communication/editor";
+import { type ImaginateGenerationParameters } from "@/wasm-communication/messages";
+
+const MAX_POLLING_RETRIES = 4;
+const SERVER_STATUS_CHECK_TIMEOUT = 5000;
+const SAMPLING_MODES_POLLING_UNSUPPORTED = ["DPM fast", "DPM adaptive"];
+
+let timer: NodeJS.Timeout | undefined;
+let terminated = false;
+
+let generatingAbortRequest: XMLHttpRequest | undefined;
+let pollingAbortController = new AbortController();
+let statusAbortController = new AbortController();
+
+// PUBLICLY CALLABLE FUNCTIONS
+
+export async function imaginateGenerate(
+	parameters: ImaginateGenerationParameters,
+	image: Blob | undefined,
+	hostname: string,
+	refreshFrequency: number,
+	documentId: bigint,
+	layerPath: BigUint64Array,
+	editor: Editor
+): Promise<void> {
+	// Ignore a request to generate a new image while another is already being generated
+	if (generatingAbortRequest !== undefined) return;
+
+	terminated = false;
+
+	// Immediately set the progress to 0% so the backend knows to update its layout
+	editor.instance.setImaginateGeneratingStatus(documentId, layerPath, 0, "Beginning");
+
+	// Initiate a request to the computation server
+	const discloseUploadingProgress = (progress: number): void => {
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, progress * 100, "Uploading");
+	};
+	const { uploaded, result, xhr } = await generate(discloseUploadingProgress, hostname, image, parameters);
+	generatingAbortRequest = xhr;
+
+	try {
+		// Wait until the request is fully uploaded, which could be slow if the img2img source is large and the user is on a slow connection
+		await uploaded;
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, 0, "Generating");
+
+		// Begin polling for updates to the in-progress image generation at the specified interval
+		// Don't poll if the chosen interval is 0, or if the chosen sampling method does not support polling
+		if (refreshFrequency > 0 && !SAMPLING_MODES_POLLING_UNSUPPORTED.includes(parameters.samplingMethod)) {
+			const interval = Math.max(refreshFrequency * 1000, 500);
+			scheduleNextPollingUpdate(interval, Date.now(), 0, editor, hostname, documentId, layerPath, parameters.resolution);
+		}
+
+		// Wait for the final image to be returned by the initial request containing either the full image or the last frame if it was terminated by the user
+		const { body, status } = await result;
+		if (status < 200 || status > 299) {
+			throw new Error(`Request to server failed to return a 200-level status code (${status})`);
+		}
+
+		// Extract the final image from the response and convert it to a data blob
+		// Highly unstable API
+		const base64 = JSON.parse(body)?.data[0]?.[0] as string | undefined;
+		if (typeof base64 !== "string" || !base64.startsWith("data:image/png;base64,")) throw new Error("Could not read final image result from server response");
+		const blob = await (await fetch(base64)).blob();
+
+		// Send the backend an updated status
+		const percent = terminated ? undefined : 100;
+		const newStatus = terminated ? "Terminated" : "Idle";
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, percent, newStatus);
+
+		// Send the backend a blob URL for the final image
+		const blobURL = URL.createObjectURL(blob);
+		editor.instance.setImaginateBlobURL(documentId, layerPath, blobURL, parameters.resolution[0], parameters.resolution[1]);
+
+		// Send the backend the blob data to be stored persistently in the layer
+		const u8Array = new Uint8Array(await blob.arrayBuffer());
+		editor.instance.setImaginateImageData(documentId, layerPath, u8Array);
+	} catch {
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, undefined, "Terminated");
+
+		await imaginateCheckConnection(hostname, editor);
+	}
+
+	abortAndResetGenerating();
+	abortAndResetPolling();
+}
+
+export async function imaginateTerminate(hostname: string, documentId: bigint, layerPath: BigUint64Array, editor: Editor): Promise<void> {
+	terminated = true;
+	abortAndResetPolling();
+
+	try {
+		await terminate(hostname);
+
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, undefined, "Terminating");
+	} catch {
+		abortAndResetGenerating();
+		abortAndResetPolling();
+
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, undefined, "Terminated");
+
+		await imaginateCheckConnection(hostname, editor);
+	}
+}
+
+export async function imaginateCheckConnection(hostname: string, editor: Editor): Promise<void> {
+	const serverReached = await checkConnection(hostname);
+	editor.instance.setImaginateServerStatus(serverReached);
+}
+
+// ABORTING AND RESETTING HELPERS
+
+function abortAndResetGenerating(): void {
+	generatingAbortRequest?.abort();
+	generatingAbortRequest = undefined;
+}
+
+function abortAndResetPolling(): void {
+	pollingAbortController.abort();
+	pollingAbortController = new AbortController();
+	clearTimeout(timer);
+}
+
+// POLLING IMPLEMENTATION DETAILS
+
+function scheduleNextPollingUpdate(
+	interval: number,
+	timeoutBegan: number,
+	pollingRetries: number,
+	editor: Editor,
+	hostname: string,
+	documentId: bigint,
+	layerPath: BigUint64Array,
+	resolution: [number, number]
+): void {
+	// Pick a future time that keeps to the user-requested interval if possible, but on slower connections will go as fast as possible without overlapping itself
+	const nextPollTimeGoal = timeoutBegan + interval;
+	const timeFromNow = Math.max(0, nextPollTimeGoal - Date.now());
+
+	timer = setTimeout(async () => {
+		const nextTimeoutBegan = Date.now();
+
+		try {
+			const [blob, percentComplete] = await pollImage(hostname);
+			if (terminated) return;
+
+			const blobURL = URL.createObjectURL(blob);
+			editor.instance.setImaginateBlobURL(documentId, layerPath, blobURL, resolution[0], resolution[1]);
+			editor.instance.setImaginateGeneratingStatus(documentId, layerPath, percentComplete, "Generating");
+
+			scheduleNextPollingUpdate(interval, nextTimeoutBegan, 0, editor, hostname, documentId, layerPath, resolution);
+		} catch {
+			if (generatingAbortRequest === undefined) return;
+
+			if (pollingRetries + 1 > MAX_POLLING_RETRIES) {
+				abortAndResetGenerating();
+				abortAndResetPolling();
+
+				await imaginateCheckConnection(hostname, editor);
+			} else {
+				scheduleNextPollingUpdate(interval, nextTimeoutBegan, pollingRetries + 1, editor, hostname, documentId, layerPath, resolution);
+			}
+		}
+	}, timeFromNow);
+}
+
+// API COMMUNICATION FUNCTIONS
+// These are highly unstable APIs that will need to be updated very frequently, so we currently assume usage of this exact commit from the server:
+// https://github.com/AUTOMATIC1111/stable-diffusion-webui/commit/7d6042b908c064774ee10961309d396eabdc6c4a
+
+function endpoint(hostname: string): string {
+	// Highly unstable API
+	return `${hostname}api/predict/`;
+}
+
+async function pollImage(hostname: string): Promise<[Blob, number]> {
+	// Highly unstable API
+	const result = await fetch(endpoint(hostname), {
+		signal: pollingAbortController.signal,
+		headers: {
+			accept: "*/*",
+			"accept-language": "en-US,en;q=0.9",
+			"content-type": "application/json",
+		},
+		referrer: hostname,
+		referrerPolicy: "strict-origin-when-cross-origin",
+		body: stripIndents`
+			{
+				"fn_index":3,
+				"data":[],
+				"session_hash":"0000000000"
+			}`,
+		method: "POST",
+		mode: "cors",
+		credentials: "omit",
+	});
+	const json = await result.json();
+	// Highly unstable API
+	const percentComplete = Math.abs(Number(json.data[0].match(/(?<="width:).*?(?=%")/)[0])); // The API sometimes returns negative values presumably due to a bug
+	// Highly unstable API
+	const base64 = json.data[2];
+
+	if (typeof base64 !== "string" || !base64.startsWith("data:image/png;base64,")) return Promise.reject();
+
+	const blob = await (await fetch(base64)).blob();
+
+	return [blob, percentComplete];
+}
+
+async function generate(
+	discloseUploadingProgress: (progress: number) => void,
+	hostname: string,
+	image: Blob | undefined,
+	parameters: ImaginateGenerationParameters
+): Promise<{
+	uploaded: Promise<void>;
+	result: Promise<RequestResult>;
+	xhr?: XMLHttpRequest;
+}> {
+	let body;
+	if (image === undefined || parameters.denoisingStrength === undefined) {
+		// Highly unstable API
+		body = stripIndents`
+		{
+			"fn_index":13,
+			"data":[
+				"${escapeJSON(parameters.prompt)}",
+				"${escapeJSON(parameters.negativePrompt)}",
+				"None",
+				"None",
+				${parameters.samples},
+				"${parameters.samplingMethod}",
+				${parameters.restoreFaces},
+				${parameters.tiling},
+				1,
+				1,
+				${parameters.cfgScale},
+				${parameters.seed},
+				-1,
+				0,
+				0,
+				0,
+				false,
+				${parameters.resolution[1]},
+				${parameters.resolution[0]},
+				false,
+				0.7,
+				0,
+				0,
+				"None",
+				false,
+				false,
+				null,
+				"",
+				"Seed",
+				"",
+				"Nothing",
+				"",
+				true,
+				false,
+				false,
+				null,
+				""
+			],
+			"session_hash":"0000000000"
+		}`;
+	} else {
+		const sourceImageBase64 = await blobToBase64(image);
+
+		// Highly unstable API
+		body = stripIndents`
+		{
+			"fn_index":33,
+			"data":[
+				0,
+				"${escapeJSON(parameters.prompt)}",
+				"${escapeJSON(parameters.negativePrompt)}",
+				"None",
+				"None",
+				"${sourceImageBase64}",
+				null,
+				null,
+				null,
+				"Draw mask",
+				${parameters.samples},
+				"${parameters.samplingMethod}",
+				4,
+				"fill",
+				${parameters.restoreFaces},
+				${parameters.tiling},
+				1,
+				1,
+				${parameters.cfgScale},
+				${parameters.denoisingStrength},
+				${parameters.seed},
+				-1,
+				0,
+				0,
+				0,
+				false,
+				${parameters.resolution[1]},
+				${parameters.resolution[0]},
+				"Just resize",
+				false,
+				32,
+				"Inpaint masked",
+				"",
+				"",
+				"None",
+				"",
+				true,
+				true,
+				"",
+				"",
+				true,
+				50,
+				true,
+				1,
+				0,
+				false,
+				4,
+				1,
+				"",
+				128,
+				8,
+				["left","right","up","down"],
+				1,
+				0.05,
+				128,
+				4,
+				"fill",
+				["left","right","up","down"],
+				false,
+				false,
+				null,
+				"",
+				"",
+				64,
+				"None",
+				"Seed",
+				"",
+				"Nothing",
+				"",
+				true,
+				false,
+				false,
+				null,
+				"",
+				""
+			],
+			"session_hash":"0000000000"
+		}`;
+	}
+
+	// Prepare a promise that will resolve after the outbound request upload is complete
+	let uploadedResolve: () => void;
+	let uploadedReject: () => void;
+	const uploaded = new Promise<void>((resolve, reject): void => {
+		uploadedResolve = resolve;
+		uploadedReject = reject;
+	});
+
+	// Fire off the request and, once the outbound request upload is complete, resolve the promise we defined above
+	const uploadProgress = (progress: number): void => {
+		if (progress < 1) {
+			discloseUploadingProgress(progress);
+		} else {
+			uploadedResolve();
+		}
+	};
+	const [result, xhr] = requestWithUploadDownloadProgress(endpoint(hostname), "POST", body, uploadProgress, abortAndResetPolling);
+	result.catch(() => uploadedReject());
+
+	// Return the promise that resolves when the request upload is complete, the promise that resolves when the response download is complete, and the XHR so it can be aborted
+	return { uploaded, result, xhr };
+}
+
+async function terminate(hostname: string): Promise<void> {
+	const body = stripIndents`
+		{
+			"fn_index":2,
+			"data":[],
+			"session_hash":"0000000000"
+		}`;
+
+	await fetch(endpoint(hostname), {
+		headers: {
+			accept: "*/*",
+			"accept-language": "en-US,en;q=0.9",
+			"content-type": "application/json",
+		},
+		referrer: hostname,
+		referrerPolicy: "strict-origin-when-cross-origin",
+		body,
+		method: "POST",
+		mode: "cors",
+		credentials: "omit",
+	});
+}
+
+async function checkConnection(hostname: string): Promise<boolean> {
+	statusAbortController.abort();
+	statusAbortController = new AbortController();
+
+	const timeout = setTimeout(() => statusAbortController.abort(), SERVER_STATUS_CHECK_TIMEOUT);
+
+	const body = stripIndents`
+		{
+			"fn_index":100,
+			"data":[],
+			"session_hash":"0000000000"
+		}`;
+
+	try {
+		await fetch(endpoint(hostname), {
+			signal: statusAbortController.signal,
+			headers: {
+				accept: "*/*",
+				"accept-language": "en-US,en;q=0.9",
+				"content-type": "application/json",
+			},
+			referrer: hostname,
+			referrerPolicy: "strict-origin-when-cross-origin",
+			body,
+			method: "POST",
+			mode: "cors",
+			credentials: "omit",
+		});
+
+		clearTimeout(timeout);
+
+		return true;
+	} catch (_) {
+		return false;
+	}
+}

frontend/src/utility-functions/network.ts

@@ -0,0 +1,33 @@
+export type RequestResult = { body: string; status: number };
+
+// Special implementation using the legacy XMLHttpRequest API that provides callbacks to get:
+// - Calls with the percent progress uploading the request to the server
+// - Calls when downloading the result from the server, after the server has begun streaming back the response data
+// It returns a tuple of the promise as well as the XHR which can be used to call the `.abort()` method on it.
+export function requestWithUploadDownloadProgress(
+	url: string,
+	method: "GET" | "HEAD" | "POST" | "PUT" | "DELETE" | "CONNECT" | "OPTIONS" | "TRACE" | "PATCH",
+	body: string,
+	uploadProgress: (progress: number) => void,
+	downloadOccurring: () => void
+): [Promise<RequestResult>, XMLHttpRequest | undefined] {
+	let xhrValue: XMLHttpRequest | undefined;
+	const promise = new Promise<RequestResult>((resolve, reject) => {
+		const xhr = new XMLHttpRequest();
+		xhr.upload.addEventListener("progress", (e) => uploadProgress(e.loaded / e.total));
+		xhr.addEventListener("progress", () => downloadOccurring());
+		xhr.addEventListener("load", () => resolve({ status: xhr.status, body: xhr.responseText }));
+		xhr.addEventListener("abort", () => resolve({ status: xhr.status, body: xhr.responseText }));
+		xhr.addEventListener("error", () => reject(new Error("Request error")));
+		xhr.open(method, url, true);
+		xhr.setRequestHeader("accept", "*/*");
+		xhr.setRequestHeader("accept-language", "en-US,en;q=0.9");
+		xhr.setRequestHeader("content-type", "application/json");
+
+		xhrValue = xhr;
+
+		xhr.send(body);
+	});
+
+	return [promise, xhrValue];
+}

frontend/src/utility-functions/rasterization.ts

@@ -1,5 +1,7 @@
+import { replaceBlobURLsWithBase64 } from "@/utility-functions/files";
+
 // Rasterize the string of an SVG document at a given width and height and turn it into the blob data of an image file matching the given MIME type
-export function rasterizeSVG(svg: string, width: number, height: number, mime: string, backgroundColor?: string): Promise<Blob> {
+export async function rasterizeSVG(svg: string, width: number, height: number, mime: string, backgroundColor?: string): Promise<Blob> {
 	let promiseResolve: (value: Blob | PromiseLike<Blob>) => void | undefined;
 	let promiseReject: () => void | undefined;
 	const promise = new Promise<Blob>((resolve, reject) => {
@@ -21,9 +23,12 @@ export function rasterizeSVG(svg: string, width: number, height: number, mime: s
 		context.fillRect(0, 0, width, height);
 	}
 
+	// This SVG rasterization scheme has the limitation that it cannot access blob URLs, so they must be inlined to base64 URLs
+	const svgWithBase64Images = await replaceBlobURLsWithBase64(svg);
+
 	// Create a blob URL for our SVG
 	const image = new Image();
-	const svgBlob = new Blob([svg], { type: "image/svg+xml;charset=utf-8" });
+	const svgBlob = new Blob([svgWithBase64Images], { type: "image/svg+xml;charset=utf-8" });
 	const url = URL.createObjectURL(svgBlob);
 	image.onload = (): void => {
 		// Draw our SVG to the canvas