Integrate Stable Diffusion with the Imaginate layer (#784)

* Add AI Artist layer

* WIP add a button to download the rendered folder under an AI Artist layer

* Successfully download the correct image

* Break out image downloading JS into helper function

* Change file download from using data URLs to blob URLs

* WIP rasterize to blob

* Remove dimensions from AI Artist layer

* Successfully draw rasterized image on layer after calculation

* Working txt2img generation based on user prompt

* Add img2img and the main parameters

* Fix ability to rasterize multi-depth documents with blob URL images by switching them to base64

* Fix test

* Rasterize with artboard background color

* Allow aspect ratio stretch of AI Artist images

* Add automatic resolution choosing

* Add a terminate button, and make the lifecycle more robust

* Add negative prompt

* Add range bounds for parameter inputs

* Add seed

* Add tiling and restore faces

* Add server status check, server hostname customization, and resizing layer to fit AI Artist resolution

* Fix background color of infinite canvas rasterization

* Escape prompt text sent in the JSON

* Revoke blob URLs when cleared/replaced to reduce memory leak

* Fix welcome screen logo color

* Add PreferencesMessageHandler

* Add persistent storage of preferences

* Fix crash introduced in previous commit when moving mouse on page load

* Add tooltips to the AI Artist layer properties

* Integrate AI Artist tool into the raster section of the tool shelf

* Add a refresh button to the connection status

* Fix crash when generating and switching to a different document tab

* Add persistent image storage to AI Artist layers and fix duplication bugs

* Add a generate with random seed button

* Simplify and standardize message names

* Majorly improve robustness of networking code

* Fix race condition causing default server hostname to show disconnected when app loads with AI Artist layer selected (probably, not confirmed fixed)

* Clean up messages and function calls by changing arguments into structs

* Update API to more recent server commit

* Add support for picking the sampling method

* Add machinery for filtering selected layers with type

* Replace placeholder button icons

* Improve the random icon by tilting the dice

* Use selected_layers() instead of repeating that code

* Fix borrow error

* Change message flow in progress towards fixing #797

* Allow loading image on non-active document (fixes #797)

* Reduce code duplication with rasterization

* Add AI Artist tool and layer icons, and remove ugly node layer icon style

* Rename "AI Artist" codename to "Imaginate" feature name

Co-authored-by: otdavies <oliver@psyfer.io>
Co-authored-by: 0hypercube <0hypercube@gmail.com>

frontend/src/state-providers/portfolio.ts

@@ -2,6 +2,7 @@
 import { reactive, readonly } from "vue";
 
 import { downloadFileText, downloadFileBlob, upload } from "@/utility-functions/files";
+import { imaginateGenerate, imaginateCheckConnection, imaginateTerminate } from "@/utility-functions/imaginate";
 import { rasterizeSVG } from "@/utility-functions/rasterization";
 import { type Editor } from "@/wasm-communication/editor";
 import {
@@ -10,8 +11,13 @@ import {
 	TriggerImport,
 	TriggerOpenDocument,
 	TriggerRasterDownload,
+	TriggerImaginateGenerate,
+	TriggerImaginateTerminate,
+	TriggerImaginateCheckServerStatus,
 	UpdateActiveDocument,
 	UpdateOpenDocumentsList,
+	UpdateImageData,
+	TriggerRevokeBlobUrl,
 } from "@/wasm-communication/messages";
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
@@ -55,6 +61,47 @@ export function createPortfolioState(editor: Editor) {
 		// Have the browser download the file to the user's disk
 		downloadFileBlob(name, blob);
 	});
+	editor.subscriptions.subscribeJsMessage(TriggerImaginateCheckServerStatus, async (triggerImaginateCheckServerStatus) => {
+		const { hostname } = triggerImaginateCheckServerStatus;
+
+		imaginateCheckConnection(hostname, editor);
+	});
+	editor.subscriptions.subscribeJsMessage(TriggerImaginateGenerate, async (triggerImaginateGenerate) => {
+		const { documentId, layerPath, hostname, refreshFrequency, baseImage, parameters } = triggerImaginateGenerate;
+
+		// Handle img2img mode
+		let image: Blob | undefined;
+		if (parameters.denoisingStrength !== undefined && baseImage !== undefined) {
+			// Rasterize the SVG to an image file
+			image = await rasterizeSVG(baseImage.svg, baseImage.size[0], baseImage.size[1], "image/png");
+
+			const blobURL = URL.createObjectURL(image);
+
+			editor.instance.setImaginateBlobURL(documentId, layerPath, blobURL, baseImage.size[0], baseImage.size[1]);
+		}
+
+		imaginateGenerate(parameters, image, hostname, refreshFrequency, documentId, layerPath, editor);
+	});
+	editor.subscriptions.subscribeJsMessage(TriggerImaginateTerminate, async (triggerImaginateTerminate) => {
+		const { documentId, layerPath, hostname } = triggerImaginateTerminate;
+
+		imaginateTerminate(hostname, documentId, layerPath, editor);
+	});
+	editor.subscriptions.subscribeJsMessage(UpdateImageData, (updateImageData) => {
+		updateImageData.imageData.forEach(async (element) => {
+			const buffer = new Uint8Array(element.imageData.values()).buffer;
+			const blob = new Blob([buffer], { type: element.mime });
+
+			const blobURL = URL.createObjectURL(blob);
+
+			const image = await createImageBitmap(blob);
+
+			editor.instance.setImageBlobURL(updateImageData.documentId, element.path, blobURL, image.width, image.height);
+		});
+	});
+	editor.subscriptions.subscribeJsMessage(TriggerRevokeBlobUrl, async (triggerRevokeBlobUrl) => {
+		URL.revokeObjectURL(triggerRevokeBlobUrl.url);
+	});
 
 	return {
 		state: readonly(state) as typeof state,