Reimplement notice file generation for third-party licenses through Rust, now with CEF credits (#3808)

tools/third-party-licenses/.gitignore

@@ -0,0 +1 @@
+*.hash

tools/third-party-licenses/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "third-party-licenses"
+edition.workspace = true
+version.workspace = true
+license.workspace = true
+authors.workspace = true
+
+[features]
+desktop = ["dep:cef-dll-sys", "dep:scraper"]
+
+[dependencies]
+# Workspace dependencies
+serde = { workspace = true }
+serde_json = { workspace = true }
+lzma-rust2 = { workspace = true }
+
+# Optional workspace dependencies
+cef-dll-sys = { workspace = true, optional = true }
+scraper = { workspace = true, optional = true }

tools/third-party-licenses/build.rs

@@ -0,0 +1,15 @@
+fn main() {
+	println!("cargo:rerun-if-changed=src");
+
+	println!("cargo:rerun-if-env-changed=DEP_CEF_DLL_WRAPPER_CEF_DIR");
+	if let Ok(cef_dir) = std::env::var("DEP_CEF_DLL_WRAPPER_CEF_DIR") {
+		println!("cargo:rustc-env=CEF_PATH={cef_dir}");
+	}
+
+	let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+	if std::env::var("CARGO_FEATURE_DESKTOP").is_ok() {
+		let _ = std::fs::remove_file(manifest_dir.join("desktop.hash"));
+	} else {
+		let _ = std::fs::remove_file(manifest_dir.join("web.hash"));
+	}
+}

tools/third-party-licenses/src/cargo.rs

@@ -0,0 +1,106 @@
+use crate::{LicenceSource, LicenseEntry, Package};
+use serde::Deserialize;
+use std::fs;
+use std::hash::{Hash, Hasher};
+use std::path::PathBuf;
+use std::process::{self, Command};
+
+pub struct CargoLicenseSource {}
+
+impl CargoLicenseSource {
+	pub fn new() -> Self {
+		Self {}
+	}
+}
+
+impl LicenceSource for CargoLicenseSource {
+	fn licenses(&self) -> Vec<LicenseEntry> {
+		parse(run())
+	}
+}
+
+impl Hash for CargoLicenseSource {
+	fn hash<H: Hasher>(&self, state: &mut H) {
+		let lock_path = PathBuf::from(env!("CARGO_WORKSPACE_DIR")).join("Cargo.lock");
+		fs::read_to_string(lock_path).unwrap().hash(state)
+	}
+}
+
+#[derive(Deserialize)]
+struct Output {
+	licenses: Vec<License>,
+}
+
+#[derive(Deserialize)]
+struct License {
+	name: Option<String>,
+	text: Option<String>,
+	used_by: Vec<UsedBy>,
+}
+
+#[derive(Deserialize)]
+struct UsedBy {
+	#[serde(rename = "crate")]
+	crate_info: Crate,
+}
+
+#[derive(Deserialize)]
+struct Crate {
+	name: Option<String>,
+	version: Option<String>,
+	authors: Option<Vec<String>>,
+	repository: Option<String>,
+}
+
+fn parse(parsed: Output) -> Vec<LicenseEntry> {
+	parsed
+		.licenses
+		.into_iter()
+		.map(|license| {
+			let packages = license
+				.used_by
+				.into_iter()
+				.map(|used| {
+					let name = used.crate_info.name.as_deref().unwrap_or_default();
+					let version = used.crate_info.version.as_deref().unwrap_or_default();
+					let display_name = if version.is_empty() { name.to_string() } else { format!("{name}@{version}") };
+
+					let repository = used.crate_info.repository.filter(|s| !s.is_empty());
+
+					Package {
+						name: display_name,
+						authors: used.crate_info.authors.unwrap_or_default(),
+						url: repository,
+					}
+				})
+				.collect();
+
+			LicenseEntry {
+				name: license.name,
+				text: license.text.as_deref().unwrap_or_default().to_string(),
+				packages,
+			}
+		})
+		.collect()
+}
+
+fn run() -> Output {
+	let output = Command::new("cargo")
+		.args(["about", "generate", "--format", "json", "--frozen"])
+		.current_dir(env!("CARGO_WORKSPACE_DIR"))
+		.output()
+		.unwrap_or_else(|e| {
+			eprintln!("Failed to run cargo about generate: {e}");
+			process::exit(1)
+		});
+
+	if !output.status.success() {
+		eprintln!("cargo about generate failed:\n{}", String::from_utf8_lossy(&output.stderr));
+		process::exit(1)
+	}
+
+	serde_json::from_str(&String::from_utf8(output.stdout).expect("cargo about generate should return valid UTF-8")).unwrap_or_else(|e| {
+		eprintln!("Failed to parse cargo about generate JSON: {e}");
+		process::exit(1)
+	})
+}

tools/third-party-licenses/src/cef.rs

@@ -0,0 +1,105 @@
+use lzma_rust2::XzReader;
+use scraper::{Html, Selector};
+use std::hash::Hash;
+use std::io::Read;
+use std::path::PathBuf;
+use std::{fs, process};
+
+use crate::{LicenceSource, LicenseEntry, Package};
+
+pub struct CefLicenseSource;
+
+impl CefLicenseSource {
+	pub fn new() -> Self {
+		Self {}
+	}
+}
+
+impl LicenceSource for CefLicenseSource {
+	fn licenses(&self) -> Vec<LicenseEntry> {
+		let html = read();
+		parse(&html)
+	}
+}
+
+impl Hash for CefLicenseSource {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		read().hash(state)
+	}
+}
+
+fn parse(html: &str) -> Vec<LicenseEntry> {
+	let document = Html::parse_document(html);
+
+	let product_sel = Selector::parse("div.product").unwrap();
+	let title_sel = Selector::parse("span.title").unwrap();
+	let homepage_sel = Selector::parse("span.homepage a").unwrap();
+	let license_sel = Selector::parse("div.license pre").unwrap();
+
+	document
+		.select(&product_sel)
+		.filter_map(|product| {
+			let name: String = product.select(&title_sel).next().map(|el| el.text().collect()).unwrap_or_default();
+
+			if name.is_empty() {
+				return None;
+			}
+
+			let homepage = product.select(&homepage_sel).next().and_then(|el| el.value().attr("href").map(String::from));
+
+			let license_text: String = product.select(&license_sel).next().map(|el| el.text().collect::<String>()).unwrap_or_default().trim().to_string();
+
+			let pkg = Package {
+				name,
+				url: homepage,
+				authors: Vec::new(),
+			};
+
+			Some(LicenseEntry {
+				name: None,
+				text: license_text,
+				packages: vec![pkg],
+			})
+		})
+		.collect()
+}
+
+fn read() -> String {
+	let cef_path = PathBuf::from(env!("CEF_PATH"));
+	let cef_credits = std::fs::read_dir(&cef_path)
+		.unwrap_or_else(|e| {
+			eprintln!("Failed to read CEF_PATH directory {}: {e}", cef_path.display());
+			process::exit(1);
+		})
+		.filter_map(|entry| entry.ok())
+		.find(|entry| {
+			let name = entry.file_name();
+			name.eq_ignore_ascii_case("credits.html") || name.eq_ignore_ascii_case("credits.html.xz")
+		})
+		.map(|entry| entry.path())
+		.unwrap_or_else(|| {
+			eprintln!("Could not find CREDITS.html or CREDITS.html.xz in {}", cef_path.display());
+			process::exit(1);
+		});
+
+	let decompress_xz = cef_credits.extension().map(|ext| ext.eq_ignore_ascii_case("xz")).unwrap_or(false);
+
+	if decompress_xz {
+		let file = fs::File::open(&cef_credits).unwrap_or_else(|e| {
+			eprintln!("Failed to open CEF credits file {}: {e}", cef_credits.display());
+			process::exit(1);
+		});
+		let mut reader = XzReader::new(file, false);
+		let mut html = String::new();
+		reader.read_to_string(&mut html).unwrap_or_else(|e| {
+			eprintln!("Failed to decompress CEF credits file {}: {e}", cef_credits.display());
+			process::exit(1);
+		});
+		html
+	} else {
+		fs::read_to_string(&cef_credits).unwrap_or_else(|e| {
+			eprintln!("Failed to read CEF credits file {}: {e}", cef_credits.display());
+			process::exit(1);
+		})
+	}
+}

tools/third-party-licenses/src/main.rs

@@ -0,0 +1,229 @@
+use std::collections::HashMap;
+use std::hash::{DefaultHasher, Hash, Hasher};
+use std::path::PathBuf;
+use std::{fs, process};
+
+mod cargo;
+#[cfg(feature = "desktop")]
+mod cef;
+mod npm;
+
+use crate::cargo::CargoLicenseSource;
+#[cfg(feature = "desktop")]
+use crate::cef::CefLicenseSource;
+use crate::npm::NpmLicenseSource;
+
+pub trait LicenceSource: std::hash::Hash {
+	fn licenses(&self) -> Vec<LicenseEntry>;
+}
+
+pub struct LicenseEntry {
+	name: Option<String>,
+	text: String,
+	packages: Vec<Package>,
+}
+
+pub struct Package {
+	name: String,
+	authors: Vec<String>,
+	url: Option<String>,
+}
+
+#[derive(Hash)]
+struct Run<'a> {
+	output: &'a Vec<u8>,
+	cargo: &'a CargoLicenseSource,
+	npm: &'a NpmLicenseSource,
+	#[cfg(feature = "desktop")]
+	cef: &'a CefLicenseSource,
+}
+
+fn main() {
+	let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+	let workspace_dir = PathBuf::from(env!("CARGO_WORKSPACE_DIR"));
+
+	#[cfg(feature = "desktop")]
+	let output_path = workspace_dir.join("desktop/third-party-licenses.txt.xz");
+	#[cfg(not(feature = "desktop"))]
+	let output_path = workspace_dir.join("frontend/third-party-licenses.txt");
+
+	#[cfg(feature = "desktop")]
+	let current_hash_path = manifest_dir.join("desktop.hash");
+	#[cfg(not(feature = "desktop"))]
+	let current_hash_path = manifest_dir.join("web.hash");
+
+	let cargo_source = CargoLicenseSource::new();
+	let npm_source = NpmLicenseSource::new(workspace_dir.join("frontend"));
+	#[cfg(feature = "desktop")]
+	let cef_source = CefLicenseSource::new();
+
+	let mut run = Run {
+		cargo: &cargo_source,
+		npm: &npm_source,
+		#[cfg(feature = "desktop")]
+		cef: &cef_source,
+		output: &fs::read(&output_path).unwrap_or_default(),
+	};
+
+	let mut hasher = DefaultHasher::new();
+	run.hash(&mut hasher);
+	let current_hash = format!("{:016x}", hasher.finish());
+
+	if current_hash == fs::read_to_string(&current_hash_path).unwrap_or_default() {
+		eprintln!("No changes in licenses detected, skipping generation.");
+		return;
+	}
+	eprintln!("Changes in licenses detected, generating new license file.");
+
+	let licenses = merge_filter_dedup_and_sort(vec![
+		cargo_source.licenses(),
+		npm_source.licenses(),
+		#[cfg(feature = "desktop")]
+		cef_source.licenses(),
+	]);
+	let formatted = format_credits(&licenses);
+
+	#[cfg(feature = "desktop")]
+	let output = compress(&formatted);
+	#[cfg(not(feature = "desktop"))]
+	let output = formatted.as_bytes().to_vec();
+	if let Some(parent) = output_path.parent() {
+		fs::create_dir_all(parent).unwrap_or_else(|e| {
+			eprintln!("Failed to create directory {}: {e}", parent.display());
+			std::process::exit(1);
+		});
+	}
+	fs::write(&output_path, &output).unwrap_or_else(|e| {
+		eprintln!("Failed to write {}: {e}", &output_path.display());
+		std::process::exit(1);
+	});
+	run.output = &output;
+
+	let hash = {
+		let mut hasher = DefaultHasher::new();
+		run.hash(&mut hasher);
+		format!("{:016x}", hasher.finish())
+	};
+
+	fs::write(&current_hash_path, hash).unwrap_or_else(|e| {
+		eprintln!("Failed to write hash file {}: {e}", current_hash_path.display());
+		process::exit(1);
+	});
+}
+
+fn format_credits(licenses: &Vec<LicenseEntry>) -> String {
+	let mut out = String::new();
+
+	out.push_str("▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐\n");
+	out.push_str("▐▐                                                   ▐▐\n");
+	out.push_str("▐▐   GRAPHITE THIRD-PARTY SOFTWARE LICENSE NOTICES   ▐▐\n");
+	out.push_str("▐▐                                                   ▐▐\n");
+	out.push_str("▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐▐\n");
+
+	for license in licenses {
+		let package_lines: Vec<String> = license
+			.packages
+			.iter()
+			.map(|pkg| match &pkg {
+				Package { name, authors, url: Some(url) } if !authors.is_empty() => format!("{} - [{}] - {}", name, authors.join(", "), url),
+				Package { name, authors: _, url: Some(url) } => format!("{} - {}", name, url),
+				Package { name, authors, url: None } if !authors.is_empty() => format!("{} - [{}]", name, authors.join(", ")),
+				_ => pkg.name.clone(),
+			})
+			.collect();
+
+		let multi = package_lines.len() > 1;
+
+		let header = format!(
+			"The package{} listed here {} licensed under the terms of the {} printed beneath",
+			if multi { "s" } else { "" },
+			if multi { "are" } else { "is" },
+			if let Some(license) = license.name.as_ref() { license.to_string() } else { "license".to_string() }
+		);
+
+		let max_len = std::iter::once(header.len()).chain(package_lines.iter().map(|l| l.chars().count())).max().unwrap_or(0);
+
+		let padded_packages: Vec<String> = package_lines
+			.iter()
+			.map(|line| {
+				let pad = max_len - line.chars().count();
+				format!("│ {}{} │", line, " ".repeat(pad))
+			})
+			.collect();
+
+		out.push_str(&format!("\n {}\n", "_".repeat(max_len + 2)));
+		out.push_str(&format!("│ {} │\n", " ".repeat(max_len)));
+		out.push_str(&format!("│ {}{} │\n", header, " ".repeat(max_len - header.len())));
+		out.push_str(&format!("│{}│\n", "_".repeat(max_len + 2)));
+		out.push_str(&padded_packages.join("\n"));
+		out.push('\n');
+		out.push_str(&format!(" {}", "\u{203e}".repeat(max_len + 2)));
+		for line in license.text.lines() {
+			if line.is_empty() {
+				out.push('\n');
+				continue;
+			}
+			out.push('\n');
+			out.push_str("    ");
+			out.push_str(line);
+		}
+		out.truncate(out.trim_end().len());
+		out.push('\n');
+	}
+
+	out
+}
+
+fn merge_filter_dedup_and_sort(sources: Vec<Vec<LicenseEntry>>) -> Vec<LicenseEntry> {
+	let mut all = Vec::new();
+	for source in sources {
+		all.extend(source);
+	}
+	filter(&mut all);
+	let mut all = dedup_by_licence_text(all);
+	all.sort_by(|a, b| b.packages.len().cmp(&a.packages.len()).then(a.text.len().cmp(&b.text.len())));
+	all
+}
+
+fn filter(licenses: &mut Vec<LicenseEntry>) {
+	licenses.iter_mut().for_each(|l| {
+		l.packages.retain(|p| !(p.authors.len() == 1 && p.authors[0].contains("contact@graphite.art")));
+	});
+	licenses.retain(|l| !l.packages.is_empty());
+}
+
+fn dedup_by_licence_text(vec: Vec<LicenseEntry>) -> Vec<LicenseEntry> {
+	let mut map: HashMap<String, LicenseEntry> = HashMap::new();
+
+	for entry in vec {
+		match map.entry(entry.text.clone()) {
+			std::collections::hash_map::Entry::Occupied(mut e) => {
+				e.get_mut().packages.extend(entry.packages);
+			}
+			std::collections::hash_map::Entry::Vacant(e) => {
+				e.insert(entry);
+			}
+		}
+	}
+
+	map.into_values().collect()
+}
+
+#[cfg(feature = "desktop")]
+fn compress(content: &str) -> Vec<u8> {
+	use std::io::Write;
+	let mut buf = Vec::new();
+	let mut writer = lzma_rust2::XzWriter::new(&mut buf, lzma_rust2::XzOptions::default()).unwrap_or_else(|e| {
+		eprintln!("Failed to create XZ writer: {e}");
+		std::process::exit(1);
+	});
+	writer.write_all(content.as_bytes()).unwrap_or_else(|e| {
+		eprintln!("Failed to write compressed credits: {e}");
+		std::process::exit(1);
+	});
+	writer.finish().unwrap_or_else(|e| {
+		eprintln!("Failed to finish XZ compression: {e}");
+		std::process::exit(1);
+	});
+	buf
+}

tools/third-party-licenses/src/npm.rs

@@ -0,0 +1,93 @@
+use std::collections::HashMap;
+use std::fs;
+use std::path::PathBuf;
+use std::process;
+use std::process::Command;
+
+use crate::{LicenceSource, LicenseEntry, Package};
+
+pub struct NpmLicenseSource {
+	dir: PathBuf,
+}
+impl NpmLicenseSource {
+	pub fn new(dir: PathBuf) -> Self {
+		Self { dir }
+	}
+}
+
+impl LicenceSource for NpmLicenseSource {
+	fn licenses(&self) -> Vec<LicenseEntry> {
+		parse(run(&self.dir))
+	}
+}
+
+impl std::hash::Hash for NpmLicenseSource {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		let lock_path = self.dir.join("package-lock.json");
+		fs::read_to_string(lock_path).unwrap().hash(state)
+	}
+}
+
+type Output = HashMap<String, NpmEntry>;
+
+#[derive(serde::Deserialize)]
+struct NpmEntry {
+	licenses: Option<String>,
+	repository: Option<String>,
+	#[serde(rename = "licenseFile")]
+	license_file: Option<String>,
+	publisher: Option<String>,
+	email: Option<String>,
+}
+
+fn parse(parsed: Output) -> Vec<LicenseEntry> {
+	parsed
+		.iter()
+		.map(|(name, entry)| {
+			let publisher_info = entry.publisher.as_ref().map(|p| {
+				let email_part = entry.email.as_ref().map(|e| format!(" <{}>", e)).unwrap_or_default();
+				format!("{}{}", p, email_part)
+			});
+
+			let pkg = Package {
+				name: name.to_string(),
+				url: entry.repository.clone(),
+				authors: publisher_info.into_iter().collect(),
+			};
+
+			let license_text = entry.license_file.as_ref().and_then(|p| fs::read_to_string(p).ok()).map(|s| s.to_string()).unwrap_or_default();
+
+			LicenseEntry {
+				name: entry.licenses.clone(),
+				text: license_text,
+				packages: vec![pkg],
+			}
+		})
+		.collect()
+}
+
+fn run(dir: &std::path::Path) -> Output {
+	#[cfg(not(target_os = "windows"))]
+	let mut cmd = Command::new("npx");
+	#[cfg(target_os = "windows")]
+	let mut cmd = Command::new("npx.cmd");
+	cmd.args(["license-checker-rseidelsohn", "--production", "--json"]);
+	cmd.current_dir(dir);
+
+	let output = cmd.output().unwrap_or_else(|e| {
+		eprintln!("Failed to run npx license-checker-rseidelsohn: {e}");
+		process::exit(1);
+	});
+
+	if !output.status.success() {
+		eprintln!("npx license-checker-rseidelsohn failed:\n{}", String::from_utf8_lossy(&output.stderr));
+		process::exit(1);
+	}
+
+	let json_str = String::from_utf8(output.stdout).expect("Invalid UTF-8 from license-checker");
+
+	serde_json::from_str(&json_str).unwrap_or_else(|e| {
+		eprintln!("Failed to parse license-checker JSON: {e}");
+		process::exit(1)
+	})
+}