Auto-recover stuck npm installs blocked on files locked by zombie processes (#4460)

frontend/.gitignore

@@ -3,3 +3,4 @@ wrapper/pkg/
 wrapper/pkg-native/
 public/build/
 dist/
+node_modules.locked-*/

tools/cargo-run/src/frontend.rs

@@ -1,10 +1,11 @@
 use crate::cmd::prelude::*;
 use crate::*;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 const WRAPPER_CRATE: &str = "graphite-wasm-wrapper";
 const WASM_TARGET: &str = "wasm32-unknown-unknown";
 const OUT_NAME: &str = "graphite_wasm_wrapper";
+const NODE_MODULES_LOCKED_PREFIX: &str = "node_modules.locked-";
 
 pub fn frontend_dir() -> PathBuf {
 	workspace_dir().join("frontend")
@@ -23,6 +24,8 @@ pub fn setup() -> Result<(), Error> {
 	let node_modules = frontend.join("node_modules");
 	let timestamp_path = node_modules.join(".install-timestamp");
 
+	sweep_locked_leftovers(frontend.clone());
+
 	let mtime = |p: PathBuf| std::fs::metadata(p).and_then(|m| m.modified()).ok();
 
 	if let Some(install_time) = mtime(timestamp_path.clone())
@@ -35,11 +38,18 @@ pub fn setup() -> Result<(), Error> {
 	}
 
 	eprintln!("Installing npm packages...");
-	let install = || utils::npm(["ci", "--include=dev", "--prefer-offline", "--no-audit", "--no-fund"]).dir(&frontend).run();
-	if install().is_err() {
-		eprintln!("Failed to install npm packages. Wiping `frontend/node_modules` and retrying...");
-		let _ = std::fs::remove_dir_all(&node_modules);
-		install()?;
+	let install = || utils::npm(["ci", "--include=dev", "--prefer-offline", "--no-audit", "--no-fund"]).dir(&frontend);
+
+	// The first attempt's output is captured, keeping npm's error dump off the screen when the failure gets recovered below
+	if !install().output_unchecked()?.status.success() {
+		eprintln!("Failed to install npm packages. Clearing `frontend/node_modules` and retrying...");
+		force_remove_node_modules(&node_modules);
+
+		// The retry streams live, so a real failure shows npm's errors in full, right above the banner
+		if let Err(e) = install().run() {
+			eprintln!("\n\n--------------------> Failed to install npm packages, even after clearing `frontend/node_modules`. Check npm's output above for the cause.\n");
+			return Err(e);
+		}
 	}
 
 	std::fs::write(&timestamp_path, "").map_err(|e| Error::Io(e, format!("writing '{}'", timestamp_path.display())))?;
@@ -47,6 +57,56 @@ pub fn setup() -> Result<(), Error> {
 	Ok(())
 }
 
+// Clears `node_modules` for a fresh install. Windows refuses to unlink a file while a running process has it memory-mapped
+// (typically a native module held by an orphaned Node.js instance), but renaming still works. So a failed delete renames
+// the directory to a sibling and deletes what it can of that in the background; anything still locked is moved and git-ignored.
+fn force_remove_node_modules(node_modules: &Path) {
+	match std::fs::remove_dir_all(node_modules) {
+		Ok(()) => return,
+		Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,
+		Err(_) => {}
+	}
+
+	let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|elapsed| elapsed.as_nanos()).unwrap_or(0);
+	let relocated = node_modules.with_file_name(format!("{NODE_MODULES_LOCKED_PREFIX}{}-{nanos}", std::process::id()));
+
+	if std::fs::rename(node_modules, &relocated).is_ok() {
+		std::thread::spawn(move || best_effort_remove_dir_all(&relocated));
+	} else {
+		eprintln!("warning: could not remove or relocate `frontend/node_modules`");
+	}
+}
+
+// Silently sweeps `node_modules.locked-*` leftovers from previous runs, whose locks are potentially gone by now (such as after a reboot)
+fn sweep_locked_leftovers(frontend: PathBuf) {
+	std::thread::spawn(move || {
+		let Ok(entries) = std::fs::read_dir(frontend) else { return };
+
+		for entry in entries.flatten() {
+			if entry.file_name().to_string_lossy().starts_with(NODE_MODULES_LOCKED_PREFIX) {
+				best_effort_remove_dir_all(&entry.path());
+			}
+		}
+	});
+}
+
+// Recursively deletes everything it can inside `dir` and then `dir` itself, skipping (not aborting on) locked entries
+fn best_effort_remove_dir_all(dir: &Path) {
+	if let Ok(entries) = std::fs::read_dir(dir) {
+		for entry in entries.flatten() {
+			let path = entry.path();
+			if entry.file_type().map(|file_type| file_type.is_dir()).unwrap_or(false) {
+				best_effort_remove_dir_all(&path);
+			} else if std::fs::remove_file(&path).is_err() {
+				// A symlink or junction to a directory must be removed as a directory
+				let _ = std::fs::remove_dir(&path);
+			}
+		}
+	}
+
+	let _ = std::fs::remove_dir(dir);
+}
+
 pub fn build_wasm(release: bool, native: bool) -> Result<(), Error> {
 	sequence(build_wasm_steps(release, native)).wait();
 	Ok(())