More resilient error handling in SymlinkWatcher

apps/cli/lib/symlinks.ts

@@ -7,22 +7,72 @@ import { FSWatcher, watch as watchPaths } from 'chokidar';
 type SymlinkWatcherEvents = {
 	symlink: [ target: string, symlinkPath: string ];
 	error: [ error: unknown ];
+	unrecoverable: [ error: unknown ];
+	restart: [];
 };
 
+// Backoff schedule for self-heal attempts after the chokidar watcher errors.
+// The 0th index is used for the first attempt; subsequent attempts use later
+// indexes, with the last value repeated if attempts exceed the array length.
+const RESTART_BACKOFF_MS = [ 500, 1000, 2000, 4000, 8000 ];
+// Max number of self-heal attempts allowed within RESTART_BUDGET_WINDOW_MS
+// before we stop trying and emit an 'unrecoverable' event. Tuned to absorb a
+// blueprint-apply burst (which typically resolves within a few seconds) without
+// looping forever against a permanently broken watch handle.
+const RESTART_BUDGET = 5;
+const RESTART_BUDGET_WINDOW_MS = 60_000;
+
 export class SymlinkWatcher extends EventEmitter< SymlinkWatcherEvents > {
 	private watcher: FSWatcher | null = null;
+	private watchPath: string | null = null;
+	private depth: number = 2;
+	private restartTimestamps: number[] = [];
+	private restartTimer: NodeJS.Timeout | null = null;
+	private stopped = false;
 
-	start( sitePath: string, depth: number = 2 ): void {
+	start( watchPath: string, depth: number = 2 ): void {
 		if ( this.watcher ) {
 			return;
 		}
+		this.watchPath = watchPath;
+		this.depth = depth;
+		this.stopped = false;
+		this.restartTimestamps = [];
+		this.attachWatcher();
+	}
+
+	async stop(): Promise< void > {
+		this.stopped = true;
+		if ( this.restartTimer ) {
+			clearTimeout( this.restartTimer );
+			this.restartTimer = null;
+		}
+		this.restartTimestamps = [];
+		this.watchPath = null;
+		const watcher = this.watcher;
+		this.watcher = null;
+		if ( watcher ) {
+			try {
+				await watcher.close();
+			} catch {
+				// Best effort — the underlying handle may already be dead.
+			}
+		}
+	}
+
+	private attachWatcher(): void {
+		if ( ! this.watchPath || this.stopped ) {
+			return;
+		}
 
-		this.watcher = watchPaths( sitePath, {
+		const watcher = watchPaths( this.watchPath, {
 			// Deep watchers can easily lead to performance issues. That's why we default to a depth of 2.
-			depth,
+			depth: this.depth,
 			ignoreInitial: true,
 			followSymlinks: false,
-			persistent: false,
+			// `persistent: true` is required so chokidar wires an 'error' listener on the
+			// underlying native FSWatcher (see chokidar handler.js setFsWatchListener)
+			persistent: true,
 			ignorePermissionErrors: true,
 			// Skip directories that produce noise and never host plugin/theme symlinks.
 			// chokidar v4+ no longer accepts globs by default — match against the path.
@@ -49,16 +99,86 @@ export class SymlinkWatcher extends EventEmitter< SymlinkWatcherEvents > {
 			this.emit( 'symlink', target, entryPath );
 		};
 
-		this.watcher.on( 'add', onMaybeSymlink );
-		this.watcher.on( 'addDir', onMaybeSymlink );
-		this.watcher.on( 'error', ( error ) => {
+		watcher.on( 'add', onMaybeSymlink );
+		watcher.on( 'addDir', onMaybeSymlink );
+		watcher.on( 'error', ( error ) => {
+			// Surface the error so the caller can log it, then try to recover. The
+			// most common trigger on Windows is a transient EPERM during heavy
+			// directory churn (e.g. blueprint apply extracting plugins/themes).
 			this.emit( 'error', error );
+			this.scheduleRestart();
 		} );
+
+		this.watcher = watcher;
 	}
 
-	async stop(): Promise< void > {
-		await this.watcher?.close();
+	private scheduleRestart(): void {
+		if ( this.stopped || this.restartTimer ) {
+			return;
+		}
+
+		const now = Date.now();
+		this.restartTimestamps = this.restartTimestamps.filter(
+			( ts ) => now - ts < RESTART_BUDGET_WINDOW_MS
+		);
+
+		if ( this.restartTimestamps.length >= RESTART_BUDGET ) {
+			const error = new Error(
+				`Symlink watcher gave up after ${ RESTART_BUDGET } restart attempts within ${
+					RESTART_BUDGET_WINDOW_MS / 1000
+				}s. Future symlinks under the site directory will not be auto-allowed; restart the site to recover.`
+			);
+			this.emit( 'unrecoverable', error );
+			// Tear down so we stop reacting to further events. start() must be
+			// called again to resume watching.
+			this.stopped = true;
+			const watcher = this.watcher;
+			this.watcher = null;
+			if ( watcher ) {
+				watcher.close().catch( () => {} );
+			}
+			return;
+		}
+
+		const attempt = this.restartTimestamps.length;
+		this.restartTimestamps.push( now );
+
+		const delay = RESTART_BACKOFF_MS[ Math.min( attempt, RESTART_BACKOFF_MS.length - 1 ) ];
+		this.restartTimer = setTimeout( () => {
+			this.restartTimer = null;
+			void this.restartNow();
+		}, delay );
+	}
+
+	private async restartNow(): Promise< void > {
+		if ( this.stopped ) {
+			return;
+		}
+
+		const previousWatcher = this.watcher;
 		this.watcher = null;
+		if ( previousWatcher ) {
+			try {
+				// Awaiting close lets chokidar release its cached FsWatchInstance for
+				// this path. If we re-watched while close was still pending, chokidar
+				// would attach our new listener to the existing (already-broken)
+				// instance and we would never recover.
+				await previousWatcher.close();
+			} catch {
+				// Best effort.
+			}
+		}
+
+		if ( this.stopped ) {
+			return;
+		}
+
+		this.attachWatcher();
+
+		// Notify the caller that the watcher was reattached. Events fired during
+		// the dead window are lost, so the caller is responsible for re-scanning
+		// the tree and reconciling any new state.
+		this.emit( 'restart' );
 	}
 }
 

apps/cli/php-server-child.ts

@@ -334,6 +334,7 @@ function startSymlinkWatcher( sitePath: string ): void {
 		return;
 	}
 
+	const wpContentPath = path.join( sitePath, 'wp-content' );
 	const watcher = new SymlinkWatcher();
 	watcher.on( 'symlink', ( target, symlinkPath ) => {
 		if ( currentOpenBasedirAllowlist.has( target ) ) {
@@ -346,14 +347,50 @@ function startSymlinkWatcher( sitePath: string ): void {
 	} );
 
 	watcher.on( 'error', ( error ) => {
-		errorToConsole( 'Symlink watcher error:', error );
+		errorToConsole( 'Symlink watcher error (will attempt to recover):', error );
+	} );
+
+	watcher.on( 'unrecoverable', ( error ) => {
+		errorToConsole(
+			'Symlink watcher gave up. New plugin/theme symlinks under wp-content will not be auto-allowed until the site is restarted.',
+			error
+		);
+	} );
+
+	watcher.on( 'restart', () => {
+		// Events fired while the watcher was dead are lost. Re-scan wp-content and
+		// fold any newly discovered symlink targets into the allowlist.
+		void reconcileSymlinkAllowlist( wpContentPath );
 	} );
 
 	// Watch wp-content and its subdirectories for symlinks
-	watcher.start( path.join( sitePath, 'wp-content' ), 2 );
+	watcher.start( wpContentPath, 2 );
 	symlinkWatcher = watcher;
 }
 
+async function reconcileSymlinkAllowlist( wpContentPath: string ): Promise< void > {
+	let entries: string[];
+	try {
+		entries = await collectSymlinkAllowlistEntries( wpContentPath );
+	} catch ( error ) {
+		errorToConsole( 'Failed to reconcile symlink allowlist after watcher restart:', error );
+		return;
+	}
+
+	let added = false;
+	for ( const target of entries ) {
+		if ( ! currentOpenBasedirAllowlist.has( target ) ) {
+			logToConsole( `Discovered symlink target after watcher restart: ${ target }` );
+			currentOpenBasedirAllowlist.add( target );
+			added = true;
+		}
+	}
+
+	if ( added ) {
+		scheduleAllowlistRestart();
+	}
+}
+
 async function stopSymlinkWatcher(): Promise< void > {
 	if ( symlinkRestartTimer ) {
 		clearTimeout( symlinkRestartTimer );