Improve visual annotation workflows

apps/cli/ai/inspector/annotation-result.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest';
+import { validateAnnotationDoneResult } from './annotation-result';
+
+describe( 'validateAnnotationDoneResult', () => {
+	it( 'accepts a valid browser result', () => {
+		const result = {
+			capturedAt: Date.now(),
+			url: 'http://example.test/',
+			annotations: [ { id: 'a_1', comment: 'Make this smaller', selector: '#hero' } ],
+		};
+
+		expect( validateAnnotationDoneResult( result ) ).toBe( result );
+	} );
+
+	it.each( [
+		null,
+		{},
+		{ capturedAt: Date.now(), url: 'http://example.test/', annotations: [] },
+		{
+			capturedAt: Date.now(),
+			url: 'http://example.test/',
+			annotations: [ { id: 'a_1', comment: '' } ],
+		},
+	] )( 'rejects an invalid browser result', ( value ) => {
+		expect( () => validateAnnotationDoneResult( value ) ).toThrow();
+	} );
+
+	it( 'rejects an oversized browser result', () => {
+		expect( () =>
+			validateAnnotationDoneResult( {
+				capturedAt: Date.now(),
+				url: 'http://example.test/',
+				annotations: [ { id: 'a_1', comment: 'Valid', extra: 'x'.repeat( 1_000_000 ) } ],
+			} )
+		).toThrow( 'too much data' );
+	} );
+} );

apps/cli/ai/inspector/annotation-result.ts

@@ -0,0 +1,28 @@
+import { validateStudioInspectorAnnotations } from '@studio/common/ai/inspector-annotations';
+import type { StudioInspectorAnnotation } from '@studio/common/ai/inspector-annotations';
+
+export interface AnnotationDoneResult {
+	capturedAt: number;
+	url: string;
+	annotations: StudioInspectorAnnotation[];
+}
+
+export function validateAnnotationDoneResult( value: unknown ): AnnotationDoneResult {
+	if ( typeof value !== 'object' || value === null ) {
+		throw new Error( 'The annotation browser returned an invalid result.' );
+	}
+	const result = value as Partial< AnnotationDoneResult >;
+	if (
+		typeof result.capturedAt !== 'number' ||
+		! Number.isFinite( result.capturedAt ) ||
+		typeof result.url !== 'string' ||
+		result.url.length > 10_000 ||
+		! Array.isArray( result.annotations )
+	) {
+		throw new Error( 'The annotation browser returned an invalid result.' );
+	}
+
+	validateStudioInspectorAnnotations( result.annotations );
+
+	return result as AnnotationDoneResult;
+}

apps/cli/ai/inspector/inspector-inject.ts

@@ -9,6 +9,10 @@
  */
 
 import { launchChromiumWithInstall } from 'cli/ai/browser-utils';
+import {
+	validateAnnotationDoneResult,
+	type AnnotationDoneResult,
+} from 'cli/ai/inspector/annotation-result';
 import { INSPECTOR_PAGE_SCRIPT } from 'cli/ai/inspector/page-script';
 
 type Browser = Awaited< ReturnType< ( typeof import('playwright') )[ 'chromium' ][ 'launch' ] > >;
@@ -65,12 +69,6 @@ async function injectInspector( page: Page ): Promise< void > {
 	} );
 }
 
-export interface AnnotationDoneResult {
-	capturedAt: number;
-	url: string;
-	annotations: unknown[];
-}
-
 /**
  * Block until the user clicks "Done" in the inspector toolbar, then return
  * the annotations. Reads from `window.__studioAnnotateDone` which the page
@@ -96,10 +94,11 @@ export async function waitForAnnotationsDone(
 		{ timeout: timeoutMs, polling: 500 }
 	);
 
-	const value = ( await handle.jsonValue() ) as AnnotationDoneResult | null;
+	const value = await handle.jsonValue();
 	if ( ! value ) {
 		throw new Error( 'Annotation submission was cleared before it could be read.' );
 	}
+	const result = validateAnnotationDoneResult( value );
 
 	// Auto-close the browser once we've captured the annotations. This
 	// makes the lifecycle unambiguous from the user's point of view —
@@ -121,7 +120,7 @@ export async function waitForAnnotationsDone(
 		}, 10_000 );
 	}
 
-	return value;
+	return result;
 }
 
 export async function openAnnotationBrowser( siteUrl: string ): Promise< string > {

apps/cli/ai/inspector/page-script.ts

@@ -24,6 +24,7 @@ export const INSPECTOR_PAGE_SCRIPT =
 
 	const STORAGE_KEY = 'studio-inspector-annotations-v1';
 	const HOST_ID = '__studio-inspector-host';
+	const MAX_ANNOTATIONS = 100;
 
 	/* --------------------------------------------------------------------
 	 * Storage
@@ -32,7 +33,17 @@ export const INSPECTOR_PAGE_SCRIPT =
 		try {
 			const raw = localStorage.getItem( STORAGE_KEY );
 			const parsed = raw ? JSON.parse( raw ) : [];
-			return Array.isArray( parsed ) ? parsed : [];
+			return Array.isArray( parsed )
+				? parsed
+						.filter(
+							( annotation ) =>
+								typeof annotation === 'object' &&
+								annotation !== null &&
+								typeof annotation.comment === 'string' &&
+								annotation.comment.trim()
+						)
+						.slice( 0, MAX_ANNOTATIONS )
+				: [];
 		} catch {
 			return [];
 		}
@@ -419,6 +430,7 @@ export const INSPECTOR_PAGE_SCRIPT =
 
 		const ta = document.createElement( 'textarea' );
 		ta.placeholder = 'What should change about this element?';
+		ta.maxLength = 10000;
 		ta.value = state.comment || '';
 		ta.addEventListener( 'input', () => {
 			state.comment = ta.value;
@@ -473,6 +485,10 @@ export const INSPECTOR_PAGE_SCRIPT =
 		save.addEventListener( 'click', () => {
 			const trimmed = ( state.comment || '' ).trim();
 			if ( ! trimmed ) return;
+			if ( ! state.id && annotations.length >= MAX_ANNOTATIONS ) {
+				showToast( 'You can add up to 100 annotations.' );
+				return;
+			}
 			if ( state.id ) {
 				annotations = annotations.map( ( a ) =>
 					a.id === state.id ? { ...a, comment: trimmed, updatedAt: Date.now() } : a

apps/cli/commands/ai/index.ts

@@ -51,6 +51,7 @@ import type {
 	StudioCustomEntryType,
 } from '@studio/common/ai/sessions/entry-types';
 import type { LoadedAiSession, TurnStatus } from '@studio/common/ai/sessions/types';
+import type { StudioVisualAnnotationSummary } from '@studio/common/ai/visual-annotations';
 import type { AskUserQuestion } from 'cli/ai/types';
 
 const logger = new Logger< string >();
@@ -95,6 +96,7 @@ export async function runCommand( options: {
 	initialDisplayMessage?: string;
 	initialImages?: StudioChatImage[];
 	initialFiles?: StudioChatFileAttachment[];
+	initialVisualAnnotations?: StudioVisualAnnotationSummary[];
 	resumeSession?: LoadedAiSession;
 	resumeSessionId?: string;
 	showLegacyCommandNotice?: boolean;
@@ -467,7 +469,8 @@ export async function runCommand( options: {
 		prompt: string,
 		displayMessage = prompt,
 		images: StudioChatImage[] = [],
-		files: StudioChatFileAttachment[] = []
+		files: StudioChatFileAttachment[] = [],
+		visualAnnotations?: StudioVisualAnnotationSummary[]
 	): Promise< { status: TurnStatus; sessionId: string } > {
 		await maybeAutoSwitchProvider();
 		const sm = await ensureSession();
@@ -535,6 +538,7 @@ export async function runCommand( options: {
 				source: 'prompt',
 				sitePath: site?.path,
 				attachments: buildChatAttachmentSummaries( images, files ),
+				visualAnnotations,
 			} )
 		);
 
@@ -614,7 +618,8 @@ export async function runCommand( options: {
 				options.initialMessage,
 				displayMessage,
 				options.initialImages,
-				options.initialFiles
+				options.initialFiles,
+				options.initialVisualAnnotations
 			);
 			const jsonStatus = result.status === 'interrupted' ? 'error' : result.status;
 			( ui as JsonAdapter ).emitTurnCompleted( jsonStatus, result.sessionId );
@@ -639,7 +644,8 @@ export async function runCommand( options: {
 				options.initialMessage,
 				displayMessage,
 				options.initialImages,
-				options.initialFiles
+				options.initialFiles,
+				options.initialVisualAnnotations
 			);
 		} catch ( error ) {
 			handleAgentTurnError( error );

apps/cli/commands/ai/sessions/resume.ts

@@ -3,6 +3,7 @@ import { validateStudioChatFiles } from '@studio/common/ai/chat-files';
 import { validateStudioChatImages } from '@studio/common/ai/chat-images';
 import { resolveActiveSiteFromEntries } from '@studio/common/ai/sessions/active-site';
 import { listAiSessions, loadAiSession } from '@studio/common/ai/sessions/store';
+import { validateStudioVisualAnnotations } from '@studio/common/ai/visual-annotations';
 import { getSessionsDirectory } from '@studio/common/lib/well-known-paths';
 import { __ } from '@wordpress/i18n';
 import { JsonAdapter } from 'cli/ai/output-adapter';
@@ -24,6 +25,7 @@ async function readInputPayload( path: string ): Promise< StudioAiSessionInputPa
 		displayMessage: parsed.displayMessage,
 		images: validateStudioChatImages( parsed.images ),
 		files: validateStudioChatFiles( parsed.files ),
+		visualAnnotations: validateStudioVisualAnnotations( parsed.visualAnnotations ),
 	};
 }
 
@@ -79,6 +81,7 @@ export async function runCommand(
 		initialDisplayMessage: inputPayload?.displayMessage ?? options.displayMessage,
 		initialImages: inputPayload?.images,
 		initialFiles: inputPayload?.files,
+		initialVisualAnnotations: inputPayload?.visualAnnotations,
 		activeSite: resolvedSite,
 	} );
 }

apps/cli/commands/ai/sessions/tests/resume.test.ts

@@ -1,3 +1,6 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
 import { loadAiSession } from '@studio/common/ai/sessions/store';
 import { vi, type Mock } from 'vitest';
 import { runCommand as runAiCommand } from 'cli/commands/ai';
@@ -84,4 +87,41 @@ describe( 'sessions resume — active site hydration (JSON mode)', () => {
 			expect.objectContaining( { activeSite: undefined } )
 		);
 	} );
+
+	it( 'passes validated visual annotations from an input payload', async () => {
+		( loadAiSession as Mock ).mockResolvedValue( { entries: [] } );
+		const directory = await fs.mkdtemp( path.join( os.tmpdir(), 'studio-annotations-' ) );
+		const inputPayloadPath = path.join( directory, 'input.json' );
+		await fs.writeFile(
+			inputPayloadPath,
+			JSON.stringify( {
+				prompt: 'Update the selected heading',
+				displayMessage: '1 annotation submitted',
+				visualAnnotations: [
+					{ comment: '  Make it smaller  ', tag: ' h1 ', nearbyText: 'Welcome' },
+				],
+			} )
+		);
+
+		try {
+			await runCommand( 'session-1', { json: true, inputPayloadPath } );
+		} finally {
+			await fs.rm( directory, { recursive: true, force: true } );
+		}
+
+		expect( runAiCommand ).toHaveBeenCalledWith(
+			expect.objectContaining( {
+				initialMessage: 'Update the selected heading',
+				initialDisplayMessage: '1 annotation submitted',
+				initialVisualAnnotations: [
+					{
+						comment: 'Make it smaller',
+						tag: 'h1',
+						elementLabel: undefined,
+						nearbyText: 'Welcome',
+					},
+				],
+			} )
+		);
+	} );
 } );

apps/hosted/src/agent-runs.ts

@@ -2,6 +2,7 @@ import crypto from 'node:crypto';
 import { stubRuntime } from './stub-runtime';
 import type { AgentProcess, AgentRuntime } from './runtime';
 import type { ActiveAgentRun, AgentEvent, AgentRunEvent } from '@studio/common/ai/agent-events';
+import type { StudioVisualAnnotationSummary } from '@studio/common/ai/visual-annotations';
 
 /**
  * Headless analog of the desktop `run-manager` (apps/studio/src/modules/
@@ -53,10 +54,11 @@ export interface StartAgentRunOptions {
 	sessionId: string;
 	prompt: string;
 	displayMessage?: string;
+	visualAnnotations?: StudioVisualAnnotationSummary[];
 }
 
 export function startAgentRun( options: StartAgentRunOptions ): { runId: string } {
-	const { sessionId, prompt, displayMessage } = options;
+	const { sessionId, prompt, displayMessage, visualAnnotations } = options;
 
 	if ( runsBySessionId.has( sessionId ) ) {
 		throw new Error( `A run is already in progress for session ${ sessionId }` );
@@ -69,6 +71,7 @@ export function startAgentRun( options: StartAgentRunOptions ): { runId: string
 		sessionId,
 		prompt,
 		displayMessage,
+		visualAnnotations,
 		onSpawn: () => emit( runId, sessionId, { type: 'run.started', timestamp: nowIso() } ),
 		onEvent: ( event ) => emit( runId, sessionId, event ),
 		onError: ( message ) =>

apps/hosted/src/index.ts

@@ -9,6 +9,7 @@ import {
 	listAiSessions,
 	loadAiSession,
 } from '@studio/common/ai/sessions/store';
+import { validateStudioVisualAnnotations } from '@studio/common/ai/visual-annotations';
 import {
 	readAuthToken,
 	readSharedSessions,
@@ -298,12 +299,28 @@ api.post(
 );
 
 api.post( '/sessions/:id/messages', ( req: Request, res: Response ) => {
-	const { prompt, displayMessage } = req.body as { prompt?: string; displayMessage?: string };
+	const { prompt, displayMessage, visualAnnotations } = req.body as {
+		prompt?: string;
+		displayMessage?: string;
+		visualAnnotations?: unknown;
+	};
 	if ( ! prompt ) {
 		res.status( 400 ).json( { error: 'prompt is required' } );
 		return;
 	}
-	const { runId } = startAgentRun( { sessionId: req.params.id, prompt, displayMessage } );
+	let validatedVisualAnnotations;
+	try {
+		validatedVisualAnnotations = validateStudioVisualAnnotations( visualAnnotations );
+	} catch {
+		res.status( 400 ).json( { error: 'visualAnnotations is invalid' } );
+		return;
+	}
+	const { runId } = startAgentRun( {
+		sessionId: req.params.id,
+		prompt,
+		displayMessage,
+		visualAnnotations: validatedVisualAnnotations,
+	} );
 	res.json( { runId } );
 } );
 

apps/hosted/src/runtime.ts

@@ -1,4 +1,5 @@
 import type { JsonEvent } from '@studio/common/ai/json-events';
+import type { StudioVisualAnnotationSummary } from '@studio/common/ai/visual-annotations';
 
 /**
  * Where the agent actually runs, behind a seam.
@@ -14,6 +15,7 @@ export interface AgentProcessOptions {
 	sessionId: string;
 	prompt: string;
 	displayMessage?: string;
+	visualAnnotations?: StudioVisualAnnotationSummary[];
 	// The process has started.
 	onSpawn: () => void;
 	// A JSON transport event the agent emitted.