Skip to content

Commit 4bbec3c

Browse files
shaunandrewsclaude
shaunandrews
and
claude
committed
Stop embedding screenshot payload text now that artifacts emit structurally
Removes the mediaWidgetPayload= lines from take_screenshot output and replaces the screenshot-local-media presentation rule, which told the model to re-present the auto-emitted screenshot via studio_present and produced duplicate images in the conversation. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent cf66b73 commit 4bbec3c

4 files changed

Lines changed: 83 additions & 153 deletions

File tree

‎apps/cli/ai/tests/system-prompt.test.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,8 @@ describe( 'buildSystemPrompt', () => {
4141
expect( prompt ).toContain( 'For generated SVGs, write a complete .svg file' );
4242
expect( prompt ).toContain( 'Do not present generated SVG code as a drawing widget' );
4343
expect( prompt ).not.toContain( '- drawing:' );
44-
expect( prompt ).toContain( '- screenshot-local-media:' );
45-
expect( prompt ).toContain( 'present the actual captured PNG' );
46-
expect( prompt ).toContain( 'Do not substitute a site-preview widget for a screenshot' );
44+
expect( prompt ).toContain( '- screenshot-auto-artifact:' );
45+
expect( prompt ).toContain( 'Never call studio_present for a screenshot' );
4746
expect( prompt ).toContain( 'site-preview is for live previews, not captured screenshots' );
4847
expect( prompt ).toContain( '- theme:' );
4948
expect( prompt ).toContain( '- theme-template:' );
@@ -147,7 +146,7 @@ describe( 'buildSystemPrompt', () => {
147146
expect( prompt ).not.toContain( '## Visual artifacts' );
148147
expect( prompt ).not.toContain( '- site-code-scratchpad:' );
149148
expect( prompt ).not.toContain( '- saved-local-media:' );
150-
expect( prompt ).not.toContain( '- screenshot-local-media:' );
149+
expect( prompt ).not.toContain( '- screenshot-auto-artifact:' );
151150
expect( prompt ).not.toContain( 'studio_present' );
152151
} );
153152
} );

‎apps/cli/ai/tests/tools.test.ts‎

Lines changed: 75 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,44 @@ describe( 'Studio AI MCP tools', () => {
139139
tool: ReturnType< typeof resolveStudioToolDefinitions >[ number ],
140140
args: Record< string, unknown >
141141
) => tool.execute( 'tool-call-1', args as never, new AbortController().signal, () => {} );
142+
const createMockPage = ( {
143+
buffer,
144+
documentHeight,
145+
}: {
146+
buffer: Buffer;
147+
documentHeight?: number;
148+
} ) => ( {
149+
emulateMedia: vi.fn(),
150+
goto: vi.fn(),
151+
waitForLoadState: vi.fn().mockResolvedValue( undefined ),
152+
evaluate: vi.fn().mockResolvedValue( documentHeight ),
153+
addStyleTag: vi.fn(),
154+
screenshot: vi.fn().mockResolvedValue( buffer ),
155+
close: vi.fn(),
156+
} );
157+
const mockScreenshotBrowser = ( ...pages: Array< ReturnType< typeof createMockPage > > ) => {
158+
const newPage = vi.fn();
159+
for ( const page of pages ) {
160+
newPage.mockResolvedValueOnce( page );
161+
}
162+
vi.mocked( getSharedBrowser ).mockResolvedValue( { newPage } as never );
163+
return { newPage };
164+
};
165+
type ScreenshotArtifact = {
166+
widgetProps: { alt: string; source: { path: string; name: string } };
167+
};
168+
const getScreenshotArtifacts = ( result: {
169+
studioArtifacts?: Array< { widgetProps: Record< string, unknown > } >;
170+
} ): ScreenshotArtifact[] => {
171+
expect( result.studioArtifacts?.length ).toBeGreaterThan( 0 );
172+
return result.studioArtifacts as unknown as ScreenshotArtifact[];
173+
};
174+
const cleanUpScreenshotArtifacts = ( artifacts: ScreenshotArtifact[] ) =>
175+
Promise.all(
176+
artifacts.map( ( artifact ) =>
177+
rm( path.dirname( artifact.widgetProps.source.path ), { recursive: true, force: true } )
178+
)
179+
);
142180
const mockWpCliResponse = ( {
143181
stdout = '',
144182
stderr = '',
@@ -315,41 +353,27 @@ describe( 'Studio AI MCP tools', () => {
315353
const studioPresent = resolveStudioToolDefinitions( {
316354
emitChatArtifacts: true,
317355
} ).find( ( tool ) => tool.name === 'studio_present' );
318-
expect( takeScreenshot?.description ).toContain( 'ready-to-use media widget payload' );
356+
expect( takeScreenshot?.description ).not.toContain( 'ready-to-use media widget payload' );
319357
expect( takeScreenshot?.description ).not.toContain(
320358
'This does not automatically show the screenshot to the user'
321359
);
322360
expect( takeScreenshot?.description ).not.toContain(
323361
'Do not use a site-preview widget as a substitute for the screenshot'
324362
);
325-
expect( studioPresent?.description ).toContain(
326-
'Do not substitute a site-preview widget for a screenshot'
327-
);
363+
expect( studioPresent?.description ).toContain( 'Never call studio_present for a screenshot' );
328364
} );
329365

330-
it( 'keeps take_screenshot output compact while returning a media payload', async () => {
366+
it( 'keeps take_screenshot output compact while returning artifacts structurally', async () => {
331367
const screenshotBuffer = Buffer.from( 'fake-jpeg' );
332-
const page = {
333-
emulateMedia: vi.fn(),
334-
goto: vi.fn(),
335-
waitForLoadState: vi.fn().mockResolvedValue( undefined ),
336-
evaluate: vi.fn().mockResolvedValue( 2400 ),
337-
addStyleTag: vi.fn(),
338-
screenshot: vi.fn().mockResolvedValue( screenshotBuffer ),
339-
close: vi.fn(),
340-
};
341-
const browser = {
342-
newPage: vi.fn().mockResolvedValue( page ),
343-
};
344-
vi.mocked( getSharedBrowser ).mockResolvedValue( browser as never );
368+
mockScreenshotBrowser( createMockPage( { buffer: screenshotBuffer, documentHeight: 2400 } ) );
345369

346370
const result = await getTool( 'take_screenshot' ).rawHandler( {
347371
url: 'http://localhost:8903/story-time',
348372
} as never );
349373
const text = getTextContent( result );
350374
expect( text ).toContain( 'Screenshot captured' );
351375
expect( text ).toContain( 'desktop: captured full page (2400px tall)' );
352-
expect( text ).toContain( 'mediaWidgetPayload=' );
376+
expect( text ).not.toContain( 'mediaWidgetPayload' );
353377
expect( text ).not.toContain( 'When this screenshot is useful to show the user' );
354378
expect( text ).not.toContain( 'Path:' );
355379
expect( text ).not.toContain( 'File URL:' );
@@ -359,28 +383,14 @@ describe( 'Studio AI MCP tools', () => {
359383
mimeType: 'image/jpeg',
360384
} );
361385

362-
const payload = JSON.parse( text!.split( 'mediaWidgetPayload=' )[ 1 ] ) as {
363-
widgetProps: { source: { path: string } };
364-
};
365-
expect( result.studioArtifacts ).toEqual( [ payload ] );
366-
await rm( path.dirname( payload.widgetProps.source.path ), { recursive: true, force: true } );
386+
const artifacts = getScreenshotArtifacts( result );
387+
expect( artifacts[ 0 ].widgetProps.source.name ).toBe( 'screenshot-desktop.jpg' );
388+
await cleanUpScreenshotArtifacts( artifacts );
367389
} );
368390

369391
it( 'can force dark mode for take_screenshot captures', async () => {
370-
const screenshotBuffer = Buffer.from( 'dark-jpeg' );
371-
const page = {
372-
emulateMedia: vi.fn(),
373-
goto: vi.fn(),
374-
waitForLoadState: vi.fn().mockResolvedValue( undefined ),
375-
evaluate: vi.fn().mockResolvedValue( 1800 ),
376-
addStyleTag: vi.fn(),
377-
screenshot: vi.fn().mockResolvedValue( screenshotBuffer ),
378-
close: vi.fn(),
379-
};
380-
const browser = {
381-
newPage: vi.fn().mockResolvedValue( page ),
382-
};
383-
vi.mocked( getSharedBrowser ).mockResolvedValue( browser as never );
392+
const page = createMockPage( { buffer: Buffer.from( 'dark-jpeg' ), documentHeight: 1800 } );
393+
mockScreenshotBrowser( page );
384394

385395
const result = await getTool( 'take_screenshot' ).rawHandler( {
386396
url: 'http://localhost:8903/story-time',
@@ -394,37 +404,21 @@ describe( 'Studio AI MCP tools', () => {
394404
} );
395405
expect( text ).toContain( 'desktop dark: captured full page (1800px tall)' );
396406

397-
const payload = JSON.parse( text!.split( 'mediaWidgetPayload=' )[ 1 ] ) as {
398-
widgetProps: { alt: string; source: { path: string; name: string } };
399-
};
407+
const artifacts = getScreenshotArtifacts( result );
400408
try {
401-
expect( payload.widgetProps.alt ).toBe(
409+
expect( artifacts[ 0 ].widgetProps.alt ).toBe(
402410
'Screenshot of http://localhost:8903/story-time (desktop dark)'
403411
);
404-
expect( payload.widgetProps.source.name ).toBe( 'screenshot-desktop-dark.jpg' );
412+
expect( artifacts[ 0 ].widgetProps.source.name ).toBe( 'screenshot-desktop-dark.jpg' );
405413
} finally {
406-
await rm( path.dirname( payload.widgetProps.source.path ), {
407-
recursive: true,
408-
force: true,
409-
} );
414+
await cleanUpScreenshotArtifacts( artifacts );
410415
}
411416
} );
412417

413418
it( 'emits a media artifact when take_screenshot runs with chat artifacts enabled', async () => {
414-
const screenshotBuffer = Buffer.from( 'artifact-jpeg' );
415-
const page = {
416-
emulateMedia: vi.fn(),
417-
goto: vi.fn(),
418-
waitForLoadState: vi.fn().mockResolvedValue( undefined ),
419-
evaluate: vi.fn().mockResolvedValue( 1200 ),
420-
addStyleTag: vi.fn(),
421-
screenshot: vi.fn().mockResolvedValue( screenshotBuffer ),
422-
close: vi.fn(),
423-
};
424-
const browser = {
425-
newPage: vi.fn().mockResolvedValue( page ),
426-
};
427-
vi.mocked( getSharedBrowser ).mockResolvedValue( browser as never );
419+
mockScreenshotBrowser(
420+
createMockPage( { buffer: Buffer.from( 'artifact-jpeg' ), documentHeight: 1200 } )
421+
);
428422
const tool = resolveStudioToolDefinitions( {
429423
emitChatArtifacts: true,
430424
} ).find( ( definition ) => definition.name === 'take_screenshot' );
@@ -455,31 +449,19 @@ describe( 'Studio AI MCP tools', () => {
455449
} ),
456450
} )
457451
);
452+
expect( getTextContent( result ) ).not.toContain( 'mediaWidgetPayload' );
458453

459-
const payload = JSON.parse( getTextContent( result )!.split( 'mediaWidgetPayload=' )[ 1 ] ) as {
460-
widgetProps: { source: { path: string } };
461-
};
462-
await rm( path.dirname( payload.widgetProps.source.path ), { recursive: true, force: true } );
454+
const details = result.details as { studioArtifacts: ScreenshotArtifact[] };
455+
await cleanUpScreenshotArtifacts( details.studioArtifacts );
463456
} );
464457

465458
it( 'can capture desktop and mobile screenshots in one take_screenshot call', async () => {
466459
const desktopBuffer = Buffer.from( 'desktop-jpeg' );
467460
const mobileBuffer = Buffer.from( 'mobile-jpeg' );
468-
const createPage = ( buffer: Buffer ) => ( {
469-
emulateMedia: vi.fn(),
470-
goto: vi.fn(),
471-
waitForLoadState: vi.fn().mockResolvedValue( undefined ),
472-
evaluate: vi.fn().mockResolvedValue( 2400 ),
473-
addStyleTag: vi.fn(),
474-
screenshot: vi.fn().mockResolvedValue( buffer ),
475-
close: vi.fn(),
476-
} );
477-
const desktopPage = createPage( desktopBuffer );
478-
const mobilePage = createPage( mobileBuffer );
479-
const browser = {
480-
newPage: vi.fn().mockResolvedValueOnce( desktopPage ).mockResolvedValueOnce( mobilePage ),
481-
};
482-
vi.mocked( getSharedBrowser ).mockResolvedValue( browser as never );
461+
const { newPage } = mockScreenshotBrowser(
462+
createMockPage( { buffer: desktopBuffer, documentHeight: 2400 } ),
463+
createMockPage( { buffer: mobileBuffer, documentHeight: 2400 } )
464+
);
483465

484466
const result = await getTool( 'take_screenshot' ).rawHandler( {
485467
url: 'http://localhost:8903/story-time',
@@ -490,8 +472,8 @@ describe( 'Studio AI MCP tools', () => {
490472
expect( text ).toContain( 'Screenshots captured:' );
491473
expect( text ).toContain( '- desktop: captured full page (2400px tall)' );
492474
expect( text ).toContain( '- mobile: captured full page (2400px tall)' );
493-
expect( text ).toContain( 'mediaWidgetPayloads=' );
494-
expect( browser.newPage ).toHaveBeenCalledTimes( 2 );
475+
expect( text ).not.toContain( 'mediaWidgetPayload' );
476+
expect( newPage ).toHaveBeenCalledTimes( 2 );
495477
expect( result.content.slice( 1 ) ).toEqual( [
496478
{
497479
type: 'image',
@@ -505,44 +487,23 @@ describe( 'Studio AI MCP tools', () => {
505487
},
506488
] );
507489

508-
const payloads = JSON.parse( text!.split( 'mediaWidgetPayloads=' )[ 1 ] ) as Array< {
509-
widgetProps: { source: { path: string; name: string } };
510-
} >;
490+
const artifacts = getScreenshotArtifacts( result );
511491
try {
512-
expect( payloads.map( ( payload ) => payload.widgetProps.source.name ) ).toEqual( [
492+
expect( artifacts.map( ( artifact ) => artifact.widgetProps.source.name ) ).toEqual( [
513493
'screenshot-desktop.jpg',
514494
'screenshot-mobile.jpg',
515495
] );
516-
expect(
517-
result.studioArtifacts?.map( ( payload ) => payload.widgetProps.source )
518-
).toMatchObject( [ { name: 'screenshot-desktop.jpg' }, { name: 'screenshot-mobile.jpg' } ] );
519496
} finally {
520-
await Promise.all(
521-
payloads.map( ( payload ) =>
522-
rm( path.dirname( payload.widgetProps.source.path ), { recursive: true, force: true } )
523-
)
524-
);
497+
await cleanUpScreenshotArtifacts( artifacts );
525498
}
526499
} );
527500

528501
it( 'can capture light and dark screenshots in one take_screenshot call', async () => {
529502
const lightBuffer = Buffer.from( 'light-jpeg' );
530503
const darkBuffer = Buffer.from( 'dark-jpeg' );
531-
const createPage = ( buffer: Buffer ) => ( {
532-
emulateMedia: vi.fn(),
533-
goto: vi.fn(),
534-
waitForLoadState: vi.fn().mockResolvedValue( undefined ),
535-
evaluate: vi.fn().mockResolvedValue( 1600 ),
536-
addStyleTag: vi.fn(),
537-
screenshot: vi.fn().mockResolvedValue( buffer ),
538-
close: vi.fn(),
539-
} );
540-
const lightPage = createPage( lightBuffer );
541-
const darkPage = createPage( darkBuffer );
542-
const browser = {
543-
newPage: vi.fn().mockResolvedValueOnce( lightPage ).mockResolvedValueOnce( darkPage ),
544-
};
545-
vi.mocked( getSharedBrowser ).mockResolvedValue( browser as never );
504+
const lightPage = createMockPage( { buffer: lightBuffer, documentHeight: 1600 } );
505+
const darkPage = createMockPage( { buffer: darkBuffer, documentHeight: 1600 } );
506+
mockScreenshotBrowser( lightPage, darkPage );
546507

547508
const result = await getTool( 'take_screenshot' ).rawHandler( {
548509
url: 'http://localhost:8903/story-time',
@@ -573,23 +534,14 @@ describe( 'Studio AI MCP tools', () => {
573534
},
574535
] );
575536

576-
const payloads = JSON.parse( text!.split( 'mediaWidgetPayloads=' )[ 1 ] ) as Array< {
577-
widgetProps: { source: { path: string; name: string } };
578-
} >;
537+
const artifacts = getScreenshotArtifacts( result );
579538
try {
580-
expect( payloads.map( ( payload ) => payload.widgetProps.source.name ) ).toEqual( [
539+
expect( artifacts.map( ( artifact ) => artifact.widgetProps.source.name ) ).toEqual( [
581540
'screenshot-desktop-light.jpg',
582541
'screenshot-desktop-dark.jpg',
583542
] );
584543
} finally {
585-
await Promise.all(
586-
payloads.map( ( payload ) =>
587-
rm( path.dirname( payload.widgetProps.source.path ), {
588-
recursive: true,
589-
force: true,
590-
} )
591-
)
592-
);
544+
await cleanUpScreenshotArtifacts( artifacts );
593545
}
594546
} );
595547

@@ -946,17 +898,8 @@ describe( 'Studio AI MCP tools', () => {
946898

947899
it( 'can force dark mode when sharing a screenshot', async () => {
948900
const screenshotBuffer = Buffer.from( 'shared-png' );
949-
const page = {
950-
emulateMedia: vi.fn(),
951-
goto: vi.fn(),
952-
waitForLoadState: vi.fn().mockResolvedValue( undefined ),
953-
evaluate: vi.fn().mockResolvedValue( undefined ),
954-
addStyleTag: vi.fn(),
955-
screenshot: vi.fn().mockResolvedValue( screenshotBuffer ),
956-
close: vi.fn(),
957-
};
958-
const browser = { newPage: vi.fn().mockResolvedValue( page ) };
959-
vi.mocked( getSharedBrowser ).mockResolvedValue( browser as never );
901+
const page = createMockPage( { buffer: screenshotBuffer } );
902+
mockScreenshotBrowser( page );
960903

961904
const result = await getTool( 'share_screenshot' ).rawHandler( {
962905
url: 'http://localhost:8903/',

‎apps/cli/ai/tools/take-screenshot.ts‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,12 @@ function getCaptureLabel( target: {
4949
function getCaptureListLabel(
5050
targets: Array< { viewportType: ScreenshotViewportType; colorScheme?: ScreenshotColorScheme } >
5151
): string {
52-
return targets.length === 1
53-
? getCaptureLabel( targets[ 0 ] )
54-
: targets.map( getCaptureLabel ).join( ', ' );
52+
return targets.map( getCaptureLabel ).join( ', ' );
5553
}
5654

5755
export const takeScreenshotTool = defineTool(
5856
'take_screenshot',
5957
'Takes a full-page screenshot of a URL. Returns the screenshot as an image that you can analyze visually. ' +
60-
'Also saves the screenshot as a temporary local image and returns a ready-to-use media widget payload. ' +
6158
'Supports desktop and mobile viewports; pass `viewport: "all"` when you need both for design verification. ' +
6259
'Pass `colorScheme: "light"`, `colorScheme: "dark"`, or `colorScheme: "all"` to verify pages that respond to prefers-color-scheme. ' +
6360
'Long pages are clipped at 8000 vertical pixels (a vision-model limit); the response reports the document height and whether more remains, and you can call again with `offset` to fetch the next slice. ' +
@@ -141,17 +138,8 @@ export const takeScreenshotTool = defineTool(
141138
const captureLines = captures.map( describeCapture );
142139
const textLines =
143140
captures.length === 1
144-
? [
145-
`Screenshot captured — ${ captureLines[ 0 ] }`,
146-
`mediaWidgetPayload=${ JSON.stringify( captures[ 0 ].mediaWidgetPayload ) }`,
147-
]
148-
: [
149-
'Screenshots captured:',
150-
...captureLines.map( ( line ) => `- ${ line }` ),
151-
`mediaWidgetPayloads=${ JSON.stringify(
152-
captures.map( ( capture ) => capture.mediaWidgetPayload )
153-
) }`,
154-
];
141+
? [ `Screenshot captured — ${ captureLines[ 0 ] }` ]
142+
: [ 'Screenshots captured:', ...captureLines.map( ( line ) => `- ${ line }` ) ];
155143
emitProgress( `Screenshot captured (${ captureLabel })` );
156144
return {
157145
content: [

‎packages/common/ai/studio-widgets.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,9 @@ export const STUDIO_PRESENTATION_RULES: StudioPresentationRule[] = [
6161
'When an image, video, SVG, logo, icon, illustration, or other visual asset is generated, written to, or discovered on disk, present it as a media widget using a file:// URL and local source metadata. For generated SVGs, write a complete .svg file to a local path first, then present that file. Use a temporary file when the user only wants to see the asset; save under the site or project only when they ask for a durable file. Do not present generated SVG code as a drawing widget.',
6262
},
6363
{
64-
id: 'screenshot-local-media',
64+
id: 'screenshot-auto-artifact',
6565
description:
66-
'When the user asks to take, show, or capture a screenshot, present the actual captured PNG from take_screenshot as a media widget using the returned local-file payload. Do not substitute a site-preview widget for a screenshot; site-preview is for live previews, not captured screenshots. Do not present every internal verification screenshot.',
66+
'take_screenshot captures are already shown to the user as inline media in the conversation. Never call studio_present for a screenshot, and do not substitute a site-preview widget for one; site-preview is for live previews, not captured screenshots.',
6767
},
6868
];
6969

0 commit comments

Comments
 (0)