Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
c6172aa
Add AI screenshot artifacts and color scheme capture
shaunandrews Jun 28, 2026
e160e95
Improve local screenshot data URL conversion
shaunandrews Jun 29, 2026
63ce0e8
Hide screenshot payloads in Studio Code transcript
shaunandrews Jun 29, 2026
df716ff
Forward tool execute args and emit chat artifacts for all remote stud…
shaunandrews Jul 3, 2026
cf66b73
Add shared media artifact helpers and payload-line stripper to @studi…
shaunandrews Jul 3, 2026
4bbec3c
Stop embedding screenshot payload text now that artifacts emit struct…
shaunandrews Jul 3, 2026
686cb25
Store screenshots in a per-session sidecar directory instead of os tmp
shaunandrews Jul 3, 2026
7cd5632
Harden inline artifact rendering in the classic conversation view
shaunandrews Jul 3, 2026
85b2b81
Render screenshot artifacts inline in the legacy Studio Code conversa…
shaunandrews Jul 3, 2026
e007735
Share the colorScheme schema and media emulation across screenshot tools
shaunandrews Jul 3, 2026
a6c3f8f
Load screenshot artifacts as data URLs instead of object URLs
shaunandrews Jul 3, 2026
ae49653
Handle take_screenshot in terminal sessions: file link, inline images…
bcotrim Jul 6, 2026
c4c70d6
Address review feedback: shared media helpers, sturdier schema consta…
bcotrim Jul 6, 2026
d2563cc
Merge branch 'trunk' into extract-screenshot-artifacts
bcotrim Jul 6, 2026
2094470
Skip payload-line stripping when no marker is present
bcotrim Jul 6, 2026
a2598eb
Never let a failed artifact emit error a successful tool result
bcotrim Jul 6, 2026
c812fcb
Log when screenshot storage falls back to the temp directory
bcotrim Jul 6, 2026
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions apps/cli/ai/runtimes/pi/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import {
} from '@studio/common/lib/site-runtime';
import { getAiPayloadsPath, getConfigDirectory } from '@studio/common/lib/well-known-paths';
import { buildSystemPrompt } from 'cli/ai/system-prompt';
import { resolveStudioToolDefinitions } from 'cli/ai/tools';
import { resolveStudioToolDefinitions, withChatArtifactEmission } from 'cli/ai/tools';
import { createAskUserQuestionTool } from 'cli/ai/tools/ask-user-question';
import { createSiteTool } from 'cli/ai/tools/create-site';
import { pullSiteTool } from 'cli/ai/tools/pull-site';
Expand Down Expand Up @@ -527,11 +527,12 @@ function buildAgentTools(
];

if ( isRemoteSite ) {
const remoteStudioTools = [ takeScreenshotTool, createSiteTool, pullSiteTool ].map( ( tool ) =>
withChatArtifactEmission( tool, chatArtifactsEnabled )
);
return [
createWpcomRequestTool( config.wpcomAccessToken!, config.activeSite!.wpcomSiteId! ),
takeScreenshotTool,
createSiteTool,
pullSiteTool,
...remoteStudioTools,
...remoteScratchTools,
...askUserTool,
...skillTool,
Expand Down
38 changes: 38 additions & 0 deletions apps/cli/ai/screenshot-storage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import { mkdir, mkdtemp } from 'fs/promises';
import os from 'os';
import path from 'path';

/**
* Ambient provider for the directory screenshot captures are saved to,
* mirroring `setChatArtifactCallback`. The `studio code` command points it at
* a per-session sidecar directory (`<session>.screenshots/` next to the
* session JSONL) so screenshots referenced by persisted chat artifacts
* survive OS temp cleanup and are removed together with the session.
* Standalone CLI/MCP runs never set a provider and fall back to a throwaway
* temp directory.
*/
type ScreenshotDirectoryProvider = () => Promise< string | null > | string | null;

let screenshotDirectoryProvider: ScreenshotDirectoryProvider | null = null;

export function setScreenshotDirectoryProvider(
provider: ScreenshotDirectoryProvider | null
): void {
screenshotDirectoryProvider = provider;
}

export async function resolveScreenshotDirectory(): Promise< string > {
try {
const directory = await screenshotDirectoryProvider?.();
if ( directory ) {
await mkdir( directory, { recursive: true } );
return directory;
}
} catch ( error ) {
// Fall through to the temp directory so the capture still succeeds, but
// say so: files there are purged by the OS, so persisted artifacts will
// eventually show as unavailable and this line is the only clue why.
console.warn( '[screenshots] falling back to a temporary directory:', error );
}
return mkdtemp( path.join( os.tmpdir(), 'studio-screenshot-' ) );
}
14 changes: 13 additions & 1 deletion apps/cli/ai/system-prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ ${ REMOTE_DESIGN_GUIDELINES }${ remoteSessionAddendum }

return `${ buildLocalIntro( {
chatArtifactsEnabled: options?.chatArtifactsEnabled ?? false,
remoteSession: options?.remoteSession ?? false,
runtime: options?.runtime,
} ) }

Expand Down Expand Up @@ -97,9 +98,20 @@ function getPostContentGuidance( runtime?: SiteRuntime ): string {

function buildLocalIntro( options: {
chatArtifactsEnabled: boolean;
remoteSession: boolean;
runtime?: SiteRuntime;
} ): string {
const postContentGuidance = getPostContentGuidance( options.runtime );
// Remote-bridge sessions also run without chat artifacts, but their user is
// on the other end of a messaging bridge: local file paths are unreachable
// and REMOTE_SESSION_GUIDANCE (share_screenshot) already covers delivery.
const terminalScreenshotSection = options.remoteSession
? ''
: `

## Screenshots

This session runs in a terminal, which may not be able to display images. Screenshots you capture are for your own visual verification; the user may only see a link to the saved image file in the transcript. Do not respond as though the user is looking at the capture (e.g. "Here's your site!") — instead, state what you verified and describe notable findings, and point to the saved screenshot file when it helps.`;
const automaticArtifactSection = options.chatArtifactsEnabled
? `

Expand All @@ -114,7 +126,7 @@ ${ getStudioPresentationRulesPrompt() }

Available desks widget types:
${ getStudioWidgetPromptManifest() }`
: '';
: terminalScreenshotSection;
const studioPresentToolBullet = options.chatArtifactsEnabled
? `
- studio_present: Show one or more Studio desks widgets as inline visual artifacts.`
Expand Down
67 changes: 61 additions & 6 deletions apps/cli/ai/tests/screenshot-helpers.test.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,77 @@
import { readFile, rm } from 'node:fs/promises';
import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
import { saveScreenshotToTempFile } from '../tools/screenshot-helpers';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { setScreenshotDirectoryProvider } from '../screenshot-storage';
import { saveScreenshotFile } from '../tools/screenshot-helpers';

describe( 'screenshot helpers', () => {
it( 'saves screenshots to a temporary local file', async () => {
afterEach( () => {
setScreenshotDirectoryProvider( null );
} );

it( 'falls back to a temporary directory when no provider is set', async () => {
const buffer = Buffer.from( 'not-really-a-png' );
const result = await saveScreenshotToTempFile( buffer, { viewportType: 'desktop' } );
const result = await saveScreenshotFile( buffer, { viewportType: 'desktop' } );

try {
expect( result.path.startsWith( os.tmpdir() ) ).toBe( true );
expect( result.fileUrl.startsWith( 'file://' ) ).toBe( true );
expect( result.name ).toBe( 'screenshot-desktop.png' );
expect( result.name ).toMatch( /^screenshot-desktop-[0-9a-f]{8}\.png$/ );
expect( result.mimeType ).toBe( 'image/png' );
await expect( readFile( result.path ) ).resolves.toEqual( buffer );
} finally {
await rm( path.dirname( result.path ), { recursive: true, force: true } );
}
} );

it( 'saves captures into the provided session directory with unique names', async () => {
const sessionRoot = await mkdtemp( path.join( os.tmpdir(), 'studio-session-' ) );
const screenshotsDirectory = path.join( sessionRoot, 'session.screenshots' );
setScreenshotDirectoryProvider( () => screenshotsDirectory );

try {
const first = await saveScreenshotFile( Buffer.from( 'first' ), {
viewportType: 'desktop',
format: 'jpeg',
colorScheme: 'dark',
} );
const second = await saveScreenshotFile( Buffer.from( 'second' ), {
viewportType: 'desktop',
format: 'jpeg',
colorScheme: 'dark',
} );

expect( path.dirname( first.path ) ).toBe( screenshotsDirectory );
expect( first.name ).toMatch( /^screenshot-desktop-dark-[0-9a-f]{8}\.jpg$/ );
expect( first.name ).not.toBe( second.name );
await expect( readFile( first.path, 'utf8' ) ).resolves.toBe( 'first' );
await expect( readFile( second.path, 'utf8' ) ).resolves.toBe( 'second' );
await expect( readdir( screenshotsDirectory ) ).resolves.toHaveLength( 2 );
} finally {
await rm( sessionRoot, { recursive: true, force: true } );
}
} );

it( 'falls back to a temporary directory when the provider throws, and says so', async () => {
const warnSpy = vi.spyOn( console, 'warn' ).mockImplementation( () => {} );
setScreenshotDirectoryProvider( () => {
throw new Error( 'no session' );
} );

const result = await saveScreenshotFile( Buffer.from( 'fallback' ), {
viewportType: 'mobile',
} );

try {
expect( result.path.startsWith( os.tmpdir() ) ).toBe( true );
expect( warnSpy ).toHaveBeenCalledWith(
expect.stringContaining( '[screenshots] falling back to a temporary directory' ),
expect.any( Error )
);
} finally {
warnSpy.mockRestore();
await rm( path.dirname( result.path ), { recursive: true, force: true } );
}
} );
} );
30 changes: 26 additions & 4 deletions apps/cli/ai/tests/system-prompt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,8 @@ describe( 'buildSystemPrompt', () => {
expect( prompt ).toContain( 'For generated SVGs, write a complete .svg file' );
expect( prompt ).toContain( 'Do not present generated SVG code as a drawing widget' );
expect( prompt ).not.toContain( '- drawing:' );
expect( prompt ).toContain( '- screenshot-local-media:' );
expect( prompt ).toContain( 'present the actual captured PNG' );
expect( prompt ).toContain( 'Do not substitute a site-preview widget for a screenshot' );
expect( prompt ).toContain( '- screenshot-auto-artifact:' );
expect( prompt ).toContain( 'Never call studio_present for a screenshot' );
expect( prompt ).toContain( 'site-preview is for live previews, not captured screenshots' );
expect( prompt ).toContain( '- theme:' );
expect( prompt ).toContain( '- theme-template:' );
Expand Down Expand Up @@ -147,7 +146,30 @@ describe( 'buildSystemPrompt', () => {
expect( prompt ).not.toContain( '## Visual artifacts' );
expect( prompt ).not.toContain( '- site-code-scratchpad:' );
expect( prompt ).not.toContain( '- saved-local-media:' );
expect( prompt ).not.toContain( '- screenshot-local-media:' );
expect( prompt ).not.toContain( '- screenshot-auto-artifact:' );
expect( prompt ).not.toContain( 'studio_present' );
} );

it( 'warns that terminal users may not see screenshots when chat artifacts are disabled', () => {
const prompt = buildSystemPrompt( { chatArtifactsEnabled: false } );

expect( prompt ).toContain( '## Screenshots' );
expect( prompt ).toContain( 'Do not respond as though the user is looking at the capture' );
} );

it( 'omits the terminal screenshot caveat when chat artifacts are enabled', () => {
const prompt = buildSystemPrompt( { chatArtifactsEnabled: true } );

expect( prompt ).not.toContain( 'Do not respond as though the user is looking at the capture' );
} );

it( 'omits the terminal screenshot caveat for remote-bridge sessions', () => {
// The Telegram user cannot open local file paths; delivery is covered
// by the remote-session share_screenshot guidance instead.
const prompt = buildSystemPrompt( { chatArtifactsEnabled: false, remoteSession: true } );

expect( prompt ).not.toContain( '## Screenshots' );
expect( prompt ).not.toContain( 'Do not respond as though the user is looking at the capture' );
expect( prompt ).toContain( '## Telegram remote session' );
} );
} );
Loading
Loading