Skip to content
152 changes: 60 additions & 92 deletions apps/cli/ai/agent.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
import fs from 'fs';
import path from 'path';
import { query, type HookCallback, type Query } from '@anthropic-ai/claude-agent-sdk';
import { AI_MODELS, DEFAULT_MODEL, type AiModelId } from '@studio/common/ai/models';
import { buildSystemPrompt } from 'cli/ai/system-prompt';
import { createRemoteSiteTools, createStudioTools } from 'cli/ai/tools';
import {
AI_MODELS,
DEFAULT_MODEL,
getAiModelFamily,
getAiModelLabel,
type AiModelId,
} from '@studio/common/ai/models';
import { anthropicRuntime } from 'cli/ai/runtimes/anthropic';
import { isOpenAIRuntimeSession, openaiRuntime } from 'cli/ai/runtimes/openai';
import { STUDIO_SITES_ROOT } from 'cli/lib/site-paths';
import type { AgentRuntime, AgentRuntimeHandle } from 'cli/ai/runtimes/types';
import type { SiteInfo } from 'cli/ai/ui';

export { AI_MODELS, DEFAULT_MODEL, type AiModelId };
export { AI_MODELS, DEFAULT_MODEL, getAiModelLabel, type AiModelId };

export interface AskUserQuestion {
question: string;
Expand All @@ -27,6 +32,14 @@ export interface AiAgentConfig {
activeSite?: SiteInfo | null;
wpcomAccessToken?: string;
onAskUser?: AskUserHandler;
/**
* Absolute path to the JSONL the recorder is writing to. Optional —
* the OpenAI runtime uses it to load/save its sidecar transcript so
* pi-agent-core memory survives across CLI process forks (the desktop
* UI forks per turn). When omitted, the OpenAI runtime keeps state in
* memory only and forgets across forks.
*/
sessionFilePath?: string;
}

// The Claude Agent SDK rejects internal pending promises (e.g. control
Expand All @@ -49,10 +62,24 @@ process.on( 'unhandledRejection', ( reason ) => {
} );

/**
* Start the AI agent and return the Query object.
* Caller can iterate messages with `for await` and call `interrupt()` to stop.
* Pick the runtime based on the selected model's family. GPT-* models route to
* the OpenAI runtime; everything else defaults to the Claude Agent SDK path.
*
* Matches the user-facing mental model: `/model` is the control, and whichever
* model you pick dictates which backend handles the turn — independent of
* which provider supplied the credentials.
*/
export function startAiAgent( config: AiAgentConfig ): Query {
function pickRuntime( model: AiModelId ): AgentRuntime {
return getAiModelFamily( model ) === 'openai' ? openaiRuntime : anthropicRuntime;
}

/**
* Start the AI agent and return a handle that streams messages until the
* turn completes. Caller can iterate with `for await` and call `interrupt()`
* to stop. The concrete runtime (Anthropic vs OpenAI) is chosen based on the
* resolved environment.
*/
export function startAiAgent( config: AiAgentConfig ): AgentRuntimeHandle {
const {
prompt,
env,
Expand All @@ -61,97 +88,38 @@ export function startAiAgent( config: AiAgentConfig ): Query {
activeSite,
wpcomAccessToken,
onAskUser,
sessionFilePath,
} = config;
const resolvedEnv = env ?? { ...( process.env as Record< string, string > ) };

const isRemoteSite = activeSite?.remote && activeSite?.wpcomSiteId && wpcomAccessToken;

// Preview-steering tools only belong in the toolset when the Studio
// desktop UI is on the other end of the IPC channel — otherwise the
// agent's navigate/reload calls render as noise in the terminal
// transcript. `process.send` is the same signal `emitEvent` uses to
// pick between IPC and stdout NDJSON.
const isForkedByDesktop = typeof process.send === 'function';

// Configure MCP servers based on site type:
// Remote sites get WP.com REST API tools + screenshot; local sites get the full Studio toolset.
const mcpServers = {
studio: isRemoteSite
? createRemoteSiteTools( wpcomAccessToken, activeSite.wpcomSiteId! )
: createStudioTools( { enablePreviewSteering: isForkedByDesktop } ),
};

// The remote-session controller sets STUDIO_REMOTE_SESSION=1 when it spawns
// `studio code --json` so the agent knows it's driving Telegram and should
// favor screenshot replies.
const remoteSession = resolvedEnv.STUDIO_REMOTE_SESSION === '1';

// Build site-aware system prompt
const systemPromptOptions = isRemoteSite
? {
remoteSite: {
name: activeSite.name,
url: activeSite.url ?? '',
id: activeSite.wpcomSiteId!,
},
remoteSession,
}
: { previewSteering: isForkedByDesktop, remoteSession };

if ( ! fs.existsSync( STUDIO_SITES_ROOT ) ) {
fs.mkdirSync( STUDIO_SITES_ROOT, { recursive: true } );
}

// Intercept the built-in AskUserQuestion tool so the agent's questions
// render in our chat UI (via onAskUser) instead of the SDK's default
// prompt. PreToolUse with `matcher: 'AskUserQuestion'` lets us inject
// answers via `updatedInput` and approve the call, while leaving every
// other tool to the 'auto' permission classifier.
const askUserQuestionHook: HookCallback | undefined = onAskUser
? async ( input ) => {
if ( input.hook_event_name !== 'PreToolUse' ) {
return {};
}
const toolInput = input.tool_input as {
questions?: AskUserQuestion[];
answers?: Record< string, string >;
};
const questions = ( toolInput.questions ?? [] ).map( ( q ) => ( {
...q,
allowFreeForm: true,
} ) );
const answers = await onAskUser( questions );
return {
hookSpecificOutput: {
hookEventName: 'PreToolUse' as const,
permissionDecision: 'allow' as const,
updatedInput: { ...toolInput, answers },
},
};
}
: undefined;
const runtime = pickRuntime( model );
// Cross-family resume guard. If the caller passes a `resume` id that was
// issued by the OpenAI runtime but we're now routing to the Anthropic
// runtime (e.g. user picked a different family in the model dropdown
// mid-session), the Claude Agent SDK doesn't recognize it and errors with
// "No conversation found". Drop the resume hint instead — the new runtime
// starts a fresh agent thread, the session record on disk continues
// accumulating turns. The OpenAI direction handles unknown resume ids
// silently (see runtimes/openai/index.ts), so we only need to gate the
// Anthropic side. In-memory tracking; survives a single process lifetime
// only (see comment on isOpenAIRuntimeSession).
const safeResume =
resume && getAiModelFamily( model ) === 'anthropic' && isOpenAIRuntimeSession( resume )
? undefined
: resume;

return query( {
return runtime.run( {
prompt,
options: {
env: resolvedEnv,
systemPrompt: {
type: 'preset',
preset: 'claude_code',
append: buildSystemPrompt( systemPromptOptions ),
},
mcpServers,
cwd: STUDIO_SITES_ROOT,
tools: { type: 'preset', preset: 'claude_code' },
permissionMode: 'auto',
...( askUserQuestionHook && {
hooks: {
PreToolUse: [ { matcher: 'AskUserQuestion', hooks: [ askUserQuestionHook ] } ],
},
} ),
plugins: [ { type: 'local' as const, path: path.resolve( import.meta.dirname, 'plugin' ) } ],
model,
resume,
},
env: resolvedEnv,
model,
resume: safeResume,
activeSite,
wpcomAccessToken,
onAskUser,
sessionFilePath,
} );
}
9 changes: 9 additions & 0 deletions apps/cli/ai/eval-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import { writeFileSync, writeSync as fsWriteSync } from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { isAiModelId, type AiModelId } from '@studio/common/ai/models';
import { startAiAgent } from 'cli/ai/agent';
import {
resolveAiEnvironment,
Expand All @@ -21,6 +22,7 @@ import type { AiProviderId } from 'cli/ai/providers';
interface EvalRunnerInput {
prompt: string;
timeoutMs?: number;
model?: AiModelId;
}

function normalizeToolName( name: string ): string {
Expand Down Expand Up @@ -131,9 +133,15 @@ function readInput(): EvalRunnerInput {
}
}

const envModel = process.env.STUDIO_EVAL_MODEL?.trim();
const varModel = typeof vars.model === 'string' ? vars.model.trim() : undefined;
const rawModel = varModel || envModel;
const model = rawModel && isAiModelId( rawModel ) ? rawModel : undefined;

return {
prompt: ( vars.prompt as string ) ?? prompt,
timeoutMs: typeof vars.timeoutMs === 'number' ? vars.timeoutMs : undefined,
model,
};
}

Expand Down Expand Up @@ -185,6 +193,7 @@ async function runEval( input: EvalRunnerInput ) {
const query = startAiAgent( {
prompt: input.prompt.trim(),
env,
...( input.model ? { model: input.model } : {} ),
} );
phaseTimingsMs.start_ai_agent_ms = Date.now() - phaseStartedAt;

Expand Down
88 changes: 78 additions & 10 deletions apps/cli/ai/providers.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
import { password } from '@inquirer/prompts';
import {
AI_MODELS,
DEFAULT_MODEL,
type AiModelFamily,
type AiModelId,
} from '@studio/common/ai/models';
import { readAuthToken } from '@studio/common/lib/shared-config';
import { __ } from '@wordpress/i18n';
import { readCliConfig, updateCliConfigWithPartial } from 'cli/lib/cli-config/core';
Expand All @@ -15,7 +21,12 @@ export const DEFAULT_AI_PROVIDER: AiProviderId = 'wpcom';
export const AI_PROVIDER_PRIORITY: AiProviderId[] = [ 'wpcom', 'anthropic-api-key' ];

const DEFAULT_WPCOM_AI_GATEWAY_BASE_URL = 'https://public-api.wordpress.com/wpcom/v2/ai-api-proxy';
const WPCOM_AI_FEATURE_HEADER = 'studio-assistant-anthropic';
// The wpcom AI proxy maps feature slugs to upstream providers. Historically
// `studio-assistant` was wired for OpenAI (OPENAI_TOKEN); when Claude support
// landed a parallel `studio-assistant-anthropic` slug was added. Keep using
// the existing slugs so no server-side allowlist change is required.
const WPCOM_AI_FEATURE_HEADER_ANTHROPIC = 'studio-assistant-anthropic';
const WPCOM_AI_FEATURE_HEADER_OPENAI = 'studio-assistant';

export interface ResolveAiEnvironmentOptions {
sessionId?: string;
Expand All @@ -24,12 +35,44 @@ export interface ResolveAiEnvironmentOptions {
export interface AiProviderDefinition {
id: AiProviderId;
autoFallbackWhenUnavailable: boolean;
/**
* Which model families this provider can service. `wpcom` relays both
* Anthropic and OpenAI wire formats through the same proxy; direct-API
* providers are restricted to their own family. `availableModels` and
* `defaultModel` are derived from this and kept on the definition so
* callers don't have to filter AI_MODELS themselves.
*/
readonly supportedModelFamilies: readonly AiModelFamily[];
readonly availableModels: readonly AiModelId[];
readonly defaultModel: AiModelId;
supportsModel( model: AiModelId ): boolean;
isVisible: () => Promise< boolean >;
isReady: () => Promise< boolean >;
prepare: ( options?: { force?: boolean } ) => Promise< void >;
resolveEnv: ( options?: ResolveAiEnvironmentOptions ) => Promise< Record< string, string > >;
}

/**
* Fills in `availableModels`, `defaultModel`, and `supportsModel` from the
* declared `supportedModelFamilies` so each provider literal below only has to
* state its family allowlist.
*/
function defineProvider(
partial: Omit< AiProviderDefinition, 'availableModels' | 'defaultModel' | 'supportsModel' >
): AiProviderDefinition {
const availableModels: AiModelId[] = AI_MODELS.filter( ( model ) =>
partial.supportedModelFamilies.includes( model.family )
).map( ( model ) => model.id );
return {
...partial,
availableModels,
defaultModel: availableModels[ 0 ] ?? DEFAULT_MODEL,
supportsModel( model ) {
return availableModels.includes( model );
},
};
}

async function resolveAnthropicApiKey( options?: {
force?: boolean;
} ): Promise< string | undefined > {
Expand Down Expand Up @@ -84,6 +127,9 @@ function createBaseEnvironment(): Record< string, string > {
delete env.ANTHROPIC_AUTH_TOKEN;
delete env.ANTHROPIC_BASE_URL;
delete env.ANTHROPIC_CUSTOM_HEADERS;
delete env.OPENAI_API_KEY;
delete env.OPENAI_BASE_URL;
delete env.STUDIO_OPENAI_DEFAULT_HEADERS;
delete env.CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS;
delete env.CLAUDE_CODE_MAX_RETRIES;

Expand All @@ -97,9 +143,10 @@ function createBaseEnvironment(): Record< string, string > {
}

const AI_PROVIDER_DEFINITIONS: Record< AiProviderId, AiProviderDefinition > = {
wpcom: {
wpcom: defineProvider( {
id: 'wpcom',
autoFallbackWhenUnavailable: true,
supportedModelFamilies: [ 'anthropic', 'openai' ],
isVisible: async () => true,
isReady: async () => hasInlineWpcomAuth() || ( await hasValidWpcomAuth() ),
prepare: async () => {
Expand All @@ -116,27 +163,48 @@ const AI_PROVIDER_DEFINITIONS: Record< AiProviderId, AiProviderDefinition > = {
throw new LoggerError( __( 'WordPress.com login required. Use /login to authenticate.' ) );
}
const env = createBaseEnvironment();
env.ANTHROPIC_BASE_URL = getWpcomAiGatewayBaseUrl();
const gatewayBaseUrl = getWpcomAiGatewayBaseUrl();

// Anthropic path — consumed by the Claude Agent SDK runtime.
env.ANTHROPIC_BASE_URL = gatewayBaseUrl;
env.ANTHROPIC_AUTH_TOKEN = accessToken;
const customHeaders: Record< string, string > = {
'X-WPCOM-AI-Feature': WPCOM_AI_FEATURE_HEADER,
const anthropicHeaders: Record< string, string > = {
'X-WPCOM-AI-Feature': WPCOM_AI_FEATURE_HEADER_ANTHROPIC,
};
if ( options?.sessionId ) {
customHeaders[ 'X-WPCOM-Session-ID' ] = options.sessionId;
anthropicHeaders[ 'X-WPCOM-Session-ID' ] = options.sessionId;
}
env.ANTHROPIC_CUSTOM_HEADERS = buildAnthropicCustomHeaders( customHeaders );
env.ANTHROPIC_CUSTOM_HEADERS = buildAnthropicCustomHeaders( anthropicHeaders );
env.CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS = '1';
// The default agent retry count (10) causes the CLI to hang for
// minutes when the WPCOM proxy returns a 429 (e.g. usage cap
// reached). Retries don't recover the cap, so fail fast and let
// the UI surface the error.
env.CLAUDE_CODE_MAX_RETRIES = '0';

// OpenAI path — consumed by the Vercel AI SDK runtime. The wpcom
// proxy accepts the same bearer token for both protocols; it
// dispatches to the right upstream based on the request path.
// @ai-sdk/openai expects baseURL to include /v1 (like the real
// OpenAI API), so the request path becomes /v1/chat/completions —
// mirroring the Anthropic path's /v1/messages.
env.OPENAI_BASE_URL = `${ gatewayBaseUrl.replace( /\/+$/, '' ) }/v1`;
env.OPENAI_API_KEY = accessToken;
const openaiHeaders: Record< string, string > = {
'X-WPCOM-AI-Feature': WPCOM_AI_FEATURE_HEADER_OPENAI,
};
if ( options?.sessionId ) {
openaiHeaders[ 'X-WPCOM-Session-ID' ] = options.sessionId;
}
env.STUDIO_OPENAI_DEFAULT_HEADERS = JSON.stringify( openaiHeaders );

return env;
},
},
'anthropic-api-key': {
} ),
'anthropic-api-key': defineProvider( {
id: 'anthropic-api-key',
autoFallbackWhenUnavailable: false,
supportedModelFamilies: [ 'anthropic' ],
isVisible: async () => true,
isReady: async () => {
const { anthropicApiKey } = await readCliConfig();
Expand All @@ -159,7 +227,7 @@ const AI_PROVIDER_DEFINITIONS: Record< AiProviderId, AiProviderDefinition > = {
env.ANTHROPIC_API_KEY = apiKey;
return env;
},
},
} ),
};

export function getAiProviderDefinition( provider: AiProviderId ): AiProviderDefinition {
Expand Down
Loading
Loading