Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';
import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService';
import { MockEndpoint } from '../../../../platform/endpoint/test/node/mockEndpoint';
import { CUSTOM_TOOL_SEARCH_NAME } from '../../../../platform/networking/common/anthropic';
import { IChatEndpoint } from '../../../../platform/networking/common/networking';
import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
import { ITestingServicesAccessor } from '../../../../platform/test/node/services';
Expand Down Expand Up @@ -102,6 +103,21 @@ describe('getAgentTools background todo enablement', () => {
return tools.some(t => t.name === ToolName.CoreManageTodoList);
}

function hasTool(tools: readonly { name: string }[], name: string): boolean {
return tools.some(tool => tool.name === name);
}

function createToolSearchEndpoint(model: string, modelProvider = 'openai', supportsToolSearch = true): IChatEndpoint {
return {
...mockEndpoint,
model,
family: model,
modelProvider,
// Pin endpoint capability so this test isolates getAgentTools gating from endpoint capability derivation.
supportsToolSearch,
} as IChatEndpoint;
}

test('background todo agent is enabled only when experiment is on and todo is not explicit', () => {
const request = new TestChatRequest('fix the bug');
configService.setConfig(ConfigKey.Advanced.BackgroundTodoAgentEnabled, false);
Expand Down Expand Up @@ -137,4 +153,20 @@ describe('getAgentTools background todo enablement', () => {
const tools = await instantiationService.invokeFunction(getAgentTools, request, mockEndpoint);
expect(hasTodoTool(tools)).toBe(false);
});

test('supported Custom OAI Responses gpt-5.4 and gpt-5.5 endpoints surface tool_search when endpoint capability is enabled', async () => {
const request = new TestChatRequest('find the right tool');
for (const model of ['gpt-5.4', 'gpt-5.5']) {
const tools = await instantiationService.invokeFunction(getAgentTools, request, createToolSearchEndpoint(model, 'CustomOAI'));
expect(hasTool(tools, CUSTOM_TOOL_SEARCH_NAME)).toBe(true);
}
});

test('supported Custom OAI Responses endpoints omit tool_search when endpoint capability is disabled', async () => {
const request = new TestChatRequest('find the right tool');
for (const model of ['gpt-5.4', 'gpt-5.5']) {
const tools = await instantiationService.invokeFunction(getAgentTools, request, createToolSearchEndpoint(model, 'CustomOAI', false));
expect(hasTool(tools, CUSTOM_TOOL_SEARCH_NAME)).toBe(false);
}
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -320,3 +320,91 @@ testFamilies.forEach(family => {
});
});
});

suite('AgentPrompt tool search alignment', () => {
let accessor: ITestingServicesAccessor;
const fileTsUri = URI.file('/workspace/file.ts');

beforeAll(() => {
const testDoc = createTextDocumentData(fileTsUri, 'line 1\nline 2\n\nline 4\nline 5', 'ts').document;
const services = createExtensionUnitTestingServices();
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[
[URI.file('/workspace')],
[testDoc]
]
));
services.define(IChatMLFetcher, new StaticChatMLFetcher([]));
accessor = services.createTestingAccessor();
accessor.get(IConfigurationService).setConfig(ConfigKey.CodeGenerationInstructions, [{
text: 'This is a test custom instruction file',
} satisfies CodeGenerationTextInstruction]);
});

afterAll(() => {
accessor.dispose();
});

async function renderAgentPromptWithToolSearchSupport(family: 'gpt-5.4' | 'gpt-5.5', supportsToolSearch: boolean): Promise<string> {
const instaService = accessor.get(IInstantiationService);
const endpoint = instaService.createInstance(MockEndpoint, family);
endpoint.supportsToolSearch = supportsToolSearch;
const conversation = new Conversation('sessionId', [new Turn('turnId', { type: 'user', message: 'hello' })]);
const toolsService = accessor.get(IToolsService);
const customizations = await PromptRegistry.resolveAllCustomizations(instaService, endpoint);
const renderer = PromptRenderer.create(instaService, endpoint, AgentPrompt, {
priority: 1,
endpoint,
location: ChatLocation.Panel,
promptContext: {
chatVariables: new ChatVariablesCollection(),
history: [],
query: 'hello',
conversation,
tools: {
availableTools: toolsService.tools,
toolInvocationToken: null as never,
toolReferences: [],
}
},
customizations,
});

const rendered = await renderer.render();
addCacheBreakpoints(rendered.messages);
return rendered.messages
.map(m => messageToMarkdown(m))
.join('\n\n')
.replace(/\\+/g, '/');
}

test.each(['gpt-5.4', 'gpt-5.5'] as const)('gates deferred-tool guidance and reminders for %s on endpoint.supportsToolSearch', async family => {
const enabledPrompt = await renderAgentPromptWithToolSearchSupport(family, true);
const disabledPrompt = await renderAgentPromptWithToolSearchSupport(family, false);

expect({
enabled: {
hasToolSearchInstructions: enabledPrompt.includes('<toolSearchInstructions>'),
hasDeferredToolReminder: enabledPrompt.includes('IMPORTANT: Before calling any deferred tool'),
hasDeferredToolList: enabledPrompt.includes('<availableDeferredTools>'),
},
disabled: {
hasToolSearchInstructions: disabledPrompt.includes('<toolSearchInstructions>'),
hasDeferredToolReminder: disabledPrompt.includes('IMPORTANT: Before calling any deferred tool'),
hasDeferredToolList: disabledPrompt.includes('<availableDeferredTools>'),
},
}).toEqual({
enabled: {
hasToolSearchInstructions: true,
hasDeferredToolReminder: true,
hasDeferredToolList: true,
},
disabled: {
hasToolSearchInstructions: false,
hasDeferredToolReminder: false,
hasDeferredToolList: false,
},
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { autorunIterableDelta } from '../../../../util/vs/base/common/observable
import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation';
import { LanguageModelToolInformation, LanguageModelToolResult2 } from '../../../../vscodeTypes';
import { getContributedToolName, getToolName, mapContributedToolNamesInSchema, mapContributedToolNamesInString, ToolName } from '../../common/toolNames';
import { ICopilotTool, ICopilotToolCtor, ToolRegistry } from '../../common/toolsRegistry';
import { ICopilotTool, ICopilotToolCtor, modelSpecificToolApplies, ToolRegistry } from '../../common/toolsRegistry';
import { BaseToolsService, IToolsService } from '../../common/toolsService';

export class TestToolsService extends BaseToolsService implements IToolsService {
Expand Down Expand Up @@ -177,11 +177,26 @@ export class TestToolsService extends BaseToolsService implements IToolsService
}

getEnabledTools(request: vscode.ChatRequest, endpoint: IChatEndpoint, filter?: (tool: LanguageModelToolInformation) => boolean | undefined): LanguageModelToolInformation[] {
const toolMap = new Map(this.tools.map(t => [t.name, t]));
const availableTools = [...this.tools];
for (const { definition } of this.getModelSpecificTools().values()) {
if (!modelSpecificToolApplies(definition, endpoint) || availableTools.some(tool => tool.name === definition.name)) {
continue;
}

availableTools.push({
name: definition.name,
description: mapContributedToolNamesInString(definition.description),
inputSchema: definition.inputSchema && mapContributedToolNamesInSchema(definition.inputSchema),
tags: definition.tags,
source: definition.source,
});
}

const toolMap = new Map(availableTools.map(t => [t.name, t]));
const requestToolsByName = new Map(Iterable.map(request.tools, ([t, enabled]) => [t.name, enabled]));

const packageJsonTools = getPackagejsonToolsForTest();
return this.tools
return availableTools
.map(tool => {
// Apply model-specific alternative if available via alternativeDefinition
const owned = this._copilotTools.get(getToolName(tool.name) as ToolName);
Expand Down
2 changes: 2 additions & 0 deletions extensions/copilot/src/extension/tools/node/toolSearchTool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ ToolRegistry.registerModelSpecificTool(
required: ['query'],
},
models: [
{ id: 'gpt-5.4' },
{ id: 'gpt-5.5' },
{ family: 'claude-sonnet-4.5' },
{ family: 'claude-sonnet-4.6' },
{ family: 'claude-opus-4.5' },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ function createMockEndpoint(model: string): IChatEndpoint {
} as unknown as IChatEndpoint;
}

function createCustomOAIResponsesEndpoint(model: string): IChatEndpoint {
return {
...createMockEndpoint(model),
modelProvider: 'CustomOAI',
urlOrRequestMetadata: 'https://custom.example.test/v1/responses',
} as IChatEndpoint;
}

function createMockOptions(overrides: Partial<ICreateEndpointBodyOptions> = {}): ICreateEndpointBodyOptions {
return {
debugName: 'test',
Expand Down Expand Up @@ -154,6 +162,21 @@ describe('createResponsesRequestBody tools', () => {
expect(tools.find(t => t.name === 'another_deferred_tool')).toBeUndefined();
});

it.each(['gpt-5.4', 'gpt-5.5'])('adds client tool_search for supported Custom OAI Responses %s requests when the upstream tool list includes tool_search', model => {
const endpoint = createCustomOAIResponsesEndpoint(model);
const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService;
configService.setConfig(ConfigKey.ResponsesApiToolSearchEnabled, true);

const body = accessor.get(IInstantiationService).invokeFunction(
createResponsesRequestBody, createMockOptions(), endpoint.model, endpoint
);

const tools = body.tools as any[];
const toolSearchTool = tools.find(t => t.type === 'tool_search');
expect(toolSearchTool).toBeDefined();
expect(toolSearchTool.execution).toBe('client');
});

it('does not defer tools for unsupported models', () => {
const endpoint = createMockEndpoint('gpt-4o');
const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService;
Expand Down Expand Up @@ -212,6 +235,27 @@ describe('createResponsesRequestBody tools', () => {
expect(tools.find(t => t.name === 'another_mcp_tool')).toBeDefined();
});

it('keeps native Responses tool_search unavailable for supported Custom OAI requests when the upstream tool list omits tool_search', () => {
const endpoint = createCustomOAIResponsesEndpoint('gpt-5.4');
const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService;
configService.setConfig(ConfigKey.ResponsesApiToolSearchEnabled, true);

const options = createMockOptions({
requestOptions: {
tools: [
createFunctionTool('some_mcp_tool', 'An MCP tool', { input: { type: 'string' } }, ['input']),
]
}
});
const body = accessor.get(IInstantiationService).invokeFunction(
createResponsesRequestBody, options, endpoint.model, endpoint
);

const tools = body.tools as any[];
expect(tools.find(t => t.type === 'tool_search')).toBeUndefined();
expect(tools.find(t => t.name === 'some_mcp_tool')).toBeDefined();
});

it('always filters tool_search function tool from tools array', () => {
const endpoint = createMockEndpoint('gpt-5.4');
const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ describe('modelSupportsPDFDocuments', () => {
});

describe('modelSupportsToolSearch', () => {
const experimentationService = {} as IExperimentationService;

function createToolSearchConfigurationService(enabled: boolean): IConfigurationService {
return {
getExperimentBasedConfig: (key: unknown) => key === ConfigKey.ResponsesApiToolSearchEnabled && enabled,
} as unknown as IConfigurationService;
}

test('supports Claude Sonnet/Opus 4.5 and up', () => {
expect(modelSupportsToolSearch('claude-sonnet-4-5')).toBe(true);
expect(modelSupportsToolSearch('claude-sonnet-4.5')).toBe(true);
Expand Down Expand Up @@ -66,28 +74,29 @@ describe('modelSupportsToolSearch', () => {
expect(modelSupportsToolSearch('claude-3-opus')).toBe(false);
});

test('supports OpenAI gpt-5.4 and gpt-5.5 models when the setting is enabled', () => {
const configurationService = {
getExperimentBasedConfig: (key: unknown) => key === ConfigKey.ResponsesApiToolSearchEnabled,
} as unknown as IConfigurationService;
const experimentationService = {} as IExperimentationService;
test('supports exact gpt-5.4 and gpt-5.5 models when the feature flag is enabled', () => {
const configurationService = createToolSearchConfigurationService(true);

expect(modelSupportsToolSearch('gpt-5.4', configurationService, experimentationService)).toBe(true);
expect(modelSupportsToolSearch('gpt-5.5', configurationService, experimentationService)).toBe(true);
expect(modelSupportsToolSearch('gpt-5.4')).toBe(false);
expect(modelSupportsToolSearch('gpt-5.5')).toBe(false);
for (const model of ['gpt-5.4', 'gpt-5.5']) {
expect(modelSupportsToolSearch(model, configurationService, experimentationService)).toBe(true);
}
});

test('rejects exact gpt-5.4 and gpt-5.5 models when the feature flag is disabled', () => {
const configurationService = createToolSearchConfigurationService(false);

for (const model of ['gpt-5.4', 'gpt-5.5']) {
expect(modelSupportsToolSearch(model, configurationService, experimentationService)).toBe(false);
}
});

test('rejects suffixed gpt-5.4/5.5 variants (exact match only)', () => {
const configurationService = {
getExperimentBasedConfig: (key: unknown) => key === ConfigKey.ResponsesApiToolSearchEnabled,
} as unknown as IConfigurationService;
const experimentationService = {} as IExperimentationService;
const configurationService = createToolSearchConfigurationService(true);

expect(modelSupportsToolSearch('gpt-5.4-mini', configurationService, experimentationService)).toBe(false);
expect(modelSupportsToolSearch('gpt-5.4-preview', configurationService, experimentationService)).toBe(false);
expect(modelSupportsToolSearch('gpt-5.5-mini', configurationService, experimentationService)).toBe(false);
expect(modelSupportsToolSearch('gpt-5.5-preview', configurationService, experimentationService)).toBe(false);
expect(modelSupportsToolSearch('gpt5.5-preview', configurationService, experimentationService)).toBe(false);
});

test('rejects other non-Claude models', () => {
Expand Down