diff --git a/extensions/copilot/src/extension/intents/node/test/backgroundTodoEnablement.spec.ts b/extensions/copilot/src/extension/intents/node/test/backgroundTodoEnablement.spec.ts index 3e40d1a63975f..6c74097a2a9f3 100644 --- a/extensions/copilot/src/extension/intents/node/test/backgroundTodoEnablement.spec.ts +++ b/extensions/copilot/src/extension/intents/node/test/backgroundTodoEnablement.spec.ts @@ -6,6 +6,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest'; import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService'; import { MockEndpoint } from '../../../../platform/endpoint/test/node/mockEndpoint'; +import { CUSTOM_TOOL_SEARCH_NAME } from '../../../../platform/networking/common/anthropic'; import { IChatEndpoint } from '../../../../platform/networking/common/networking'; import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; import { ITestingServicesAccessor } from '../../../../platform/test/node/services'; @@ -102,6 +103,21 @@ describe('getAgentTools background todo enablement', () => { return tools.some(t => t.name === ToolName.CoreManageTodoList); } + function hasTool(tools: readonly { name: string }[], name: string): boolean { + return tools.some(tool => tool.name === name); + } + + function createToolSearchEndpoint(model: string, modelProvider = 'openai', supportsToolSearch = true): IChatEndpoint { + return { + ...mockEndpoint, + model, + family: model, + modelProvider, + // Pin endpoint capability so this test isolates getAgentTools gating from endpoint capability derivation. + supportsToolSearch, + } as IChatEndpoint; + } + test('background todo agent is enabled only when experiment is on and todo is not explicit', () => { const request = new TestChatRequest('fix the bug'); configService.setConfig(ConfigKey.Advanced.BackgroundTodoAgentEnabled, false); @@ -137,4 +153,20 @@ describe('getAgentTools background todo enablement', () => { const tools = await instantiationService.invokeFunction(getAgentTools, request, mockEndpoint); expect(hasTodoTool(tools)).toBe(false); }); + + test('supported Custom OAI Responses gpt-5.4 and gpt-5.5 endpoints surface tool_search when endpoint capability is enabled', async () => { + const request = new TestChatRequest('find the right tool'); + for (const model of ['gpt-5.4', 'gpt-5.5']) { + const tools = await instantiationService.invokeFunction(getAgentTools, request, createToolSearchEndpoint(model, 'CustomOAI')); + expect(hasTool(tools, CUSTOM_TOOL_SEARCH_NAME)).toBe(true); + } + }); + + test('supported Custom OAI Responses endpoints omit tool_search when endpoint capability is disabled', async () => { + const request = new TestChatRequest('find the right tool'); + for (const model of ['gpt-5.4', 'gpt-5.5']) { + const tools = await instantiationService.invokeFunction(getAgentTools, request, createToolSearchEndpoint(model, 'CustomOAI', false)); + expect(hasTool(tools, CUSTOM_TOOL_SEARCH_NAME)).toBe(false); + } + }); }); diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/agentPrompt.spec.tsx b/extensions/copilot/src/extension/prompts/node/agent/test/agentPrompt.spec.tsx index 226d68a702584..749a354426682 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/agentPrompt.spec.tsx +++ b/extensions/copilot/src/extension/prompts/node/agent/test/agentPrompt.spec.tsx @@ -320,3 +320,91 @@ testFamilies.forEach(family => { }); }); }); + +suite('AgentPrompt tool search alignment', () => { + let accessor: ITestingServicesAccessor; + const fileTsUri = URI.file('/workspace/file.ts'); + + beforeAll(() => { + const testDoc = createTextDocumentData(fileTsUri, 'line 1\nline 2\n\nline 4\nline 5', 'ts').document; + const services = createExtensionUnitTestingServices(); + services.define(IWorkspaceService, new SyncDescriptor( + TestWorkspaceService, + [ + [URI.file('/workspace')], + [testDoc] + ] + )); + services.define(IChatMLFetcher, new StaticChatMLFetcher([])); + accessor = services.createTestingAccessor(); + accessor.get(IConfigurationService).setConfig(ConfigKey.CodeGenerationInstructions, [{ + text: 'This is a test custom instruction file', + } satisfies CodeGenerationTextInstruction]); + }); + + afterAll(() => { + accessor.dispose(); + }); + + async function renderAgentPromptWithToolSearchSupport(family: 'gpt-5.4' | 'gpt-5.5', supportsToolSearch: boolean): Promise { + const instaService = accessor.get(IInstantiationService); + const endpoint = instaService.createInstance(MockEndpoint, family); + endpoint.supportsToolSearch = supportsToolSearch; + const conversation = new Conversation('sessionId', [new Turn('turnId', { type: 'user', message: 'hello' })]); + const toolsService = accessor.get(IToolsService); + const customizations = await PromptRegistry.resolveAllCustomizations(instaService, endpoint); + const renderer = PromptRenderer.create(instaService, endpoint, AgentPrompt, { + priority: 1, + endpoint, + location: ChatLocation.Panel, + promptContext: { + chatVariables: new ChatVariablesCollection(), + history: [], + query: 'hello', + conversation, + tools: { + availableTools: toolsService.tools, + toolInvocationToken: null as never, + toolReferences: [], + } + }, + customizations, + }); + + const rendered = await renderer.render(); + addCacheBreakpoints(rendered.messages); + return rendered.messages + .map(m => messageToMarkdown(m)) + .join('\n\n') + .replace(/\\+/g, '/'); + } + + test.each(['gpt-5.4', 'gpt-5.5'] as const)('gates deferred-tool guidance and reminders for %s on endpoint.supportsToolSearch', async family => { + const enabledPrompt = await renderAgentPromptWithToolSearchSupport(family, true); + const disabledPrompt = await renderAgentPromptWithToolSearchSupport(family, false); + + expect({ + enabled: { + hasToolSearchInstructions: enabledPrompt.includes(''), + hasDeferredToolReminder: enabledPrompt.includes('IMPORTANT: Before calling any deferred tool'), + hasDeferredToolList: enabledPrompt.includes(''), + }, + disabled: { + hasToolSearchInstructions: disabledPrompt.includes(''), + hasDeferredToolReminder: disabledPrompt.includes('IMPORTANT: Before calling any deferred tool'), + hasDeferredToolList: disabledPrompt.includes(''), + }, + }).toEqual({ + enabled: { + hasToolSearchInstructions: true, + hasDeferredToolReminder: true, + hasDeferredToolList: true, + }, + disabled: { + hasToolSearchInstructions: false, + hasDeferredToolReminder: false, + hasDeferredToolList: false, + }, + }); + }); +}); diff --git a/extensions/copilot/src/extension/tools/node/test/testToolsService.ts b/extensions/copilot/src/extension/tools/node/test/testToolsService.ts index 7724505fdd906..543e9b47c7284 100644 --- a/extensions/copilot/src/extension/tools/node/test/testToolsService.ts +++ b/extensions/copilot/src/extension/tools/node/test/testToolsService.ts @@ -17,7 +17,7 @@ import { autorunIterableDelta } from '../../../../util/vs/base/common/observable import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation'; import { LanguageModelToolInformation, LanguageModelToolResult2 } from '../../../../vscodeTypes'; import { getContributedToolName, getToolName, mapContributedToolNamesInSchema, mapContributedToolNamesInString, ToolName } from '../../common/toolNames'; -import { ICopilotTool, ICopilotToolCtor, ToolRegistry } from '../../common/toolsRegistry'; +import { ICopilotTool, ICopilotToolCtor, modelSpecificToolApplies, ToolRegistry } from '../../common/toolsRegistry'; import { BaseToolsService, IToolsService } from '../../common/toolsService'; export class TestToolsService extends BaseToolsService implements IToolsService { @@ -177,11 +177,26 @@ export class TestToolsService extends BaseToolsService implements IToolsService } getEnabledTools(request: vscode.ChatRequest, endpoint: IChatEndpoint, filter?: (tool: LanguageModelToolInformation) => boolean | undefined): LanguageModelToolInformation[] { - const toolMap = new Map(this.tools.map(t => [t.name, t])); + const availableTools = [...this.tools]; + for (const { definition } of this.getModelSpecificTools().values()) { + if (!modelSpecificToolApplies(definition, endpoint) || availableTools.some(tool => tool.name === definition.name)) { + continue; + } + + availableTools.push({ + name: definition.name, + description: mapContributedToolNamesInString(definition.description), + inputSchema: definition.inputSchema && mapContributedToolNamesInSchema(definition.inputSchema), + tags: definition.tags, + source: definition.source, + }); + } + + const toolMap = new Map(availableTools.map(t => [t.name, t])); const requestToolsByName = new Map(Iterable.map(request.tools, ([t, enabled]) => [t.name, enabled])); const packageJsonTools = getPackagejsonToolsForTest(); - return this.tools + return availableTools .map(tool => { // Apply model-specific alternative if available via alternativeDefinition const owned = this._copilotTools.get(getToolName(tool.name) as ToolName); diff --git a/extensions/copilot/src/extension/tools/node/toolSearchTool.ts b/extensions/copilot/src/extension/tools/node/toolSearchTool.ts index fb35e4a0f2ae2..335e6bf475d7e 100644 --- a/extensions/copilot/src/extension/tools/node/toolSearchTool.ts +++ b/extensions/copilot/src/extension/tools/node/toolSearchTool.ts @@ -75,6 +75,8 @@ ToolRegistry.registerModelSpecificTool( required: ['query'], }, models: [ + { id: 'gpt-5.4' }, + { id: 'gpt-5.5' }, { family: 'claude-sonnet-4.5' }, { family: 'claude-sonnet-4.6' }, { family: 'claude-opus-4.5' }, diff --git a/extensions/copilot/src/platform/endpoint/node/test/responsesApiToolSearch.spec.ts b/extensions/copilot/src/platform/endpoint/node/test/responsesApiToolSearch.spec.ts index 41b106f1381fa..dfc1675ae56b9 100644 --- a/extensions/copilot/src/platform/endpoint/node/test/responsesApiToolSearch.spec.ts +++ b/extensions/copilot/src/platform/endpoint/node/test/responsesApiToolSearch.spec.ts @@ -43,6 +43,14 @@ function createMockEndpoint(model: string): IChatEndpoint { } as unknown as IChatEndpoint; } +function createCustomOAIResponsesEndpoint(model: string): IChatEndpoint { + return { + ...createMockEndpoint(model), + modelProvider: 'CustomOAI', + urlOrRequestMetadata: 'https://custom.example.test/v1/responses', + } as IChatEndpoint; +} + function createMockOptions(overrides: Partial = {}): ICreateEndpointBodyOptions { return { debugName: 'test', @@ -154,6 +162,21 @@ describe('createResponsesRequestBody tools', () => { expect(tools.find(t => t.name === 'another_deferred_tool')).toBeUndefined(); }); + it.each(['gpt-5.4', 'gpt-5.5'])('adds client tool_search for supported Custom OAI Responses %s requests when the upstream tool list includes tool_search', model => { + const endpoint = createCustomOAIResponsesEndpoint(model); + const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService; + configService.setConfig(ConfigKey.ResponsesApiToolSearchEnabled, true); + + const body = accessor.get(IInstantiationService).invokeFunction( + createResponsesRequestBody, createMockOptions(), endpoint.model, endpoint + ); + + const tools = body.tools as any[]; + const toolSearchTool = tools.find(t => t.type === 'tool_search'); + expect(toolSearchTool).toBeDefined(); + expect(toolSearchTool.execution).toBe('client'); + }); + it('does not defer tools for unsupported models', () => { const endpoint = createMockEndpoint('gpt-4o'); const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService; @@ -212,6 +235,27 @@ describe('createResponsesRequestBody tools', () => { expect(tools.find(t => t.name === 'another_mcp_tool')).toBeDefined(); }); + it('keeps native Responses tool_search unavailable for supported Custom OAI requests when the upstream tool list omits tool_search', () => { + const endpoint = createCustomOAIResponsesEndpoint('gpt-5.4'); + const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService; + configService.setConfig(ConfigKey.ResponsesApiToolSearchEnabled, true); + + const options = createMockOptions({ + requestOptions: { + tools: [ + createFunctionTool('some_mcp_tool', 'An MCP tool', { input: { type: 'string' } }, ['input']), + ] + } + }); + const body = accessor.get(IInstantiationService).invokeFunction( + createResponsesRequestBody, options, endpoint.model, endpoint + ); + + const tools = body.tools as any[]; + expect(tools.find(t => t.type === 'tool_search')).toBeUndefined(); + expect(tools.find(t => t.name === 'some_mcp_tool')).toBeDefined(); + }); + it('always filters tool_search function tool from tools array', () => { const endpoint = createMockEndpoint('gpt-5.4'); const configService = accessor.get(IConfigurationService) as InMemoryConfigurationService; diff --git a/extensions/copilot/src/platform/endpoint/test/node/chatModelCapabilities.spec.ts b/extensions/copilot/src/platform/endpoint/test/node/chatModelCapabilities.spec.ts index d438c31649638..1865db1cc7deb 100644 --- a/extensions/copilot/src/platform/endpoint/test/node/chatModelCapabilities.spec.ts +++ b/extensions/copilot/src/platform/endpoint/test/node/chatModelCapabilities.spec.ts @@ -34,6 +34,14 @@ describe('modelSupportsPDFDocuments', () => { }); describe('modelSupportsToolSearch', () => { + const experimentationService = {} as IExperimentationService; + + function createToolSearchConfigurationService(enabled: boolean): IConfigurationService { + return { + getExperimentBasedConfig: (key: unknown) => key === ConfigKey.ResponsesApiToolSearchEnabled && enabled, + } as unknown as IConfigurationService; + } + test('supports Claude Sonnet/Opus 4.5 and up', () => { expect(modelSupportsToolSearch('claude-sonnet-4-5')).toBe(true); expect(modelSupportsToolSearch('claude-sonnet-4.5')).toBe(true); @@ -66,28 +74,29 @@ describe('modelSupportsToolSearch', () => { expect(modelSupportsToolSearch('claude-3-opus')).toBe(false); }); - test('supports OpenAI gpt-5.4 and gpt-5.5 models when the setting is enabled', () => { - const configurationService = { - getExperimentBasedConfig: (key: unknown) => key === ConfigKey.ResponsesApiToolSearchEnabled, - } as unknown as IConfigurationService; - const experimentationService = {} as IExperimentationService; + test('supports exact gpt-5.4 and gpt-5.5 models when the feature flag is enabled', () => { + const configurationService = createToolSearchConfigurationService(true); - expect(modelSupportsToolSearch('gpt-5.4', configurationService, experimentationService)).toBe(true); - expect(modelSupportsToolSearch('gpt-5.5', configurationService, experimentationService)).toBe(true); - expect(modelSupportsToolSearch('gpt-5.4')).toBe(false); - expect(modelSupportsToolSearch('gpt-5.5')).toBe(false); + for (const model of ['gpt-5.4', 'gpt-5.5']) { + expect(modelSupportsToolSearch(model, configurationService, experimentationService)).toBe(true); + } + }); + + test('rejects exact gpt-5.4 and gpt-5.5 models when the feature flag is disabled', () => { + const configurationService = createToolSearchConfigurationService(false); + + for (const model of ['gpt-5.4', 'gpt-5.5']) { + expect(modelSupportsToolSearch(model, configurationService, experimentationService)).toBe(false); + } }); test('rejects suffixed gpt-5.4/5.5 variants (exact match only)', () => { - const configurationService = { - getExperimentBasedConfig: (key: unknown) => key === ConfigKey.ResponsesApiToolSearchEnabled, - } as unknown as IConfigurationService; - const experimentationService = {} as IExperimentationService; + const configurationService = createToolSearchConfigurationService(true); expect(modelSupportsToolSearch('gpt-5.4-mini', configurationService, experimentationService)).toBe(false); expect(modelSupportsToolSearch('gpt-5.4-preview', configurationService, experimentationService)).toBe(false); + expect(modelSupportsToolSearch('gpt-5.5-mini', configurationService, experimentationService)).toBe(false); expect(modelSupportsToolSearch('gpt-5.5-preview', configurationService, experimentationService)).toBe(false); - expect(modelSupportsToolSearch('gpt5.5-preview', configurationService, experimentationService)).toBe(false); }); test('rejects other non-Claude models', () => {