diff --git a/extensions/copilot/package.json b/extensions/copilot/package.json index cdb978e76ad..a9d3ba0e477 100644 --- a/extensions/copilot/package.json +++ b/extensions/copilot/package.json @@ -82,6 +82,7 @@ "onLanguageModelChat:copilot", "onUri", "onCommand:_github.copilot.chat.reportModelFeedbackSurvey", + "onCommand:github.copilot.survey.signalUsage", "onSpeech", "onFileSystem:ccreq", "onFileSystem:ccsettings" diff --git a/extensions/copilot/src/extension/survey/vscode-node/surveyCommands.ts b/extensions/copilot/src/extension/survey/vscode-node/surveyCommands.ts index 3f1a3838a36..40f46b3b453 100644 --- a/extensions/copilot/src/extension/survey/vscode-node/surveyCommands.ts +++ b/extensions/copilot/src/extension/survey/vscode-node/surveyCommands.ts @@ -13,7 +13,7 @@ export class SurveyCommandContribution extends Disposable { constructor(@ISurveyService private readonly _surveyService: ISurveyService) { super(); this._register(vscode.commands.registerCommand(SURVEY_SIGNAL_USAGE_ID, (source: string, languageId?: string) => { - this._surveyService.signalUsage(source, languageId); + return this._surveyService.signalUsage(source, languageId); })); } } diff --git a/extensions/copilot/src/extension/survey/vscode-node/test/surveyCommands.spec.ts b/extensions/copilot/src/extension/survey/vscode-node/test/surveyCommands.spec.ts new file mode 100644 index 00000000000..178d8e25d71 --- /dev/null +++ b/extensions/copilot/src/extension/survey/vscode-node/test/surveyCommands.spec.ts @@ -0,0 +1,112 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { IVSCodeExtensionContext } from '../../../../platform/extContext/common/extensionContext'; +import { SurveyService } from '../../../../platform/survey/vscode/surveyServiceImpl'; +import { NullExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; +import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService'; +import { MockExtensionContext } from '../../../../platform/test/node/extensionContext'; +import { mock } from '../../../../util/common/test/simpleMock'; +import { SURVEY_SIGNAL_USAGE_ID, SurveyCommandContribution } from '../surveyCommands'; + +const vscodeMock = vi.hoisted(() => { + const commands = new Map Promise>(); + return { + commands, + registerCommand: (id: string, handler: (source: string, languageId?: string) => Promise) => { + commands.set(id, handler); + return { dispose: () => commands.delete(id) }; + }, + showInformationMessage: vi.fn().mockResolvedValue(undefined), + }; +}); + +vi.mock('vscode', async importOriginal => ({ + ...await importOriginal(), + commands: { registerCommand: vscodeMock.registerCommand }, + window: { showInformationMessage: vscodeMock.showInformationMessage }, +})); + +class TestExtensionContext extends mock() { + override readonly globalState = Object.assign(new MockExtensionContext().globalState, { setKeysForSync() { } }); +} + +describe('Copilot survey harness usage bridge', () => { + const day = 24 * 60 * 60 * 1000; + const now = new Date(2026, 9, 19, 12).getTime(); + const today = new Date(now).setHours(0, 0, 0, 0); + let context: TestExtensionContext; + let telemetry: NullTelemetryService; + let service: SurveyService; + let contribution: SurveyCommandContribution; + + async function signalHarnessUsage() { + await vscodeMock.commands.get(SURVEY_SIGNAL_USAGE_ID)!('agent.copilot'); + } + + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(now); + vi.spyOn(Math, 'random').mockReturnValue(0.5); + vscodeMock.showInformationMessage.mockClear(); + context = new TestExtensionContext(); + telemetry = new NullTelemetryService(); + const experimentation = new NullExperimentationService(); + vi.spyOn(experimentation, 'getTreatmentVariable').mockReturnValue(100); + service = new SurveyService(telemetry, context, experimentation); + contribution = new SurveyCommandContribution(service); + }); + + afterEach(() => { + contribution.dispose(); + service.dispose(); + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it('refreshes activity before the startup inactivity check and does not prompt a one-day Copilot harness user', async () => { + await context.globalState.update('survey.usage', { firstActive: now - 30 * day, activeDays: [today - 20 * day] }); + await vi.advanceTimersByTimeAsync(4 * 60 * 1000); + await signalHarnessUsage(); + await vi.advanceTimersByTimeAsync(4 * 60 * 1000); + expect({ + usage: context.globalState.get('survey.usage'), + inactive: await service.checkInactiveUserHeuristic(), + prompts: vscodeMock.showInformationMessage.mock.calls, + }).toEqual({ + usage: { firstActive: now - 30 * day, activeDays: [today] }, + inactive: false, + prompts: [], + }); + }); + + it('schedules the normal debounced usage check with Copilot harness attribution and the existing cooldown', async () => { + const shown = vi.spyOn(telemetry, 'sendMSFTTelemetryEvent'); + await context.globalState.update('survey.usage', { firstActive: now - 30 * day, activeDays: [today - day] }); + await signalHarnessUsage(); + await signalHarnessUsage(); + await vi.advanceTimersByTimeAsync(3 * 60 * 1000 - 1); + expect(vscodeMock.showInformationMessage).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(3 * 60 * 1000); + expect({ + shown: shown.mock.calls.filter(([event]) => event === 'survey.show'), + promptCount: vscodeMock.showInformationMessage.mock.calls.length, + nextSurveyDate: context.globalState.get('survey.nextSurveyDate'), + }).toEqual({ + shown: [['survey.show', { source: 'agent.copilot', language: '', surveyType: 'usage' }, { activeDays: 2, firstActive: 30 }]], + promptCount: 1, + nextSurveyDate: now + 3 * 60 * 1000 + 90 * day, + }); + }); + + it('returns signal completion and propagates persistence failures to the workbench caller', async () => { + const error = new Error('Survey state could not be saved'); + vi.spyOn(context.globalState, 'update').mockRejectedValue(error); + await expect(signalHarnessUsage()).rejects.toBe(error); + expect(vscodeMock.showInformationMessage).not.toHaveBeenCalled(); + }); +}); diff --git a/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts b/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts index c6ce114c441..eb83e42ff90 100644 --- a/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts @@ -53,6 +53,7 @@ import { ChatMcpServersStarting, ChatPendingRequestChangeClassification, ChatPen import { ChatRequestTelemetry, ChatServiceTelemetry, getChatSessionTelemetryIds } from './chatServiceTelemetry.js'; import { IRemoteChatHistoryScheduler, IRemoteChatSessionRestoreOptions, IRemoteChatSessionRestoreResult, RemoteChatSessionRestorer } from './remoteChatSessionRestorer.js'; import { IAgentHostConnectionsService } from '../../../../../platform/agentHost/common/agentHostConnectionsService.js'; +import { isCopilotAgentHostProvider } from '../../../../../platform/agentHost/common/agentHostSessionType.js'; import { IChatSession, IChatSessionsService, IChatSessionServerRequest, isAgentHostTarget, isTerminalCommandPrompt, localChatSessionType } from '../chatSessionsService.js'; import { ChatSessionStore, IChatSessionEntryMetadata } from '../model/chatSessionStore.js'; import { IChatSlashCommandService } from '../participants/chatSlashCommands.js'; @@ -1602,6 +1603,12 @@ export class ChatService extends Disposable implements IChatService { if (isSubmission && !options?.isSystemInitiated && !options?.hideFromTranscript) { this._modelsWithAcceptedRequests.add(model); this._onDidAcceptRequest.fire({ chatSessionResource: sessionResource, isNewSession }); + const provider = this.chatSessionService.getChatSessionContribution(getChatSessionType(sessionResource))?.agentHostProviderId; + if (isCopilotAgentHostProvider(provider)) { + // Copilot harness submissions bypass the extension's legacy usage signal, including queued sends in either window. + this.commandService.executeCommand('github.copilot.survey.signalUsage', 'agent.copilot') + .catch(error => this.logService.warn('[CopilotSurvey] Failed to signal Copilot harness usage', error)); + } } }; diff --git a/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts b/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts index 8f7521f651a..0d81b721a9c 100644 --- a/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts @@ -662,6 +662,77 @@ suite('ChatService', () => { assert.deepStrictEqual(accepted, [true, false, false]); }); + async function createSurveySession(sessionType: string, provider?: string) { + const sessions = new MockChatSessionsService(); + sessions.setContributions([{ type: sessionType, name: 'Test', displayName: 'Test', description: '', agentHostProviderId: provider }]); + testDisposables.add(sessions.registerChatSessionContentProvider(sessionType, { + provideChatSessionContent: async resource => ({ sessionResource: resource, history: [], onWillDispose: Event.None, dispose() { } }), + })); + instantiationService.stub(IChatSessionsService, sessions); + const commands = stub(instantiationService.get(ICommandService), 'executeCommand').resolves(undefined); + testDisposables.add(toDisposable(() => commands.restore())); + const service = createChatService(); + const model = sessionType === SessionType.Local ? startSessionModel(service).object + : await getOrRestoreModel(service, URI.from({ scheme: sessionType, path: '/survey' })); + assert.ok(model); + assert.strictEqual(commands.callCount, 0); + return { service, model, commands }; + } + + for (const { sessionType, provider, signals } of [ + { sessionType: SessionType.AgentHostCopilot, provider: 'copilotcli', signals: true }, + { sessionType: 'remote-ssh-machine-copilot', provider: 'copilot', signals: true }, + { sessionType: 'opaque-host-session', provider: 'copilot', signals: true }, + { sessionType: SessionType.CloudSandbox, provider: 'copilot', signals: true }, + { sessionType: SessionType.CopilotCLI, provider: undefined, signals: false }, + { sessionType: SessionType.Local, provider: undefined, signals: false }, + { sessionType: SessionType.AgentHostClaude, provider: 'claude', signals: false }, + { sessionType: SessionType.AgentHostCodex, provider: 'codex', signals: false }, + { sessionType: 'remote-unknown-copilotcli', provider: undefined, signals: false }, + ]) { + test(`survey usage filters by registered provider: ${sessionType}`, async () => { + const { service, model, commands } = await createSurveySession(sessionType, provider); + const sent = await service.sendRequest(model.sessionResource, 'hello'); + ChatSendResult.assertSent(sent); + await sent.data.responseCompletePromise; + assert.deepStrictEqual(commands.args, signals ? [['github.copilot.survey.signalUsage', 'agent.copilot']] : []); + }); + } + + test('survey usage counts accepted Agent Host submissions, not retries, system messages or queue drains', async () => { + const { service, model, commands } = await createSurveySession(SessionType.AgentHostCopilot, 'copilotcli'); + await service.sendRequest(model.sessionResource, ''); + const first = await service.sendRequest(model.sessionResource, 'first'); + ChatSendResult.assertSent(first); + await first.data.responseCompletePromise; + await service.resendRequest(model.getRequests()[0]); + for (const options of [{ isSystemInitiated: true }, { hideFromTranscript: true }]) { + const sent = await service.sendRequest(model.sessionResource, 'background', options); + ChatSendResult.assertSent(sent); + await sent.data.responseCompletePromise; + } + await service.sendRequest(model.sessionResource, 'queued', { queue: ChatRequestQueueKind.Queued, pauseQueue: true }); + await service.sendRequest(model.sessionResource, 'steering', { queue: ChatRequestQueueKind.Steering, pauseQueue: true }); + service.processPendingRequests(model.sessionResource); + await timeout(10); + assert.deepStrictEqual(commands.args, Array.from({ length: 3 }, () => ['github.copilot.survey.signalUsage', 'agent.copilot'])); + }); + + test('survey usage bridge failure is logged without failing or retrying the submission', async () => { + const { service, model, commands } = await createSurveySession(SessionType.AgentHostCopilot, 'copilotcli'); + const error = new Error('Extension unavailable'); + commands.rejects(error); + const warning = spy(instantiationService.get(ILogService), 'warn'); + testDisposables.add(toDisposable(() => warning.restore())); + const sent = await service.sendRequest(model.sessionResource, 'hello'); + ChatSendResult.assertSent(sent); + await sent.data.responseCompletePromise; + assert.deepStrictEqual({ + calls: commands.callCount, + logged: warning.calledWith('[CopilotSurvey] Failed to signal Copilot harness usage', error), + }, { calls: 1, logged: true }); + }); + test('only the first accepted message starts a session even when queued or removed', async () => { const service = createChatService(); const model = startSessionModel(service).object;