chat: record Copilot harness usage for PMF surveys (#342745)

Signal the extension-owned survey service for accepted Copilot Agent Host submissions, including remote and sandbox sessions. Preserve legacy usage paths and existing survey eligibility and pacing.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

extensions/copilot/package.json

@@ -82,6 +82,7 @@
     "onLanguageModelChat:copilot",
     "onUri",
     "onCommand:_github.copilot.chat.reportModelFeedbackSurvey",
+    "onCommand:github.copilot.survey.signalUsage",
     "onSpeech",
     "onFileSystem:ccreq",
     "onFileSystem:ccsettings"

extensions/copilot/src/extension/survey/vscode-node/surveyCommands.ts

@@ -13,7 +13,7 @@ export class SurveyCommandContribution extends Disposable {
 	constructor(@ISurveyService private readonly _surveyService: ISurveyService) {
 		super();
 		this._register(vscode.commands.registerCommand(SURVEY_SIGNAL_USAGE_ID, (source: string, languageId?: string) => {
-			this._surveyService.signalUsage(source, languageId);
+			return this._surveyService.signalUsage(source, languageId);
 		}));
 	}
 }

extensions/copilot/src/extension/survey/vscode-node/test/surveyCommands.spec.ts

@@ -0,0 +1,112 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { IVSCodeExtensionContext } from '../../../../platform/extContext/common/extensionContext';
+import { SurveyService } from '../../../../platform/survey/vscode/surveyServiceImpl';
+import { NullExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
+import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService';
+import { MockExtensionContext } from '../../../../platform/test/node/extensionContext';
+import { mock } from '../../../../util/common/test/simpleMock';
+import { SURVEY_SIGNAL_USAGE_ID, SurveyCommandContribution } from '../surveyCommands';
+
+const vscodeMock = vi.hoisted(() => {
+	const commands = new Map<string, (source: string, languageId?: string) => Promise<void>>();
+	return {
+		commands,
+		registerCommand: (id: string, handler: (source: string, languageId?: string) => Promise<void>) => {
+			commands.set(id, handler);
+			return { dispose: () => commands.delete(id) };
+		},
+		showInformationMessage: vi.fn().mockResolvedValue(undefined),
+	};
+});
+
+vi.mock('vscode', async importOriginal => ({
+	...await importOriginal<typeof import('vscode')>(),
+	commands: { registerCommand: vscodeMock.registerCommand },
+	window: { showInformationMessage: vscodeMock.showInformationMessage },
+}));
+
+class TestExtensionContext extends mock<IVSCodeExtensionContext>() {
+	override readonly globalState = Object.assign(new MockExtensionContext().globalState, { setKeysForSync() { } });
+}
+
+describe('Copilot survey harness usage bridge', () => {
+	const day = 24 * 60 * 60 * 1000;
+	const now = new Date(2026, 9, 19, 12).getTime();
+	const today = new Date(now).setHours(0, 0, 0, 0);
+	let context: TestExtensionContext;
+	let telemetry: NullTelemetryService;
+	let service: SurveyService;
+	let contribution: SurveyCommandContribution;
+
+	async function signalHarnessUsage() {
+		await vscodeMock.commands.get(SURVEY_SIGNAL_USAGE_ID)!('agent.copilot');
+	}
+
+	beforeEach(() => {
+		vi.useFakeTimers();
+		vi.setSystemTime(now);
+		vi.spyOn(Math, 'random').mockReturnValue(0.5);
+		vscodeMock.showInformationMessage.mockClear();
+		context = new TestExtensionContext();
+		telemetry = new NullTelemetryService();
+		const experimentation = new NullExperimentationService();
+		vi.spyOn(experimentation, 'getTreatmentVariable').mockReturnValue(100);
+		service = new SurveyService(telemetry, context, experimentation);
+		contribution = new SurveyCommandContribution(service);
+	});
+
+	afterEach(() => {
+		contribution.dispose();
+		service.dispose();
+		vi.restoreAllMocks();
+		vi.useRealTimers();
+	});
+
+	it('refreshes activity before the startup inactivity check and does not prompt a one-day Copilot harness user', async () => {
+		await context.globalState.update('survey.usage', { firstActive: now - 30 * day, activeDays: [today - 20 * day] });
+		await vi.advanceTimersByTimeAsync(4 * 60 * 1000);
+		await signalHarnessUsage();
+		await vi.advanceTimersByTimeAsync(4 * 60 * 1000);
+		expect({
+			usage: context.globalState.get('survey.usage'),
+			inactive: await service.checkInactiveUserHeuristic(),
+			prompts: vscodeMock.showInformationMessage.mock.calls,
+		}).toEqual({
+			usage: { firstActive: now - 30 * day, activeDays: [today] },
+			inactive: false,
+			prompts: [],
+		});
+	});
+
+	it('schedules the normal debounced usage check with Copilot harness attribution and the existing cooldown', async () => {
+		const shown = vi.spyOn(telemetry, 'sendMSFTTelemetryEvent');
+		await context.globalState.update('survey.usage', { firstActive: now - 30 * day, activeDays: [today - day] });
+		await signalHarnessUsage();
+		await signalHarnessUsage();
+		await vi.advanceTimersByTimeAsync(3 * 60 * 1000 - 1);
+		expect(vscodeMock.showInformationMessage).not.toHaveBeenCalled();
+		await vi.advanceTimersByTimeAsync(1);
+		await vi.advanceTimersByTimeAsync(3 * 60 * 1000);
+		expect({
+			shown: shown.mock.calls.filter(([event]) => event === 'survey.show'),
+			promptCount: vscodeMock.showInformationMessage.mock.calls.length,
+			nextSurveyDate: context.globalState.get('survey.nextSurveyDate'),
+		}).toEqual({
+			shown: [['survey.show', { source: 'agent.copilot', language: '', surveyType: 'usage' }, { activeDays: 2, firstActive: 30 }]],
+			promptCount: 1,
+			nextSurveyDate: now + 3 * 60 * 1000 + 90 * day,
+		});
+	});
+
+	it('returns signal completion and propagates persistence failures to the workbench caller', async () => {
+		const error = new Error('Survey state could not be saved');
+		vi.spyOn(context.globalState, 'update').mockRejectedValue(error);
+		await expect(signalHarnessUsage()).rejects.toBe(error);
+		expect(vscodeMock.showInformationMessage).not.toHaveBeenCalled();
+	});
+});

src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts

@@ -53,6 +53,7 @@ import { ChatMcpServersStarting, ChatPendingRequestChangeClassification, ChatPen
 import { ChatRequestTelemetry, ChatServiceTelemetry, getChatSessionTelemetryIds } from './chatServiceTelemetry.js';
 import { IRemoteChatHistoryScheduler, IRemoteChatSessionRestoreOptions, IRemoteChatSessionRestoreResult, RemoteChatSessionRestorer } from './remoteChatSessionRestorer.js';
 import { IAgentHostConnectionsService } from '../../../../../platform/agentHost/common/agentHostConnectionsService.js';
+import { isCopilotAgentHostProvider } from '../../../../../platform/agentHost/common/agentHostSessionType.js';
 import { IChatSession, IChatSessionsService, IChatSessionServerRequest, isAgentHostTarget, isTerminalCommandPrompt, localChatSessionType } from '../chatSessionsService.js';
 import { ChatSessionStore, IChatSessionEntryMetadata } from '../model/chatSessionStore.js';
 import { IChatSlashCommandService } from '../participants/chatSlashCommands.js';
@@ -1602,6 +1603,12 @@ export class ChatService extends Disposable implements IChatService {
 			if (isSubmission && !options?.isSystemInitiated && !options?.hideFromTranscript) {
 				this._modelsWithAcceptedRequests.add(model);
 				this._onDidAcceptRequest.fire({ chatSessionResource: sessionResource, isNewSession });
+				const provider = this.chatSessionService.getChatSessionContribution(getChatSessionType(sessionResource))?.agentHostProviderId;
+				if (isCopilotAgentHostProvider(provider)) {
+					// Copilot harness submissions bypass the extension's legacy usage signal, including queued sends in either window.
+					this.commandService.executeCommand('github.copilot.survey.signalUsage', 'agent.copilot')
+						.catch(error => this.logService.warn('[CopilotSurvey] Failed to signal Copilot harness usage', error));
+				}
 			}
 		};
 

src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts

@@ -662,6 +662,77 @@ suite('ChatService', () => {
 		assert.deepStrictEqual(accepted, [true, false, false]);
 	});
 
+	async function createSurveySession(sessionType: string, provider?: string) {
+		const sessions = new MockChatSessionsService();
+		sessions.setContributions([{ type: sessionType, name: 'Test', displayName: 'Test', description: '', agentHostProviderId: provider }]);
+		testDisposables.add(sessions.registerChatSessionContentProvider(sessionType, {
+			provideChatSessionContent: async resource => ({ sessionResource: resource, history: [], onWillDispose: Event.None, dispose() { } }),
+		}));
+		instantiationService.stub(IChatSessionsService, sessions);
+		const commands = stub(instantiationService.get(ICommandService), 'executeCommand').resolves(undefined);
+		testDisposables.add(toDisposable(() => commands.restore()));
+		const service = createChatService();
+		const model = sessionType === SessionType.Local ? startSessionModel(service).object
+			: await getOrRestoreModel(service, URI.from({ scheme: sessionType, path: '/survey' }));
+		assert.ok(model);
+		assert.strictEqual(commands.callCount, 0);
+		return { service, model, commands };
+	}
+
+	for (const { sessionType, provider, signals } of [
+		{ sessionType: SessionType.AgentHostCopilot, provider: 'copilotcli', signals: true },
+		{ sessionType: 'remote-ssh-machine-copilot', provider: 'copilot', signals: true },
+		{ sessionType: 'opaque-host-session', provider: 'copilot', signals: true },
+		{ sessionType: SessionType.CloudSandbox, provider: 'copilot', signals: true },
+		{ sessionType: SessionType.CopilotCLI, provider: undefined, signals: false },
+		{ sessionType: SessionType.Local, provider: undefined, signals: false },
+		{ sessionType: SessionType.AgentHostClaude, provider: 'claude', signals: false },
+		{ sessionType: SessionType.AgentHostCodex, provider: 'codex', signals: false },
+		{ sessionType: 'remote-unknown-copilotcli', provider: undefined, signals: false },
+	]) {
+		test(`survey usage filters by registered provider: ${sessionType}`, async () => {
+			const { service, model, commands } = await createSurveySession(sessionType, provider);
+			const sent = await service.sendRequest(model.sessionResource, 'hello');
+			ChatSendResult.assertSent(sent);
+			await sent.data.responseCompletePromise;
+			assert.deepStrictEqual(commands.args, signals ? [['github.copilot.survey.signalUsage', 'agent.copilot']] : []);
+		});
+	}
+
+	test('survey usage counts accepted Agent Host submissions, not retries, system messages or queue drains', async () => {
+		const { service, model, commands } = await createSurveySession(SessionType.AgentHostCopilot, 'copilotcli');
+		await service.sendRequest(model.sessionResource, '');
+		const first = await service.sendRequest(model.sessionResource, 'first');
+		ChatSendResult.assertSent(first);
+		await first.data.responseCompletePromise;
+		await service.resendRequest(model.getRequests()[0]);
+		for (const options of [{ isSystemInitiated: true }, { hideFromTranscript: true }]) {
+			const sent = await service.sendRequest(model.sessionResource, 'background', options);
+			ChatSendResult.assertSent(sent);
+			await sent.data.responseCompletePromise;
+		}
+		await service.sendRequest(model.sessionResource, 'queued', { queue: ChatRequestQueueKind.Queued, pauseQueue: true });
+		await service.sendRequest(model.sessionResource, 'steering', { queue: ChatRequestQueueKind.Steering, pauseQueue: true });
+		service.processPendingRequests(model.sessionResource);
+		await timeout(10);
+		assert.deepStrictEqual(commands.args, Array.from({ length: 3 }, () => ['github.copilot.survey.signalUsage', 'agent.copilot']));
+	});
+
+	test('survey usage bridge failure is logged without failing or retrying the submission', async () => {
+		const { service, model, commands } = await createSurveySession(SessionType.AgentHostCopilot, 'copilotcli');
+		const error = new Error('Extension unavailable');
+		commands.rejects(error);
+		const warning = spy(instantiationService.get(ILogService), 'warn');
+		testDisposables.add(toDisposable(() => warning.restore()));
+		const sent = await service.sendRequest(model.sessionResource, 'hello');
+		ChatSendResult.assertSent(sent);
+		await sent.data.responseCompletePromise;
+		assert.deepStrictEqual({
+			calls: commands.callCount,
+			logged: warning.calledWith('[CopilotSurvey] Failed to signal Copilot harness usage', error),
+		}, { calls: 1, logged: true });
+	});
+
 	test('only the first accepted message starts a session even when queued or removed', async () => {
 		const service = createChatService();
 		const model = startSessionModel(service).object;