From 18b4f9d0784f96055f23c3d6a335ac7bf950b6c2 Mon Sep 17 00:00:00 2001 From: lucaronin Date: Thu, 30 Apr 2026 02:58:36 +0200 Subject: [PATCH] feat: retrofit safe product analytics events --- docs/ABSTRACTIONS.md | 5 + docs/ARCHITECTURE.md | 1 + src/components/AiPanel.test.tsx | 13 +++ src/components/FilePreview.test.tsx | 40 ++++++- src/components/FilePreview.tsx | 21 +++- src/components/SettingsPanel.test.tsx | 26 +++++ src/components/SettingsPanel.tsx | 4 +- src/components/useAiPanelController.ts | 4 +- src/lib/aiAgentSession.test.ts | 139 ++++++++++++++++--------- src/lib/aiAgentSession.ts | 15 +++ src/lib/aiAgentStreamCallbacks.test.ts | 24 ++++- src/lib/aiAgentStreamCallbacks.ts | 8 ++ src/lib/productAnalytics.ts | 103 ++++++++++++++++++ 13 files changed, 345 insertions(+), 58 deletions(-) create mode 100644 src/lib/productAnalytics.ts diff --git a/docs/ABSTRACTIONS.md b/docs/ABSTRACTIONS.md index 0e791990..62c8042e 100644 --- a/docs/ABSTRACTIONS.md +++ b/docs/ABSTRACTIONS.md @@ -781,6 +781,11 @@ Managed by `useSettings` hook and `SettingsPanel` component. `theme_mode` is ins - **`src/main.tsx`** — React root error callbacks (`onCaughtError`, `onUncaughtError`, `onRecoverableError`) forward component-stack context to `Sentry.reactErrorHandler()` for debuggable production React errors. - **`src-tauri/src/telemetry.rs`** — Rust-side Sentry init with `beforeSend` path scrubber. `init_sentry_from_settings()` reads settings and conditionally initializes; stable calendar `CARGO_PKG_VERSION` values become Sentry releases, while alpha/prerelease/internal versions are kept as diagnostic tags only. `reinit_sentry()` for runtime toggle. +### Product Events +- **File previews** — `file_preview_opened`, `file_preview_action`, and `file_preview_failed` report only preview/action categories such as `image`, `pdf`, `unsupported`, `open_external`, `copy_path`, and `reveal`. +- **AI agent sessions** — `ai_agent_message_sent`, `ai_agent_message_blocked`, `ai_agent_response_completed`, `ai_agent_response_failed`, and `ai_agent_permission_mode_changed` use only agent ids, permission modes, counts, and coarse status categories. +- **All Notes visibility** — `all_notes_visibility_changed` records only the toggled category and enabled state. + ### Tauri Commands - **`reinit_telemetry`** — Re-reads settings and toggles Rust Sentry on/off. Called from frontend when user changes crash reporting setting. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6139e71b..440132f2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -961,6 +961,7 @@ sequenceDiagram - `anonymous_id` is a locally-generated UUID, never tied to identity - `send_default_pii: false` on both SDKs - PostHog: `autocapture: false`, `persistence: 'memory'`, no cookies +- Product events use categorical metadata only: file preview kind/action, AI agent id/permission mode/counts/status, and All Notes visibility category/enabled state. **Architecture:** - **Rust:** `sentry` crate initialized in `lib.rs::setup()` via `telemetry::init_sentry_from_settings()` diff --git a/src/components/AiPanel.test.tsx b/src/components/AiPanel.test.tsx index ef31269d..aaae70e2 100644 --- a/src/components/AiPanel.test.tsx +++ b/src/components/AiPanel.test.tsx @@ -7,6 +7,14 @@ import type { VaultEntry } from '../types' import { queueAiPrompt } from '../utils/aiPromptBridge' import { bindVaultConfigStore, getVaultConfig, resetVaultConfigStore } from '../utils/vaultConfigStore' +const { trackEventMock } = vi.hoisted(() => ({ + trackEventMock: vi.fn(), +})) + +vi.mock('../lib/telemetry', () => ({ + trackEvent: trackEventMock, +})) + // Mock the hooks and utils to isolate component tests let mockMessages: ReturnType['messages'] = [] let mockStatus: ReturnType['status'] = 'idle' @@ -80,6 +88,7 @@ describe('AiPanel', () => { mockClearConversation.mockReset() mockAddLocalMarker.mockReset() mockUseCliAiAgent.mockReset() + trackEventMock.mockClear() resetVaultConfigStore() bindVaultConfigStore({ zoom: null, @@ -141,6 +150,10 @@ describe('AiPanel', () => { expect(mockAddLocalMarker).toHaveBeenCalledWith( 'AI permission mode changed to Power User. It will apply to the next message.', ) + expect(trackEventMock).toHaveBeenCalledWith('ai_agent_permission_mode_changed', { + agent: 'claude_code', + permission_mode: 'power_user', + }) }) it('disables permission mode changes while the AI agent is running', () => { diff --git a/src/components/FilePreview.test.tsx b/src/components/FilePreview.test.tsx index 5ef7d12c..bdc5772e 100644 --- a/src/components/FilePreview.test.tsx +++ b/src/components/FilePreview.test.tsx @@ -1,12 +1,20 @@ import { fireEvent, render, screen } from '@testing-library/react' -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { FilePreview } from './FilePreview' import type { VaultEntry } from '../types' +const { trackEventMock } = vi.hoisted(() => ({ + trackEventMock: vi.fn(), +})) + vi.mock('@tauri-apps/api/core', () => ({ convertFileSrc: (path: string) => `asset://${path}`, })) +vi.mock('../lib/telemetry', () => ({ + trackEvent: trackEventMock, +})) + const imageEntry: VaultEntry = { path: '/vault/Attachments/photo.png', filename: 'photo.png', @@ -48,6 +56,10 @@ const pdfEntry: VaultEntry = { } describe('FilePreview', () => { + beforeEach(() => { + trackEventMock.mockClear() + }) + it('routes header file actions to the active file path', () => { const onRevealFile = vi.fn() const onCopyFilePath = vi.fn() @@ -62,6 +74,8 @@ describe('FilePreview', () => { />, ) + expect(trackEventMock).toHaveBeenCalledWith('file_preview_opened', { preview_kind: 'image' }) + fireEvent.click(screen.getByRole('button', { name: 'Reveal' })) fireEvent.click(screen.getByRole('button', { name: 'Copy path' })) fireEvent.click(screen.getByRole('button', { name: 'Open' })) @@ -69,6 +83,18 @@ describe('FilePreview', () => { expect(onRevealFile).toHaveBeenCalledWith('/vault/Attachments/photo.png') expect(onCopyFilePath).toHaveBeenCalledWith('/vault/Attachments/photo.png') expect(onOpenExternalFile).toHaveBeenCalledWith('/vault/Attachments/photo.png') + expect(trackEventMock).toHaveBeenCalledWith('file_preview_action', { + action: 'reveal', + preview_kind: 'image', + }) + expect(trackEventMock).toHaveBeenCalledWith('file_preview_action', { + action: 'copy_path', + preview_kind: 'image', + }) + expect(trackEventMock).toHaveBeenCalledWith('file_preview_action', { + action: 'open_external', + preview_kind: 'image', + }) }) it('renders supported PDF files through the asset preview path', () => { @@ -90,4 +116,16 @@ describe('FilePreview', () => { expect(screen.getByTestId('file-preview-fallback')).toHaveTextContent('PDF preview failed') expect(screen.getByRole('button', { name: 'Open in default app' })).toBeInTheDocument() }) + + it('tracks image preview failures without leaking the file path', () => { + render() + + fireEvent.error(screen.getByTestId('image-file-preview')) + + expect(trackEventMock).toHaveBeenCalledWith('file_preview_failed', { preview_kind: 'image' }) + expect(trackEventMock).not.toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ path: expect.any(String) }), + ) + }) }) diff --git a/src/components/FilePreview.tsx b/src/components/FilePreview.tsx index 3c16562d..174a0f0a 100644 --- a/src/components/FilePreview.tsx +++ b/src/components/FilePreview.tsx @@ -1,7 +1,8 @@ -import { useCallback, useMemo, useState, type KeyboardEvent } from 'react' +import { useCallback, useEffect, useMemo, useState, type KeyboardEvent } from 'react' import { convertFileSrc } from '@tauri-apps/api/core' import { ArrowSquareOut, ClipboardText, FileDashed, FilePdf, FolderOpen, ImageSquare, WarningCircle } from '@phosphor-icons/react' import type { VaultEntry } from '../types' +import { trackFilePreviewAction, trackFilePreviewFailed, trackFilePreviewOpened } from '../lib/productAnalytics' import { filePreviewKind, previewFileTypeLabel, type FilePreviewKind } from '../utils/filePreview' import { focusNoteListContainer } from '../utils/neighborhoodHistory' import { openLocalFile } from '../utils/url' @@ -228,9 +229,17 @@ export function FilePreview({ const assetSrc = useMemo(() => (previewKind ? convertFileSrc(entry.path) : null), [entry.path, previewKind]) const fileTypeLabel = previewFileTypeLabel(entry) const imageFailed = failedImagePath === entry.path - const handleImageError = useCallback(() => setFailedImagePath(entry.path), [entry.path]) + const handleImageError = useCallback(() => { + setFailedImagePath(entry.path) + trackFilePreviewFailed('image') + }, [entry.path]) + + useEffect(() => { + trackFilePreviewOpened(previewKind) + }, [entry.path, previewKind]) const handleOpenExternal = useCallback(() => { + trackFilePreviewAction('open_external', previewKind) if (onOpenExternalFile) { onOpenExternalFile(entry.path) return @@ -239,15 +248,17 @@ export function FilePreview({ void openLocalFile(entry.path).catch((error) => { console.warn('Failed to open file with default app:', error) }) - }, [entry.path, onOpenExternalFile]) + }, [entry.path, onOpenExternalFile, previewKind]) const handleRevealFile = useCallback(() => { + trackFilePreviewAction('reveal', previewKind) onRevealFile?.(entry.path) - }, [entry.path, onRevealFile]) + }, [entry.path, onRevealFile, previewKind]) const handleCopyFilePath = useCallback(() => { + trackFilePreviewAction('copy_path', previewKind) onCopyFilePath?.(entry.path) - }, [entry.path, onCopyFilePath]) + }, [entry.path, onCopyFilePath, previewKind]) const handleKeyDown = useCallback((event: KeyboardEvent) => { if (event.key !== 'Escape') return diff --git a/src/components/SettingsPanel.test.tsx b/src/components/SettingsPanel.test.tsx index 780465db..270ee781 100644 --- a/src/components/SettingsPanel.test.tsx +++ b/src/components/SettingsPanel.test.tsx @@ -4,6 +4,14 @@ import { SettingsPanel } from './SettingsPanel' import type { Settings } from '../types' import { THEME_MODE_STORAGE_KEY } from '../lib/themeMode' +const { trackEventMock } = vi.hoisted(() => ({ + trackEventMock: vi.fn(), +})) + +vi.mock('../lib/telemetry', () => ({ + trackEvent: trackEventMock, +})) + const emptySettings: Settings = { auto_pull_interval_minutes: null, autogit_enabled: null, @@ -55,6 +63,7 @@ describe('SettingsPanel', () => { beforeEach(() => { vi.clearAllMocks() + trackEventMock.mockClear() Object.defineProperty(window, 'localStorage', { value: localStorageMock, configurable: true }) window.localStorage.clear() document.documentElement.removeAttribute('data-theme') @@ -181,6 +190,23 @@ describe('SettingsPanel', () => { expect(onClose).toHaveBeenCalled() }) + it('tracks All Notes visibility toggles with categorical metadata only', () => { + render( + + ) + + fireEvent.click(within(screen.getByTestId('settings-all-notes-show-images')).getByRole('checkbox')) + + expect(trackEventMock).toHaveBeenCalledWith('all_notes_visibility_changed', { + category: 'images', + enabled: 1, + }) + expect(trackEventMock).not.toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ path: expect.any(String) }), + ) + }) + it('defaults the color mode control to light', () => { render( diff --git a/src/components/SettingsPanel.tsx b/src/components/SettingsPanel.tsx index aa953dda..99346c95 100644 --- a/src/components/SettingsPanel.tsx +++ b/src/components/SettingsPanel.tsx @@ -39,6 +39,7 @@ import { import { normalizeReleaseChannel, serializeReleaseChannel, type ReleaseChannel } from '../lib/releaseChannel' import { shouldHideGitignoredFiles } from '../lib/gitignoredVisibility' import { trackEvent } from '../lib/telemetry' +import { trackAllNotesVisibilityChanged } from '../lib/productAnalytics' import { resolveAllNotesFileVisibility, settingsWithAllNotesFileVisibility, @@ -308,9 +309,10 @@ function SettingsPanelInner({ }, [onSave, settings, updateDraft]) const handleAllNotesFileVisibilityChange = useCallback((value: AllNotesFileVisibility) => { + trackAllNotesVisibilityChanged(draft.allNotesFileVisibility, value) updateDraft('allNotesFileVisibility', value) onSave(settingsWithAllNotesFileVisibility(settings, value)) - }, [onSave, settings, updateDraft]) + }, [draft.allNotesFileVisibility, onSave, settings, updateDraft]) const handleThemeModeChange = useCallback((value: ThemeMode) => { updateDraft('themeMode', value) diff --git a/src/components/useAiPanelController.ts b/src/components/useAiPanelController.ts index 33906616..e54a708d 100644 --- a/src/components/useAiPanelController.ts +++ b/src/components/useAiPanelController.ts @@ -1,6 +1,7 @@ import { useCallback, useMemo, useState, useSyncExternalStore } from 'react' import type { AiAgentId, AiAgentReadiness } from '../lib/aiAgents' import type { AppLocale } from '../lib/i18n' +import { trackAiAgentPermissionModeChanged } from '../lib/productAnalytics' import { aiAgentPermissionModeMarker, normalizeAiAgentPermissionMode, @@ -132,8 +133,9 @@ export function useAiPanelController({ if (isActive || nextMode === permissionMode) return updateVaultConfigField('ai_agent_permission_mode', nextMode) + trackAiAgentPermissionModeChanged(defaultAiAgent, nextMode) agent.addLocalMarker(aiAgentPermissionModeMarker(nextMode, locale)) - }, [agent, isActive, locale, permissionMode]) + }, [agent, defaultAiAgent, isActive, locale, permissionMode]) const handleNewChat = useCallback(() => { agent.clearConversation() diff --git a/src/lib/aiAgentSession.test.ts b/src/lib/aiAgentSession.test.ts index 7884d983..c5d3f78e 100644 --- a/src/lib/aiAgentSession.test.ts +++ b/src/lib/aiAgentSession.test.ts @@ -7,6 +7,7 @@ const { formatMessageWithHistoryMock, nextMessageIdMock, streamAiAgentMock, + trackEventMock, trimHistoryMock, } = vi.hoisted(() => ({ buildAgentSystemPromptMock: vi.fn(() => 'SYSTEM'), @@ -14,6 +15,7 @@ const { formatMessageWithHistoryMock: vi.fn((_history: unknown, prompt: string) => `formatted:${prompt}`), nextMessageIdMock: vi.fn(), streamAiAgentMock: vi.fn(async () => {}), + trackEventMock: vi.fn(), trimHistoryMock: vi.fn((history: unknown) => history), })) @@ -36,6 +38,10 @@ vi.mock('../utils/streamAiAgent', () => ({ streamAiAgent: streamAiAgentMock, })) +vi.mock('./telemetry', () => ({ + trackEvent: trackEventMock, +})) + import { clearAgentConversation, sendAgentMessage, @@ -79,6 +85,61 @@ function createRuntime( } } +type RuntimeFixture = ReturnType + +const completedHistory: AiAgentMessage = { + id: 'msg-1', + userMessage: 'Previous question', + actions: [], + response: 'Previous answer', +} +const streamingHistory: AiAgentMessage = { + id: 'msg-2', + userMessage: 'Ignored streaming question', + actions: [], + isStreaming: true, +} +const expectedChatHistory = [ + { role: 'user', content: 'Previous question', id: 'msg-1' }, + { role: 'assistant', content: 'Previous answer', id: 'msg-1-resp' }, +] + +function expectStreamingRuntimeState(session: RuntimeFixture): void { + expect(session.runtime.abortRef.current).toEqual({ aborted: false }) + expect(session.runtime.responseAccRef.current).toBe('') + expect(session.runtime.toolInputMapRef.current.size).toBe(0) + expect(session.getStatus()).toBe('thinking') + expect(session.getMessages().at(-1)).toEqual({ + userMessage: 'Latest question', + references: [{ path: '/vault/ref.md', title: 'Ref' }], + actions: [], + isStreaming: true, + id: 'msg-stream', + }) +} + +function expectFormattedHistoryUsed(): void { + expect(trimHistoryMock).toHaveBeenCalledWith(expectedChatHistory, 100_000) + expect(formatMessageWithHistoryMock).toHaveBeenCalledWith(expectedChatHistory, 'Latest question') +} + +function expectStreamingRequest(runtime: RuntimeFixture['runtime']): void { + expect(createStreamCallbacksMock).toHaveBeenCalledWith(expect.objectContaining({ + messageId: 'msg-stream', + vaultPath: '/vault', + setMessages: runtime.setMessages, + setStatus: runtime.setStatus, + })) + expect(streamAiAgentMock).toHaveBeenCalledWith({ + agent: 'codex', + message: 'formatted:Latest question', + systemPrompt: 'OVERRIDE', + vaultPath: '/vault', + permissionMode: 'power_user', + callbacks: { stream: 'callbacks' }, + }) +} + describe('aiAgentSession', () => { beforeEach(() => { vi.clearAllMocks() @@ -87,12 +148,19 @@ describe('aiAgentSession', () => { formatMessageWithHistoryMock.mockImplementation((_history: unknown, prompt: string) => `formatted:${prompt}`) trimHistoryMock.mockImplementation((history: unknown) => history) streamAiAgentMock.mockResolvedValue(undefined) + trackEventMock.mockClear() }) async function expectLocalResponse(options: { messageId: string - context: { agent: string; ready: boolean; vaultPath: string } + context: { + agent: 'claude_code' | 'codex' | 'opencode' | 'pi' | 'gemini' + ready: boolean + vaultPath: string + permissionMode: 'safe' | 'power_user' + } prompt: { text: string; references?: [] } + reason: 'agent_unavailable' | 'missing_vault' response: string }) { nextMessageIdMock.mockReturnValue(options.messageId) @@ -114,20 +182,24 @@ describe('aiAgentSession', () => { }, ]) expect(streamAiAgentMock).not.toHaveBeenCalled() + expect(trackEventMock).toHaveBeenCalledWith('ai_agent_message_blocked', { + agent: options.context.agent, + reason: options.reason, + }) } it('ignores blank prompts and busy runtimes', async () => { const idleRuntime = createRuntime() await sendAgentMessage({ runtime: idleRuntime.runtime, - context: { agent: 'codex', ready: true, vaultPath: '/vault' }, + context: { agent: 'codex', ready: true, vaultPath: '/vault', permissionMode: 'safe' }, prompt: { text: ' ' }, }) const busyRuntime = createRuntime([], 'thinking') await sendAgentMessage({ runtime: busyRuntime.runtime, - context: { agent: 'codex', ready: true, vaultPath: '/vault' }, + context: { agent: 'codex', ready: true, vaultPath: '/vault', permissionMode: 'safe' }, prompt: { text: 'Question' }, }) @@ -140,14 +212,16 @@ describe('aiAgentSession', () => { const fallbackCases = [ { messageId: 'msg-local', - context: { agent: 'codex', ready: true, vaultPath: '' }, + context: { agent: 'codex', ready: true, vaultPath: '', permissionMode: 'safe' }, prompt: { text: 'Open a note' }, + reason: 'missing_vault', response: 'No vault loaded. Open a vault first.', }, { messageId: 'msg-missing', - context: { agent: 'codex', ready: false, vaultPath: '/vault' }, + context: { agent: 'codex', ready: false, vaultPath: '/vault', permissionMode: 'safe' }, prompt: { text: 'Open a note', references: [] }, + reason: 'agent_unavailable', response: 'Codex is not available on this machine. Install it or switch the default AI agent in Settings.', }, @@ -160,29 +234,18 @@ describe('aiAgentSession', () => { it('starts a streaming session with formatted history and fresh refs', async () => { nextMessageIdMock.mockReturnValue('msg-stream') - const completedHistory: AiAgentMessage = { - id: 'msg-1', - userMessage: 'Previous question', - actions: [], - response: 'Previous answer', - } - const streamingHistory: AiAgentMessage = { - id: 'msg-2', - userMessage: 'Ignored streaming question', - actions: [], - isStreaming: true, - } - const { runtime, getMessages, getStatus } = createRuntime([ + const session = createRuntime([ completedHistory, streamingHistory, ]) await sendAgentMessage({ - runtime, + runtime: session.runtime, context: { agent: 'codex', ready: true, vaultPath: '/vault', + permissionMode: 'power_user', systemPromptOverride: 'OVERRIDE', }, prompt: { @@ -191,37 +254,15 @@ describe('aiAgentSession', () => { }, }) - expect(runtime.abortRef.current).toEqual({ aborted: false }) - expect(runtime.responseAccRef.current).toBe('') - expect(runtime.toolInputMapRef.current.size).toBe(0) - expect(getStatus()).toBe('thinking') - expect(getMessages().at(-1)).toEqual({ - userMessage: 'Latest question', - references: [{ path: '/vault/ref.md', title: 'Ref' }], - actions: [], - isStreaming: true, - id: 'msg-stream', - }) - expect(trimHistoryMock).toHaveBeenCalledWith([ - { role: 'user', content: 'Previous question', id: 'msg-1' }, - { role: 'assistant', content: 'Previous answer', id: 'msg-1-resp' }, - ], 100_000) - expect(formatMessageWithHistoryMock).toHaveBeenCalledWith([ - { role: 'user', content: 'Previous question', id: 'msg-1' }, - { role: 'assistant', content: 'Previous answer', id: 'msg-1-resp' }, - ], 'Latest question') - expect(createStreamCallbacksMock).toHaveBeenCalledWith(expect.objectContaining({ - messageId: 'msg-stream', - vaultPath: '/vault', - setMessages: runtime.setMessages, - setStatus: runtime.setStatus, - })) - expect(streamAiAgentMock).toHaveBeenCalledWith({ + expectStreamingRuntimeState(session) + expectFormattedHistoryUsed() + expectStreamingRequest(session.runtime) + expect(trackEventMock).toHaveBeenCalledWith('ai_agent_message_sent', { agent: 'codex', - message: 'formatted:Latest question', - systemPrompt: 'OVERRIDE', - vaultPath: '/vault', - callbacks: { stream: 'callbacks' }, + permission_mode: 'power_user', + has_context: 1, + reference_count: 1, + history_message_count: 1, }) }) diff --git a/src/lib/aiAgentSession.ts b/src/lib/aiAgentSession.ts index dbc0210d..03bd334f 100644 --- a/src/lib/aiAgentSession.ts +++ b/src/lib/aiAgentSession.ts @@ -13,6 +13,7 @@ import { import type { AgentFileCallbacks } from './aiAgentFileOperations' import { createStreamCallbacks } from './aiAgentStreamCallbacks' import type { ToolInvocation } from './aiAgentMessageState' +import { trackAiAgentMessageBlocked, trackAiAgentMessageSent } from './productAnalytics' import { streamAiAgent } from '../utils/streamAiAgent' export interface AiAgentSessionRuntime { @@ -39,6 +40,10 @@ function normalizePrompt(prompt: PendingUserPrompt): PendingUserPrompt { } } +function completedMessageCount(messages: AiAgentMessage[]): number { + return messages.filter((message) => !message.isStreaming && !message.localMarker).length +} + export async function sendAgentMessage({ runtime, context, @@ -50,11 +55,13 @@ export async function sendAgentMessage({ if (!normalizedPrompt.text || currentStatus === 'thinking' || currentStatus === 'tool-executing') return if (!context.vaultPath) { + trackAiAgentMessageBlocked(context.agent, 'missing_vault') appendLocalResponse(runtime.setMessages, normalizedPrompt, 'No vault loaded. Open a vault first.') return } if (!context.ready) { + trackAiAgentMessageBlocked(context.agent, 'agent_unavailable') appendLocalResponse( runtime.setMessages, normalizedPrompt, @@ -63,6 +70,14 @@ export async function sendAgentMessage({ return } + trackAiAgentMessageSent({ + agent: context.agent, + permissionMode: context.permissionMode, + hasContext: !!context.systemPromptOverride, + referenceCount: normalizedPrompt.references?.length ?? 0, + historyMessageCount: completedMessageCount(runtime.messagesRef.current), + }) + runtime.abortRef.current = { aborted: false } runtime.responseAccRef.current = '' runtime.toolInputMapRef.current = new Map() diff --git a/src/lib/aiAgentStreamCallbacks.test.ts b/src/lib/aiAgentStreamCallbacks.test.ts index eed632ab..0a09646c 100644 --- a/src/lib/aiAgentStreamCallbacks.test.ts +++ b/src/lib/aiAgentStreamCallbacks.test.ts @@ -1,8 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentStatus, AiAgentMessage } from './aiAgentConversation' -const { detectFileOperationMock } = vi.hoisted(() => ({ +const { detectFileOperationMock, trackEventMock } = vi.hoisted(() => ({ detectFileOperationMock: vi.fn(), + trackEventMock: vi.fn(), })) vi.mock('./aiAgentFileOperations', async (importOriginal) => ({ @@ -10,6 +11,10 @@ vi.mock('./aiAgentFileOperations', async (importOriginal) => ({ detectFileOperation: detectFileOperationMock, })) +vi.mock('./telemetry', () => ({ + trackEvent: trackEventMock, +})) + import { createStreamCallbacks } from './aiAgentStreamCallbacks' function createMessageStore(initialMessages: AiAgentMessage[]) { @@ -37,6 +42,7 @@ function createStatusStore(initialStatus: AgentStatus = 'idle') { describe('aiAgentStreamCallbacks', () => { beforeEach(() => { vi.clearAllMocks() + trackEventMock.mockClear() }) it('handles the happy-path lifecycle and refreshes the vault at the end', () => { @@ -85,6 +91,11 @@ describe('aiAgentStreamCallbacks', () => { callbacks: fileCallbacks, }) expect(fileCallbacks.onVaultChanged).toHaveBeenCalledTimes(1) + expect(trackEventMock).toHaveBeenCalledWith('ai_agent_response_completed', { + agent: 'claude_code', + had_text: 1, + tool_count: 1, + }) expect(messages.getMessages()).toEqual([ { id: 'msg-1', @@ -137,6 +148,12 @@ describe('aiAgentStreamCallbacks', () => { callbacks.onError('boom') expect(status.getStatus()).toBe('error') + expect(trackEventMock).toHaveBeenCalledWith('ai_agent_response_failed', { + agent: 'claude_code', + error_kind: 'stream_error', + had_partial_response: 1, + tool_count: 0, + }) expect(messages.getMessages()).toEqual([ { id: 'msg-1', @@ -183,6 +200,11 @@ describe('aiAgentStreamCallbacks', () => { callbacks.onDone() expect(status.getStatus()).toBe('done') + expect(trackEventMock).toHaveBeenCalledWith('ai_agent_response_completed', { + agent, + had_text: 0, + tool_count: 0, + }) expect(messages.getMessages()).toEqual([ { id: 'msg-1', diff --git a/src/lib/aiAgentStreamCallbacks.ts b/src/lib/aiAgentStreamCallbacks.ts index 9d1d47dc..6fc2f9c3 100644 --- a/src/lib/aiAgentStreamCallbacks.ts +++ b/src/lib/aiAgentStreamCallbacks.ts @@ -8,6 +8,10 @@ import { type ToolInvocation, } from './aiAgentMessageState' import { getAiAgentDefinition, type AiAgentId } from './aiAgents' +import { + trackAiAgentResponseCompleted, + trackAiAgentResponseFailed, +} from './productAnalytics' export interface StreamMutationContext { agent: AiAgentId @@ -39,6 +43,7 @@ export function createStreamCallbacks(context: StreamMutationContext) { toolInputMapRef, fileCallbacksRef, } = context + let failureTracked = false return { onThinking: (chunk: string) => { @@ -93,6 +98,8 @@ export function createStreamCallbacks(context: StreamMutationContext) { setStatus('error') const partial = responseAccRef.current + failureTracked = true + trackAiAgentResponseFailed(agent, partial, toolInputMapRef.current.size) updateMessage(setMessages, messageId, (message) => ({ ...message, isStreaming: false, @@ -109,6 +116,7 @@ export function createStreamCallbacks(context: StreamMutationContext) { setStatus('done') const finalResponse = finalResponseText(responseAccRef.current, agent) + trackAiAgentResponseCompleted(agent, responseAccRef.current, toolInputMapRef.current.size, failureTracked) updateMessage(setMessages, messageId, (message) => ({ ...message, isStreaming: false, diff --git a/src/lib/productAnalytics.ts b/src/lib/productAnalytics.ts new file mode 100644 index 00000000..3b1d6936 --- /dev/null +++ b/src/lib/productAnalytics.ts @@ -0,0 +1,103 @@ +import type { AiAgentId } from './aiAgents' +import type { AiAgentPermissionMode } from './aiAgentPermissionMode' +import { trackEvent } from './telemetry' +import type { AllNotesFileVisibility } from '../utils/allNotesFileVisibility' +import type { FilePreviewKind } from '../utils/filePreview' + +type TrackedPreviewKind = FilePreviewKind | 'unsupported' +type FilePreviewAction = 'copy_path' | 'open_external' | 'reveal' +type AgentBlockedReason = 'agent_unavailable' | 'missing_vault' + +const ALL_NOTES_VISIBILITY_CATEGORIES: ReadonlyArray = [ + 'pdfs', + 'images', + 'unsupported', +] + +function trackedPreviewKind(previewKind: FilePreviewKind | null): TrackedPreviewKind { + return previewKind ?? 'unsupported' +} + +function numericFlag(value: boolean): number { + return value ? 1 : 0 +} + +export function trackFilePreviewOpened(previewKind: FilePreviewKind | null): void { + trackEvent('file_preview_opened', { + preview_kind: trackedPreviewKind(previewKind), + }) +} + +export function trackFilePreviewAction(action: FilePreviewAction, previewKind: FilePreviewKind | null): void { + trackEvent('file_preview_action', { + action, + preview_kind: trackedPreviewKind(previewKind), + }) +} + +export function trackFilePreviewFailed(previewKind: FilePreviewKind): void { + trackEvent('file_preview_failed', { preview_kind: previewKind }) +} + +export function trackAllNotesVisibilityChanged( + previous: AllNotesFileVisibility, + next: AllNotesFileVisibility, +): void { + for (const category of ALL_NOTES_VISIBILITY_CATEGORIES) { + if (previous[category] === next[category]) continue + trackEvent('all_notes_visibility_changed', { + category, + enabled: numericFlag(next[category]), + }) + } +} + +export function trackAiAgentMessageBlocked(agent: AiAgentId, reason: AgentBlockedReason): void { + trackEvent('ai_agent_message_blocked', { agent, reason }) +} + +export function trackAiAgentMessageSent(params: { + agent: AiAgentId + permissionMode: AiAgentPermissionMode + hasContext: boolean + referenceCount: number + historyMessageCount: number +}): void { + trackEvent('ai_agent_message_sent', { + agent: params.agent, + permission_mode: params.permissionMode, + has_context: numericFlag(params.hasContext), + reference_count: params.referenceCount, + history_message_count: params.historyMessageCount, + }) +} + +export function trackAiAgentResponseCompleted( + agent: AiAgentId, + response: string, + toolCount: number, + skipped: boolean, +): void { + if (skipped) return + trackEvent('ai_agent_response_completed', { + agent, + had_text: numericFlag(response.trim().length > 0), + tool_count: toolCount, + }) +} + +export function trackAiAgentResponseFailed(agent: AiAgentId, response: string, toolCount: number): void { + trackEvent('ai_agent_response_failed', { + agent, + error_kind: 'stream_error', + had_partial_response: numericFlag(response.trim().length > 0), + tool_count: toolCount, + }) +} + +export function trackAiAgentPermissionModeChanged(agent: AiAgentId, permissionMode: AiAgentPermissionMode): void { + trackEvent('ai_agent_permission_mode_changed', { + agent, + permission_mode: permissionMode, + }) +}