From ecbb94ae83641e2ae0de38541b5fe886746dd545 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 24 Mar 2026 16:18:05 +0100 Subject: [PATCH] =?UTF-8?q?refactor:=20remove=20QMD=20semantic=20indexing?= =?UTF-8?q?=20=E2=80=94=20keep=20keyword=20search=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the entire QMD search engine (semantic indexing, hybrid search, vector embeddings) and replace with a simple walkdir-based keyword search. QMD never worked reliably in production, added bundle bloat, and showed "Index failed" in the status bar. What changed: - Rust: deleted indexing.rs, rewrote search.rs as pure keyword search - Frontend: removed useIndexing hook, IndexingBadge, hybrid search - Menu: removed "Reindex Vault" command from palette and menu bar - Config: removed qmd from bundle resources and build script - Deleted tools/qmd/ (QMD CLI tool, MCP server, tests) - Updated docs (ARCHITECTURE, ABSTRACTIONS, GETTING-STARTED) Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/ABSTRACTIONS.md | 25 +- docs/ARCHITECTURE.md | 69 +- docs/GETTING-STARTED.md | 8 +- scripts/bundle-qmd.sh | 155 - src-tauri/build.rs | 4 +- src-tauri/src/commands.rs | 79 +- src-tauri/src/indexing.rs | 914 ------ src-tauri/src/lib.rs | 4 - src-tauri/src/menu.rs | 7 - src-tauri/src/search.rs | 292 +- src-tauri/tauri.conf.json | 5 +- src/App.tsx | 11 +- src/components/SearchPanel.test.tsx | 64 +- src/components/StatusBar.test.tsx | 183 -- src/components/StatusBar.tsx | 75 +- src/hooks/useAppCommands.ts | 3 - src/hooks/useCommandRegistry.test.ts | 40 - src/hooks/useCommandRegistry.ts | 5 +- src/hooks/useIndexing.test.ts | 145 - src/hooks/useIndexing.ts | 159 - src/hooks/useMenuEvents.test.ts | 7 - src/hooks/useMenuEvents.ts | 4 +- src/hooks/useUnifiedSearch.ts | 25 +- src/mock-tauri/mock-handlers.ts | 3 - src/utils/indexingHelpers.ts | 10 - .../smoke/fresh-install-regression-qa.spec.ts | 3 +- tests/smoke/indexing-reindex-status.spec.ts | 28 - tools/qmd/bun.lock | 637 ---- tools/qmd/package.json | 53 - tools/qmd/src/cli.test.ts | 963 ------ tools/qmd/src/collections.ts | 390 --- tools/qmd/src/eval.test.ts | 412 --- tools/qmd/src/formatter.ts | 427 --- tools/qmd/src/llm.test.ts | 559 ---- tools/qmd/src/llm.ts | 1208 -------- tools/qmd/src/mcp.test.ts | 889 ------ tools/qmd/src/mcp.ts | 626 ---- tools/qmd/src/qmd.ts | 2699 ----------------- tools/qmd/src/store-paths.test.ts | 395 --- tools/qmd/src/store.test.ts | 2483 --------------- tools/qmd/src/store.ts | 2571 ---------------- tools/qmd/tsconfig.json | 29 - 42 files changed, 148 insertions(+), 16520 deletions(-) delete mode 100755 scripts/bundle-qmd.sh delete mode 100644 src-tauri/src/indexing.rs delete mode 100644 src/hooks/useIndexing.test.ts delete mode 100644 src/hooks/useIndexing.ts delete mode 100644 src/utils/indexingHelpers.ts delete mode 100644 tests/smoke/indexing-reindex-status.spec.ts delete mode 100644 tools/qmd/bun.lock delete mode 100644 tools/qmd/package.json delete mode 100644 tools/qmd/src/cli.test.ts delete mode 100644 tools/qmd/src/collections.ts delete mode 100644 tools/qmd/src/eval.test.ts delete mode 100644 tools/qmd/src/formatter.ts delete mode 100644 tools/qmd/src/llm.test.ts delete mode 100644 tools/qmd/src/llm.ts delete mode 100644 tools/qmd/src/mcp.test.ts delete mode 100644 tools/qmd/src/mcp.ts delete mode 100755 tools/qmd/src/qmd.ts delete mode 100644 tools/qmd/src/store-paths.test.ts delete mode 100644 tools/qmd/src/store.test.ts delete mode 100644 tools/qmd/src/store.ts delete mode 100644 tools/qmd/tsconfig.json diff --git a/docs/ABSTRACTIONS.md b/docs/ABSTRACTIONS.md index 475f4f17..7cfae9d5 100644 --- a/docs/ABSTRACTIONS.md +++ b/docs/ABSTRACTIONS.md @@ -490,42 +490,29 @@ The Inspector panel (`src/components/Inspector.tsx`) is composed of sub-panels: `useClosedTabHistory` hook (`src/hooks/useClosedTabHistory.ts`) provides a LIFO stack for closed tab entries, used by `useTabManagement` to support Cmd+Shift+T reopen. Each entry stores the note's path, tab index, and full `VaultEntry`. The stack is in-memory only (resets on restart), capped at 20 entries, and deduplicates by path. -## Search & Indexing +## Search -### Search Modes +### Search + +Keyword-based search scans all vault `.md` files using `walkdir`: ```typescript -type SearchMode = 'keyword' | 'semantic' | 'hybrid' - interface SearchResult { title: string path: string snippet: string score: number } - -interface SearchResponse { - results: SearchResult[] - elapsedMs: number -} ``` ### Search Integration `SearchPanel` component provides the search UI: -- Mode selector (keyword/semantic/hybrid) -- Real-time results as user types +- Real-time results as user types (300ms debounce) - Click result to open note in editor - Shows relevance score and snippet -### Indexing - -Managed by `useIndexing` hook: -- Checks index status on vault load -- Two-phase indexing: scanning (parse files) → embedding (generate vectors) -- Progress streamed via Tauri events -- Incremental updates after git sync -- Metadata persisted in `.laputa-index.json` +No indexing step required — search runs directly against the filesystem. ## Vault Management diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5c2dbcc3..6cdba254 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -79,7 +79,7 @@ flowchart LR | Frontmatter parsing | gray_matter | 0.2 | | AI (in-app chat) | Anthropic Claude API (Haiku 3.5 default) | - | | AI (agent panel) | Claude CLI subprocess (streaming NDJSON) | - | -| Search | qmd (keyword + semantic + hybrid) | - | +| Search | Keyword (walkdir-based file scan) | - | | MCP | @modelcontextprotocol/sdk | 1.0 | | Tests | Vitest (unit), Playwright (E2E/smoke), cargo test (Rust) | - | | Package manager | pnpm | - | @@ -113,7 +113,7 @@ flowchart TD GIT["git/"] GH["github/"] THEME["theme/"] - SEARCH["search.rs + indexing.rs"] + SEARCH["search.rs"] CLI["claude_cli.rs"] end @@ -121,7 +121,6 @@ flowchart TD ANTH["Anthropic API\n(Claude chat)"] CCLI["Claude CLI\n(agent subprocess)"] MCP["MCP Server\n(ws://9710, 9711)"] - QMD["qmd\n(search engine)"] GHAPI["GitHub API\n(OAuth, repos, clone)"] end @@ -359,53 +358,16 @@ flowchart LR The `WsBridgeChild` state wrapper in `lib.rs` ensures the bridge process is killed on app exit via `RunEvent::Exit` handler. -## Search & Indexing +## Search -### Search Engine +Search is keyword-based, using `walkdir` to scan all `.md` files in the vault directory. No external binary or indexing step required. -Search uses the external `qmd` binary (semantic search engine) with three modes: +- Matches query against file titles and content (case-insensitive) +- Scores results: title matches ranked higher than content-only matches +- Extracts contextual snippets around the first match +- Skips trashed and hidden files -| Mode | Command | Description | -|------|---------|-------------| -| `keyword` | `qmd search` | Term matching (default) | -| `semantic` | `qmd vsearch` | Vector similarity search | -| `hybrid` | `qmd query` | Combined keyword + semantic | - -### Indexing Flow - -```mermaid -flowchart TD - A([Vault opened]) --> B[check_index_status] - B --> C{Index status?} - C -->|Fresh| D[run_incremental_update\ngit diff since last commit] - C -->|Stale / Missing| E - - subgraph E[Full Indexing — start_indexing] - E1["Phase 1: qmd update\n(scan all .md files)"] - E2["Phase 2: qmd embed\n(generate vector embeddings)"] - E1 --> E2 - end - - E --> F[Save .laputa-index.json\nlast_indexed_commit + timestamp] - D --> G([Search ready]) - F --> G - - E2 -.->|failure is non-fatal| G - G --> H{Search mode} - H -->|keyword| I[qmd search] - H -->|semantic| J[qmd vsearch] - H -->|hybrid| K[qmd query] -``` - -Embedding failure is non-fatal — keyword search still works. - -### qmd Binary Resolution - -1. Bundled macOS app resource: `/Contents/Resources/qmd/qmd` -2. Dev mode: `CARGO_MANIFEST_DIR/resources/qmd/qmd` -3. System locations: `~/.bun/bin/qmd`, `/usr/local/bin/qmd`, `/opt/homebrew/bin/qmd` -4. PATH lookup via `which qmd` -5. Auto-install via `bun install -g qmd` if missing +The `search_vault` Tauri command runs the scan in a blocking Tokio task and returns results sorted by relevance score. ## Vault Cache System @@ -533,7 +495,6 @@ sequenceDiagram VL->>T: invoke('get_modified_files') VL->>T: useMcpStatus — register if needed VL->>T: useThemeManager — load active theme - VL->>T: useIndexing — incremental update if stale VL-->>A: entries ready end @@ -619,8 +580,7 @@ The vault backend (`src-tauri/src/vault/`) is split into focused submodules: | `git/` | Git operations (`commit.rs`, `status.rs`, `history.rs`, `conflict.rs`, `remote.rs`, `pulse.rs`) | | `github/` | GitHub OAuth + API (`auth.rs`, `api.rs`, `clone.rs`) | | `theme/` | Theme management (`mod.rs`, `create.rs`, `defaults.rs`, `seed.rs`) | -| `search.rs` | qmd search integration (keyword/semantic/hybrid) | -| `indexing.rs` | qmd indexing with progress streaming | +| `search.rs` | Keyword search — walkdir-based vault file scan | | `claude_cli.rs` | Claude CLI subprocess spawning + NDJSON stream parsing | | `ai_chat.rs` | Direct Anthropic API client (non-streaming, for Tauri builds) | | `mcp.rs` | MCP server spawning + config registration | @@ -689,14 +649,11 @@ The vault backend (`src-tauri/src/vault/`) is split into focused submodules: | `github_create_repo` | Create new repo | | `clone_repo` | Clone repo with token auth | -### Search & Indexing +### Search | Command | Description | |---------|-------------| -| `search_vault` | Search via qmd (keyword/semantic/hybrid) | -| `get_index_status` | Check qmd index state | -| `start_indexing` | Full index with progress streaming | -| `trigger_incremental_index` | Incremental index update | +| `search_vault` | Keyword search across vault files | ### Theme @@ -772,7 +729,7 @@ No Redux or global context. State lives in the root `App.tsx` and custom hooks: | `useAIChat` | `messages`, `isStreaming` | AI chat conversation | | `useAiAgent` | `messages`, `status`, tool actions | AI agent conversation | | `useAutoSync` | Sync interval, pull/push state | Git auto-sync | -| `useIndexing` | Index status, progress | Search indexing | +| `useUnifiedSearch` | Query, results, loading state | Keyword search | | `useSettings` | App settings (API keys, GitHub token) | Persistent settings | | `useVaultConfig` | Per-vault UI preferences | Vault-specific config | diff --git a/docs/GETTING-STARTED.md b/docs/GETTING-STARTED.md index b157d72d..9cd8777e 100644 --- a/docs/GETTING-STARTED.md +++ b/docs/GETTING-STARTED.md @@ -7,7 +7,6 @@ How to navigate the codebase, run the app, and find what you need. - **Node.js** 18+ and **pnpm** - **Rust** 1.77.2+ (for the Tauri backend) - **git** CLI (required by the git integration features) -- **qmd** (optional — for search indexing; auto-installed if missing) ## Quick Start @@ -97,7 +96,7 @@ laputa-app/ │ │ ├── useEditorSave.ts # Auto-save with debounce │ │ ├── useTheme.ts # Flatten theme.json → CSS vars │ │ ├── useThemeManager.ts # Vault theme lifecycle -│ │ ├── useIndexing.ts # Search indexing management +│ │ ├── useUnifiedSearch.ts # Keyword search │ │ ├── useNoteSearch.ts # Note search │ │ ├── useCommandRegistry.ts # Command palette registry │ │ ├── useAppCommands.ts # App-level commands @@ -158,8 +157,7 @@ laputa-app/ │ │ │ ├── mod.rs, auth.rs, api.rs, clone.rs │ │ ├── theme/ # Theme module │ │ │ ├── mod.rs, create.rs, defaults.rs, seed.rs -│ │ ├── search.rs # qmd search integration -│ │ ├── indexing.rs # qmd indexing + progress streaming +│ │ ├── search.rs # Keyword search (walkdir-based) │ │ ├── claude_cli.rs # Claude CLI subprocess management │ │ ├── ai_chat.rs # Direct Anthropic API client │ │ ├── mcp.rs # MCP server lifecycle + registration @@ -220,7 +218,7 @@ laputa-app/ | `src-tauri/src/frontmatter/ops.rs` | YAML manipulation — how properties are updated/deleted in files. | | `src-tauri/src/git/` | All git operations (commit, pull, push, conflicts, pulse). | | `src-tauri/src/github/` | GitHub OAuth device flow + repo clone/create. | -| `src-tauri/src/search.rs` | qmd search integration (keyword/semantic/hybrid). | +| `src-tauri/src/search.rs` | Keyword search — scans vault files with walkdir. | | `src-tauri/src/claude_cli.rs` | Claude CLI subprocess spawning + NDJSON stream parsing. | ### Editor diff --git a/scripts/bundle-qmd.sh b/scripts/bundle-qmd.sh deleted file mode 100755 index 8bbbe5f9..00000000 --- a/scripts/bundle-qmd.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash -# Bundle qmd into a self-contained directory for Tauri resource embedding. -# -# Output: src-tauri/resources/qmd/ -# qmd — compiled standalone binary -# node_modules/sqlite-vec/ — JS shim for sqlite-vec -# node_modules/sqlite-vec-darwin-arm64/ — native .dylib (arm64) -# node_modules/sqlite-vec-darwin-x64/ — native .dylib (x64) -# node_modules/node-llama-cpp/ — stub (keyword search only) -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -ROOT="$SCRIPT_DIR/.." -OUT="$ROOT/src-tauri/resources/qmd" - -# ---------- locate tools ---------- -find_bun() { - for c in \ - "$HOME/.bun/bin/bun" \ - "/opt/homebrew/bin/bun" \ - "/usr/local/bin/bun"; do - [[ -x "$c" ]] && { echo "$c"; return 0; } - done - command -v bun 2>/dev/null && return 0 - return 1 -} - -BUN=$(find_bun) || { echo "ERROR: bun not found — install from https://bun.sh" >&2; exit 1; } -echo "Using bun: $BUN" - -# ---------- locate qmd source ---------- -# Prefer bundled source in tools/qmd/ (works in CI and dev), -# then fall back to globally installed qmd on dev machines. -QMD_SRC="" -for c in \ - "$ROOT/tools/qmd" \ - "$HOME/.bun/install/global/node_modules/qmd" \ - "/opt/homebrew/lib/node_modules/qmd" \ - "/usr/local/lib/node_modules/qmd"; do - [[ -f "$c/src/qmd.ts" ]] && { QMD_SRC="$c"; break; } -done - -[[ -n "$QMD_SRC" ]] || { echo "ERROR: qmd source not found. tools/qmd/ is missing or incomplete." >&2; exit 1; } -echo "Using qmd source: $QMD_SRC" - -# Install qmd dependencies if needed (for CI where node_modules don't exist yet) -if [[ ! -d "$QMD_SRC/node_modules" ]]; then - echo "Installing qmd dependencies..." - (cd "$QMD_SRC" && "$BUN" install --frozen-lockfile) -fi - -# ---------- compile ---------- -echo "Compiling qmd with bun build --compile..." -mkdir -p "$OUT" - -(cd "$QMD_SRC" && "$BUN" build --compile \ - "src/qmd.ts" \ - --outfile "$OUT/qmd" \ - --external node-llama-cpp \ - --external sqlite-vec \ - --external sqlite-vec-darwin-arm64 \ - --external sqlite-vec-darwin-x64) - -chmod +x "$OUT/qmd" - -# ---------- bundle sqlite-vec ---------- -echo "Bundling sqlite-vec native extensions..." - -# Find sqlite-vec packages — prefer node_modules in QMD_SRC (after bun install), -# fall back to bun global cache for dev machines. -NM="$QMD_SRC/node_modules" - -find_pkg() { - local pkg="$1" - # Check node_modules from bun install in QMD_SRC first - if [[ -d "$NM/$pkg" ]]; then - echo "$NM/$pkg"; return 0 - fi - # Fall back to bun global cache - local cache_dir - cache_dir=$(find "$HOME/.bun/install/cache" -maxdepth 1 -name "${pkg}@*" -type d 2>/dev/null | head -1) - [[ -n "$cache_dir" ]] && echo "$cache_dir" && return 0 - return 1 -} - -# sqlite-vec JS shim -SQLVEC_DIR=$(find_pkg "sqlite-vec") || { echo "ERROR: sqlite-vec not found" >&2; exit 1; } -mkdir -p "$OUT/node_modules/sqlite-vec" -cp "$SQLVEC_DIR/index.mjs" "$OUT/node_modules/sqlite-vec/index.mjs" -cp "$SQLVEC_DIR/package.json" "$OUT/node_modules/sqlite-vec/package.json" -[[ -f "$SQLVEC_DIR/index.cjs" ]] && cp "$SQLVEC_DIR/index.cjs" "$OUT/node_modules/sqlite-vec/index.cjs" - -# sqlite-vec-darwin-arm64 -ARM64_DIR=$(find_pkg "sqlite-vec-darwin-arm64") || true -if [[ -n "$ARM64_DIR" ]]; then - mkdir -p "$OUT/node_modules/sqlite-vec-darwin-arm64" - cp "$ARM64_DIR/vec0.dylib" "$OUT/node_modules/sqlite-vec-darwin-arm64/vec0.dylib" - cp "$ARM64_DIR/package.json" "$OUT/node_modules/sqlite-vec-darwin-arm64/package.json" - echo " ✓ arm64 dylib" -fi - -# sqlite-vec-darwin-x64 -X64_DIR=$(find_pkg "sqlite-vec-darwin-x64") || true -if [[ -n "$X64_DIR" ]]; then - mkdir -p "$OUT/node_modules/sqlite-vec-darwin-x64" - cp "$X64_DIR/vec0.dylib" "$OUT/node_modules/sqlite-vec-darwin-x64/vec0.dylib" - cp "$X64_DIR/package.json" "$OUT/node_modules/sqlite-vec-darwin-x64/package.json" - echo " ✓ x64 dylib" -fi - -# ---------- stub node-llama-cpp ---------- -echo "Creating node-llama-cpp stub (keyword search only)..." -mkdir -p "$OUT/node_modules/node-llama-cpp" - -cat > "$OUT/node_modules/node-llama-cpp/package.json" << 'PJSON' -{"name":"node-llama-cpp","version":"0.0.0-stub","type":"module","main":"index.js"} -PJSON - -cat > "$OUT/node_modules/node-llama-cpp/index.js" << 'STUB' -// Stub: node-llama-cpp not bundled — semantic search unavailable, keyword search works. -const unavailable = (name) => (...args) => { - throw new Error(`${name}() unavailable: node-llama-cpp not bundled. Keyword search still works.`); -}; -export const getLlama = unavailable("getLlama"); -export const resolveModelFile = unavailable("resolveModelFile"); -export class LlamaChatSession { - constructor() { throw new Error("LlamaChatSession unavailable"); } -} -export const LlamaLogLevel = { Error: 0, Warn: 1, Info: 2, Debug: 3 }; -STUB - -# ---------- code signing (macOS) ---------- -# In CI (APPLE_SIGNING_IDENTITY set): sign with Developer ID + hardened runtime (required for notarization) -# In dev (no identity): ad-hoc sign to remove quarantine -if [[ "$(uname)" == "Darwin" ]] && command -v codesign &>/dev/null; then - SIGN_ID="${APPLE_SIGNING_IDENTITY:--}" - if [[ "$SIGN_ID" != "-" ]]; then - echo "Signing bundled binaries with Developer ID: $SIGN_ID" - SIGN_OPTS=(--force --sign "$SIGN_ID" --options runtime --timestamp) - else - echo "Ad-hoc signing bundled binaries (dev mode)..." - SIGN_OPTS=(--force --sign -) - fi - codesign "${SIGN_OPTS[@]}" "$OUT/qmd" 2>/dev/null && echo " ✓ qmd signed" || echo " ⚠ qmd signing failed (non-fatal)" - while IFS= read -r -d '' dylib; do - codesign "${SIGN_OPTS[@]}" "$dylib" 2>/dev/null && echo " ✓ $(basename "$dylib") signed" || echo " ⚠ $(basename "$dylib") signing failed (non-fatal)" - done < <(find "$OUT/node_modules" -name "*.dylib" -print0) -fi - -# ---------- summary ---------- -echo "" -echo "qmd bundled → $OUT/" -du -sh "$OUT/qmd" -du -sh "$OUT/node_modules" -echo "Done." diff --git a/src-tauri/build.rs b/src-tauri/build.rs index dc4d9fe2..66379918 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -1,8 +1,8 @@ fn main() { // Ensure resource directories exist for the Tauri build. - // These are gitignored and populated by scripts (bundle-qmd.sh, bundle-mcp-server.mjs). + // These are gitignored and populated by scripts (bundle-mcp-server.mjs). // Without a placeholder, `tauri build` / `cargo test` fails if the scripts haven't run. - for dir in ["resources/qmd", "resources/mcp-server"] { + for dir in ["resources/mcp-server"] { let path = std::path::Path::new(dir); if !path.exists() { std::fs::create_dir_all(path).ok(); diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 55ead010..72a83c2d 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -10,12 +10,11 @@ use crate::git::{ PulseCommit, }; use crate::github::{DeviceFlowPollResult, DeviceFlowStart, GitHubUser, GithubRepo}; -use crate::indexing::{IndexStatus, IndexingProgress}; use crate::search::SearchResponse; use crate::settings::Settings; use crate::vault::{RenameResult, VaultEntry}; use crate::vault_list::VaultList; -use crate::{frontmatter, git, github, indexing, menu, search, vault, vault_list}; +use crate::{frontmatter, git, github, menu, search, vault, vault_list}; /// Expand a leading `~` or `~/` in a path string to the user's home directory. /// Returns the original string unchanged if it doesn't start with `~` or if the @@ -42,20 +41,6 @@ pub fn parse_build_label(version: &str) -> String { } } -pub fn emit_unavailable(app_handle: &tauri::AppHandle) { - use tauri::Emitter; - let _ = app_handle.emit( - "indexing-progress", - IndexingProgress { - phase: "unavailable".to_string(), - current: 0, - total: 0, - done: true, - error: Some("qmd not available".to_string()), - }, - ); -} - // ── Vault commands ────────────────────────────────────────────────────────── #[tauri::command] @@ -423,7 +408,7 @@ pub async fn stream_claude_agent( .map_err(|e| format!("Task failed: {e}"))? } -// ── Search & indexing commands ────────────────────────────────────────────── +// ── Search commands ───────────────────────────────────────────────────────── #[tauri::command] pub async fn search_vault( @@ -439,66 +424,6 @@ pub async fn search_vault( .map_err(|e| format!("Search task failed: {}", e))? } -#[tauri::command] -pub fn get_index_status(vault_path: String) -> IndexStatus { - let vault_path = expand_tilde(&vault_path); - indexing::check_index_status(&vault_path) -} - -#[tauri::command] -pub async fn start_indexing( - app_handle: tauri::AppHandle, - vault_path: String, -) -> Result<(), String> { - use tauri::Emitter; - let vault_path = expand_tilde(&vault_path).into_owned(); - tokio::task::spawn_blocking(move || { - if indexing::find_qmd_binary().is_none() { - log::info!("qmd binary not found — attempting auto-install via bun"); - let _ = app_handle.emit( - "indexing-progress", - IndexingProgress { - phase: "installing".to_string(), - current: 0, - total: 0, - done: false, - error: None, - }, - ); - - match indexing::try_auto_install_qmd() { - Ok(()) if indexing::find_qmd_binary().is_some() => { - log::info!("qmd auto-installed successfully, proceeding with indexing"); - } - Ok(()) => { - log::warn!("qmd auto-install reported success but binary still not found"); - emit_unavailable(&app_handle); - return Err("qmd not available after install".to_string()); - } - Err(e) => { - log::info!("qmd auto-install failed: {e}"); - emit_unavailable(&app_handle); - return Err(format!("qmd not available: {e}")); - } - } - } - - indexing::run_full_index(&vault_path, |progress| { - let _ = app_handle.emit("indexing-progress", &progress); - }) - }) - .await - .map_err(|e| format!("Indexing task failed: {e}"))? -} - -#[tauri::command] -pub async fn trigger_incremental_index(vault_path: String) -> Result<(), String> { - let vault_path = expand_tilde(&vault_path).into_owned(); - tokio::task::spawn_blocking(move || indexing::run_incremental_update(&vault_path)) - .await - .map_err(|e| format!("Incremental index failed: {e}"))? -} - // ── MCP commands ──────────────────────────────────────────────────────────── #[tauri::command] diff --git a/src-tauri/src/indexing.rs b/src-tauri/src/indexing.rs deleted file mode 100644 index 67427186..00000000 --- a/src-tauri/src/indexing.rs +++ /dev/null @@ -1,914 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::Mutex; - -/// Resolved qmd binary location: path + optional working directory. -/// The working dir is required for the bundled binary to find its node_modules. -#[derive(Debug, Clone)] -pub struct QmdBinary { - pub path: String, - pub work_dir: Option, -} - -impl QmdBinary { - /// Create a `Command` pre-configured with the correct working directory. - pub fn command(&self) -> Command { - let mut cmd = Command::new(&self.path); - if let Some(ref dir) = self.work_dir { - cmd.current_dir(dir); - } - cmd - } -} - -static QMD_CACHE: Mutex> = Mutex::new(None); - -/// Locate the qmd binary, checking bundled resource first, then known locations. -/// Caches the result for subsequent calls. -pub fn find_qmd_binary() -> Option { - if let Ok(guard) = QMD_CACHE.lock() { - if let Some(ref cached) = *guard { - return Some(cached.clone()); - } - } - - let result = find_qmd_binary_uncached(); - - if let Some(ref bin) = result { - if let Ok(mut guard) = QMD_CACHE.lock() { - *guard = Some(bin.clone()); - } - } - - result -} - -fn find_qmd_binary_uncached() -> Option { - // 1. Check bundled binary (Tauri resource) - if let Some(bin) = find_bundled_qmd() { - return Some(bin); - } - - // 2. Check known system locations - let candidates = [ - dirs::home_dir().map(|h| h.join(".bun/bin/qmd").to_string_lossy().to_string()), - Some("/usr/local/bin/qmd".to_string()), - Some("/opt/homebrew/bin/qmd".to_string()), - ]; - for candidate in candidates.into_iter().flatten() { - if Path::new(&candidate).exists() { - return Some(QmdBinary { - path: candidate, - work_dir: None, - }); - } - } - - // 3. Fallback: try PATH - Command::new("which") - .arg("qmd") - .output() - .ok() - .and_then(|o| { - if o.status.success() { - Some(QmdBinary { - path: String::from_utf8_lossy(&o.stdout).trim().to_string(), - work_dir: None, - }) - } else { - None - } - }) -} - -/// Look for the bundled qmd binary inside the app bundle or dev resources. -fn find_bundled_qmd() -> Option { - let exe = std::env::current_exe().ok()?; - let exe_dir = exe.parent()?; - - // macOS app bundle: /Contents/MacOS/laputa → /Contents/Resources/qmd/qmd - let bundle_dir = exe_dir.parent()?.join("Resources").join("qmd"); - if let Some(bin) = prepare_bundled_dir(&bundle_dir) { - return Some(bin); - } - - // Dev mode: use compile-time CARGO_MANIFEST_DIR for reliable path resolution - let dev_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources") - .join("qmd"); - if let Some(bin) = prepare_bundled_dir(&dev_dir) { - return Some(bin); - } - - // Dev mode fallback: walk up from exe_dir to find the project root - let mut dir = exe_dir.to_path_buf(); - for _ in 0..6 { - let qmd_dir = dir.join("resources").join("qmd"); - if let Some(bin) = prepare_bundled_dir(&qmd_dir) { - return Some(bin); - } - if !dir.pop() { - break; - } - } - - None -} - -/// Validate a bundled qmd directory and prepare the binary for execution. -/// Sets execute permissions and removes macOS quarantine attributes. -fn prepare_bundled_dir(qmd_dir: &Path) -> Option { - let qmd_path = qmd_dir.join("qmd"); - if !qmd_path.exists() { - return None; - } - - ensure_executable(&qmd_path); - - // Remove macOS quarantine attributes that block execution of bundled binaries - #[cfg(target_os = "macos")] - { - let _ = Command::new("xattr") - .args(["-rd", "com.apple.quarantine"]) - .arg(qmd_dir) - .output(); - } - - Some(QmdBinary { - path: qmd_path.to_string_lossy().to_string(), - work_dir: Some(qmd_dir.to_path_buf()), - }) -} - -/// Ensure a file has execute permission. -#[cfg(unix)] -fn ensure_executable(path: &Path) { - use std::os::unix::fs::PermissionsExt; - if let Ok(metadata) = path.metadata() { - let mode = metadata.permissions().mode(); - if mode & 0o111 == 0 { - let mut perms = metadata.permissions(); - perms.set_mode(mode | 0o755); - let _ = std::fs::set_permissions(path, perms); - } - } -} - -#[cfg(not(unix))] -fn ensure_executable(_path: &Path) {} - -/// Try to install qmd globally using bun. Returns Ok if installation succeeded. -pub fn try_auto_install_qmd() -> Result<(), String> { - let bun = find_bun().ok_or("bun not found — cannot auto-install qmd")?; - - log::info!("Auto-installing qmd via bun..."); - let output = Command::new(&bun) - .args(["install", "-g", "qmd"]) - .output() - .map_err(|e| format!("Failed to run bun install: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("bun install -g qmd failed: {stderr}")); - } - - // Clear cache so the newly installed binary is discovered - clear_qmd_cache(); - log::info!("qmd auto-install succeeded"); - Ok(()) -} - -/// Locate bun binary for auto-installing qmd. -fn find_bun() -> Option { - let candidates = [ - dirs::home_dir().map(|h| h.join(".bun/bin/bun")), - Some(PathBuf::from("/opt/homebrew/bin/bun")), - Some(PathBuf::from("/usr/local/bin/bun")), - ]; - for candidate in candidates.into_iter().flatten() { - if candidate.exists() { - return Some(candidate); - } - } - - // Fallback: try PATH - Command::new("which") - .arg("bun") - .output() - .ok() - .and_then(|o| { - if o.status.success() { - Some(PathBuf::from( - String::from_utf8_lossy(&o.stdout).trim().to_string(), - )) - } else { - None - } - }) -} - -/// Clear the cached qmd binary (e.g. after path changes or installation). -pub fn clear_qmd_cache() { - if let Ok(mut guard) = QMD_CACHE.lock() { - *guard = None; - } -} - -#[derive(Debug, Serialize, Clone)] -pub struct IndexStatus { - pub available: bool, - pub qmd_installed: bool, - pub collection_exists: bool, - pub indexed_count: usize, - pub embedded_count: usize, - pub pending_embed: usize, - pub last_indexed_commit: Option, - pub last_indexed_at: Option, -} - -// --- Index metadata persistence --- - -#[derive(Debug, Serialize, Deserialize, Default)] -struct IndexMetadata { - #[serde(default)] - last_indexed_commit: Option, - #[serde(default)] - last_indexed_at: Option, -} - -fn index_metadata_path(vault_path: &str) -> PathBuf { - Path::new(vault_path).join(".laputa-index.json") -} - -fn load_index_metadata(vault_path: &str) -> IndexMetadata { - let path = index_metadata_path(vault_path); - std::fs::read_to_string(&path) - .ok() - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default() -} - -fn save_index_metadata(vault_path: &str, meta: &IndexMetadata) -> Result<(), String> { - let path = index_metadata_path(vault_path); - let json = - serde_json::to_string_pretty(meta).map_err(|e| format!("Failed to serialize: {e}"))?; - std::fs::write(&path, json).map_err(|e| format!("Failed to write index metadata: {e}")) -} - -/// Get the current HEAD commit hash for a vault. -fn get_head_commit(vault_path: &str) -> Option { - let output = Command::new("git") - .args(["rev-parse", "HEAD"]) - .current_dir(vault_path) - .output() - .ok()?; - if output.status.success() { - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) - } else { - None - } -} - -/// Record the current HEAD as the last indexed commit. -fn stamp_index_commit(vault_path: &str) { - if let Some(commit) = get_head_commit(vault_path) { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - let meta = IndexMetadata { - last_indexed_commit: Some(commit), - last_indexed_at: Some(now), - }; - let _ = save_index_metadata(vault_path, &meta); - } -} - -/// Check whether the vault has a qmd index and its status. -pub fn check_index_status(vault_path: &str) -> IndexStatus { - let meta = load_index_metadata(vault_path); - - let qmd = match find_qmd_binary() { - Some(b) => b, - None => { - return IndexStatus { - available: false, - qmd_installed: false, - collection_exists: false, - indexed_count: 0, - embedded_count: 0, - pending_embed: 0, - last_indexed_commit: meta.last_indexed_commit, - last_indexed_at: meta.last_indexed_at, - } - } - }; - - let vault_name = vault_dir_name(vault_path); - let output = qmd.command().args(["status"]).output(); - - let mut status = match output { - Ok(o) if o.status.success() => { - let stdout = String::from_utf8_lossy(&o.stdout); - parse_status_for_vault(&stdout, &vault_name) - } - _ => IndexStatus { - available: false, - qmd_installed: true, - collection_exists: false, - indexed_count: 0, - embedded_count: 0, - pending_embed: 0, - last_indexed_commit: None, - last_indexed_at: None, - }, - }; - - status.last_indexed_commit = meta.last_indexed_commit; - status.last_indexed_at = meta.last_indexed_at; - status -} - -fn vault_dir_name(vault_path: &str) -> String { - Path::new(vault_path) - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("laputa") - .to_lowercase() -} - -fn parse_status_for_vault(status_output: &str, vault_name: &str) -> IndexStatus { - let mut collection_exists = false; - let mut indexed_count = 0; - let mut embedded_count = 0; - let mut pending_embed = 0; - - // Look for collection section matching vault name - let mut in_vault_section = false; - for line in status_output.lines() { - let trimmed = line.trim(); - // Collection headers look like: " laputa (qmd://laputa/)" - if trimmed.contains(&format!("qmd://{vault_name}/")) { - collection_exists = true; - in_vault_section = true; - continue; - } - // New collection section starts - if trimmed.contains("qmd://") && !trimmed.contains(vault_name) { - in_vault_section = false; - continue; - } - if in_vault_section { - if let Some(count_str) = extract_count_from_line(trimmed, "Files:") { - indexed_count = count_str; - } - } - } - - // Global counts from the Documents section - for line in status_output.lines() { - let trimmed = line.trim(); - if trimmed.starts_with("Total:") { - if let Some(n) = extract_first_number(trimmed) { - if embedded_count == 0 && indexed_count == 0 { - indexed_count = n; - } - } - } else if trimmed.starts_with("Vectors:") { - if let Some(n) = extract_first_number(trimmed) { - embedded_count = n; - } - } else if trimmed.starts_with("Pending:") { - if let Some(n) = extract_first_number(trimmed) { - pending_embed = n; - } - } - } - - IndexStatus { - available: true, - qmd_installed: true, - collection_exists, - indexed_count, - embedded_count, - pending_embed, - last_indexed_commit: None, - last_indexed_at: None, - } -} - -fn extract_count_from_line(line: &str, prefix: &str) -> Option { - if !line.starts_with(prefix) { - return None; - } - extract_first_number(line) -} - -fn extract_first_number(s: &str) -> Option { - s.split_whitespace() - .find_map(|word| word.parse::().ok()) -} - -/// Ensure a qmd collection exists for this vault. Creates one if missing. -pub fn ensure_collection(vault_path: &str) -> Result<(), String> { - let qmd = find_qmd_binary().ok_or("qmd not installed")?; - let vault_name = vault_dir_name(vault_path); - - // Check if collection already exists - let output = qmd - .command() - .args(["collection", "list"]) - .output() - .map_err(|e| format!("Failed to list collections: {e}"))?; - - if output.status.success() { - let stdout = String::from_utf8_lossy(&output.stdout); - if stdout.contains(&format!("qmd://{vault_name}/")) { - return Ok(()); - } - } - - // Create collection - qmd.command() - .args([ - "collection", - "add", - vault_path, - "--name", - &vault_name, - "--mask", - "**/*.md", - ]) - .output() - .map_err(|e| format!("Failed to create collection: {e}"))?; - - Ok(()) -} - -#[derive(Debug, Serialize, Clone)] -pub struct IndexingProgress { - pub phase: String, - pub current: usize, - pub total: usize, - pub done: bool, - pub error: Option, -} - -/// Run full indexing: update + embed. Returns progress updates via callback. -pub fn run_full_index(vault_path: &str, on_progress: F) -> Result<(), String> -where - F: Fn(IndexingProgress), -{ - let qmd = find_qmd_binary().ok_or("qmd not installed")?; - - ensure_collection(vault_path)?; - - let vault_name = vault_dir_name(vault_path); - - // Phase 1: update (scan files) — scoped to this vault's collection only - on_progress(IndexingProgress { - phase: "scanning".to_string(), - current: 0, - total: 0, - done: false, - error: None, - }); - - let update_output = qmd - .command() - .args(["update", &vault_name]) - .output() - .map_err(|e| format!("qmd update failed: {e}"))?; - - if !update_output.status.success() { - let stderr = String::from_utf8_lossy(&update_output.stderr); - let err = format!("qmd update failed: {stderr}"); - on_progress(IndexingProgress { - phase: "error".to_string(), - current: 0, - total: 0, - done: true, - error: Some(err.clone()), - }); - return Err(err); - } - - // Parse update output for counts - let update_stdout = String::from_utf8_lossy(&update_output.stdout); - let total = parse_indexed_count(&update_stdout); - - on_progress(IndexingProgress { - phase: "scanning".to_string(), - current: total, - total, - done: false, - error: None, - }); - - // Phase 2: embed (generate vectors) — scoped to this vault's collection only - on_progress(IndexingProgress { - phase: "embedding".to_string(), - current: 0, - total, - done: false, - error: None, - }); - - let embed_output = qmd - .command() - .args(["embed", "-c", &vault_name]) - .output() - .map_err(|e| format!("qmd embed failed: {e}"))?; - - if !embed_output.status.success() { - let stderr = String::from_utf8_lossy(&embed_output.stderr); - // Embedding failure is non-fatal — keyword search still works - log::warn!("qmd embed failed (keyword search still works): {stderr}"); - stamp_index_commit(vault_path); - on_progress(IndexingProgress { - phase: "complete".to_string(), - current: total, - total, - done: true, - error: Some("Embedding failed — keyword search only".to_string()), - }); - return Ok(()); - } - - stamp_index_commit(vault_path); - - on_progress(IndexingProgress { - phase: "complete".to_string(), - current: total, - total, - done: true, - error: None, - }); - - Ok(()) -} - -fn parse_indexed_count(update_output: &str) -> usize { - // qmd update output typically contains lines like "Indexed 9078 files" - for line in update_output.lines() { - if let Some(n) = extract_first_number(line) { - return n; - } - } - 0 -} - -/// Run incremental update for a single file change. -pub fn run_incremental_update(vault_path: &str) -> Result<(), String> { - let qmd = find_qmd_binary().ok_or("qmd not installed")?; - - // Verify collection exists - let vault_name = vault_dir_name(vault_path); - let list_output = qmd - .command() - .args(["collection", "list"]) - .output() - .map_err(|e| format!("Failed to list collections: {e}"))?; - - if list_output.status.success() { - let stdout = String::from_utf8_lossy(&list_output.stdout); - if !stdout.contains(&format!("qmd://{vault_name}/")) { - // Collection doesn't exist yet — skip incremental, full index needed - return Ok(()); - } - } - - let output = qmd - .command() - .args(["update", &vault_name]) - .output() - .map_err(|e| format!("qmd incremental update failed: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("qmd update failed: {stderr}")); - } - - stamp_index_commit(vault_path); - - Ok(()) -} - -/// Check if HEAD has advanced past the last indexed commit. -#[cfg(test)] -fn needs_reindex_after_sync(vault_path: &str) -> bool { - let meta = load_index_metadata(vault_path); - let head = get_head_commit(vault_path); - match (meta.last_indexed_commit, head) { - (Some(last), Some(current)) => last != current, - (None, Some(_)) => true, - _ => false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn qmd_binary_command_sets_work_dir() { - let qmd = QmdBinary { - path: "/bin/echo".to_string(), - work_dir: Some(PathBuf::from("/tmp")), - }; - let cmd = qmd.command(); - let dbg = format!("{:?}", cmd); - assert!(dbg.contains("/bin/echo")); - } - - #[test] - fn qmd_binary_command_no_work_dir() { - let qmd = QmdBinary { - path: "/bin/echo".to_string(), - work_dir: None, - }; - let output = qmd.command().arg("hello").output().unwrap(); - assert!(output.status.success()); - } - - #[test] - fn vault_dir_name_extracts_last_segment() { - assert_eq!(vault_dir_name("/Users/luca/Laputa"), "laputa"); - assert_eq!(vault_dir_name("/home/user/MyVault"), "myvault"); - } - - #[test] - fn vault_dir_name_fallback() { - assert_eq!(vault_dir_name(""), "laputa"); - } - - #[test] - fn extract_first_number_works() { - assert_eq!( - extract_first_number("Total: 9078 files indexed"), - Some(9078) - ); - assert_eq!(extract_first_number("Vectors: 14676 embedded"), Some(14676)); - assert_eq!(extract_first_number("no numbers here"), None); - } - - #[test] - fn parse_status_finds_collection() { - let status = r#" -QMD Status - -Index: /Users/luca/.cache/qmd/index.sqlite -Size: 100.9 MB - -Documents - Total: 9115 files indexed - Vectors: 14676 embedded - Pending: 26 need embedding - -Collections - laputa (qmd://laputa/) - Pattern: **/*.md - Files: 9078 (updated 20d ago) -"#; - let result = parse_status_for_vault(status, "laputa"); - assert!(result.collection_exists); - assert_eq!(result.indexed_count, 9078); - assert_eq!(result.embedded_count, 14676); - assert_eq!(result.pending_embed, 26); - } - - #[test] - fn parse_status_missing_collection() { - let status = r#" -QMD Status - -Documents - Total: 100 files indexed - Vectors: 50 embedded - Pending: 0 - -Collections - other (qmd://other/) - Files: 100 -"#; - let result = parse_status_for_vault(status, "laputa"); - assert!(!result.collection_exists); - } - - #[test] - fn extract_count_from_line_works() { - assert_eq!( - extract_count_from_line("Files: 9078 (updated 20d ago)", "Files:"), - Some(9078) - ); - assert_eq!(extract_count_from_line("Pattern: **/*.md", "Files:"), None); - } - - #[test] - fn parse_indexed_count_from_output() { - assert_eq!(parse_indexed_count("Indexed 342 files in 1.2s"), 342); - assert_eq!(parse_indexed_count("No output"), 0); - } - - #[test] - fn ensure_executable_sets_permission() { - use std::fs; - let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("test-bin"); - fs::write(&file, "#!/bin/sh\necho ok").unwrap(); - - // Start with no execute permission - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&file, fs::Permissions::from_mode(0o644)).unwrap(); - assert_eq!(fs::metadata(&file).unwrap().permissions().mode() & 0o111, 0); - - ensure_executable(&file); - assert_ne!(fs::metadata(&file).unwrap().permissions().mode() & 0o111, 0); - } - } - - #[test] - fn ensure_executable_noop_when_already_executable() { - use std::fs; - let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("test-bin"); - fs::write(&file, "#!/bin/sh\necho ok").unwrap(); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&file, fs::Permissions::from_mode(0o755)).unwrap(); - ensure_executable(&file); - let mode = fs::metadata(&file).unwrap().permissions().mode(); - assert_ne!(mode & 0o111, 0); - } - } - - #[test] - fn prepare_bundled_dir_returns_none_for_missing_binary() { - let dir = tempfile::tempdir().unwrap(); - assert!(prepare_bundled_dir(dir.path()).is_none()); - } - - #[test] - fn prepare_bundled_dir_finds_and_prepares_binary() { - use std::fs; - let dir = tempfile::tempdir().unwrap(); - let qmd_path = dir.path().join("qmd"); - fs::write(&qmd_path, "#!/bin/sh\necho ok").unwrap(); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&qmd_path, fs::Permissions::from_mode(0o644)).unwrap(); - } - - let result = prepare_bundled_dir(dir.path()); - assert!(result.is_some()); - let bin = result.unwrap(); - assert!(bin.path.ends_with("qmd")); - assert_eq!(bin.work_dir.unwrap(), dir.path()); - - // Verify execute permission was set - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - assert_ne!( - fs::metadata(&qmd_path).unwrap().permissions().mode() & 0o111, - 0 - ); - } - } - - #[test] - fn find_bun_returns_some_if_available() { - // This test may succeed or fail depending on the system. - // It verifies the function doesn't panic. - let _ = find_bun(); - } - - #[test] - fn index_metadata_roundtrip() { - let dir = tempfile::tempdir().unwrap(); - let vault = dir.path().to_str().unwrap(); - - // Default when no file exists - let meta = load_index_metadata(vault); - assert!(meta.last_indexed_commit.is_none()); - assert!(meta.last_indexed_at.is_none()); - - // Write and read back - let meta = IndexMetadata { - last_indexed_commit: Some("abc123def456".to_string()), - last_indexed_at: Some(1709000000), - }; - save_index_metadata(vault, &meta).unwrap(); - - let loaded = load_index_metadata(vault); - assert_eq!(loaded.last_indexed_commit.as_deref(), Some("abc123def456")); - assert_eq!(loaded.last_indexed_at, Some(1709000000)); - } - - #[test] - fn index_metadata_survives_malformed_json() { - let dir = tempfile::tempdir().unwrap(); - let vault = dir.path().to_str().unwrap(); - - // Write garbage - std::fs::write(dir.path().join(".laputa-index.json"), "not json").unwrap(); - - let meta = load_index_metadata(vault); - assert!(meta.last_indexed_commit.is_none()); - } - - #[test] - fn needs_reindex_after_sync_no_metadata() { - let dir = tempfile::tempdir().unwrap(); - let vault = dir.path().to_str().unwrap(); - - // Init a git repo so get_head_commit works - Command::new("git") - .args(["init"]) - .current_dir(dir.path()) - .output() - .unwrap(); - Command::new("git") - .args(["commit", "--allow-empty", "-m", "init"]) - .current_dir(dir.path()) - .output() - .unwrap(); - - // No metadata → needs reindex - assert!(needs_reindex_after_sync(vault)); - } - - #[test] - fn needs_reindex_after_sync_same_commit() { - let dir = tempfile::tempdir().unwrap(); - let vault = dir.path().to_str().unwrap(); - - Command::new("git") - .args(["init"]) - .current_dir(dir.path()) - .output() - .unwrap(); - Command::new("git") - .args(["commit", "--allow-empty", "-m", "init"]) - .current_dir(dir.path()) - .output() - .unwrap(); - - let head = get_head_commit(vault).unwrap(); - let meta = IndexMetadata { - last_indexed_commit: Some(head), - last_indexed_at: Some(1709000000), - }; - save_index_metadata(vault, &meta).unwrap(); - - assert!(!needs_reindex_after_sync(vault)); - } - - #[test] - fn needs_reindex_after_sync_different_commit() { - let dir = tempfile::tempdir().unwrap(); - let vault = dir.path().to_str().unwrap(); - - Command::new("git") - .args(["init"]) - .current_dir(dir.path()) - .output() - .unwrap(); - Command::new("git") - .args(["commit", "--allow-empty", "-m", "init"]) - .current_dir(dir.path()) - .output() - .unwrap(); - - let meta = IndexMetadata { - last_indexed_commit: Some("old_commit_hash".to_string()), - last_indexed_at: Some(1709000000), - }; - save_index_metadata(vault, &meta).unwrap(); - - assert!(needs_reindex_after_sync(vault)); - } - - #[test] - fn check_index_status_includes_metadata() { - let dir = tempfile::tempdir().unwrap(); - let vault = dir.path().to_str().unwrap(); - - let meta = IndexMetadata { - last_indexed_commit: Some("abc123".to_string()), - last_indexed_at: Some(1709000000), - }; - save_index_metadata(vault, &meta).unwrap(); - - let status = check_index_status(vault); - assert_eq!(status.last_indexed_commit.as_deref(), Some("abc123")); - assert_eq!(status.last_indexed_at, Some(1709000000)); - } -} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 18b5f664..014f2efe 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -4,7 +4,6 @@ mod commands; pub mod frontmatter; pub mod git; pub mod github; -pub mod indexing; pub mod mcp; pub mod menu; pub mod search; @@ -149,9 +148,6 @@ pub fn run() { commands::github_device_flow_poll, commands::github_get_user, commands::search_vault, - commands::get_index_status, - commands::start_indexing, - commands::trigger_incremental_index, commands::create_getting_started_vault, commands::check_vault_exists, commands::get_default_vault_path, diff --git a/src-tauri/src/menu.rs b/src-tauri/src/menu.rs index d162a574..564a3ed6 100644 --- a/src-tauri/src/menu.rs +++ b/src-tauri/src/menu.rs @@ -51,7 +51,6 @@ const VAULT_PULL: &str = "vault-pull"; const VAULT_RESOLVE_CONFLICTS: &str = "vault-resolve-conflicts"; const VAULT_VIEW_CHANGES: &str = "vault-view-changes"; const VAULT_INSTALL_MCP: &str = "vault-install-mcp"; -const VAULT_REINDEX: &str = "vault-reindex"; const VAULT_RELOAD: &str = "vault-reload"; const VAULT_REPAIR: &str = "vault-repair"; @@ -96,7 +95,6 @@ const CUSTOM_IDS: &[&str] = &[ VAULT_RESOLVE_CONFLICTS, VAULT_VIEW_CHANGES, VAULT_INSTALL_MCP, - VAULT_REINDEX, VAULT_RELOAD, VAULT_REPAIR, ]; @@ -358,9 +356,6 @@ fn build_vault_menu(app: &App) -> MenuResult { let install_mcp = MenuItemBuilder::new("Restore MCP Server") .id(VAULT_INSTALL_MCP) .build(app)?; - let reindex = MenuItemBuilder::new("Reindex Vault") - .id(VAULT_REINDEX) - .build(app)?; let reload = MenuItemBuilder::new("Reload Vault") .id(VAULT_RELOAD) .build(app)?; @@ -378,7 +373,6 @@ fn build_vault_menu(app: &App) -> MenuResult { .item(&resolve_conflicts) .item(&view_changes) .separator() - .item(&reindex) .item(&reload) .item(&repair) .item(&install_mcp) @@ -499,7 +493,6 @@ mod tests { VAULT_RESOLVE_CONFLICTS, VAULT_VIEW_CHANGES, VAULT_INSTALL_MCP, - VAULT_REINDEX, VAULT_RELOAD, ]; for id in &expected { diff --git a/src-tauri/src/search.rs b/src-tauri/src/search.rs index ba60fc8e..e9c0b95b 100644 --- a/src-tauri/src/search.rs +++ b/src-tauri/src/search.rs @@ -1,12 +1,10 @@ -use crate::indexing; use crate::vault; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use serde::Serialize; use std::path::Path; -use std::sync::Mutex; use std::time::Instant; +use walkdir::WalkDir; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Clone)] pub struct SearchResult { pub title: String, pub path: String, @@ -23,159 +21,108 @@ pub struct SearchResponse { pub mode: String, } -#[derive(Debug, Deserialize)] -struct QmdResult { - pub file: String, - pub title: String, - pub snippet: String, - pub score: f64, -} - -fn qmd_uri_to_vault_path(uri: &str, vault_path: &str) -> String { - // qmd://laputa/essay/foo.md → essay/foo.md - let relative = uri - .strip_prefix("qmd://") - .and_then(|s| s.find('/').map(|i| &s[i + 1..])) - .unwrap_or(uri); - format!("{}/{}", vault_path, relative) -} - -fn extract_clean_snippet(raw_snippet: &str) -> String { - // qmd snippets start with "@@ -N,N @@ (N before, N after)\n" - // We want just the content lines - let lines: Vec<&str> = raw_snippet.lines().collect(); - let content_start = lines.iter().position(|l| !l.starts_with("@@")).unwrap_or(0); - let content: String = lines[content_start..] - .iter() - .filter(|l| !l.starts_with("---")) - .take(3) - .copied() - .collect::>() - .join(" "); - // Trim to reasonable length - if content.len() > 200 { - format!("{}...", &content[..200]) - } else { - content - } -} - -static COLLECTION_CACHE: Mutex>> = Mutex::new(None); - -fn detect_collection_name(vault_path: &str) -> String { - // Check cache first - if let Ok(guard) = COLLECTION_CACHE.lock() { - if let Some(ref cache) = *guard { - if let Some(name) = cache.get(vault_path) { - return name.clone(); - } - } - } - - let result = detect_collection_name_uncached(vault_path); - - // Store in cache - if let Ok(mut guard) = COLLECTION_CACHE.lock() { - let cache = guard.get_or_insert_with(HashMap::new); - cache.insert(vault_path.to_string(), result.clone()); - } - - result -} - -fn detect_collection_name_uncached(vault_path: &str) -> String { - let qmd = match indexing::find_qmd_binary() { - Some(b) => b, - None => return "laputa".to_string(), +fn extract_snippet(content: &str, query_lower: &str) -> String { + let content_lower = content.to_lowercase(); + let pos = match content_lower.find(query_lower) { + Some(p) => p, + None => return String::new(), }; - - let output = qmd.command().args(["collection", "list"]).output(); - - match output { - Ok(o) if o.status.success() => { - let stdout = String::from_utf8_lossy(&o.stdout); - let vault_name = Path::new(vault_path) - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("laputa") - .to_lowercase(); - for line in stdout.lines() { - let trimmed = line.trim(); - if trimmed.contains(&vault_name) && trimmed.contains("qmd://") { - if let Some(name) = trimmed.split_whitespace().next() { - return name.to_string(); - } - } - } - vault_name - } - _ => Path::new(vault_path) - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("laputa") - .to_lowercase(), + let start = content[..pos] + .rfind('\n') + .map(|i| i + 1) + .unwrap_or(pos.saturating_sub(60)); + let end = content[pos..] + .find('\n') + .map(|i| pos + i) + .unwrap_or_else(|| (pos + 120).min(content.len())); + let snippet = &content[start..end]; + if snippet.len() > 200 { + format!("{}…", &snippet[..200]) + } else { + snippet.to_string() } } +fn score_match(title_lower: &str, content_lower: &str, query_lower: &str) -> f64 { + let title_exact = title_lower.contains(query_lower); + let title_word = title_lower + .split_whitespace() + .any(|w| w == query_lower); + let content_count = content_lower.matches(query_lower).count(); + + let mut score = 0.0; + if title_word { + score += 10.0; + } else if title_exact { + score += 5.0; + } + score += (content_count as f64).min(20.0) * 0.5; + score +} + pub fn search_vault( vault_path: &str, query: &str, - mode: &str, + _mode: &str, limit: usize, ) -> Result { let start = Instant::now(); + let query_lower = query.to_lowercase(); + let vault_dir = Path::new(vault_path); - let qmd = indexing::find_qmd_binary().ok_or_else(|| "qmd binary not found".to_string())?; + let mut results: Vec = Vec::new(); - let collection = detect_collection_name(vault_path); + for entry in WalkDir::new(vault_dir) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if !path.extension().is_some_and(|ext| ext == "md") { + continue; + } + if vault::is_file_trashed(path) { + continue; + } + // Skip hidden dirs and .laputa config + if path + .components() + .any(|c| c.as_os_str().to_string_lossy().starts_with('.')) + { + continue; + } - let search_cmd = match mode { - "semantic" => "vsearch", - "hybrid" => "query", - _ => "search", // "keyword" default - }; + let content = match std::fs::read_to_string(path) { + Ok(c) => c, + Err(_) => continue, + }; - let limit_str = limit.to_string(); - let output = qmd - .command() - .args([ - search_cmd, - query, - "--collection", - &collection, - "--json", - "-n", - &limit_str, - ]) - .output() - .map_err(|e| format!("Failed to run qmd: {}", e))?; + let content_lower = content.to_lowercase(); + let title = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(); + let title_lower = title.to_lowercase(); - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("qmd search failed: {}", stderr)); + if !title_lower.contains(&query_lower) && !content_lower.contains(&query_lower) { + continue; + } + + let score = score_match(&title_lower, &content_lower, &query_lower); + let snippet = extract_snippet(&content, &query_lower); + let full_path = path.to_string_lossy().to_string(); + + results.push(SearchResult { + title, + path: full_path, + snippet, + score, + note_type: None, + }); } - let stdout = String::from_utf8_lossy(&output.stdout); - let qmd_results: Vec = - serde_json::from_str(&stdout).map_err(|e| format!("Failed to parse qmd output: {}", e))?; - - let results: Vec = qmd_results - .into_iter() - .filter_map(|r| { - let path = qmd_uri_to_vault_path(&r.file, vault_path); - if vault::is_file_trashed(Path::new(&path)) { - return None; - } - let snippet = extract_clean_snippet(&r.snippet); - Some(SearchResult { - title: r.title, - path, - snippet, - score: r.score, - note_type: None, - }) - }) - .collect(); + results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + results.truncate(limit); let elapsed_ms = start.elapsed().as_millis() as u64; @@ -183,7 +130,7 @@ pub fn search_vault( results, elapsed_ms, query: query.to_string(), - mode: mode.to_string(), + mode: "keyword".to_string(), }) } @@ -192,59 +139,36 @@ mod tests { use super::*; #[test] - fn test_qmd_uri_to_vault_path() { - assert_eq!( - qmd_uri_to_vault_path("qmd://laputa/essay/foo.md", "/Users/luca/Laputa"), - "/Users/luca/Laputa/essay/foo.md" - ); - assert_eq!( - qmd_uri_to_vault_path( - "qmd://laputa/event/2025-10-15-retreat.md", - "/Users/luca/Laputa" - ), - "/Users/luca/Laputa/event/2025-10-15-retreat.md" - ); + fn test_extract_snippet_basic() { + let content = "line one\nline with keyword here\nline three"; + let snippet = extract_snippet(content, "keyword"); + assert!(snippet.contains("keyword")); } #[test] - fn test_extract_clean_snippet() { - let raw = "@@ -2,4 @@ (1 before, 9 after)\naliases:\n - \"Refactoring Retreat\"\n\"Is A\":\n - Event"; - let clean = extract_clean_snippet(raw); - assert!(clean.starts_with("aliases:")); - assert!(clean.contains("Refactoring Retreat")); + fn test_extract_snippet_no_match() { + let snippet = extract_snippet("nothing here", "missing"); + assert!(snippet.is_empty()); } #[test] - fn test_extract_clean_snippet_long() { - let raw = format!("@@ -1,1 @@\n{}", "a".repeat(300)); - let clean = extract_clean_snippet(&raw); - assert!(clean.len() <= 203); // 200 + "..." - assert!(clean.ends_with("...")); + fn test_score_match_title_word() { + let score = score_match("my keyword", "", "keyword"); + assert!(score >= 10.0); } #[test] - fn test_detect_collection_fallback() { - // With a non-existent vault path, should return either the lowercase dir name - // (if qmd is available and collection list succeeds) or "laputa" (if qmd is not installed). - // Both are valid fallbacks — this test verifies the function doesn't panic. - let name = detect_collection_name("/tmp/test-vault"); - assert!( - name == "test-vault" || name == "laputa", - "Expected 'test-vault' or 'laputa', got '{}'", - name - ); - } - #[test] - fn test_qmd_uri_fallback() { - // Covers fallback branch when URI doesn't start with "qmd://" - let result = qmd_uri_to_vault_path("invalid-uri", "/vault"); - assert!(result.contains("invalid-uri")); + fn test_score_match_content_only() { + let score = score_match("unrelated", "some keyword text keyword", "keyword"); + assert!(score > 0.0); + assert!(score < 10.0); } #[test] - fn test_extract_clean_snippet_no_header() { - // No @@ header — content_start = 0 - let snippet = extract_clean_snippet("plain content line"); - assert_eq!(snippet, "plain content line"); + fn test_extract_snippet_long() { + let long_line = "a".repeat(300); + let content = format!("start\n{}keyword{}\nend", long_line, long_line); + let snippet = extract_snippet(&content, "keyword"); + assert!(snippet.len() <= 203); // 200 + "…" (3 bytes UTF-8) } } diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index b50e13ba..3fafb59d 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -7,7 +7,7 @@ "frontendDist": "../dist", "devUrl": "http://localhost:5202", "beforeDevCommand": "pnpm dev", - "beforeBuildCommand": "pnpm build && pnpm bundle-mcp && bash scripts/bundle-qmd.sh" + "beforeBuildCommand": "pnpm build && pnpm bundle-mcp" }, "app": { "withGlobalTauri": true, @@ -38,8 +38,7 @@ "targets": "all", "createUpdaterArtifacts": true, "resources": { - "resources/mcp-server/**/*": "mcp-server/", - "resources/qmd/**/*": "qmd/" + "resources/mcp-server/**/*": "mcp-server/" }, "icon": [ "icons/32x32.png", diff --git a/src/App.tsx b/src/App.tsx index 51990b86..379caa3c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -30,7 +30,6 @@ import { useGitHistory } from './hooks/useGitHistory' import { useUpdater, restartApp } from './hooks/useUpdater' import { useAutoSync } from './hooks/useAutoSync' import { useConflictResolver } from './hooks/useConflictResolver' -import { useIndexing } from './hooks/useIndexing' import { useZoom } from './hooks/useZoom' import { useVaultConfig } from './hooks/useVaultConfig' import { useBuildNumber } from './hooks/useBuildNumber' @@ -98,13 +97,10 @@ function App() { const flatVaultMigration = useFlatVaultMigration(resolvedPath, vault.entries.length > 0, vault.reloadVault) const { mcpStatus, installMcp } = useMcpStatus(resolvedPath, setToastMessage) - const indexing = useIndexing(resolvedPath) - const autoSync = useAutoSync({ vaultPath: resolvedPath, intervalMinutes: settings.auto_pull_interval_minutes, onVaultUpdated: vault.reloadVault, - onSyncUpdated: indexing.triggerIncrementalIndex, onConflict: (files) => { const names = files.map((f) => f.split('/').pop()).join(', ') setToastMessage(`Conflict in ${names} — click to resolve`) @@ -294,11 +290,9 @@ function App() { openNoteInNewWindow(entry.path, resolvedPath, entry.title) }, [resolvedPath]) - const { triggerIncrementalIndex } = indexing const onAfterSave = useCallback(() => { vault.loadModifiedFiles() - triggerIncrementalIndex() - }, [vault, triggerIncrementalIndex]) + }, [vault]) // eslint-disable-next-line @typescript-eslint/no-unused-vars -- signature required by useEditorSave const onNotePersisted = useCallback((path: string, _content: string) => { @@ -507,7 +501,6 @@ function App() { onEmptyTrash: deleteActions.handleEmptyTrash, trashedCount: deleteActions.trashedCount, onReopenClosedTab: notes.handleReopenClosedTab, - onReindexVault: indexing.triggerFullReindex, onReloadVault: vault.reloadVault, onRepairVault: handleRepairVault, onSetNoteIcon: handleSetNoteIconCommand, @@ -640,7 +633,7 @@ function App() { /> )} - handleSetSelection({ kind: 'filter', filter: 'changes' })} hasGitHub={!!settings.github_token} syncStatus={autoSync.syncStatus} lastSyncTime={autoSync.lastSyncTime} conflictCount={autoSync.conflictFiles.length} lastCommitInfo={autoSync.lastCommitInfo} remoteStatus={autoSync.remoteStatus} onTriggerSync={autoSync.triggerSync} onPullAndPush={autoSync.pullAndPush} onOpenConflictResolver={handleOpenConflictResolver} zoomLevel={zoom.zoomLevel} onZoomReset={zoom.zoomReset} buildNumber={buildNumber} onCheckForUpdates={handleCheckForUpdates} indexingProgress={indexing.progress} lastIndexedTime={indexing.lastIndexedTime} onRetryIndexing={indexing.retryIndexing} onReindexVault={indexing.triggerFullReindex} onRemoveVault={vaultSwitcher.removeVault} mcpStatus={mcpStatus} onInstallMcp={installMcp} /> + handleSetSelection({ kind: 'filter', filter: 'changes' })} hasGitHub={!!settings.github_token} syncStatus={autoSync.syncStatus} lastSyncTime={autoSync.lastSyncTime} conflictCount={autoSync.conflictFiles.length} lastCommitInfo={autoSync.lastCommitInfo} remoteStatus={autoSync.remoteStatus} onTriggerSync={autoSync.triggerSync} onPullAndPush={autoSync.pullAndPush} onOpenConflictResolver={handleOpenConflictResolver} zoomLevel={zoom.zoomLevel} onZoomReset={zoom.zoomReset} buildNumber={buildNumber} onCheckForUpdates={handleCheckForUpdates} onRemoveVault={vaultSwitcher.removeVault} mcpStatus={mcpStatus} onInstallMcp={installMcp} /> setToastMessage(null)} /> diff --git a/src/components/SearchPanel.test.tsx b/src/components/SearchPanel.test.tsx index 2cdf8994..22c19f63 100644 --- a/src/components/SearchPanel.test.tsx +++ b/src/components/SearchPanel.test.tsx @@ -125,7 +125,7 @@ describe('SearchPanel', () => { expect(onClose).toHaveBeenCalled() }) - it('performs unified search with keyword then hybrid', async () => { + it('performs keyword search', async () => { mockInvokeFn.mockResolvedValue({ results: [ { title: 'How to Design AI-first APIs', path: '/vault/essay/ai-apis.md', snippet: '...designing APIs for AI...', score: 0.87, note_type: 'Essay' }, @@ -140,7 +140,6 @@ describe('SearchPanel', () => { const input = screen.getByPlaceholderText('Search in all notes...') fireEvent.change(input, { target: { value: 'api design' } }) - // Should call keyword search first await waitFor(() => { expect(mockInvokeFn).toHaveBeenCalledWith('search_vault', { vaultPath: '/vault', @@ -150,20 +149,9 @@ describe('SearchPanel', () => { }) }) - // Results should appear await waitFor(() => { expect(screen.getByText('How to Design AI-first APIs')).toBeInTheDocument() }) - - // Should also call hybrid search - await waitFor(() => { - expect(mockInvokeFn).toHaveBeenCalledWith('search_vault', { - vaultPath: '/vault', - query: 'api design', - mode: 'hybrid', - limit: 20, - }) - }) }) it('shows no results message when search returns empty', async () => { @@ -331,7 +319,7 @@ describe('SearchPanel', () => { fireEvent.change(screen.getByPlaceholderText('Search in all notes...'), { target: { value: 'test' } }) - // Spinner appears when keyword search starts (after debounce) + // Spinner appears when search starts (after debounce) await waitFor(() => { expect(screen.getByTestId('search-spinner')).toBeInTheDocument() }) @@ -342,21 +330,9 @@ describe('SearchPanel', () => { elapsed_ms: 30, }) - // Keyword results appear, spinner still visible (hybrid in progress) + // Spinner disappears after search completes await waitFor(() => { expect(screen.getByText('Result')).toBeInTheDocument() - expect(screen.getByTestId('search-spinner')).toBeInTheDocument() - }) - - // Wait for hybrid call then resolve it - await waitFor(() => { expect(resolvers).toHaveLength(2) }) - resolvers[1]({ - results: [{ title: 'Result', path: '/vault/essay/ai-apis.md', snippet: '', score: 0.9, note_type: null }], - elapsed_ms: 150, - }) - - // Spinner disappears after hybrid completes - await waitFor(() => { expect(screen.queryByTestId('search-spinner')).not.toBeInTheDocument() }) }) @@ -388,44 +364,12 @@ describe('SearchPanel', () => { }) }) - it('keeps keyword results when hybrid search fails', async () => { - mockInvokeFn.mockImplementation(async (_cmd: string, args?: Record) => { - const mode = (args as Record)?.mode - if (mode === 'keyword') { - return { - results: [{ title: 'Keyword Only', path: '/vault/essay/ai-apis.md', snippet: '', score: 0.9, note_type: null }], - elapsed_ms: 30, - } - } - throw new Error('qmd unavailable') - }) - - render( - , - ) - - fireEvent.change(screen.getByPlaceholderText('Search in all notes...'), { target: { value: 'test' } }) - - // Keyword results appear - await waitFor(() => { - expect(screen.getByText('Keyword Only')).toBeInTheDocument() - }) - - // Spinner disappears after hybrid fails - await waitFor(() => { - expect(screen.queryByTestId('search-spinner')).not.toBeInTheDocument() - }) - - // Keyword results remain - expect(screen.getByText('Keyword Only')).toBeInTheDocument() - }) - it('deduplicates results when backend returns same note twice', async () => { mockInvokeFn.mockResolvedValue({ results: [ { title: 'How to Design AI-first APIs', path: '/vault/essay/ai-apis.md', snippet: 'keyword hit', score: 0.7, note_type: 'Essay' }, { title: 'Refactoring Retreat', path: '/vault/event/retreat.md', snippet: 'unique', score: 0.6, note_type: 'Event' }, - { title: 'How to Design AI-first APIs', path: '/vault/essay/ai-apis.md', snippet: 'semantic hit', score: 0.9, note_type: 'Essay' }, + { title: 'How to Design AI-first APIs', path: '/vault/essay/ai-apis.md', snippet: 'duplicate hit', score: 0.9, note_type: 'Essay' }, ], elapsed_ms: 48, }) diff --git a/src/components/StatusBar.test.tsx b/src/components/StatusBar.test.tsx index 202118b6..bc6d32cc 100644 --- a/src/components/StatusBar.test.tsx +++ b/src/components/StatusBar.test.tsx @@ -2,8 +2,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { render, screen, fireEvent } from '@testing-library/react' import { StatusBar } from './StatusBar' import type { VaultOption } from './StatusBar' -import { formatIndexedElapsed } from '../utils/indexingHelpers' - vi.mock('../utils/url', async () => { const actual = await vi.importActual('../utils/url') return { ...actual, openExternalUrl: vi.fn().mockResolvedValue(undefined) } @@ -226,121 +224,6 @@ describe('StatusBar', () => { expect(screen.getByTitle('View pending changes')).toBeInTheDocument() }) - it('shows indexing badge when indexing is in progress', () => { - render( - - ) - expect(screen.getByTestId('status-indexing')).toBeInTheDocument() - expect(screen.getByText(/Indexing… 342\/1,057/)).toBeInTheDocument() - }) - - it('shows embedding phase in indexing badge', () => { - render( - - ) - expect(screen.getByText(/Embedding… 50\/200/)).toBeInTheDocument() - }) - - it('shows index ready when indexing is complete', () => { - render( - - ) - expect(screen.getByText('Index ready')).toBeInTheDocument() - }) - - it('shows error state in indexing badge with retry label', () => { - render( - - ) - expect(screen.getByText('Index failed — retry')).toBeInTheDocument() - }) - - it('hides indexing badge when phase is unavailable', () => { - render( - - ) - expect(screen.queryByTestId('status-indexing')).not.toBeInTheDocument() - }) - - it('calls onRetryIndexing when clicking error badge', () => { - const onRetryIndexing = vi.fn() - render( - - ) - fireEvent.click(screen.getByTestId('status-indexing')) - expect(onRetryIndexing).toHaveBeenCalledOnce() - }) - - it('hides indexing badge when phase is idle', () => { - render( - - ) - expect(screen.queryByTestId('status-indexing')).not.toBeInTheDocument() - }) - - it('hides indexing badge when no progress prop provided', () => { - render( - - ) - expect(screen.queryByTestId('status-indexing')).not.toBeInTheDocument() - }) - - it('shows installing phase in indexing badge', () => { - render( - - ) - expect(screen.getByText('Installing search…')).toBeInTheDocument() - }) - it('shows MCP warning badge when status is not_installed', () => { render( @@ -396,38 +279,6 @@ describe('StatusBar', () => { expect(onInstallMcp).not.toHaveBeenCalled() }) - it('shows "Indexed just now" when lastIndexedTime is recent and phase is idle', () => { - render( - - ) - expect(screen.getByText(/Indexed just now/)).toBeInTheDocument() - expect(screen.getByTestId('status-indexed-time')).toBeInTheDocument() - }) - - it('calls onReindexVault when clicking the indexed time badge', () => { - const onReindexVault = vi.fn() - render( - - ) - fireEvent.click(screen.getByTestId('status-indexed-time')) - expect(onReindexVault).toHaveBeenCalledOnce() - }) - it('shows Pull required label when syncStatus is pull_required', () => { render( @@ -462,38 +313,4 @@ describe('StatusBar', () => { expect(screen.getByText(/1 behind/)).toBeInTheDocument() }) - it('hides indexed time badge when no lastIndexedTime', () => { - render( - - ) - expect(screen.queryByTestId('status-indexed-time')).not.toBeInTheDocument() - }) -}) - -describe('formatIndexedElapsed', () => { - it('returns empty string for null', () => { - expect(formatIndexedElapsed(null)).toBe('') - }) - - it('returns "Indexed just now" for < 60s', () => { - expect(formatIndexedElapsed(Date.now() - 30_000)).toBe('Indexed just now') - }) - - it('returns minutes for < 60min', () => { - expect(formatIndexedElapsed(Date.now() - 5 * 60_000)).toBe('Indexed 5m ago') - }) - - it('returns hours for < 24h', () => { - expect(formatIndexedElapsed(Date.now() - 3 * 3600_000)).toBe('Indexed 3h ago') - }) - - it('returns days for >= 24h', () => { - expect(formatIndexedElapsed(Date.now() - 48 * 3600_000)).toBe('Indexed 2d ago') - }) }) diff --git a/src/components/StatusBar.tsx b/src/components/StatusBar.tsx index 8d35bb1c..23b5946e 100644 --- a/src/components/StatusBar.tsx +++ b/src/components/StatusBar.tsx @@ -1,10 +1,8 @@ import { useState, useRef, useEffect } from 'react' -import { Package, RefreshCw, FileText, Bell, Settings, FolderOpen, Check, Github, CircleDot, AlertTriangle, Loader2, GitCommitHorizontal, Search, X, Cpu, ArrowDown, GitBranch } from 'lucide-react' +import { Package, RefreshCw, FileText, Bell, Settings, FolderOpen, Check, Github, CircleDot, AlertTriangle, Loader2, GitCommitHorizontal, X, Cpu, ArrowDown, GitBranch } from 'lucide-react' import type { GitRemoteStatus, LastCommitInfo, SyncStatus } from '../types' -import type { IndexingProgress } from '../hooks/useIndexing' import type { McpStatus } from '../hooks/useMcpStatus' import { openExternalUrl } from '../utils/url' -import { formatIndexedElapsed } from '../utils/indexingHelpers' export interface VaultOption { label: string @@ -35,10 +33,6 @@ interface StatusBarProps { onZoomReset?: () => void buildNumber?: string onCheckForUpdates?: () => void - indexingProgress?: IndexingProgress - lastIndexedTime?: number | null - onRetryIndexing?: () => void - onReindexVault?: () => void onRemoveVault?: (path: string) => void mcpStatus?: McpStatus onInstallMcp?: () => void @@ -334,70 +328,6 @@ function ConflictBadge({ count, onClick }: { count: number; onClick?: () => void ) } -const INDEXING_LABELS: Record = { - installing: 'Installing search…', - scanning: 'Indexing…', - embedding: 'Embedding…', - complete: 'Index ready', - error: 'Index failed — retry', - unavailable: 'Search unavailable', -} - -function IndexingBadge({ progress, lastIndexedTime, onRetry, onReindex }: { progress: IndexingProgress; lastIndexedTime?: number | null; onRetry?: () => void; onReindex?: () => void }) { - const isIdle = progress.phase === 'idle' || progress.phase === 'unavailable' - - // When idle, show "Indexed Xm ago" if we have a timestamp - if (isIdle) { - if (!lastIndexedTime) return null - const elapsed = formatIndexedElapsed(lastIndexedTime) - if (!elapsed) return null - return ( - <> - | - { e.currentTarget.style.background = 'var(--hover)' } : undefined} - onMouseLeave={onReindex ? (e) => { e.currentTarget.style.background = 'transparent' } : undefined} - > - {elapsed} - - - ) - } - - const label = INDEXING_LABELS[progress.phase] ?? progress.phase - const isActive = !progress.done - const isError = progress.phase === 'error' - const showCount = progress.total > 0 && isActive - const displayText = showCount - ? `${label} ${progress.current.toLocaleString()}/${progress.total.toLocaleString()}` - : label - const color = isError ? 'var(--accent-orange)' : 'var(--accent-blue, #3b82f6)' - - return ( - <> - | - - {isActive - ? - : - } - {displayText} - - - ) -} - function PendingBadge({ count, onClick }: { count: number; onClick?: () => void }) { if (count <= 0) return null return ( @@ -451,7 +381,7 @@ function McpBadge({ status, onInstall }: { status: McpStatus; onInstall?: () => ) } -export function StatusBar({ noteCount, modifiedCount = 0, vaultPath, vaults, onSwitchVault, onOpenSettings, onOpenLocalFolder, onConnectGitHub, onClickPending, hasGitHub, syncStatus = 'idle', lastSyncTime = null, conflictCount = 0, lastCommitInfo, remoteStatus, onTriggerSync, onPullAndPush, onOpenConflictResolver, zoomLevel = 100, onZoomReset, buildNumber, onCheckForUpdates, indexingProgress, lastIndexedTime, onRetryIndexing, onReindexVault, onRemoveVault, mcpStatus, onInstallMcp }: StatusBarProps) { +export function StatusBar({ noteCount, modifiedCount = 0, vaultPath, vaults, onSwitchVault, onOpenSettings, onOpenLocalFolder, onConnectGitHub, onClickPending, hasGitHub, syncStatus = 'idle', lastSyncTime = null, conflictCount = 0, lastCommitInfo, remoteStatus, onTriggerSync, onPullAndPush, onOpenConflictResolver, zoomLevel = 100, onZoomReset, buildNumber, onCheckForUpdates, onRemoveVault, mcpStatus, onInstallMcp }: StatusBarProps) { const [, setTick] = useState(0) useEffect(() => { const id = setInterval(() => setTick((t) => t + 1), 30_000) @@ -477,7 +407,6 @@ export function StatusBar({ noteCount, modifiedCount = 0, vaultPath, vaults, onS {lastCommitInfo && } - {indexingProgress && } {mcpStatus && }
diff --git a/src/hooks/useAppCommands.ts b/src/hooks/useAppCommands.ts index 9b60fc4b..622a7cd5 100644 --- a/src/hooks/useAppCommands.ts +++ b/src/hooks/useAppCommands.ts @@ -64,7 +64,6 @@ interface AppCommandsConfig { onEmptyTrash?: () => void trashedCount?: number onReopenClosedTab?: () => void - onReindexVault?: () => void onReloadVault?: () => void onRepairVault?: () => void onSetNoteIcon?: () => void @@ -156,7 +155,6 @@ export function useAppCommands(config: AppCommandsConfig): CommandAction[] { onResolveConflicts: config.onResolveConflicts, onViewChanges: viewChanges, onInstallMcp: config.onInstallMcp, - onReindexVault: config.onReindexVault, onReloadVault: config.onReloadVault, onRepairVault: config.onRepairVault, onEmptyTrash: config.onEmptyTrash, @@ -212,7 +210,6 @@ export function useAppCommands(config: AppCommandsConfig): CommandAction[] { onInstallMcp: config.onInstallMcp, onEmptyTrash: config.onEmptyTrash, trashedCount: config.trashedCount, - onReindexVault: config.onReindexVault, onReloadVault: config.onReloadVault, onRepairVault: config.onRepairVault, onSetNoteIcon: config.onSetNoteIcon, diff --git a/src/hooks/useCommandRegistry.test.ts b/src/hooks/useCommandRegistry.test.ts index d047a5cd..dd3e7c54 100644 --- a/src/hooks/useCommandRegistry.test.ts +++ b/src/hooks/useCommandRegistry.test.ts @@ -95,46 +95,6 @@ describe('useCommandRegistry', () => { expect(cmd!.enabled).toBe(false) }) - it('includes reindex-vault command in Settings group', () => { - const config = makeConfig({ onReindexVault: vi.fn() }) - const { result } = renderHook(() => useCommandRegistry(config)) - const cmd = findCommand(result.current, 'reindex-vault') - expect(cmd).toBeDefined() - expect(cmd!.group).toBe('Settings') - expect(cmd!.label).toBe('Reindex Vault') - }) - - it('reindex-vault is enabled when onReindexVault is provided', () => { - const config = makeConfig({ onReindexVault: vi.fn() }) - const { result } = renderHook(() => useCommandRegistry(config)) - const cmd = findCommand(result.current, 'reindex-vault') - expect(cmd!.enabled).toBe(true) - }) - - it('reindex-vault is disabled when onReindexVault is not provided', () => { - const config = makeConfig() - const { result } = renderHook(() => useCommandRegistry(config)) - const cmd = findCommand(result.current, 'reindex-vault') - expect(cmd!.enabled).toBe(false) - }) - - it('reindex-vault executes onReindexVault callback', () => { - const onReindexVault = vi.fn() - const config = makeConfig({ onReindexVault }) - const { result } = renderHook(() => useCommandRegistry(config)) - const cmd = findCommand(result.current, 'reindex-vault') - cmd!.execute() - expect(onReindexVault).toHaveBeenCalled() - }) - - it('reindex-vault has searchable keywords', () => { - const config = makeConfig({ onReindexVault: vi.fn() }) - const { result } = renderHook(() => useCommandRegistry(config)) - const cmd = findCommand(result.current, 'reindex-vault') - expect(cmd!.keywords).toContain('reindex') - expect(cmd!.keywords).toContain('search') - }) - it('resolve-conflicts stays enabled across rerenders', () => { const config = makeConfig() const { result, rerender } = renderHook( diff --git a/src/hooks/useCommandRegistry.ts b/src/hooks/useCommandRegistry.ts index ccc0eade..6c695b9c 100644 --- a/src/hooks/useCommandRegistry.ts +++ b/src/hooks/useCommandRegistry.ts @@ -25,7 +25,6 @@ interface CommandRegistryConfig { onInstallMcp?: () => void onEmptyTrash?: () => void trashedCount?: number - onReindexVault?: () => void onReloadVault?: () => void onRepairVault?: () => void onSetNoteIcon?: () => void @@ -169,7 +168,6 @@ export function useCommandRegistry(config: CommandRegistryConfig): CommandAction onRemoveActiveVault, onRestoreGettingStarted, isGettingStartedHidden, vaultCount, mcpStatus, onInstallMcp, onEmptyTrash, trashedCount, - onReindexVault, onReloadVault, onRepairVault, onSetNoteIcon, @@ -256,7 +254,6 @@ export function useCommandRegistry(config: CommandRegistryConfig): CommandAction { id: 'restore-getting-started', label: 'Restore Getting Started Vault', group: 'Settings', keywords: ['vault', 'restore', 'demo', 'getting started', 'reset'], enabled: !!isGettingStartedHidden && !!onRestoreGettingStarted, execute: () => onRestoreGettingStarted?.() }, { id: 'check-updates', label: 'Check for Updates', group: 'Settings', keywords: ['update', 'version', 'upgrade', 'release'], enabled: true, execute: () => onCheckForUpdates?.() }, { id: 'install-mcp', label: mcpStatus === 'installed' ? 'Restore MCP Server' : 'Install MCP Server', group: 'Settings', keywords: ['mcp', 'claude', 'ai', 'tools', 'install', 'restore', 'fix', 'repair'], enabled: true, execute: () => onInstallMcp?.() }, - { id: 'reindex-vault', label: 'Reindex Vault', group: 'Settings', keywords: ['reindex', 'index', 'search', 'rebuild', 'refresh'], enabled: !!onReindexVault, execute: () => onReindexVault?.() }, { id: 'reload-vault', label: 'Reload Vault', group: 'Settings', keywords: ['reload', 'refresh', 'rescan', 'sync', 'filesystem', 'cache'], enabled: !!onReloadVault, execute: () => onReloadVault?.() }, { id: 'repair-vault', label: 'Repair Vault', group: 'Settings', keywords: ['repair', 'fix', 'restore', 'config', 'agents', 'themes', 'missing', 'reset', 'flatten', 'structure'], enabled: !!onRepairVault, execute: () => onRepairVault?.() }, @@ -282,7 +279,7 @@ export function useCommandRegistry(config: CommandRegistryConfig): CommandAction vaultTypes, onRemoveActiveVault, onRestoreGettingStarted, isGettingStartedHidden, vaultCount, mcpStatus, onInstallMcp, onEmptyTrash, trashedCount, - onReindexVault, onReloadVault, onRepairVault, + onReloadVault, onRepairVault, onSetNoteIcon, onRemoveNoteIcon, activeNoteHasIcon, isSectionGroup, noteListFilter, onSetNoteListFilter, onOpenInNewWindow, diff --git a/src/hooks/useIndexing.test.ts b/src/hooks/useIndexing.test.ts deleted file mode 100644 index d4bb88e6..00000000 --- a/src/hooks/useIndexing.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' -import { renderHook, act } from '@testing-library/react' -import { useIndexing } from './useIndexing' - -vi.mock('@tauri-apps/api/core', () => ({ invoke: vi.fn() })) -vi.mock('@tauri-apps/api/event', () => ({ listen: vi.fn().mockResolvedValue(vi.fn()) })) -vi.mock('../mock-tauri', () => ({ - isTauri: () => false, - mockInvoke: vi.fn().mockResolvedValue({ - available: true, - qmd_installed: true, - collection_exists: true, - indexed_count: 100, - embedded_count: 80, - pending_embed: 0, - }), -})) - -const { mockInvoke } = await import('../mock-tauri') as { mockInvoke: ReturnType } - -describe('useIndexing', () => { - beforeEach(() => { - vi.useFakeTimers() - vi.clearAllMocks() - mockInvoke.mockResolvedValue({ - available: true, - qmd_installed: true, - collection_exists: true, - indexed_count: 100, - embedded_count: 80, - pending_embed: 0, - }) - }) - - afterEach(() => { - vi.useRealTimers() - }) - - it('starts with idle phase', () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - expect(result.current.progress.phase).toBe('idle') - }) - - it('auto-dismisses error phase after 15 seconds', async () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - - // Simulate setting error state via retryIndexing - mockInvoke.mockRejectedValueOnce(new Error('qmd update failed')) - await act(async () => { await result.current.retryIndexing() }) - expect(result.current.progress.phase).toBe('error') - - act(() => { vi.advanceTimersByTime(15000) }) - expect(result.current.progress.phase).toBe('idle') - }) - - it('sets unavailable phase for "not installed" errors', async () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - - mockInvoke.mockRejectedValueOnce(new Error('bun not installed')) - await act(async () => { await result.current.retryIndexing() }) - expect(result.current.progress.phase).toBe('unavailable') - }) - - it('sets unavailable phase for "not available" errors', async () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - - mockInvoke.mockRejectedValueOnce(new Error('qmd not available: bun not found')) - await act(async () => { await result.current.retryIndexing() }) - expect(result.current.progress.phase).toBe('unavailable') - }) - - it('auto-dismisses unavailable phase after 8 seconds', async () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - - mockInvoke.mockRejectedValueOnce(new Error('bun not installed')) - await act(async () => { await result.current.retryIndexing() }) - expect(result.current.progress.phase).toBe('unavailable') - - act(() => { vi.advanceTimersByTime(8000) }) - expect(result.current.progress.phase).toBe('idle') - }) - - it('exposes retryIndexing function', () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - expect(typeof result.current.retryIndexing).toBe('function') - }) - - it('exposes triggerFullReindex function', () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - expect(typeof result.current.triggerFullReindex).toBe('function') - }) - - it('retryIndexing is the same reference as triggerFullReindex', () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - expect(result.current.retryIndexing).toBe(result.current.triggerFullReindex) - }) - - it('triggerFullReindex sets scanning phase then completes', async () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - - await act(async () => { await result.current.triggerFullReindex() }) - // In non-Tauri mode, it goes to 'complete' then auto-dismisses - expect(result.current.progress.phase).toBe('complete') - }) - - it('triggerFullReindex sets lastIndexedTime on success', async () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - - expect(result.current.lastIndexedTime).toBeNull() - await act(async () => { await result.current.triggerFullReindex() }) - expect(result.current.lastIndexedTime).toBeGreaterThan(0) - }) - - it('triggerFullReindex sets error phase on failure', async () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - - mockInvoke.mockRejectedValueOnce(new Error('indexing failed')) - await act(async () => { await result.current.triggerFullReindex() }) - expect(result.current.progress.phase).toBe('error') - }) - - it('populates lastIndexedTime from backend metadata on mount', async () => { - mockInvoke.mockResolvedValue({ - available: true, - qmd_installed: true, - collection_exists: true, - indexed_count: 100, - embedded_count: 80, - pending_embed: 0, - last_indexed_commit: 'abc123', - last_indexed_at: 1709800000, - }) - - const { result } = renderHook(() => useIndexing('/test/vault')) - - // Wait for the effect to run - await act(async () => { await vi.advanceTimersByTimeAsync(10) }) - expect(result.current.lastIndexedTime).toBe(1709800000000) // seconds → ms - }) - - it('starts with lastIndexedTime as null', () => { - const { result } = renderHook(() => useIndexing('/test/vault')) - expect(result.current.lastIndexedTime).toBeNull() - }) -}) diff --git a/src/hooks/useIndexing.ts b/src/hooks/useIndexing.ts deleted file mode 100644 index a3eb5e7e..00000000 --- a/src/hooks/useIndexing.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { useState, useEffect, useCallback, useRef } from 'react' -import { invoke } from '@tauri-apps/api/core' -import { isTauri, mockInvoke } from '../mock-tauri' - -export interface IndexingProgress { - phase: 'idle' | 'installing' | 'scanning' | 'embedding' | 'complete' | 'error' | 'unavailable' - current: number - total: number - done: boolean - error: string | null -} - -interface IndexStatus { - available: boolean - qmd_installed: boolean - collection_exists: boolean - indexed_count: number - embedded_count: number - pending_embed: number - last_indexed_commit: string | null - last_indexed_at: number | null -} - -const IDLE: IndexingProgress = { phase: 'idle', current: 0, total: 0, done: false, error: null } - -function invokeCmd(cmd: string, args?: Record): Promise { - return isTauri() ? invoke(cmd, args) : mockInvoke(cmd, args) -} - -export function useIndexing(vaultPath: string) { - const [progress, setProgress] = useState(IDLE) - const [lastIndexedTime, setLastIndexedTime] = useState(null) - const indexingRef = useRef(false) - const vaultPathRef = useRef(vaultPath) - - useEffect(() => { vaultPathRef.current = vaultPath }, [vaultPath]) - - // Listen for progress events from Rust - useEffect(() => { - if (!isTauri()) return - - let cleanup: (() => void) | null = null - - import('@tauri-apps/api/event').then(({ listen }) => { - const unlisten = listen('indexing-progress', (event) => { - setProgress(event.payload) - if (event.payload.done) { - indexingRef.current = false - if (event.payload.phase === 'complete') { - setLastIndexedTime(Date.now()) - } - } - }) - cleanup = () => { unlisten.then(fn => fn()) } - }) - - return () => { cleanup?.() } - }, []) - - // Check index status and auto-trigger indexing on vault open - useEffect(() => { - if (!vaultPath) return - let cancelled = false - - async function checkAndIndex() { - try { - const status = await invokeCmd('get_index_status', { vaultPath }) - if (cancelled) return - - // Populate last indexed time from backend metadata - if (status.last_indexed_at) { - setLastIndexedTime(status.last_indexed_at * 1000) // seconds → ms - } - - // If qmd not installed or no collection or pending embeds, trigger indexing - const needsIndexing = !status.qmd_installed || !status.collection_exists || status.pending_embed > 0 - if (needsIndexing && !indexingRef.current) { - indexingRef.current = true - setProgress({ - phase: status.qmd_installed ? 'scanning' : 'installing', - current: 0, - total: status.indexed_count, - done: false, - error: null, - }) - // Fire and forget — progress updates come via events - invokeCmd('start_indexing', { vaultPath }).catch((err) => { - if (cancelled) return - const msg = String(err) - const isUnavailable = msg.includes('not installed') || msg.includes('not available') - setProgress({ - phase: isUnavailable ? 'unavailable' : 'error', - current: 0, - total: 0, - done: true, - error: msg, - }) - indexingRef.current = false - }) - } - } catch { - // get_index_status failed — likely qmd not available, non-fatal - } - } - - checkAndIndex() - return () => { cancelled = true } - }, [vaultPath]) - - // Auto-dismiss transient statuses after a delay - useEffect(() => { - if (progress.phase === 'complete') { - const timer = setTimeout(() => setProgress(IDLE), 5000) - return () => clearTimeout(timer) - } - if (progress.phase === 'unavailable') { - const timer = setTimeout(() => setProgress(IDLE), 8000) - return () => clearTimeout(timer) - } - if (progress.phase === 'error') { - const timer = setTimeout(() => setProgress(IDLE), 15000) - return () => clearTimeout(timer) - } - }, [progress.phase]) - - const triggerIncrementalIndex = useCallback(async () => { - if (indexingRef.current) return - try { - await invokeCmd('trigger_incremental_index', { vaultPath: vaultPathRef.current }) - setLastIndexedTime(Date.now()) - } catch { - // Incremental update failure is non-fatal - } - }, []) - - const triggerFullReindex = useCallback(async () => { - if (indexingRef.current || !vaultPathRef.current) return - indexingRef.current = true - setProgress({ phase: 'scanning', current: 0, total: 0, done: false, error: null }) - try { - await invokeCmd('start_indexing', { vaultPath: vaultPathRef.current }) - // In non-Tauri mode, mark complete immediately - if (!isTauri()) { - setProgress({ phase: 'complete', current: 0, total: 0, done: true, error: null }) - setLastIndexedTime(Date.now()) - } - } catch (err) { - const msg = String(err) - const isUnavailable = msg.includes('not installed') || msg.includes('not available') - setProgress({ phase: isUnavailable ? 'unavailable' : 'error', current: 0, total: 0, done: true, error: msg }) - } finally { - indexingRef.current = false - } - }, []) - - const retryIndexing = triggerFullReindex - - return { progress, lastIndexedTime, triggerIncrementalIndex, triggerFullReindex, retryIndexing } -} diff --git a/src/hooks/useMenuEvents.test.ts b/src/hooks/useMenuEvents.test.ts index b500d632..fc248dc7 100644 --- a/src/hooks/useMenuEvents.test.ts +++ b/src/hooks/useMenuEvents.test.ts @@ -33,7 +33,6 @@ function makeHandlers(): MenuEventHandlers { onResolveConflicts: vi.fn(), onViewChanges: vi.fn(), onInstallMcp: vi.fn(), - onReindexVault: vi.fn(), onReloadVault: vi.fn(), onReopenClosedTab: vi.fn(), onOpenInNewWindow: vi.fn(), @@ -296,12 +295,6 @@ describe('dispatchMenuEvent', () => { expect(h.onInstallMcp).toHaveBeenCalled() }) - it('vault-reindex triggers reindex vault', () => { - const h = makeHandlers() - dispatchMenuEvent('vault-reindex', h) - expect(h.onReindexVault).toHaveBeenCalled() - }) - it('vault-reload triggers reload vault', () => { const h = makeHandlers() dispatchMenuEvent('vault-reload', h) diff --git a/src/hooks/useMenuEvents.ts b/src/hooks/useMenuEvents.ts index e99ef94e..23a7bceb 100644 --- a/src/hooks/useMenuEvents.ts +++ b/src/hooks/useMenuEvents.ts @@ -36,7 +36,6 @@ export interface MenuEventHandlers { onInstallMcp?: () => void onReopenClosedTab?: () => void onOpenInNewWindow?: () => void - onReindexVault?: () => void onReloadVault?: () => void onRepairVault?: () => void onEmptyTrash?: () => void @@ -82,7 +81,7 @@ type OptionalHandler = | 'onGoBack' | 'onGoForward' | 'onCheckForUpdates' | 'onCreateType' | 'onToggleRawEditor' | 'onToggleDiff' | 'onToggleAIChat' | 'onOpenVault' | 'onRemoveActiveVault' | 'onRestoreGettingStarted' - | 'onCommitPush' | 'onPull' | 'onResolveConflicts' | 'onViewChanges' | 'onInstallMcp' | 'onReindexVault' | 'onReloadVault' | 'onRepairVault' + | 'onCommitPush' | 'onPull' | 'onResolveConflicts' | 'onViewChanges' | 'onInstallMcp' | 'onReloadVault' | 'onRepairVault' | 'onEmptyTrash' | 'onReopenClosedTab' | 'onOpenInNewWindow' @@ -103,7 +102,6 @@ const OPTIONAL_EVENT_MAP: Record = { 'vault-resolve-conflicts': 'onResolveConflicts', 'vault-view-changes': 'onViewChanges', 'vault-install-mcp': 'onInstallMcp', - 'vault-reindex': 'onReindexVault', 'vault-reload': 'onReloadVault', 'vault-repair': 'onRepairVault', 'note-empty-trash': 'onEmptyTrash', diff --git a/src/hooks/useUnifiedSearch.ts b/src/hooks/useUnifiedSearch.ts index a91254ed..f96ba979 100644 --- a/src/hooks/useUnifiedSearch.ts +++ b/src/hooks/useUnifiedSearch.ts @@ -17,7 +17,6 @@ interface SearchResponseData { } const DEBOUNCE_MS = 300 -const HYBRID_TIMEOUT_MS = 5000 function searchCall(args: Record): Promise { return isTauri() @@ -25,16 +24,6 @@ function searchCall(args: Record): Promise : mockInvoke('search_vault', args) } -function searchWithTimeout(args: Record, ms: number): Promise { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error('Search timeout')), ms) - searchCall(args).then( - result => { clearTimeout(timer); resolve(result) }, - err => { clearTimeout(timer); reject(err) }, - ) - }) -} - function mapResults(raw: SearchResultData[]): SearchResult[] { const seen = new Set() return raw @@ -92,19 +81,7 @@ export function useUnifiedSearch(vaultPath: string, active: boolean) { setSelectedIndex(0) } catch { if (gen !== searchGenRef.current) return - setLoading(false) - return - } - try { - const response = await searchWithTimeout( - { vaultPath, query: q, mode: 'hybrid', limit: 20 }, - HYBRID_TIMEOUT_MS, - ) - if (gen !== searchGenRef.current) return - setResults(mapResults(response.results)) - setElapsedMs(response.elapsed_ms) - setSelectedIndex(prev => Math.min(prev, Math.max(response.results.length - 1, 0))) - } catch { /* Hybrid timed out — keyword results remain */ } finally { + } finally { if (gen === searchGenRef.current) setLoading(false) } }, [vaultPath]) diff --git a/src/mock-tauri/mock-handlers.ts b/src/mock-tauri/mock-handlers.ts index 0502de7d..410dec1c 100644 --- a/src/mock-tauri/mock-handlers.ts +++ b/src/mock-tauri/mock-handlers.ts @@ -265,9 +265,6 @@ export const mockHandlers: Record any> = { create_getting_started_vault: () => '/Users/mock/Documents/Getting Started', register_mcp_tools: () => 'registered', check_mcp_status: () => 'installed', - get_index_status: () => ({ available: true, qmd_installed: true, collection_exists: true, indexed_count: 100, embedded_count: 80, pending_embed: 0, last_indexed_commit: 'abc123', last_indexed_at: Math.floor(Date.now() / 1000) - 3600 }), - start_indexing: () => null, - trigger_incremental_index: () => null, repair_vault: (): string => 'Vault repaired', } diff --git a/src/utils/indexingHelpers.ts b/src/utils/indexingHelpers.ts deleted file mode 100644 index 279ecd58..00000000 --- a/src/utils/indexingHelpers.ts +++ /dev/null @@ -1,10 +0,0 @@ -export function formatIndexedElapsed(lastIndexedTime: number | null): string { - if (!lastIndexedTime) return '' - const secs = Math.round((Date.now() - lastIndexedTime) / 1000) - if (secs < 60) return 'Indexed just now' - const mins = Math.floor(secs / 60) - if (mins < 60) return `Indexed ${mins}m ago` - const hrs = Math.floor(mins / 60) - if (hrs < 24) return `Indexed ${hrs}h ago` - return `Indexed ${Math.floor(hrs / 24)}d ago` -} diff --git a/tests/smoke/fresh-install-regression-qa.spec.ts b/tests/smoke/fresh-install-regression-qa.spec.ts index 64279050..8fdeeddb 100644 --- a/tests/smoke/fresh-install-regression-qa.spec.ts +++ b/tests/smoke/fresh-install-regression-qa.spec.ts @@ -6,8 +6,7 @@ import { sendShortcut, openCommandPalette, findCommand } from './helpers' * MacBook without pre-configured environment. * * Tasks under test: - * 1. Search/qmd bundling + auto-index - * 2. MCP server foundation + auto-registration + * 1. MCP server foundation + auto-registration * 3. AGENTS.md vault-level instructions * 4. AI Agent panel: vault-native AI * 5. Claude API wiring + full agent loop diff --git a/tests/smoke/indexing-reindex-status.spec.ts b/tests/smoke/indexing-reindex-status.spec.ts deleted file mode 100644 index d2ae43b4..00000000 --- a/tests/smoke/indexing-reindex-status.spec.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { test, expect } from '@playwright/test' -import { - openCommandPalette, - findCommand, - executeCommand, -} from './helpers' - -test.describe('Reindex Vault smoke tests', () => { - test.beforeEach(async ({ page }) => { - await page.goto('/') - await page.waitForLoadState('networkidle') - }) - - test('Reindex Vault command appears in command palette', async ({ page }) => { - await openCommandPalette(page) - const found = await findCommand(page, 'Reindex Vault') - expect(found).toBe(true) - }) - - test('Reindex Vault command triggers indexing progress', async ({ page }) => { - await openCommandPalette(page) - await executeCommand(page, 'Reindex Vault') - - // After triggering reindex, the indexing badge should appear in the status bar - const indexingBadge = page.locator('[data-testid="status-indexing"], [data-testid="status-indexed-time"]') - await expect(indexingBadge.first()).toBeVisible({ timeout: 5_000 }) - }) -}) diff --git a/tools/qmd/bun.lock b/tools/qmd/bun.lock deleted file mode 100644 index f5d7aa5d..00000000 --- a/tools/qmd/bun.lock +++ /dev/null @@ -1,637 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "2025-12-07-bm25-q", - "dependencies": { - "@modelcontextprotocol/sdk": "^1.25.1", - "node-llama-cpp": "^3.14.5", - "sqlite-vec": "^0.1.7-alpha.2", - "yaml": "^2.8.2", - "zod": "^4.2.1", - }, - "devDependencies": { - "@types/bun": "latest", - }, - "optionalDependencies": { - "sqlite-vec-darwin-arm64": "^0.1.7-alpha.2", - "sqlite-vec-darwin-x64": "^0.1.7-alpha.2", - "sqlite-vec-linux-x64": "^0.1.7-alpha.2", - "sqlite-vec-win32-x64": "^0.1.7-alpha.2", - }, - "peerDependencies": { - "typescript": "^5.9.3", - }, - }, - }, - "packages": { - "@hono/node-server": ["@hono/node-server@1.19.7", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vUcD0uauS7EU2caukW8z5lJKtoGMokxNbJtBiwHgpqxEXokaHCBkQUmCHhjFB1VUTWdqj25QoMkMKzgjq+uhrw=="], - - "@huggingface/jinja": ["@huggingface/jinja@0.5.3", "", {}, "sha512-asqfZ4GQS0hD876Uw4qiUb7Tr/V5Q+JZuo2L+BtdrD4U40QU58nIRq3ZSgAzJgT874VLjhGVacaYfrdpXtEvtA=="], - - "@kwsites/file-exists": ["@kwsites/file-exists@1.1.1", "", { "dependencies": { "debug": "^4.1.1" } }, "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw=="], - - "@kwsites/promise-deferred": ["@kwsites/promise-deferred@1.1.1", "", {}, "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="], - - "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.25.1", "", { "dependencies": { "@hono/node-server": "^1.19.7", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.0.1", "express-rate-limit": "^7.5.0", "jose": "^6.1.1", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.0" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-yO28oVFFC7EBoiKdAn+VqRm+plcfv4v0xp6osG/VsCB0NlPZWi87ajbCZZ8f/RvOFLEu7//rSRmuZZ7lMoe3gQ=="], - - "@node-llama-cpp/linux-arm64": ["@node-llama-cpp/linux-arm64@3.14.5", "", { "os": "linux", "cpu": [ "x64", "arm64", ] }, "sha512-58IcWW7EOqc/66mYWXRsoMCy1MR3pTX/YaC0HYF9Rg5XeAPKhUP7NHrglbqgjO62CkcuFZaSEiX2AtG972GQYQ=="], - - "@node-llama-cpp/linux-armv7l": ["@node-llama-cpp/linux-armv7l@3.14.5", "", { "os": "linux", "cpu": [ "arm", "x64", ] }, "sha512-mJWN0qWsn8y+r/34DC3XlSiXjjKs6wX1BTx0wwJ37fWefS/qfzuBJwQGqpfqe5xpfafib/RgQX44fsvE/9yb1w=="], - - "@node-llama-cpp/linux-x64": ["@node-llama-cpp/linux-x64@3.14.5", "", { "os": "linux", "cpu": "x64" }, "sha512-f6xCqlSqSxMP9Iwm3CpaTzFybbHrzpLkNzA18v21PwhMN8u4DP44euLoxe+BMbOpyzx4iMxU1AUsPsgcHD1Y4w=="], - - "@node-llama-cpp/linux-x64-cuda": ["@node-llama-cpp/linux-x64-cuda@3.14.5", "", { "os": "linux", "cpu": "x64" }, "sha512-yk0EGnAJ+m/paSaItigmxcqC8nNjZlkx9yZgQE51CsTip7tmnqqlj60pW1fWmhrjOJ9XnRlVVTP81fa9B+O1Hg=="], - - "@node-llama-cpp/linux-x64-cuda-ext": ["@node-llama-cpp/linux-x64-cuda-ext@3.14.5", "", { "os": "linux", "cpu": "x64" }, "sha512-AACXmXjqvAppoC6Z20UI7yeSZaFb6uP9x/2lzctVwlm42ef76SN6DNXaX1yzH7DTyzK5zYhoH4ycJUe+zOeGzw=="], - - "@node-llama-cpp/linux-x64-vulkan": ["@node-llama-cpp/linux-x64-vulkan@3.14.5", "", { "os": "linux", "cpu": "x64" }, "sha512-9wZG90CUyyO8EsqfDEh03/fK0ctbQFbKaAFa6Goh+jFLOtqPL+plLqAsW3jDFdLRF5+oAPTKt9/4Y7vHTajQbQ=="], - - "@node-llama-cpp/mac-arm64-metal": ["@node-llama-cpp/mac-arm64-metal@3.14.5", "", { "os": "darwin", "cpu": [ "x64", "arm64", ] }, "sha512-7pclj/nbQyx7gPVbyqkCn+ftlGcnw7YrewxBv1/BWWAMzBrMt2+qkjtUcUhwXH7mT5WN/+eWsszhIMXH3Uf6vQ=="], - - "@node-llama-cpp/mac-x64": ["@node-llama-cpp/mac-x64@3.14.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-iZBmLgPkLKiKS0lYAuqq8i85etGeQ9L+AjEJUhG5N6T/vCF4XSOkUTsEFMEX+iJLV3VxvY/C8R1e/UF7InUjUg=="], - - "@node-llama-cpp/win-arm64": ["@node-llama-cpp/win-arm64@3.14.5", "", { "os": "win32", "cpu": [ "x64", "arm64", ] }, "sha512-WTZJeb2JZo/qPNHf++xA2YeMXB46G7G4WsKEnHVyCpAhhslHAhe/LPgSQfNfk9rYusbsRiy9QMxeGNSOowZMVQ=="], - - "@node-llama-cpp/win-x64": ["@node-llama-cpp/win-x64@3.14.5", "", { "os": "win32", "cpu": "x64" }, "sha512-cEuhb1iLTodM+V8xc1mWKeWRYkX9tlnl0+9jUjwsv2kgnAjEob3WlTYsCXewvEe2ShSyk8AsLsBPZxv7IQaBsw=="], - - "@node-llama-cpp/win-x64-cuda": ["@node-llama-cpp/win-x64-cuda@3.14.5", "", { "os": "win32", "cpu": "x64" }, "sha512-gwBMSzUteLD765Gq/hYQ4UC21vggR7oG+DU4zAg0Mt3i34PqKJC+tBop5jsTN5Hq8RaM9+nTNrVbF/x228TLvg=="], - - "@node-llama-cpp/win-x64-cuda-ext": ["@node-llama-cpp/win-x64-cuda-ext@3.14.5", "", { "os": "win32", "cpu": "x64" }, "sha512-kBHnUmodr+n8N+sKTh1c6aNNEmvXBWM5AtaLWIEfkCb00bVHNFeqYPmLuPNtMX3dIUtD9PHdA4Jsn0RJmNZJfA=="], - - "@node-llama-cpp/win-x64-vulkan": ["@node-llama-cpp/win-x64-vulkan@3.14.5", "", { "os": "win32", "cpu": "x64" }, "sha512-rY+vr5RaGSCWEe22WZMkhUu16o9zpeqTZO/nD5G27Y0bb+xBRDLmXbxYMp2dDQTfpkNWIZ0ia3PGWwl5yhYw7A=="], - - "@octokit/app": ["@octokit/app@16.1.2", "", { "dependencies": { "@octokit/auth-app": "^8.1.2", "@octokit/auth-unauthenticated": "^7.0.3", "@octokit/core": "^7.0.6", "@octokit/oauth-app": "^8.0.3", "@octokit/plugin-paginate-rest": "^14.0.0", "@octokit/types": "^16.0.0", "@octokit/webhooks": "^14.0.0" } }, "sha512-8j7sEpUYVj18dxvh0KWj6W/l6uAiVRBl1JBDVRqH1VHKAO/G5eRVl4yEoYACjakWers1DjUkcCHyJNQK47JqyQ=="], - - "@octokit/auth-app": ["@octokit/auth-app@8.1.2", "", { "dependencies": { "@octokit/auth-oauth-app": "^9.0.3", "@octokit/auth-oauth-user": "^6.0.2", "@octokit/request": "^10.0.6", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "toad-cache": "^3.7.0", "universal-github-app-jwt": "^2.2.0", "universal-user-agent": "^7.0.0" } }, "sha512-db8VO0PqXxfzI6GdjtgEFHY9tzqUql5xMFXYA12juq8TeTgPAuiiP3zid4h50lwlIP457p5+56PnJOgd2GGBuw=="], - - "@octokit/auth-oauth-app": ["@octokit/auth-oauth-app@9.0.3", "", { "dependencies": { "@octokit/auth-oauth-device": "^8.0.3", "@octokit/auth-oauth-user": "^6.0.2", "@octokit/request": "^10.0.6", "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-+yoFQquaF8OxJSxTb7rnytBIC2ZLbLqA/yb71I4ZXT9+Slw4TziV9j/kyGhUFRRTF2+7WlnIWsePZCWHs+OGjg=="], - - "@octokit/auth-oauth-device": ["@octokit/auth-oauth-device@8.0.3", "", { "dependencies": { "@octokit/oauth-methods": "^6.0.2", "@octokit/request": "^10.0.6", "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-zh2W0mKKMh/VWZhSqlaCzY7qFyrgd9oTWmTmHaXnHNeQRCZr/CXy2jCgHo4e4dJVTiuxP5dLa0YM5p5QVhJHbw=="], - - "@octokit/auth-oauth-user": ["@octokit/auth-oauth-user@6.0.2", "", { "dependencies": { "@octokit/auth-oauth-device": "^8.0.3", "@octokit/oauth-methods": "^6.0.2", "@octokit/request": "^10.0.6", "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-qLoPPc6E6GJoz3XeDG/pnDhJpTkODTGG4kY0/Py154i/I003O9NazkrwJwRuzgCalhzyIeWQ+6MDvkUmKXjg/A=="], - - "@octokit/auth-token": ["@octokit/auth-token@6.0.0", "", {}, "sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w=="], - - "@octokit/auth-unauthenticated": ["@octokit/auth-unauthenticated@7.0.3", "", { "dependencies": { "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0" } }, "sha512-8Jb1mtUdmBHL7lGmop9mU9ArMRUTRhg8vp0T1VtZ4yd9vEm3zcLwmjQkhNEduKawOOORie61xhtYIhTDN+ZQ3g=="], - - "@octokit/core": ["@octokit/core@7.0.6", "", { "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", "@octokit/request": "^10.0.6", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "before-after-hook": "^4.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q=="], - - "@octokit/endpoint": ["@octokit/endpoint@11.0.2", "", { "dependencies": { "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-4zCpzP1fWc7QlqunZ5bSEjxc6yLAlRTnDwKtgXfcI/FxxGoqedDG8V2+xJ60bV2kODqcGB+nATdtap/XYq2NZQ=="], - - "@octokit/graphql": ["@octokit/graphql@9.0.3", "", { "dependencies": { "@octokit/request": "^10.0.6", "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA=="], - - "@octokit/oauth-app": ["@octokit/oauth-app@8.0.3", "", { "dependencies": { "@octokit/auth-oauth-app": "^9.0.2", "@octokit/auth-oauth-user": "^6.0.1", "@octokit/auth-unauthenticated": "^7.0.2", "@octokit/core": "^7.0.5", "@octokit/oauth-authorization-url": "^8.0.0", "@octokit/oauth-methods": "^6.0.1", "@types/aws-lambda": "^8.10.83", "universal-user-agent": "^7.0.0" } }, "sha512-jnAjvTsPepyUaMu9e69hYBuozEPgYqP4Z3UnpmvoIzHDpf8EXDGvTY1l1jK0RsZ194oRd+k6Hm13oRU8EoDFwg=="], - - "@octokit/oauth-authorization-url": ["@octokit/oauth-authorization-url@8.0.0", "", {}, "sha512-7QoLPRh/ssEA/HuHBHdVdSgF8xNLz/Bc5m9fZkArJE5bb6NmVkDm3anKxXPmN1zh6b5WKZPRr3697xKT/yM3qQ=="], - - "@octokit/oauth-methods": ["@octokit/oauth-methods@6.0.2", "", { "dependencies": { "@octokit/oauth-authorization-url": "^8.0.0", "@octokit/request": "^10.0.6", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0" } }, "sha512-HiNOO3MqLxlt5Da5bZbLV8Zarnphi4y9XehrbaFMkcoJ+FL7sMxH/UlUsCVxpddVu4qvNDrBdaTVE2o4ITK8ng=="], - - "@octokit/openapi-types": ["@octokit/openapi-types@27.0.0", "", {}, "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA=="], - - "@octokit/openapi-webhooks-types": ["@octokit/openapi-webhooks-types@12.1.0", "", {}, "sha512-WiuzhOsiOvb7W3Pvmhf8d2C6qaLHXrWiLBP4nJ/4kydu+wpagV5Fkz9RfQwV2afYzv3PB+3xYgp4mAdNGjDprA=="], - - "@octokit/plugin-paginate-graphql": ["@octokit/plugin-paginate-graphql@6.0.0", "", { "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-crfpnIoFiBtRkvPqOyLOsw12XsveYuY2ieP6uYDosoUegBJpSVxGwut9sxUgFFcll3VTOTqpUf8yGd8x1OmAkQ=="], - - "@octokit/plugin-paginate-rest": ["@octokit/plugin-paginate-rest@14.0.0", "", { "dependencies": { "@octokit/types": "^16.0.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw=="], - - "@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@17.0.0", "", { "dependencies": { "@octokit/types": "^16.0.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw=="], - - "@octokit/plugin-retry": ["@octokit/plugin-retry@8.0.3", "", { "dependencies": { "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": ">=7" } }, "sha512-vKGx1i3MC0za53IzYBSBXcrhmd+daQDzuZfYDd52X5S0M2otf3kVZTVP8bLA3EkU0lTvd1WEC2OlNNa4G+dohA=="], - - "@octokit/plugin-throttling": ["@octokit/plugin-throttling@11.0.3", "", { "dependencies": { "@octokit/types": "^16.0.0", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": "^7.0.0" } }, "sha512-34eE0RkFCKycLl2D2kq7W+LovheM/ex3AwZCYN8udpi6bxsyjZidb2McXs69hZhLmJlDqTSP8cH+jSRpiaijBg=="], - - "@octokit/request": ["@octokit/request@10.0.7", "", { "dependencies": { "@octokit/endpoint": "^11.0.2", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "fast-content-type-parse": "^3.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-v93h0i1yu4idj8qFPZwjehoJx4j3Ntn+JhXsdJrG9pYaX6j/XRz2RmasMUHtNgQD39nrv/VwTWSqK0RNXR8upA=="], - - "@octokit/request-error": ["@octokit/request-error@7.1.0", "", { "dependencies": { "@octokit/types": "^16.0.0" } }, "sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw=="], - - "@octokit/types": ["@octokit/types@16.0.0", "", { "dependencies": { "@octokit/openapi-types": "^27.0.0" } }, "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg=="], - - "@octokit/webhooks": ["@octokit/webhooks@14.2.0", "", { "dependencies": { "@octokit/openapi-webhooks-types": "12.1.0", "@octokit/request-error": "^7.0.0", "@octokit/webhooks-methods": "^6.0.0" } }, "sha512-da6KbdNCV5sr1/txD896V+6W0iamFWrvVl8cHkBSPT+YlvmT3DwXa4jxZnQc+gnuTEqSWbBeoSZYTayXH9wXcw=="], - - "@octokit/webhooks-methods": ["@octokit/webhooks-methods@6.0.0", "", {}, "sha512-MFlzzoDJVw/GcbfzVC1RLR36QqkTLUf79vLVO3D+xn7r0QgxnFoLZgtrzxiQErAjFUOdH6fas2KeQJ1yr/qaXQ=="], - - "@reflink/reflink": ["@reflink/reflink@0.1.19", "", { "optionalDependencies": { "@reflink/reflink-darwin-arm64": "0.1.19", "@reflink/reflink-darwin-x64": "0.1.19", "@reflink/reflink-linux-arm64-gnu": "0.1.19", "@reflink/reflink-linux-arm64-musl": "0.1.19", "@reflink/reflink-linux-x64-gnu": "0.1.19", "@reflink/reflink-linux-x64-musl": "0.1.19", "@reflink/reflink-win32-arm64-msvc": "0.1.19", "@reflink/reflink-win32-x64-msvc": "0.1.19" } }, "sha512-DmCG8GzysnCZ15bres3N5AHCmwBwYgp0As6xjhQ47rAUTUXxJiK+lLUxaGsX3hd/30qUpVElh05PbGuxRPgJwA=="], - - "@reflink/reflink-darwin-arm64": ["@reflink/reflink-darwin-arm64@0.1.19", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ruy44Lpepdk1FqDz38vExBY/PVUsjxZA+chd9wozjUH9JjuDT/HEaQYA6wYN9mf041l0yLVar6BCZuWABJvHSA=="], - - "@reflink/reflink-darwin-x64": ["@reflink/reflink-darwin-x64@0.1.19", "", { "os": "darwin", "cpu": "x64" }, "sha512-By85MSWrMZa+c26TcnAy8SDk0sTUkYlNnwknSchkhHpGXOtjNDUOxJE9oByBnGbeuIE1PiQsxDG3Ud+IVV9yuA=="], - - "@reflink/reflink-linux-arm64-gnu": ["@reflink/reflink-linux-arm64-gnu@0.1.19", "", { "os": "linux", "cpu": "arm64" }, "sha512-7P+er8+rP9iNeN+bfmccM4hTAaLP6PQJPKWSA4iSk2bNvo6KU6RyPgYeHxXmzNKzPVRcypZQTpFgstHam6maVg=="], - - "@reflink/reflink-linux-arm64-musl": ["@reflink/reflink-linux-arm64-musl@0.1.19", "", { "os": "linux", "cpu": "arm64" }, "sha512-37iO/Dp6m5DDaC2sf3zPtx/hl9FV3Xze4xoYidrxxS9bgP3S8ALroxRK6xBG/1TtfXKTvolvp+IjrUU6ujIGmA=="], - - "@reflink/reflink-linux-x64-gnu": ["@reflink/reflink-linux-x64-gnu@0.1.19", "", { "os": "linux", "cpu": "x64" }, "sha512-jbI8jvuYCaA3MVUdu8vLoLAFqC+iNMpiSuLbxlAgg7x3K5bsS8nOpTRnkLF7vISJ+rVR8W+7ThXlXlUQ93ulkw=="], - - "@reflink/reflink-linux-x64-musl": ["@reflink/reflink-linux-x64-musl@0.1.19", "", { "os": "linux", "cpu": "x64" }, "sha512-e9FBWDe+lv7QKAwtKOt6A2W/fyy/aEEfr0g6j/hWzvQcrzHCsz07BNQYlNOjTfeytrtLU7k449H1PI95jA4OjQ=="], - - "@reflink/reflink-win32-arm64-msvc": ["@reflink/reflink-win32-arm64-msvc@0.1.19", "", { "os": "win32", "cpu": "arm64" }, "sha512-09PxnVIQcd+UOn4WAW73WU6PXL7DwGS6wPlkMhMg2zlHHG65F3vHepOw06HFCq+N42qkaNAc8AKIabWvtk6cIQ=="], - - "@reflink/reflink-win32-x64-msvc": ["@reflink/reflink-win32-x64-msvc@0.1.19", "", { "os": "win32", "cpu": "x64" }, "sha512-E//yT4ni2SyhwP8JRjVGWr3cbnhWDiPLgnQ66qqaanjjnMiu3O/2tjCPQXlcGc/DEYofpDc9fvhv6tALQsMV9w=="], - - "@tinyhttp/content-disposition": ["@tinyhttp/content-disposition@2.2.2", "", {}, "sha512-crXw1txzrS36huQOyQGYFvhTeLeG0Si1xu+/l6kXUVYpE0TjFjEZRqTbuadQLfKGZ0jaI+jJoRyqaWwxOSHW2g=="], - - "@types/aws-lambda": ["@types/aws-lambda@8.10.159", "", {}, "sha512-SAP22WSGNN12OQ8PlCzGzRCZ7QDCwI85dQZbmpz7+mAk+L7j+wI7qnvmdKh+o7A5LaOp6QnOZ2NJphAZQTTHQg=="], - - "@types/bun": ["@types/bun@1.3.5", "", { "dependencies": { "bun-types": "1.3.5" } }, "sha512-RnygCqNrd3srIPEWBd5LFeUYG7plCoH2Yw9WaZGyNmdTEei+gWaHqydbaIRkIkcbXwhBT94q78QljxN0Sk838w=="], - - "@types/node": ["@types/node@24.10.1", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ=="], - - "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], - - "ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], - - "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], - - "ansi-escapes": ["ansi-escapes@6.2.1", "", {}, "sha512-4nJ3yixlEthEJ9Rk4vPcdBRkZvQZlYyu8j4/Mqz5sgIkddmEnH2Yj2ZrnP9S3tQOvSNRUIgVNF/1yPpRAGNRig=="], - - "ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], - - "ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], - - "aproba": ["aproba@2.1.0", "", {}, "sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew=="], - - "are-we-there-yet": ["are-we-there-yet@3.0.1", "", { "dependencies": { "delegates": "^1.0.0", "readable-stream": "^3.6.0" } }, "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg=="], - - "async-retry": ["async-retry@1.3.3", "", { "dependencies": { "retry": "0.13.1" } }, "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw=="], - - "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], - - "axios": ["axios@1.13.2", "", { "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA=="], - - "before-after-hook": ["before-after-hook@4.0.0", "", {}, "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ=="], - - "body-parser": ["body-parser@2.2.1", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw=="], - - "bottleneck": ["bottleneck@2.19.5", "", {}, "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="], - - "bun-types": ["bun-types@1.3.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="], - - "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], - - "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], - - "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], - - "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], - - "chmodrp": ["chmodrp@1.0.2", "", {}, "sha512-TdngOlFV1FLTzU0o1w8MB6/BFywhtLC0SzRTGJU7T9lmdjlCWeMRt1iVo0Ki+ldwNk0BqNiKoc8xpLZEQ8mY1w=="], - - "chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="], - - "ci-info": ["ci-info@4.3.1", "", {}, "sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA=="], - - "cli-cursor": ["cli-cursor@5.0.0", "", { "dependencies": { "restore-cursor": "^5.0.0" } }, "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw=="], - - "cli-spinners": ["cli-spinners@2.9.2", "", {}, "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg=="], - - "cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], - - "cmake-js": ["cmake-js@7.4.0", "", { "dependencies": { "axios": "^1.6.5", "debug": "^4", "fs-extra": "^11.2.0", "memory-stream": "^1.0.0", "node-api-headers": "^1.1.0", "npmlog": "^6.0.2", "rc": "^1.2.7", "semver": "^7.5.4", "tar": "^6.2.0", "url-join": "^4.0.1", "which": "^2.0.2", "yargs": "^17.7.2" }, "bin": { "cmake-js": "bin/cmake-js" } }, "sha512-Lw0JxEHrmk+qNj1n9W9d4IvkDdYTBn7l2BW6XmtLj7WPpIo2shvxUy+YokfjMxAAOELNonQwX3stkPhM5xSC2Q=="], - - "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], - - "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], - - "color-support": ["color-support@1.1.3", "", { "bin": { "color-support": "bin.js" } }, "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg=="], - - "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], - - "commander": ["commander@10.0.1", "", {}, "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug=="], - - "console-control-strings": ["console-control-strings@1.1.0", "", {}, "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ=="], - - "content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="], - - "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], - - "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], - - "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], - - "cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="], - - "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], - - "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - - "deep-extend": ["deep-extend@0.6.0", "", {}, "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="], - - "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], - - "delegates": ["delegates@1.0.0", "", {}, "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ=="], - - "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], - - "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], - - "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], - - "emoji-regex": ["emoji-regex@10.6.0", "", {}, "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A=="], - - "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], - - "env-var": ["env-var@7.5.0", "", {}, "sha512-mKZOzLRN0ETzau2W2QXefbFjo5EF4yWq28OyKb9ICdeNhHJlOE/pHHnz4hdYJ9cNZXcJHo5xN4OT4pzuSHSNvA=="], - - "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], - - "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], - - "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], - - "es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="], - - "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], - - "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], - - "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], - - "eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="], - - "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], - - "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], - - "express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], - - "express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="], - - "fast-content-type-parse": ["fast-content-type-parse@3.0.0", "", {}, "sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg=="], - - "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], - - "fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="], - - "filename-reserved-regex": ["filename-reserved-regex@3.0.0", "", {}, "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw=="], - - "filenamify": ["filenamify@6.0.0", "", { "dependencies": { "filename-reserved-regex": "^3.0.0" } }, "sha512-vqIlNogKeyD3yzrm0yhRMQg8hOVwYcYRfjEoODd49iCprMn4HL85gK3HcykQE53EPIpX3HcAbGA5ELQv216dAQ=="], - - "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], - - "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], - - "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], - - "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], - - "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], - - "fs-extra": ["fs-extra@11.3.3", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg=="], - - "fs-minipass": ["fs-minipass@2.1.0", "", { "dependencies": { "minipass": "^3.0.0" } }, "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg=="], - - "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], - - "gauge": ["gauge@4.0.4", "", { "dependencies": { "aproba": "^1.0.3 || ^2.0.0", "color-support": "^1.1.3", "console-control-strings": "^1.1.0", "has-unicode": "^2.0.1", "signal-exit": "^3.0.7", "string-width": "^4.2.3", "strip-ansi": "^6.0.1", "wide-align": "^1.1.5" } }, "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg=="], - - "get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="], - - "get-east-asian-width": ["get-east-asian-width@1.4.0", "", {}, "sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q=="], - - "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], - - "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], - - "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], - - "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - - "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], - - "has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="], - - "has-unicode": ["has-unicode@2.0.1", "", {}, "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ=="], - - "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], - - "hono": ["hono@4.11.1", "", {}, "sha512-KsFcH0xxHes0J4zaQgWbYwmz3UPOOskdqZmItstUG93+Wk1ePBLkLGwbP9zlmh1BFUiL8Qp+Xfu9P7feJWpGNg=="], - - "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], - - "iconv-lite": ["iconv-lite@0.7.0", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ=="], - - "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], - - "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], - - "ini": ["ini@1.3.8", "", {}, "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="], - - "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], - - "ipull": ["ipull@3.9.3", "", { "dependencies": { "@tinyhttp/content-disposition": "^2.2.0", "async-retry": "^1.3.3", "chalk": "^5.3.0", "ci-info": "^4.0.0", "cli-spinners": "^2.9.2", "commander": "^10.0.0", "eventemitter3": "^5.0.1", "filenamify": "^6.0.0", "fs-extra": "^11.1.1", "is-unicode-supported": "^2.0.0", "lifecycle-utils": "^2.0.1", "lodash.debounce": "^4.0.8", "lowdb": "^7.0.1", "pretty-bytes": "^6.1.0", "pretty-ms": "^8.0.0", "sleep-promise": "^9.1.0", "slice-ansi": "^7.1.0", "stdout-update": "^4.0.1", "strip-ansi": "^7.1.0" }, "optionalDependencies": { "@reflink/reflink": "^0.1.16" }, "bin": { "ipull": "dist/cli/cli.js" } }, "sha512-ZMkxaopfwKHwmEuGDYx7giNBdLxbHbRCWcQVA1D2eqE4crUguupfxej6s7UqbidYEwT69dkyumYkY8DPHIxF9g=="], - - "is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], - - "is-interactive": ["is-interactive@2.0.0", "", {}, "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ=="], - - "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], - - "is-unicode-supported": ["is-unicode-supported@2.1.0", "", {}, "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ=="], - - "isexe": ["isexe@3.1.1", "", {}, "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ=="], - - "jose": ["jose@6.1.3", "", {}, "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ=="], - - "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], - - "json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="], - - "jsonfile": ["jsonfile@6.2.0", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg=="], - - "lifecycle-utils": ["lifecycle-utils@3.0.1", "", {}, "sha512-Qt/Jl5dsNIsyCAZsHB6x3mbwHFn0HJbdmvF49sVX/bHgX2cW7+G+U+I67Zw+TPM1Sr21Gb2nfJMd2g6iUcI1EQ=="], - - "lodash.debounce": ["lodash.debounce@4.0.8", "", {}, "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow=="], - - "log-symbols": ["log-symbols@7.0.1", "", { "dependencies": { "is-unicode-supported": "^2.0.0", "yoctocolors": "^2.1.1" } }, "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg=="], - - "lowdb": ["lowdb@7.0.1", "", { "dependencies": { "steno": "^4.0.2" } }, "sha512-neJAj8GwF0e8EpycYIDFqEPcx9Qz4GUho20jWFR7YiFeXzF1YMLdxB36PypcTSPMA+4+LvgyMacYhlr18Zlymw=="], - - "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], - - "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], - - "memory-stream": ["memory-stream@1.0.0", "", { "dependencies": { "readable-stream": "^3.4.0" } }, "sha512-Wm13VcsPIMdG96dzILfij09PvuS3APtcKNh7M28FsCA/w6+1mjR7hhPmfFNoilX9xU7wTdhsH5lJAm6XNzdtww=="], - - "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], - - "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], - - "mime-types": ["mime-types@3.0.2", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A=="], - - "mimic-function": ["mimic-function@5.0.1", "", {}, "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA=="], - - "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], - - "minipass": ["minipass@5.0.0", "", {}, "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ=="], - - "minizlib": ["minizlib@2.1.2", "", { "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" } }, "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg=="], - - "mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="], - - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - - "nanoid": ["nanoid@5.1.6", "", { "bin": { "nanoid": "bin/nanoid.js" } }, "sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg=="], - - "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], - - "node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], - - "node-api-headers": ["node-api-headers@1.7.0", "", {}, "sha512-uJMGdkhVwu9+I3UsVvI3KW6ICAy/yDfsu5Br9rSnTtY3WpoaComXvKloiV5wtx0Md2rn0B9n29Ys2WMNwWxj9A=="], - - "node-llama-cpp": ["node-llama-cpp@3.14.5", "", { "dependencies": { "@huggingface/jinja": "^0.5.3", "async-retry": "^1.3.3", "bytes": "^3.1.2", "chalk": "^5.4.1", "chmodrp": "^1.0.2", "cmake-js": "^7.4.0", "cross-spawn": "^7.0.6", "env-var": "^7.5.0", "filenamify": "^6.0.0", "fs-extra": "^11.3.0", "ignore": "^7.0.4", "ipull": "^3.9.2", "is-unicode-supported": "^2.1.0", "lifecycle-utils": "^3.0.1", "log-symbols": "^7.0.0", "nanoid": "^5.1.5", "node-addon-api": "^8.3.1", "octokit": "^5.0.3", "ora": "^8.2.0", "pretty-ms": "^9.2.0", "proper-lockfile": "^4.1.2", "semver": "^7.7.1", "simple-git": "^3.27.0", "slice-ansi": "^7.1.0", "stdout-update": "^4.0.1", "strip-ansi": "^7.1.0", "validate-npm-package-name": "^6.0.0", "which": "^5.0.0", "yargs": "^17.7.2" }, "optionalDependencies": { "@node-llama-cpp/linux-arm64": "3.14.5", "@node-llama-cpp/linux-armv7l": "3.14.5", "@node-llama-cpp/linux-x64": "3.14.5", "@node-llama-cpp/linux-x64-cuda": "3.14.5", "@node-llama-cpp/linux-x64-cuda-ext": "3.14.5", "@node-llama-cpp/linux-x64-vulkan": "3.14.5", "@node-llama-cpp/mac-arm64-metal": "3.14.5", "@node-llama-cpp/mac-x64": "3.14.5", "@node-llama-cpp/win-arm64": "3.14.5", "@node-llama-cpp/win-x64": "3.14.5", "@node-llama-cpp/win-x64-cuda": "3.14.5", "@node-llama-cpp/win-x64-cuda-ext": "3.14.5", "@node-llama-cpp/win-x64-vulkan": "3.14.5" }, "peerDependencies": { "typescript": ">=5.0.0" }, "optionalPeers": ["typescript"], "bin": { "node-llama-cpp": "dist/cli/cli.js", "nlc": "dist/cli/cli.js" } }, "sha512-Db+RFqFMJOOVWprUINq77LVe44FaiJ6JvNiq14r2+DZRgkgyxckSZa6DcZ5Xe5MC+hGA5aqOdnNxsrudUcs74Q=="], - - "npmlog": ["npmlog@6.0.2", "", { "dependencies": { "are-we-there-yet": "^3.0.0", "console-control-strings": "^1.1.0", "gauge": "^4.0.3", "set-blocking": "^2.0.0" } }, "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg=="], - - "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], - - "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], - - "octokit": ["octokit@5.0.5", "", { "dependencies": { "@octokit/app": "^16.1.2", "@octokit/core": "^7.0.6", "@octokit/oauth-app": "^8.0.3", "@octokit/plugin-paginate-graphql": "^6.0.0", "@octokit/plugin-paginate-rest": "^14.0.0", "@octokit/plugin-rest-endpoint-methods": "^17.0.0", "@octokit/plugin-retry": "^8.0.3", "@octokit/plugin-throttling": "^11.0.3", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "@octokit/webhooks": "^14.0.0" } }, "sha512-4+/OFSqOjoyULo7eN7EA97DE0Xydj/PW5aIckxqQIoFjFwqXKuFCvXUJObyJfBF9Khu4RL/jlDRI9FPaMGfPnw=="], - - "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="], - - "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], - - "onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], - - "ora": ["ora@8.2.0", "", { "dependencies": { "chalk": "^5.3.0", "cli-cursor": "^5.0.0", "cli-spinners": "^2.9.2", "is-interactive": "^2.0.0", "is-unicode-supported": "^2.0.0", "log-symbols": "^6.0.0", "stdin-discarder": "^0.2.2", "string-width": "^7.2.0", "strip-ansi": "^7.1.0" } }, "sha512-weP+BZ8MVNnlCm8c0Qdc1WSWq4Qn7I+9CJGm7Qali6g44e/PUzbjNqJX5NJ9ljlNMosfJvg1fKEGILklK9cwnw=="], - - "parse-ms": ["parse-ms@4.0.0", "", {}, "sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw=="], - - "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], - - "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], - - "path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="], - - "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="], - - "pretty-bytes": ["pretty-bytes@6.1.1", "", {}, "sha512-mQUvGU6aUFQ+rNvTIAcZuWGRT9a6f6Yrg9bHs4ImKF+HZCEK+plBvnAZYSIQztknZF2qnzNtr6F8s0+IuptdlQ=="], - - "pretty-ms": ["pretty-ms@9.3.0", "", { "dependencies": { "parse-ms": "^4.0.0" } }, "sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ=="], - - "proper-lockfile": ["proper-lockfile@4.1.2", "", { "dependencies": { "graceful-fs": "^4.2.4", "retry": "^0.12.0", "signal-exit": "^3.0.2" } }, "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA=="], - - "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], - - "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], - - "qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="], - - "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], - - "raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="], - - "rc": ["rc@1.2.8", "", { "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" }, "bin": { "rc": "./cli.js" } }, "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw=="], - - "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], - - "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], - - "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], - - "restore-cursor": ["restore-cursor@5.1.0", "", { "dependencies": { "onetime": "^7.0.0", "signal-exit": "^4.1.0" } }, "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA=="], - - "retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="], - - "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="], - - "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], - - "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], - - "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - - "send": ["send@1.2.0", "", { "dependencies": { "debug": "^4.3.5", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.0", "mime-types": "^3.0.1", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.1" } }, "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw=="], - - "serve-static": ["serve-static@2.2.0", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ=="], - - "set-blocking": ["set-blocking@2.0.0", "", {}, "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw=="], - - "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], - - "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], - - "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - - "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], - - "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], - - "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], - - "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], - - "signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], - - "simple-git": ["simple-git@3.30.0", "", { "dependencies": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", "debug": "^4.4.0" } }, "sha512-q6lxyDsCmEal/MEGhP1aVyQ3oxnagGlBDOVSIB4XUVLl1iZh0Pah6ebC9V4xBap/RfgP2WlI8EKs0WS0rMEJHg=="], - - "sleep-promise": ["sleep-promise@9.1.0", "", {}, "sha512-UHYzVpz9Xn8b+jikYSD6bqvf754xL2uBUzDFwiU6NcdZeifPr6UfgU43xpkPu67VMS88+TI2PSI7Eohgqf2fKA=="], - - "slice-ansi": ["slice-ansi@7.1.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "is-fullwidth-code-point": "^5.0.0" } }, "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w=="], - - "sqlite-vec": ["sqlite-vec@0.1.7-alpha.2", "", { "optionalDependencies": { "sqlite-vec-darwin-arm64": "0.1.7-alpha.2", "sqlite-vec-darwin-x64": "0.1.7-alpha.2", "sqlite-vec-linux-arm64": "0.1.7-alpha.2", "sqlite-vec-linux-x64": "0.1.7-alpha.2", "sqlite-vec-windows-x64": "0.1.7-alpha.2" } }, "sha512-rNgRCv+4V4Ed3yc33Qr+nNmjhtrMnnHzXfLVPeGb28Dx5mmDL3Ngw/Wk8vhCGjj76+oC6gnkmMG8y73BZWGBwQ=="], - - "sqlite-vec-darwin-arm64": ["sqlite-vec-darwin-arm64@0.1.7-alpha.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-raIATOqFYkeCHhb/t3r7W7Cf2lVYdf4J3ogJ6GFc8PQEgHCPEsi+bYnm2JT84MzLfTlSTIdxr4/NKv+zF7oLPw=="], - - "sqlite-vec-darwin-x64": ["sqlite-vec-darwin-x64@0.1.7-alpha.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-jeZEELsQjjRsVojsvU5iKxOvkaVuE+JYC8Y4Ma8U45aAERrDYmqZoHvgSG7cg1PXL3bMlumFTAmHynf1y4pOzA=="], - - "sqlite-vec-linux-arm64": ["sqlite-vec-linux-arm64@0.1.7-alpha.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-6Spj4Nfi7tG13jsUG+W7jnT0bCTWbyPImu2M8nWp20fNrd1SZ4g3CSlDAK8GBdavX7wRlbBHCZ+BDa++rbDewA=="], - - "sqlite-vec-linux-x64": ["sqlite-vec-linux-x64@0.1.7-alpha.2", "", { "os": "linux", "cpu": "x64" }, "sha512-IcgrbHaDccTVhXDf8Orwdc2+hgDLAFORl6OBUhcvlmwswwBP1hqBTSEhovClG4NItwTOBNgpwOoQ7Qp3VDPWLg=="], - - "sqlite-vec-windows-x64": ["sqlite-vec-windows-x64@0.1.7-alpha.2", "", { "os": "win32", "cpu": "x64" }, "sha512-TRP6hTjAcwvQ6xpCZvjP00pdlda8J38ArFy1lMYhtQWXiIBmWnhMaMbq4kaeCYwvTTddfidatRS+TJrwIKB/oQ=="], - - "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], - - "stdin-discarder": ["stdin-discarder@0.2.2", "", {}, "sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ=="], - - "stdout-update": ["stdout-update@4.0.1", "", { "dependencies": { "ansi-escapes": "^6.2.0", "ansi-styles": "^6.2.1", "string-width": "^7.1.0", "strip-ansi": "^7.1.0" } }, "sha512-wiS21Jthlvl1to+oorePvcyrIkiG/6M3D3VTmDUlJm7Cy6SbFhKkAvX+YBuHLxck/tO3mrdpC/cNesigQc3+UQ=="], - - "steno": ["steno@4.0.2", "", {}, "sha512-yhPIQXjrlt1xv7dyPQg2P17URmXbuM5pdGkpiMB3RenprfiBlvK415Lctfe0eshk90oA7/tNq7WEiMK8RSP39A=="], - - "string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], - - "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - - "strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="], - - "strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="], - - "tar": ["tar@6.2.1", "", { "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", "minipass": "^5.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" } }, "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A=="], - - "toad-cache": ["toad-cache@3.7.0", "", {}, "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw=="], - - "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], - - "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], - - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - - "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], - - "universal-github-app-jwt": ["universal-github-app-jwt@2.2.2", "", {}, "sha512-dcmbeSrOdTnsjGjUfAlqNDJrhxXizjAz94ija9Qw8YkZ1uu0d+GoZzyH+Jb9tIIqvGsadUfwg+22k5aDqqwzbw=="], - - "universal-user-agent": ["universal-user-agent@7.0.3", "", {}, "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A=="], - - "universalify": ["universalify@2.0.1", "", {}, "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw=="], - - "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], - - "url-join": ["url-join@4.0.1", "", {}, "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA=="], - - "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], - - "validate-npm-package-name": ["validate-npm-package-name@6.0.2", "", {}, "sha512-IUoow1YUtvoBBC06dXs8bR8B9vuA3aJfmQNKMoaPG/OFsPmoQvw8xh+6Ye25Gx9DQhoEom3Pcu9MKHerm/NpUQ=="], - - "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], - - "which": ["which@5.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ=="], - - "wide-align": ["wide-align@1.1.5", "", { "dependencies": { "string-width": "^1.0.2 || 2 || 3 || 4" } }, "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg=="], - - "wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], - - "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], - - "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], - - "yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], - - "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], - - "yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], - - "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], - - "yoctocolors": ["yoctocolors@2.1.2", "", {}, "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug=="], - - "zod": ["zod@4.2.1", "", {}, "sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw=="], - - "zod-to-json-schema": ["zod-to-json-schema@3.25.0", "", { "peerDependencies": { "zod": "^3.25 || ^4" } }, "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ=="], - - "cliui/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - - "cliui/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - - "cmake-js/which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], - - "cross-spawn/which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], - - "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - - "fs-minipass/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], - - "gauge/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - - "gauge/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - - "ipull/lifecycle-utils": ["lifecycle-utils@2.1.0", "", {}, "sha512-AnrXnE2/OF9PHCyFg0RSqsnQTzV991XaZA/buhFDoc58xU7rhSCDgCz/09Lqpsn4MpoPHt7TRAXV1kWZypFVsA=="], - - "ipull/pretty-ms": ["pretty-ms@8.0.0", "", { "dependencies": { "parse-ms": "^3.0.0" } }, "sha512-ASJqOugUF1bbzI35STMBUpZqdfYKlJugy6JBziGi2EE+AL5JPJGSzvpeVXojxrr0ViUYoToUjb5kjSEGf7Y83Q=="], - - "minizlib/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], - - "ora/log-symbols": ["log-symbols@6.0.0", "", { "dependencies": { "chalk": "^5.3.0", "is-unicode-supported": "^1.3.0" } }, "sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw=="], - - "proper-lockfile/retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="], - - "restore-cursor/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], - - "wide-align/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - - "wrap-ansi/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], - - "wrap-ansi/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - - "wrap-ansi/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - - "yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - - "cliui/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], - - "cliui/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], - - "cliui/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - - "cmake-js/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - - "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - - "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], - - "gauge/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], - - "gauge/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], - - "gauge/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - - "ipull/pretty-ms/parse-ms": ["parse-ms@3.0.0", "", {}, "sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw=="], - - "ora/log-symbols/is-unicode-supported": ["is-unicode-supported@1.3.0", "", {}, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="], - - "wide-align/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], - - "wide-align/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], - - "wide-align/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - - "wrap-ansi/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], - - "wrap-ansi/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], - - "wrap-ansi/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - - "yargs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], - - "yargs/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], - - "yargs/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - - "wide-align/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - - "yargs/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - } -} diff --git a/tools/qmd/package.json b/tools/qmd/package.json deleted file mode 100644 index 4ac978a5..00000000 --- a/tools/qmd/package.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "name": "qmd", - "version": "1.0.0", - "description": "Quick Markdown Search - Full-text and vector search for markdown files", - "type": "module", - "bin": { - "qmd": "./qmd" - }, - "scripts": { - "test": "bun test", - "qmd": "bun src/qmd.ts", - "index": "bun src/qmd.ts index", - "vector": "bun src/qmd.ts vector", - "search": "bun src/qmd.ts search", - "vsearch": "bun src/qmd.ts vsearch", - "rerank": "bun src/qmd.ts rerank", - "link": "bun link", - "inspector": "npx @modelcontextprotocol/inspector bun src/qmd.ts mcp" - }, - "dependencies": { - "@modelcontextprotocol/sdk": "^1.25.1", - "node-llama-cpp": "^3.14.5", - "sqlite-vec": "^0.1.7-alpha.2", - "yaml": "^2.8.2", - "zod": "^4.2.1" - }, - "optionalDependencies": { - "sqlite-vec-darwin-arm64": "^0.1.7-alpha.2", - "sqlite-vec-darwin-x64": "^0.1.7-alpha.2", - "sqlite-vec-linux-x64": "^0.1.7-alpha.2", - "sqlite-vec-win32-x64": "^0.1.7-alpha.2" - }, - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5.9.3" - }, - "engines": { - "bun": ">=1.0.0" - }, - "keywords": [ - "markdown", - "search", - "fts", - "vector", - "sqlite", - "bm25", - "embeddings", - "ollama" - ], - "license": "MIT" -} diff --git a/tools/qmd/src/cli.test.ts b/tools/qmd/src/cli.test.ts deleted file mode 100644 index 89300be3..00000000 --- a/tools/qmd/src/cli.test.ts +++ /dev/null @@ -1,963 +0,0 @@ -/** - * CLI Integration Tests - * - * Tests all qmd CLI commands using a temporary test database via INDEX_PATH. - * These tests spawn actual qmd processes to verify end-to-end functionality. - */ - -import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test"; -import { mkdtemp, rm, writeFile, mkdir } from "fs/promises"; -import { tmpdir } from "os"; -import { join } from "path"; - -// Test fixtures directory and database path -let testDir: string; -let testDbPath: string; -let testConfigDir: string; -let fixturesDir: string; -let testCounter = 0; // Unique counter for each test run - -// Get the directory where this test file lives (same as qmd.ts) -const qmdDir = import.meta.dir; -const qmdScript = join(qmdDir, "qmd.ts"); - -// Helper to run qmd command with test database -async function runQmd( - args: string[], - options: { cwd?: string; env?: Record; dbPath?: string; configDir?: string } = {} -): Promise<{ stdout: string; stderr: string; exitCode: number }> { - const workingDir = options.cwd || fixturesDir; - const dbPath = options.dbPath || testDbPath; - const configDir = options.configDir || testConfigDir; - const proc = Bun.spawn(["bun", qmdScript, ...args], { - cwd: workingDir, - env: { - ...process.env, - INDEX_PATH: dbPath, - QMD_CONFIG_DIR: configDir, // Use test config directory - PWD: workingDir, // Must explicitly set PWD since getPwd() checks this - ...options.env, - }, - stdout: "pipe", - stderr: "pipe", - }); - - const stdout = await new Response(proc.stdout).text(); - const stderr = await new Response(proc.stderr).text(); - const exitCode = await proc.exited; - - return { stdout, stderr, exitCode }; -} - -// Get a fresh database path for isolated tests -function getFreshDbPath(): string { - testCounter++; - return join(testDir, `test-${testCounter}.sqlite`); -} - -// Create an isolated test environment (db + config dir) -async function createIsolatedTestEnv(prefix: string): Promise<{ dbPath: string; configDir: string }> { - testCounter++; - const dbPath = join(testDir, `${prefix}-${testCounter}.sqlite`); - const configDir = join(testDir, `${prefix}-config-${testCounter}`); - await mkdir(configDir, { recursive: true }); - await writeFile(join(configDir, "index.yml"), "collections: {}\n"); - return { dbPath, configDir }; -} - -// Setup test fixtures -beforeAll(async () => { - // Create temp directory structure - testDir = await mkdtemp(join(tmpdir(), "qmd-test-")); - testDbPath = join(testDir, "test.sqlite"); - testConfigDir = join(testDir, "config"); - fixturesDir = join(testDir, "fixtures"); - - await mkdir(testConfigDir, { recursive: true }); - await mkdir(fixturesDir, { recursive: true }); - await mkdir(join(fixturesDir, "notes"), { recursive: true }); - await mkdir(join(fixturesDir, "docs"), { recursive: true }); - - // Create empty YAML config for tests - await writeFile( - join(testConfigDir, "index.yml"), - "collections: {}\n" - ); - - // Create test markdown files - await writeFile( - join(fixturesDir, "README.md"), - `# Test Project - -This is a test project for QMD CLI testing. - -## Features - -- Full-text search with BM25 -- Vector similarity search -- Hybrid search with reranking -` - ); - - await writeFile( - join(fixturesDir, "notes", "meeting.md"), - `# Team Meeting Notes - -Date: 2024-01-15 - -## Attendees -- Alice -- Bob -- Charlie - -## Discussion Topics -- Project timeline review -- Resource allocation -- Technical debt prioritization - -## Action Items -1. Alice to update documentation -2. Bob to fix authentication bug -3. Charlie to review pull requests -` - ); - - await writeFile( - join(fixturesDir, "notes", "ideas.md"), - `# Product Ideas - -## Feature Requests -- Dark mode support -- Keyboard shortcuts -- Export to PDF - -## Technical Improvements -- Improve search performance -- Add caching layer -- Optimize database queries -` - ); - - await writeFile( - join(fixturesDir, "docs", "api.md"), - `# API Documentation - -## Endpoints - -### GET /search -Search for documents. - -Parameters: -- q: Search query (required) -- limit: Max results (default: 10) - -### GET /document/:id -Retrieve a specific document. - -### POST /index -Index new documents. -` - ); - - // Create test files for path normalization tests - await writeFile( - join(fixturesDir, "test1.md"), - `# Test Document 1 - -This is the first test document. - -It has multiple lines for testing line numbers. -Line 6 is here. -Line 7 is here. -` - ); - - await writeFile( - join(fixturesDir, "test2.md"), - `# Test Document 2 - -This is the second test document. -` - ); -}); - -// Cleanup after all tests -afterAll(async () => { - if (testDir) { - await rm(testDir, { recursive: true, force: true }); - } -}); - -// Reset YAML config before each test to ensure isolation -beforeEach(async () => { - // Reset to empty collections config - await writeFile( - join(testConfigDir, "index.yml"), - "collections: {}\n" - ); -}); - -describe("CLI Help", () => { - test("shows help with --help flag", async () => { - const { stdout, exitCode } = await runQmd(["--help"]); - expect(exitCode).toBe(0); - expect(stdout).toContain("Usage:"); - expect(stdout).toContain("qmd collection add"); - expect(stdout).toContain("qmd search"); - }); - - test("shows help with no arguments", async () => { - const { stdout, exitCode } = await runQmd([]); - expect(exitCode).toBe(1); - expect(stdout).toContain("Usage:"); - }); -}); - -describe("CLI Add Command", () => { - test("adds files from current directory", async () => { - const { stdout, exitCode } = await runQmd(["collection", "add", "."]); - expect(exitCode).toBe(0); - expect(stdout).toContain("Collection:"); - expect(stdout).toContain("Indexed:"); - }); - - test("adds files with custom glob pattern", async () => { - const { stdout, stderr, exitCode } = await runQmd(["collection", "add", ".", "--mask", "notes/*.md"]); - if (exitCode !== 0) { - console.error("Command failed:", stderr); - } - expect(exitCode).toBe(0); - expect(stdout).toContain("Collection:"); - // Should find meeting.md and ideas.md in notes/ - expect(stdout).toContain("notes/*.md"); - }); - - test("can recreate collection with remove and add", async () => { - // First add - await runQmd(["collection", "add", "."]); - // Remove it - await runQmd(["collection", "remove", "fixtures"]); - // Re-add - const { stdout, exitCode } = await runQmd(["collection", "add", "."]); - expect(exitCode).toBe(0); - expect(stdout).toContain("Collection 'fixtures' created successfully"); - }); -}); - -describe("CLI Status Command", () => { - beforeEach(async () => { - // Ensure we have indexed files - await runQmd(["collection", "add", "."]); - }); - - test("shows index status", async () => { - const { stdout, exitCode } = await runQmd(["status"]); - expect(exitCode).toBe(0); - // Should show collection info - expect(stdout).toContain("Collection"); - }); -}); - -describe("CLI Search Command", () => { - beforeEach(async () => { - // Ensure we have indexed files - await runQmd(["collection", "add", "."]); - }); - - test("searches for documents with BM25", async () => { - const { stdout, exitCode } = await runQmd(["search", "meeting"]); - expect(exitCode).toBe(0); - // Should find meeting.md - expect(stdout.toLowerCase()).toContain("meeting"); - }); - - test("searches with limit option", async () => { - const { stdout, exitCode } = await runQmd(["search", "-n", "1", "test"]); - expect(exitCode).toBe(0); - }); - - test("searches with all results option", async () => { - const { stdout, exitCode } = await runQmd(["search", "--all", "the"]); - expect(exitCode).toBe(0); - }); - - test("returns no results message for non-matching query", async () => { - const { stdout, exitCode } = await runQmd(["search", "xyznonexistent123"]); - expect(exitCode).toBe(0); - expect(stdout).toContain("No results"); - }); - - test("requires query argument", async () => { - const { stdout, stderr, exitCode } = await runQmd(["search"]); - expect(exitCode).toBe(1); - // Error message goes to stderr - expect(stderr).toContain("Usage:"); - }); -}); - -describe("CLI Get Command", () => { - beforeEach(async () => { - // Ensure we have indexed files - await runQmd(["collection", "add", "."]); - }); - - test("retrieves document content by path", async () => { - const { stdout, exitCode } = await runQmd(["get", "README.md"]); - expect(exitCode).toBe(0); - expect(stdout).toContain("Test Project"); - }); - - test("retrieves document from subdirectory", async () => { - const { stdout, exitCode } = await runQmd(["get", "notes/meeting.md"]); - expect(exitCode).toBe(0); - expect(stdout).toContain("Team Meeting"); - }); - - test("handles non-existent file", async () => { - const { stdout, exitCode } = await runQmd(["get", "nonexistent.md"]); - // Should indicate file not found - expect(exitCode).toBe(1); - }); -}); - -describe("CLI Multi-Get Command", () => { - let localDbPath: string; - - beforeEach(async () => { - // Use fresh database for each test - localDbPath = getFreshDbPath(); - // Ensure we have indexed files - const addResult = await runQmd(["collection", "add", ".", "--name", "fixtures"], { dbPath: localDbPath }); - if (addResult.exitCode !== 0) { - throw new Error(`Failed to add collection: ${addResult.stderr}`); - } - }); - - test("retrieves multiple documents by pattern", async () => { - // Test glob pattern matching - const { stdout, stderr, exitCode } = await runQmd(["multi-get", "notes/*.md"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - // Should contain content from both notes files - expect(stdout).toContain("Meeting"); - expect(stdout).toContain("Ideas"); - }); - - test("retrieves documents by comma-separated paths", async () => { - const { stdout, exitCode } = await runQmd([ - "multi-get", - "README.md,notes/meeting.md", - ], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Test Project"); - expect(stdout).toContain("Team Meeting"); - }); -}); - -describe("CLI Update Command", () => { - let localDbPath: string; - - beforeEach(async () => { - // Use a fresh database for this test suite - localDbPath = getFreshDbPath(); - // Ensure we have indexed files - await runQmd(["collection", "add", "."], { dbPath: localDbPath }); - }); - - test("updates all collections", async () => { - const { stdout, exitCode } = await runQmd(["update"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Updating"); - }); -}); - -describe("CLI Add-Context Command", () => { - let localDbPath: string; - let localConfigDir: string; - const collName = "fixtures"; - - beforeAll(async () => { - const env = await createIsolatedTestEnv("context-cmd"); - localDbPath = env.dbPath; - localConfigDir = env.configDir; - - // Add collection with known name - const { exitCode, stderr } = await runQmd( - ["collection", "add", fixturesDir, "--name", collName], - { dbPath: localDbPath, configDir: localConfigDir } - ); - if (exitCode !== 0) console.error("collection add failed:", stderr); - expect(exitCode).toBe(0); - }); - - test("adds context to a path", async () => { - // Add context to the collection root using virtual path - const { stdout, exitCode } = await runQmd([ - "context", - "add", - `qmd://${collName}/`, - "Personal notes and meeting logs", - ], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - expect(stdout).toContain("✓ Added context"); - }); - - test("requires path and text arguments", async () => { - const { stderr, exitCode } = await runQmd(["context", "add"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(1); - // Error message goes to stderr - expect(stderr).toContain("Usage:"); - }); -}); - -describe("CLI Cleanup Command", () => { - beforeEach(async () => { - // Ensure we have indexed files - await runQmd(["collection", "add", "."]); - }); - - test("cleans up orphaned entries", async () => { - const { stdout, exitCode } = await runQmd(["cleanup"]); - expect(exitCode).toBe(0); - }); -}); - -describe("CLI Error Handling", () => { - test("handles unknown command", async () => { - const { stderr, exitCode } = await runQmd(["unknowncommand"]); - expect(exitCode).toBe(1); - // Should indicate unknown command - expect(stderr).toContain("Unknown command"); - }); - - test("uses INDEX_PATH environment variable", async () => { - // Verify the test DB path is being used by creating a separate index - const customDbPath = join(testDir, "custom.sqlite"); - const { exitCode } = await runQmd(["collection", "add", "."], { - env: { INDEX_PATH: customDbPath }, - }); - expect(exitCode).toBe(0); - - // The custom database should exist - const file = Bun.file(customDbPath); - expect(await file.exists()).toBe(true); - }); -}); - -describe("CLI Output Formats", () => { - beforeEach(async () => { - await runQmd(["collection", "add", "."]); - }); - - test("search with --json flag outputs JSON", async () => { - const { stdout, exitCode } = await runQmd(["search", "--json", "test"]); - expect(exitCode).toBe(0); - // Should be valid JSON - const parsed = JSON.parse(stdout); - expect(Array.isArray(parsed)).toBe(true); - }); - - test("search with --files flag outputs file paths", async () => { - const { stdout, exitCode } = await runQmd(["search", "--files", "meeting"]); - expect(exitCode).toBe(0); - expect(stdout).toContain(".md"); - }); - - test("search output includes snippets by default", async () => { - const { stdout, exitCode } = await runQmd(["search", "API"]); - expect(exitCode).toBe(0); - // If results found, should have snippet content - if (!stdout.includes("No results")) { - expect(stdout.toLowerCase()).toContain("api"); - } - }); -}); - -describe("CLI Search with Collection Filter", () => { - let localDbPath: string; - - beforeEach(async () => { - // Use a fresh database for this test suite - localDbPath = getFreshDbPath(); - // Create multiple collections with explicit names - await runQmd(["collection", "add", ".", "--name", "notes", "--mask", "notes/*.md"], { dbPath: localDbPath }); - await runQmd(["collection", "add", ".", "--name", "docs", "--mask", "docs/*.md"], { dbPath: localDbPath }); - }); - - test("filters search by collection name", async () => { - const { stdout, stderr, exitCode } = await runQmd([ - "search", - "-c", - "notes", - "meeting", - ], { dbPath: localDbPath }); - if (exitCode !== 0) { - console.log("Collection filter search failed:"); - console.log("stdout:", stdout); - console.log("stderr:", stderr); - } - expect(exitCode).toBe(0); - }); -}); - -describe("CLI Context Management", () => { - let localDbPath: string; - - beforeEach(async () => { - // Use a fresh database for this test suite - localDbPath = getFreshDbPath(); - // Index some files first - await runQmd(["collection", "add", "."], { dbPath: localDbPath }); - }); - - test("add global context with /", async () => { - const { stdout, exitCode } = await runQmd([ - "context", - "add", - "/", - "Global system context", - ], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("✓ Set global context"); - expect(stdout).toContain("Global system context"); - }); - - test("list contexts", async () => { - // Add a global context first - await runQmd([ - "context", - "add", - "/", - "Test context", - ], { dbPath: localDbPath }); - - const { stdout, exitCode } = await runQmd([ - "context", - "list", - ], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Configured Contexts"); - expect(stdout).toContain("Test context"); - }); - - test("add context to virtual path", async () => { - // Collection name should be "fixtures" (basename of the fixtures directory) - const { stdout, exitCode } = await runQmd([ - "context", - "add", - "qmd://fixtures/notes", - "Context for notes subdirectory", - ], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("✓ Added context for: qmd://fixtures/notes"); - }); - - test("remove global context", async () => { - // Add a global context first - await runQmd([ - "context", - "add", - "/", - "Global context to remove", - ], { dbPath: localDbPath }); - - const { stdout, exitCode } = await runQmd([ - "context", - "rm", - "/", - ], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("✓ Removed"); - }); - - test("remove virtual path context", async () => { - // Add a context first - await runQmd([ - "context", - "add", - "qmd://fixtures/notes", - "Context to remove", - ], { dbPath: localDbPath }); - - const { stdout, exitCode } = await runQmd([ - "context", - "rm", - "qmd://fixtures/notes", - ], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("✓ Removed context for: qmd://fixtures/notes"); - }); - - test("fails to remove non-existent context", async () => { - const { stdout, stderr, exitCode } = await runQmd([ - "context", - "rm", - "qmd://nonexistent/path", - ], { dbPath: localDbPath }); - expect(exitCode).toBe(1); - expect(stderr || stdout).toContain("not found"); - }); -}); - -describe("CLI ls Command", () => { - let localDbPath: string; - - beforeEach(async () => { - // Use a fresh database for this test suite - localDbPath = getFreshDbPath(); - // Index some files first - await runQmd(["collection", "add", "."], { dbPath: localDbPath }); - }); - - test("lists all collections", async () => { - const { stdout, exitCode } = await runQmd(["ls"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Collections:"); - expect(stdout).toContain("qmd://fixtures/"); - }); - - test("lists files in a collection", async () => { - const { stdout, exitCode } = await runQmd(["ls", "fixtures"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - // handelize converts to lowercase - expect(stdout).toContain("qmd://fixtures/readme.md"); - expect(stdout).toContain("qmd://fixtures/notes/meeting.md"); - }); - - test("lists files with path prefix", async () => { - const { stdout, exitCode } = await runQmd(["ls", "fixtures/notes"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("qmd://fixtures/notes/meeting.md"); - expect(stdout).toContain("qmd://fixtures/notes/ideas.md"); - // Should not include files outside the prefix (handelize converts to lowercase) - expect(stdout).not.toContain("qmd://fixtures/readme.md"); - }); - - test("lists files with virtual path", async () => { - const { stdout, exitCode } = await runQmd(["ls", "qmd://fixtures/docs"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("qmd://fixtures/docs/api.md"); - }); - - test("handles non-existent collection", async () => { - const { stderr, exitCode } = await runQmd(["ls", "nonexistent"], { dbPath: localDbPath }); - expect(exitCode).toBe(1); - expect(stderr).toContain("Collection not found"); - }); -}); - -describe("CLI Collection Commands", () => { - let localDbPath: string; - - beforeEach(async () => { - // Use a fresh database for this test suite - localDbPath = getFreshDbPath(); - // Index some files first to create a collection - await runQmd(["collection", "add", "."], { dbPath: localDbPath }); - }); - - test("lists collections", async () => { - const { stdout, exitCode } = await runQmd(["collection", "list"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Collections"); - expect(stdout).toContain("fixtures"); - expect(stdout).toContain("qmd://fixtures/"); - expect(stdout).toContain("Pattern:"); - expect(stdout).toContain("Files:"); - }); - - test("removes a collection", async () => { - // First verify the collection exists - const { stdout: listBefore } = await runQmd(["collection", "list"], { dbPath: localDbPath }); - expect(listBefore).toContain("fixtures"); - - // Remove it - const { stdout, exitCode } = await runQmd(["collection", "remove", "fixtures"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("✓ Removed collection 'fixtures'"); - expect(stdout).toContain("Deleted"); - - // Verify it's gone - const { stdout: listAfter } = await runQmd(["collection", "list"], { dbPath: localDbPath }); - expect(listAfter).not.toContain("fixtures"); - }); - - test("handles removing non-existent collection", async () => { - const { stderr, exitCode } = await runQmd(["collection", "remove", "nonexistent"], { dbPath: localDbPath }); - expect(exitCode).toBe(1); - expect(stderr).toContain("Collection not found"); - }); - - test("handles missing remove argument", async () => { - const { stderr, exitCode } = await runQmd(["collection", "remove"], { dbPath: localDbPath }); - expect(exitCode).toBe(1); - expect(stderr).toContain("Usage:"); - }); - - test("handles unknown subcommand", async () => { - const { stderr, exitCode } = await runQmd(["collection", "invalid"], { dbPath: localDbPath }); - expect(exitCode).toBe(1); - expect(stderr).toContain("Unknown subcommand"); - }); - - test("renames a collection", async () => { - // First verify the collection exists - const { stdout: listBefore } = await runQmd(["collection", "list"], { dbPath: localDbPath }); - expect(listBefore).toContain("qmd://fixtures/"); - - // Rename it - const { stdout, exitCode } = await runQmd(["collection", "rename", "fixtures", "my-fixtures"], { dbPath: localDbPath }); - expect(exitCode).toBe(0); - expect(stdout).toContain("✓ Renamed collection 'fixtures' to 'my-fixtures'"); - expect(stdout).toContain("qmd://fixtures/"); - expect(stdout).toContain("qmd://my-fixtures/"); - - // Verify the new name exists and old name is gone - const { stdout: listAfter } = await runQmd(["collection", "list"], { dbPath: localDbPath }); - expect(listAfter).toContain("qmd://my-fixtures/"); - expect(listAfter).not.toContain("qmd://fixtures/"); // Old collection should not appear - }); - - test("handles renaming non-existent collection", async () => { - const { stderr, exitCode } = await runQmd(["collection", "rename", "nonexistent", "newname"], { dbPath: localDbPath }); - expect(exitCode).toBe(1); - expect(stderr).toContain("Collection not found"); - }); - - test("handles renaming to existing collection name", async () => { - // Create a second collection in a temp directory - const tempDir = await mkdtemp(join(tmpdir(), "qmd-second-")); - await writeFile(join(tempDir, "test.md"), "# Test"); - const addResult = await runQmd(["collection", "add", tempDir, "--name", "second"], { dbPath: localDbPath }); - - if (addResult.exitCode !== 0) { - console.error("Failed to add second collection:", addResult.stderr); - } - expect(addResult.exitCode).toBe(0); - - // Verify both collections exist - const { stdout: listBoth } = await runQmd(["collection", "list"], { dbPath: localDbPath }); - expect(listBoth).toContain("qmd://fixtures/"); - expect(listBoth).toContain("qmd://second/"); - - // Try to rename fixtures to second (which already exists) - const { stderr, exitCode } = await runQmd(["collection", "rename", "fixtures", "second"], { dbPath: localDbPath }); - expect(exitCode).toBe(1); - expect(stderr).toContain("Collection name already exists"); - }); - - test("handles missing rename arguments", async () => { - const { stderr: stderr1, exitCode: exitCode1 } = await runQmd(["collection", "rename"], { dbPath: localDbPath }); - expect(exitCode1).toBe(1); - expect(stderr1).toContain("Usage:"); - - const { stderr: stderr2, exitCode: exitCode2 } = await runQmd(["collection", "rename", "fixtures"], { dbPath: localDbPath }); - expect(exitCode2).toBe(1); - expect(stderr2).toContain("Usage:"); - }); -}); - -// ============================================================================= -// Output Format Tests - qmd:// URIs, context, and docid -// ============================================================================= - -describe("search output formats", () => { - let localDbPath: string; - let localConfigDir: string; - const collName = "fixtures"; - - beforeAll(async () => { - const env = await createIsolatedTestEnv("output-format"); - localDbPath = env.dbPath; - localConfigDir = env.configDir; - - // Add collection - const { exitCode, stderr } = await runQmd( - ["collection", "add", fixturesDir, "--name", collName], - { dbPath: localDbPath, configDir: localConfigDir } - ); - if (exitCode !== 0) console.error("collection add failed:", stderr); - expect(exitCode).toBe(0); - - // Add context - await runQmd(["context", "add", `qmd://${collName}/`, "Test fixtures for QMD"], { dbPath: localDbPath, configDir: localConfigDir }); - }); - - test("search --json includes qmd:// path, docid, and context", async () => { - const { stdout, exitCode } = await runQmd(["search", "test", "--json", "-n", "1"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - const results = JSON.parse(stdout); - expect(results.length).toBeGreaterThan(0); - - const result = results[0]; - expect(result.file).toMatch(new RegExp(`^qmd://${collName}/`)); - expect(result.docid).toMatch(/^#[a-f0-9]{6}$/); - expect(result.context).toBe("Test fixtures for QMD"); - // Ensure no full filesystem paths - expect(result.file).not.toMatch(/^\/Users\//); - expect(result.file).not.toMatch(/^\/home\//); - }); - - test("search --files includes qmd:// path, docid, and context", async () => { - const { stdout, exitCode } = await runQmd(["search", "test", "--files", "-n", "1"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - // Format: #docid,score,qmd://collection/path,"context" - expect(stdout).toMatch(new RegExp(`^#[a-f0-9]{6},[\\d.]+,qmd://${collName}/`, "m")); - expect(stdout).toContain("Test fixtures for QMD"); - // Ensure no full filesystem paths - expect(stdout).not.toMatch(/\/Users\//); - expect(stdout).not.toMatch(/\/home\//); - }); - - test("search --csv includes qmd:// path, docid, and context", async () => { - const { stdout, exitCode } = await runQmd(["search", "test", "--csv", "-n", "1"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - // Header should include context - expect(stdout).toMatch(/^docid,score,file,title,context,line,snippet$/m); - // Data rows should have qmd:// paths and context - expect(stdout).toMatch(new RegExp(`#[a-f0-9]{6},[\\d.]+,qmd://${collName}/`)); - expect(stdout).toContain("Test fixtures for QMD"); - // Ensure no full filesystem paths - expect(stdout).not.toMatch(/\/Users\//); - expect(stdout).not.toMatch(/\/home\//); - }); - - test("search --md includes docid and context", async () => { - const { stdout, exitCode } = await runQmd(["search", "test", "--md", "-n", "1"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - expect(stdout).toMatch(/\*\*docid:\*\* `#[a-f0-9]{6}`/); - expect(stdout).toContain("**context:** Test fixtures for QMD"); - }); - - test("search --xml includes qmd:// path, docid, and context", async () => { - const { stdout, exitCode } = await runQmd(["search", "test", "--xml", "-n", "1"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - expect(stdout).toMatch(new RegExp(` { - const { stdout, exitCode } = await runQmd(["search", "test", "-n", "1"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - // First line should have qmd:// path and docid - expect(stdout).toMatch(new RegExp(`^qmd://${collName}/.*#[a-f0-9]{6}`, "m")); - expect(stdout).toContain("Context: Test fixtures for QMD"); - // Ensure no full filesystem paths - expect(stdout).not.toMatch(/\/Users\//); - expect(stdout).not.toMatch(/\/home\//); - }); -}); - -// ============================================================================= -// Get Command Path Normalization Tests -// ============================================================================= - -describe("get command path normalization", () => { - let localDbPath: string; - let localConfigDir: string; - const collName = "fixtures"; - - beforeAll(async () => { - const env = await createIsolatedTestEnv("get-paths"); - localDbPath = env.dbPath; - localConfigDir = env.configDir; - - const { exitCode, stderr } = await runQmd( - ["collection", "add", fixturesDir, "--name", collName], - { dbPath: localDbPath, configDir: localConfigDir } - ); - if (exitCode !== 0) console.error("collection add failed:", stderr); - expect(exitCode).toBe(0); - }); - - test("get with qmd://collection/path format", async () => { - const { stdout, exitCode } = await runQmd(["get", `qmd://${collName}/test1.md`, "-l", "3"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Test Document 1"); - }); - - test("get with collection/path format (no scheme)", async () => { - const { stdout, exitCode } = await runQmd(["get", `${collName}/test1.md`, "-l", "3"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Test Document 1"); - }); - - test("get with //collection/path format", async () => { - const { stdout, exitCode } = await runQmd(["get", `//${collName}/test1.md`, "-l", "3"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Test Document 1"); - }); - - test("get with qmd:////collection/path format (extra slashes)", async () => { - const { stdout, exitCode } = await runQmd(["get", `qmd:////${collName}/test1.md`, "-l", "3"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - expect(stdout).toContain("Test Document 1"); - }); - - test("get with path:line format", async () => { - const { stdout, exitCode } = await runQmd(["get", `${collName}/test1.md:3`, "-l", "2"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - // Should start from line 3, not line 1 - expect(stdout).not.toMatch(/^# Test Document 1$/m); - }); - - test("get with qmd://path:line format", async () => { - const { stdout, exitCode } = await runQmd(["get", `qmd://${collName}/test1.md:3`, "-l", "2"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - // Should start from line 3, not line 1 - expect(stdout).not.toMatch(/^# Test Document 1$/m); - }); -}); - -// ============================================================================= -// Status and Collection List - No Full Paths -// ============================================================================= - -describe("status and collection list hide filesystem paths", () => { - let localDbPath: string; - let localConfigDir: string; - const collName = "fixtures"; - - beforeAll(async () => { - const env = await createIsolatedTestEnv("status-paths"); - localDbPath = env.dbPath; - localConfigDir = env.configDir; - - const { exitCode, stderr } = await runQmd( - ["collection", "add", fixturesDir, "--name", collName], - { dbPath: localDbPath, configDir: localConfigDir } - ); - if (exitCode !== 0) console.error("collection add failed:", stderr); - expect(exitCode).toBe(0); - }); - - test("status does not show full filesystem paths", async () => { - const { stdout, exitCode } = await runQmd(["status"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - // Should show qmd:// URIs - expect(stdout).toContain(`qmd://${collName}/`); - // Should NOT show full filesystem paths (except for the index location which is ok) - const lines = stdout.split('\n').filter(l => !l.includes('Index:')); - const pathLines = lines.filter(l => l.includes('/Users/') || l.includes('/home/') || l.includes('/tmp/')); - expect(pathLines.length).toBe(0); - }); - - test("collection list does not show full filesystem paths", async () => { - const { stdout, exitCode } = await runQmd(["collection", "list"], { dbPath: localDbPath, configDir: localConfigDir }); - expect(exitCode).toBe(0); - - // Should show qmd:// URIs - expect(stdout).toContain(`qmd://${collName}/`); - // Should NOT show Path: lines with filesystem paths - expect(stdout).not.toMatch(/Path:\s+\//); - }); -}); diff --git a/tools/qmd/src/collections.ts b/tools/qmd/src/collections.ts deleted file mode 100644 index f13ba70c..00000000 --- a/tools/qmd/src/collections.ts +++ /dev/null @@ -1,390 +0,0 @@ -/** - * Collections configuration management - * - * This module manages the YAML-based collection configuration at ~/.config/qmd/index.yml. - * Collections define which directories to index and their associated contexts. - */ - -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; -import { join } from "path"; -import { homedir } from "os"; -import YAML from "yaml"; - -// ============================================================================ -// Types -// ============================================================================ - -/** - * Context definitions for a collection - * Key is path prefix (e.g., "/", "/2024", "/Board of Directors") - * Value is the context description - */ -export type ContextMap = Record; - -/** - * A single collection configuration - */ -export interface Collection { - path: string; // Absolute path to index - pattern: string; // Glob pattern (e.g., "**/*.md") - context?: ContextMap; // Optional context definitions - update?: string; // Optional bash command to run during qmd update -} - -/** - * The complete configuration file structure - */ -export interface CollectionConfig { - global_context?: string; // Context applied to all collections - collections: Record; // Collection name -> config -} - -/** - * Collection with its name (for return values) - */ -export interface NamedCollection extends Collection { - name: string; -} - -// ============================================================================ -// Configuration paths -// ============================================================================ - -// Current index name (default: "index") -let currentIndexName: string = "index"; - -/** - * Set the current index name for config file lookup - * Config file will be ~/.config/qmd/{indexName}.yml - */ -export function setConfigIndexName(name: string): void { - currentIndexName = name; -} - -function getConfigDir(): string { - // Allow override via QMD_CONFIG_DIR for testing - if (process.env.QMD_CONFIG_DIR) { - return process.env.QMD_CONFIG_DIR; - } - return join(homedir(), ".config", "qmd"); -} - -function getConfigFilePath(): string { - return join(getConfigDir(), `${currentIndexName}.yml`); -} - -/** - * Ensure config directory exists - */ -function ensureConfigDir(): void { - const configDir = getConfigDir(); - if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); - } -} - -// ============================================================================ -// Core functions -// ============================================================================ - -/** - * Load configuration from ~/.config/qmd/index.yml - * Returns empty config if file doesn't exist - */ -export function loadConfig(): CollectionConfig { - const configPath = getConfigFilePath(); - if (!existsSync(configPath)) { - return { collections: {} }; - } - - try { - const content = readFileSync(configPath, "utf-8"); - const config = YAML.parse(content) as CollectionConfig; - - // Ensure collections object exists - if (!config.collections) { - config.collections = {}; - } - - return config; - } catch (error) { - throw new Error(`Failed to parse ${configPath}: ${error}`); - } -} - -/** - * Save configuration to ~/.config/qmd/index.yml - */ -export function saveConfig(config: CollectionConfig): void { - ensureConfigDir(); - const configPath = getConfigFilePath(); - - try { - const yaml = YAML.stringify(config, { - indent: 2, - lineWidth: 0, // Don't wrap lines - }); - writeFileSync(configPath, yaml, "utf-8"); - } catch (error) { - throw new Error(`Failed to write ${configPath}: ${error}`); - } -} - -/** - * Get a specific collection by name - * Returns null if not found - */ -export function getCollection(name: string): NamedCollection | null { - const config = loadConfig(); - const collection = config.collections[name]; - - if (!collection) { - return null; - } - - return { name, ...collection }; -} - -/** - * List all collections - */ -export function listCollections(): NamedCollection[] { - const config = loadConfig(); - return Object.entries(config.collections).map(([name, collection]) => ({ - name, - ...collection, - })); -} - -/** - * Add or update a collection - */ -export function addCollection( - name: string, - path: string, - pattern: string = "**/*.md" -): void { - const config = loadConfig(); - - config.collections[name] = { - path, - pattern, - context: config.collections[name]?.context, // Preserve existing context - }; - - saveConfig(config); -} - -/** - * Remove a collection - */ -export function removeCollection(name: string): boolean { - const config = loadConfig(); - - if (!config.collections[name]) { - return false; - } - - delete config.collections[name]; - saveConfig(config); - return true; -} - -/** - * Rename a collection - */ -export function renameCollection(oldName: string, newName: string): boolean { - const config = loadConfig(); - - if (!config.collections[oldName]) { - return false; - } - - if (config.collections[newName]) { - throw new Error(`Collection '${newName}' already exists`); - } - - config.collections[newName] = config.collections[oldName]; - delete config.collections[oldName]; - saveConfig(config); - return true; -} - -// ============================================================================ -// Context management -// ============================================================================ - -/** - * Get global context - */ -export function getGlobalContext(): string | undefined { - const config = loadConfig(); - return config.global_context; -} - -/** - * Set global context - */ -export function setGlobalContext(context: string | undefined): void { - const config = loadConfig(); - config.global_context = context; - saveConfig(config); -} - -/** - * Get all contexts for a collection - */ -export function getContexts(collectionName: string): ContextMap | undefined { - const collection = getCollection(collectionName); - return collection?.context; -} - -/** - * Add or update a context for a specific path in a collection - */ -export function addContext( - collectionName: string, - pathPrefix: string, - contextText: string -): boolean { - const config = loadConfig(); - const collection = config.collections[collectionName]; - - if (!collection) { - return false; - } - - if (!collection.context) { - collection.context = {}; - } - - collection.context[pathPrefix] = contextText; - saveConfig(config); - return true; -} - -/** - * Remove a context from a collection - */ -export function removeContext( - collectionName: string, - pathPrefix: string -): boolean { - const config = loadConfig(); - const collection = config.collections[collectionName]; - - if (!collection?.context?.[pathPrefix]) { - return false; - } - - delete collection.context[pathPrefix]; - - // Remove empty context object - if (Object.keys(collection.context).length === 0) { - delete collection.context; - } - - saveConfig(config); - return true; -} - -/** - * List all contexts across all collections - */ -export function listAllContexts(): Array<{ - collection: string; - path: string; - context: string; -}> { - const config = loadConfig(); - const results: Array<{ collection: string; path: string; context: string }> = []; - - // Add global context if present - if (config.global_context) { - results.push({ - collection: "*", - path: "/", - context: config.global_context, - }); - } - - // Add collection contexts - for (const [name, collection] of Object.entries(config.collections)) { - if (collection.context) { - for (const [path, context] of Object.entries(collection.context)) { - results.push({ - collection: name, - path, - context, - }); - } - } - } - - return results; -} - -/** - * Find best matching context for a given collection and path - * Returns the most specific matching context (longest path prefix match) - */ -export function findContextForPath( - collectionName: string, - filePath: string -): string | undefined { - const config = loadConfig(); - const collection = config.collections[collectionName]; - - if (!collection?.context) { - return config.global_context; - } - - // Find all matching prefixes - const matches: Array<{ prefix: string; context: string }> = []; - - for (const [prefix, context] of Object.entries(collection.context)) { - // Normalize paths for comparison - const normalizedPath = filePath.startsWith("/") ? filePath : `/${filePath}`; - const normalizedPrefix = prefix.startsWith("/") ? prefix : `/${prefix}`; - - if (normalizedPath.startsWith(normalizedPrefix)) { - matches.push({ prefix: normalizedPrefix, context }); - } - } - - // Return most specific match (longest prefix) - if (matches.length > 0) { - matches.sort((a, b) => b.prefix.length - a.prefix.length); - return matches[0]!.context; - } - - // Fallback to global context - return config.global_context; -} - -// ============================================================================ -// Utility functions -// ============================================================================ - -/** - * Get the config file path (useful for error messages) - */ -export function getConfigPath(): string { - return getConfigFilePath(); -} - -/** - * Check if config file exists - */ -export function configExists(): boolean { - return existsSync(getConfigFilePath()); -} - -/** - * Validate a collection name - * Collection names must be valid and not contain special characters - */ -export function isValidCollectionName(name: string): boolean { - // Allow alphanumeric, hyphens, underscores - return /^[a-zA-Z0-9_-]+$/.test(name); -} diff --git a/tools/qmd/src/eval.test.ts b/tools/qmd/src/eval.test.ts deleted file mode 100644 index ff5d7daa..00000000 --- a/tools/qmd/src/eval.test.ts +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Evaluation Tests for QMD Search Quality - * - * Tests search quality against synthetic documents with known-answer queries. - * Validates that search improvements don't regress quality. - * - * Three test suites: - * 1. BM25 (FTS) - lexical search baseline - * 2. Vector Search - semantic search with embeddings - * 3. Hybrid (RRF) - combined lexical + vector with rank fusion - */ - -import { describe, test, expect, beforeAll, afterAll } from "bun:test"; -import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs"; -import { join } from "path"; -import { tmpdir } from "os"; -import Database from "bun:sqlite"; - -// Set INDEX_PATH before importing store to prevent using global index -const tempDir = mkdtempSync(join(tmpdir(), "qmd-eval-")); -process.env.INDEX_PATH = join(tempDir, "eval.sqlite"); - -import { - createStore, - searchFTS, - searchVec, - insertDocument, - insertContent, - insertEmbedding, - chunkDocumentByTokens, - reciprocalRankFusion, - DEFAULT_EMBED_MODEL, - type RankedResult, -} from "./store"; -import { getDefaultLlamaCpp, formatDocForEmbedding, disposeDefaultLlamaCpp } from "./llm"; - -// Eval queries with expected documents -const evalQueries: { - query: string; - expectedDoc: string; - difficulty: "easy" | "medium" | "hard" | "fusion"; -}[] = [ - // EASY: Exact keyword matches - { query: "API versioning", expectedDoc: "api-design", difficulty: "easy" }, - { query: "Series A fundraising", expectedDoc: "fundraising", difficulty: "easy" }, - { query: "CAP theorem", expectedDoc: "distributed-systems", difficulty: "easy" }, - { query: "overfitting machine learning", expectedDoc: "machine-learning", difficulty: "easy" }, - { query: "remote work VPN", expectedDoc: "remote-work", difficulty: "easy" }, - { query: "Project Phoenix retrospective", expectedDoc: "product-launch", difficulty: "easy" }, - - // MEDIUM: Semantic/conceptual queries - { query: "how to structure REST endpoints", expectedDoc: "api-design", difficulty: "medium" }, - { query: "raising money for startup", expectedDoc: "fundraising", difficulty: "medium" }, - { query: "consistency vs availability tradeoffs", expectedDoc: "distributed-systems", difficulty: "medium" }, - { query: "how to prevent models from memorizing data", expectedDoc: "machine-learning", difficulty: "medium" }, - { query: "working from home guidelines", expectedDoc: "remote-work", difficulty: "medium" }, - { query: "what went wrong with the launch", expectedDoc: "product-launch", difficulty: "medium" }, - - // HARD: Vague, partial memory, indirect - { query: "nouns not verbs", expectedDoc: "api-design", difficulty: "hard" }, - { query: "Sequoia investor pitch", expectedDoc: "fundraising", difficulty: "hard" }, - { query: "Raft algorithm leader election", expectedDoc: "distributed-systems", difficulty: "hard" }, - { query: "F1 score precision recall", expectedDoc: "machine-learning", difficulty: "hard" }, - { query: "quarterly team gathering travel", expectedDoc: "remote-work", difficulty: "hard" }, - { query: "beta program 47 bugs", expectedDoc: "product-launch", difficulty: "hard" }, - - // FUSION: Multi-signal queries that need both lexical AND semantic matching - // These should have weak individual scores but strong combined RRF scores - { query: "how much runway before running out of money", expectedDoc: "fundraising", difficulty: "fusion" }, - { query: "datacenter replication sync strategy", expectedDoc: "distributed-systems", difficulty: "fusion" }, - { query: "splitting data for training and testing", expectedDoc: "machine-learning", difficulty: "fusion" }, - { query: "JSON response codes error messages", expectedDoc: "api-design", difficulty: "fusion" }, - { query: "video calls camera async messaging", expectedDoc: "remote-work", difficulty: "fusion" }, - { query: "CI/CD pipeline testing coverage", expectedDoc: "product-launch", difficulty: "fusion" }, -]; - -// Helper to check if result matches expected doc -function matchesExpected(filepath: string, expectedDoc: string): boolean { - return filepath.toLowerCase().includes(expectedDoc); -} - -// Helper to calculate hit rate -function calcHitRate( - queries: typeof evalQueries, - searchFn: (query: string) => { filepath: string }[], - topK: number -): number { - let hits = 0; - for (const { query, expectedDoc } of queries) { - const results = searchFn(query).slice(0, topK); - if (results.some(r => matchesExpected(r.filepath, expectedDoc))) hits++; - } - return hits / queries.length; -} - -// ============================================================================= -// BM25 (Lexical) Tests - Fast, no model loading needed -// ============================================================================= - -describe("BM25 Search (FTS)", () => { - let store: ReturnType; - let db: Database; - - beforeAll(() => { - store = createStore(); - db = store.db; - - // Load and index eval documents - const evalDocsDir = join(import.meta.dir, "../test/eval-docs"); - const files = readdirSync(evalDocsDir).filter(f => f.endsWith(".md")); - - for (const file of files) { - const content = readFileSync(join(evalDocsDir, file), "utf-8"); - const title = content.split("\n")[0]?.replace(/^#\s*/, "") || file; - const hash = Bun.hash(content).toString(16).slice(0, 12); - const now = new Date().toISOString(); - - insertContent(db, hash, content, now); - insertDocument(db, "eval-docs", file, title, hash, now, now); - } - }); - - afterAll(() => { - store.close(); - }); - - test("easy queries: ≥80% Hit@3", () => { - const easyQueries = evalQueries.filter(q => q.difficulty === "easy"); - const hitRate = calcHitRate(easyQueries, q => searchFTS(db, q, 5), 3); - expect(hitRate).toBeGreaterThanOrEqual(0.8); - }); - - test("medium queries: ≥15% Hit@3 (BM25 struggles with semantic)", () => { - const mediumQueries = evalQueries.filter(q => q.difficulty === "medium"); - const hitRate = calcHitRate(mediumQueries, q => searchFTS(db, q, 5), 3); - expect(hitRate).toBeGreaterThanOrEqual(0.15); - }); - - test("hard queries: ≥15% Hit@5 (BM25 baseline)", () => { - const hardQueries = evalQueries.filter(q => q.difficulty === "hard"); - const hitRate = calcHitRate(hardQueries, q => searchFTS(db, q, 5), 5); - expect(hitRate).toBeGreaterThanOrEqual(0.15); - }); - - test("overall Hit@3 ≥40% (BM25 baseline)", () => { - const hitRate = calcHitRate(evalQueries, q => searchFTS(db, q, 5), 3); - expect(hitRate).toBeGreaterThanOrEqual(0.4); - }); -}); - -// ============================================================================= -// Vector Search Tests - Requires embedding model -// ============================================================================= - -describe("Vector Search", () => { - let store: ReturnType; - let db: Database; - let hasEmbeddings = false; - - beforeAll(async () => { - store = createStore(); - db = store.db; - - // Check if embeddings already exist (from previous test run) - const vecTable = db.prepare( - `SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'` - ).get(); - - if (vecTable) { - const count = db.prepare(`SELECT COUNT(*) as cnt FROM vectors_vec`).get() as { cnt: number }; - if (count.cnt > 0) { - hasEmbeddings = true; - return; - } - } - - // Generate embeddings for test documents - const llm = getDefaultLlamaCpp(); - store.ensureVecTable(768); // embeddinggemma uses 768 dimensions - - const evalDocsDir = join(import.meta.dir, "../test/eval-docs"); - const files = readdirSync(evalDocsDir).filter(f => f.endsWith(".md")); - - for (const file of files) { - const content = readFileSync(join(evalDocsDir, file), "utf-8"); - const hash = Bun.hash(content).toString(16).slice(0, 12); - const title = content.split("\n")[0]?.replace(/^#\s*/, "") || file; - - // Chunk and embed - const chunks = await chunkDocumentByTokens(content); - for (let seq = 0; seq < chunks.length; seq++) { - const chunk = chunks[seq]; - if (!chunk) continue; - const formatted = formatDocForEmbedding(chunk.text, title); - const result = await llm.embed(formatted, { model: DEFAULT_EMBED_MODEL, isQuery: false }); - if (result?.embedding) { - // Convert to Float32Array for sqlite-vec - const embedding = new Float32Array(result.embedding); - const now = new Date().toISOString(); - insertEmbedding(db, hash, seq, chunk.pos, embedding, DEFAULT_EMBED_MODEL, now); - } - } - } - hasEmbeddings = true; - }, 120000); // 2 minute timeout for embedding generation - - afterAll(() => { - store.close(); - }); - - // Note: Don't dispose here - Hybrid tests also use llama. - // Dispose happens in the global afterAll. - - test("easy queries: ≥60% Hit@3 (vector should match keywords too)", async () => { - if (!hasEmbeddings) return; // Skip if embedding failed - - const easyQueries = evalQueries.filter(q => q.difficulty === "easy"); - let hits = 0; - for (const { query, expectedDoc } of easyQueries) { - const results = await searchVec(db, query, DEFAULT_EMBED_MODEL, 5); - if (results.slice(0, 3).some(r => matchesExpected(r.filepath, expectedDoc))) hits++; - } - expect(hits / easyQueries.length).toBeGreaterThanOrEqual(0.6); - }, 60000); - - test("medium queries: ≥40% Hit@3 (vector excels at semantic)", async () => { - if (!hasEmbeddings) return; - - const mediumQueries = evalQueries.filter(q => q.difficulty === "medium"); - let hits = 0; - for (const { query, expectedDoc } of mediumQueries) { - const results = await searchVec(db, query, DEFAULT_EMBED_MODEL, 5); - if (results.slice(0, 3).some(r => matchesExpected(r.filepath, expectedDoc))) hits++; - } - // Vector search should do better on semantic queries than BM25 - expect(hits / mediumQueries.length).toBeGreaterThanOrEqual(0.4); - }, 60000); - - test("hard queries: ≥30% Hit@5 (vector helps with vague queries)", async () => { - if (!hasEmbeddings) return; - - const hardQueries = evalQueries.filter(q => q.difficulty === "hard"); - let hits = 0; - for (const { query, expectedDoc } of hardQueries) { - const results = await searchVec(db, query, DEFAULT_EMBED_MODEL, 5); - if (results.some(r => matchesExpected(r.filepath, expectedDoc))) hits++; - } - expect(hits / hardQueries.length).toBeGreaterThanOrEqual(0.3); - }, 60000); - - test("overall Hit@3 ≥50% (vector baseline)", async () => { - if (!hasEmbeddings) return; - - let hits = 0; - for (const { query, expectedDoc } of evalQueries) { - const results = await searchVec(db, query, DEFAULT_EMBED_MODEL, 5); - if (results.slice(0, 3).some(r => matchesExpected(r.filepath, expectedDoc))) hits++; - } - expect(hits / evalQueries.length).toBeGreaterThanOrEqual(0.5); - }, 60000); -}); - -// ============================================================================= -// Hybrid Search (RRF) Tests - Combines BM25 + Vector -// ============================================================================= - -describe("Hybrid Search (RRF)", () => { - let store: ReturnType; - let db: Database; - let hasVectors = false; - - beforeAll(() => { - store = createStore(); - db = store.db; - // Check if vectors exist - const vecTable = db.prepare( - `SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'` - ).get(); - if (vecTable) { - const count = db.prepare(`SELECT COUNT(*) as cnt FROM vectors_vec`).get() as { cnt: number }; - hasVectors = count.cnt > 0; - } - }); - - afterAll(() => { - store.close(); - }); - - // Helper: run hybrid search with RRF fusion - async function hybridSearch(query: string, limit: number = 10): Promise { - const rankedLists: RankedResult[][] = []; - - // FTS results - const ftsResults = searchFTS(db, query, 20); - if (ftsResults.length > 0) { - rankedLists.push(ftsResults.map(r => ({ - file: r.filepath, - displayPath: r.displayPath, - title: r.title, - body: r.body || "", - score: r.score - }))); - } - - // Vector results - const vecResults = await searchVec(db, query, DEFAULT_EMBED_MODEL, 20); - if (vecResults.length > 0) { - rankedLists.push(vecResults.map(r => ({ - file: r.filepath, - displayPath: r.displayPath, - title: r.title, - body: r.body || "", - score: r.score - }))); - } - - if (rankedLists.length === 0) return []; - - // Apply RRF fusion - const fused = reciprocalRankFusion(rankedLists); - return fused.slice(0, limit); - } - - test("easy queries: ≥80% Hit@3 (hybrid should match BM25)", async () => { - const easyQueries = evalQueries.filter(q => q.difficulty === "easy"); - let hits = 0; - for (const { query, expectedDoc } of easyQueries) { - const results = await hybridSearch(query); - if (results.slice(0, 3).some(r => matchesExpected(r.file, expectedDoc))) hits++; - } - expect(hits / easyQueries.length).toBeGreaterThanOrEqual(0.8); - }, 60000); - - test("medium queries: ≥50% Hit@3 with vectors, ≥15% without", async () => { - const mediumQueries = evalQueries.filter(q => q.difficulty === "medium"); - let hits = 0; - for (const { query, expectedDoc } of mediumQueries) { - const results = await hybridSearch(query); - if (results.slice(0, 3).some(r => matchesExpected(r.file, expectedDoc))) hits++; - } - // With vectors: hybrid should outperform both BM25 (15%) and vector (40%) - // Without vectors: hybrid is just BM25, so use BM25 threshold - const threshold = hasVectors ? 0.5 : 0.15; - expect(hits / mediumQueries.length).toBeGreaterThanOrEqual(threshold); - }, 60000); - - test("hard queries: ≥35% Hit@5 with vectors, ≥15% without", async () => { - const hardQueries = evalQueries.filter(q => q.difficulty === "hard"); - let hits = 0; - for (const { query, expectedDoc } of hardQueries) { - const results = await hybridSearch(query); - if (results.some(r => matchesExpected(r.file, expectedDoc))) hits++; - } - const threshold = hasVectors ? 0.35 : 0.15; - expect(hits / hardQueries.length).toBeGreaterThanOrEqual(threshold); - }, 60000); - - test("fusion queries: ≥50% Hit@3 (RRF combines weak signals)", async () => { - if (!hasVectors) return; // Fusion requires both methods - - const fusionQueries = evalQueries.filter(q => q.difficulty === "fusion"); - let hybridHits = 0; - let bm25Hits = 0; - let vecHits = 0; - - for (const { query, expectedDoc } of fusionQueries) { - // Hybrid results - const hybridResults = await hybridSearch(query); - if (hybridResults.slice(0, 3).some(r => matchesExpected(r.file, expectedDoc))) hybridHits++; - - // BM25 results for comparison - const bm25Results = searchFTS(db, query, 5); - if (bm25Results.slice(0, 3).some(r => matchesExpected(r.filepath, expectedDoc))) bm25Hits++; - - // Vector results for comparison - const vecResults = await searchVec(db, query, DEFAULT_EMBED_MODEL, 5); - if (vecResults.slice(0, 3).some(r => matchesExpected(r.filepath, expectedDoc))) vecHits++; - } - - const hybridRate = hybridHits / fusionQueries.length; - const bm25Rate = bm25Hits / fusionQueries.length; - const vecRate = vecHits / fusionQueries.length; - - // Fusion should achieve at least 50% on these multi-signal queries - expect(hybridRate).toBeGreaterThanOrEqual(0.5); - - // Fusion should outperform or match the best individual method - expect(hybridRate).toBeGreaterThanOrEqual(Math.max(bm25Rate, vecRate)); - }, 60000); - - test("overall Hit@3 ≥60% with vectors, ≥40% without", async () => { - // Filter out fusion queries for overall score (they're tested separately) - const standardQueries = evalQueries.filter(q => q.difficulty !== "fusion"); - let hits = 0; - for (const { query, expectedDoc } of standardQueries) { - const results = await hybridSearch(query); - if (results.slice(0, 3).some(r => matchesExpected(r.file, expectedDoc))) hits++; - } - const threshold = hasVectors ? 0.6 : 0.4; - expect(hits / standardQueries.length).toBeGreaterThanOrEqual(threshold); - }, 60000); -}); - -// ============================================================================= -// Cleanup -// ============================================================================= - -afterAll(async () => { - // Ensure native resources are released to avoid ggml-metal asserts on process exit. - await disposeDefaultLlamaCpp(); - rmSync(tempDir, { recursive: true, force: true }); -}); diff --git a/tools/qmd/src/formatter.ts b/tools/qmd/src/formatter.ts deleted file mode 100644 index 756c8d3a..00000000 --- a/tools/qmd/src/formatter.ts +++ /dev/null @@ -1,427 +0,0 @@ -/** - * formatter.ts - Output formatting utilities for QMD - * - * Provides methods to format search results and documents into various output formats: - * JSON, CSV, XML, Markdown, files list, and CLI (colored terminal output). - */ - -import { extractSnippet } from "./store.js"; -import type { SearchResult, MultiGetResult, DocumentResult } from "./store.js"; - -// ============================================================================= -// Types -// ============================================================================= - -// Re-export store types for convenience -export type { SearchResult, MultiGetResult, DocumentResult }; - -// Flattened type for formatter convenience (extracts info from MultiGetResult) -export type MultiGetFile = { - filepath: string; - displayPath: string; - title: string; - body: string; - context?: string | null; - skipped: false; -} | { - filepath: string; - displayPath: string; - title: string; - body: string; - context?: string | null; - skipped: true; - skipReason: string; -}; - -export type OutputFormat = "cli" | "csv" | "md" | "xml" | "files" | "json"; - -export type FormatOptions = { - full?: boolean; // Show full document content instead of snippet - query?: string; // Query for snippet extraction and highlighting - useColor?: boolean; // Enable terminal colors (default: false for non-CLI) - lineNumbers?: boolean;// Add line numbers to output -}; - -// ============================================================================= -// Helper Functions -// ============================================================================= - -/** - * Add line numbers to text content. - * Each line becomes: "{lineNum}: {content}" - * @param text The text to add line numbers to - * @param startLine Optional starting line number (default: 1) - */ -export function addLineNumbers(text: string, startLine: number = 1): string { - const lines = text.split('\n'); - return lines.map((line, i) => `${startLine + i}: ${line}`).join('\n'); -} - -/** - * Extract short docid from a full hash (first 6 characters). - */ -export function getDocid(hash: string): string { - return hash.slice(0, 6); -} - -// ============================================================================= -// Escape Helpers -// ============================================================================= - -export function escapeCSV(value: string | null | number): string { - if (value === null || value === undefined) return ""; - const str = String(value); - if (str.includes(",") || str.includes('"') || str.includes("\n")) { - return `"${str.replace(/"/g, '""')}"`; - } - return str; -} - -export function escapeXml(str: string): string { - return str - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """) - .replace(/'/g, "'"); -} - -// ============================================================================= -// Search Results Formatters -// ============================================================================= - -/** - * Format search results as JSON - */ -export function searchResultsToJson( - results: SearchResult[], - opts: FormatOptions = {} -): string { - const query = opts.query || ""; - const output = results.map(row => { - const bodyStr = row.body || ""; - let body = opts.full ? bodyStr : undefined; - let snippet = !opts.full ? extractSnippet(bodyStr, query, 300, row.chunkPos).snippet : undefined; - - if (opts.lineNumbers) { - if (body) body = addLineNumbers(body); - if (snippet) snippet = addLineNumbers(snippet); - } - - return { - docid: `#${row.docid}`, - score: Math.round(row.score * 100) / 100, - file: row.displayPath, - title: row.title, - ...(row.context && { context: row.context }), - ...(body && { body }), - ...(snippet && { snippet }), - }; - }); - return JSON.stringify(output, null, 2); -} - -/** - * Format search results as CSV - */ -export function searchResultsToCsv( - results: SearchResult[], - opts: FormatOptions = {} -): string { - const query = opts.query || ""; - const header = "docid,score,file,title,context,line,snippet"; - const rows = results.map(row => { - const bodyStr = row.body || ""; - const { line, snippet } = extractSnippet(bodyStr, query, 500, row.chunkPos); - let content = opts.full ? bodyStr : snippet; - if (opts.lineNumbers && content) { - content = addLineNumbers(content); - } - return [ - `#${row.docid}`, - row.score.toFixed(4), - escapeCSV(row.displayPath), - escapeCSV(row.title), - escapeCSV(row.context || ""), - line, - escapeCSV(content), - ].join(","); - }); - return [header, ...rows].join("\n"); -} - -/** - * Format search results as simple files list (docid,score,filepath,context) - */ -export function searchResultsToFiles(results: SearchResult[]): string { - return results.map(row => { - const ctx = row.context ? `,"${row.context.replace(/"/g, '""')}"` : ""; - return `#${row.docid},${row.score.toFixed(2)},${row.displayPath}${ctx}`; - }).join("\n"); -} - -/** - * Format search results as Markdown - */ -export function searchResultsToMarkdown( - results: SearchResult[], - opts: FormatOptions = {} -): string { - const query = opts.query || ""; - return results.map(row => { - const heading = row.title || row.displayPath; - const bodyStr = row.body || ""; - let content: string; - if (opts.full) { - content = bodyStr; - } else { - content = extractSnippet(bodyStr, query, 500, row.chunkPos).snippet; - } - if (opts.lineNumbers) { - content = addLineNumbers(content); - } - return `---\n# ${heading}\n\n**docid:** \`#${row.docid}\`\n\n${content}\n`; - }).join("\n"); -} - -/** - * Format search results as XML - */ -export function searchResultsToXml( - results: SearchResult[], - opts: FormatOptions = {} -): string { - const query = opts.query || ""; - const items = results.map(row => { - const titleAttr = row.title ? ` title="${escapeXml(row.title)}"` : ""; - const bodyStr = row.body || ""; - let content = opts.full ? bodyStr : extractSnippet(bodyStr, query, 500, row.chunkPos).snippet; - if (opts.lineNumbers) { - content = addLineNumbers(content); - } - return `\n${escapeXml(content)}\n`; - }); - return items.join("\n\n"); -} - -/** - * Format search results for MCP (simpler CSV format with pre-extracted snippets) - */ -export function searchResultsToMcpCsv( - results: { docid: string; file: string; title: string; score: number; context: string | null; snippet: string }[] -): string { - const header = "docid,file,title,score,context,snippet"; - const rows = results.map(r => - [`#${r.docid}`, r.file, r.title, r.score, r.context || "", r.snippet].map(escapeCSV).join(",") - ); - return [header, ...rows].join("\n"); -} - -// ============================================================================= -// Document Formatters (for multi-get using MultiGetFile from store) -// ============================================================================= - -/** - * Format documents as JSON - */ -export function documentsToJson(results: MultiGetFile[]): string { - const output = results.map(r => ({ - file: r.displayPath, - title: r.title, - ...(r.context && { context: r.context }), - ...(r.skipped ? { skipped: true, reason: r.skipReason } : { body: r.body }), - })); - return JSON.stringify(output, null, 2); -} - -/** - * Format documents as CSV - */ -export function documentsToCsv(results: MultiGetFile[]): string { - const header = "file,title,context,skipped,body"; - const rows = results.map(r => - [ - r.displayPath, - r.title, - r.context || "", - r.skipped ? "true" : "false", - r.skipped ? (r.skipReason || "") : r.body - ].map(escapeCSV).join(",") - ); - return [header, ...rows].join("\n"); -} - -/** - * Format documents as files list - */ -export function documentsToFiles(results: MultiGetFile[]): string { - return results.map(r => { - const ctx = r.context ? `,"${r.context.replace(/"/g, '""')}"` : ""; - const status = r.skipped ? ",[SKIPPED]" : ""; - return `${r.displayPath}${ctx}${status}`; - }).join("\n"); -} - -/** - * Format documents as Markdown - */ -export function documentsToMarkdown(results: MultiGetFile[]): string { - return results.map(r => { - let md = `## ${r.displayPath}\n\n`; - if (r.title && r.title !== r.displayPath) md += `**Title:** ${r.title}\n\n`; - if (r.context) md += `**Context:** ${r.context}\n\n`; - if (r.skipped) { - md += `> ${r.skipReason}\n`; - } else { - md += "```\n" + r.body + "\n```\n"; - } - return md; - }).join("\n"); -} - -/** - * Format documents as XML - */ -export function documentsToXml(results: MultiGetFile[]): string { - const items = results.map(r => { - let xml = " \n"; - xml += ` ${escapeXml(r.displayPath)}\n`; - xml += ` ${escapeXml(r.title)}\n`; - if (r.context) xml += ` ${escapeXml(r.context)}\n`; - if (r.skipped) { - xml += ` true\n`; - xml += ` ${escapeXml(r.skipReason || "")}\n`; - } else { - xml += ` ${escapeXml(r.body)}\n`; - } - xml += " "; - return xml; - }); - return `\n\n${items.join("\n")}\n`; -} - -// ============================================================================= -// Single Document Formatters -// ============================================================================= - -/** - * Format a single DocumentResult as JSON - */ -export function documentToJson(doc: DocumentResult): string { - return JSON.stringify({ - file: doc.displayPath, - title: doc.title, - ...(doc.context && { context: doc.context }), - hash: doc.hash, - modifiedAt: doc.modifiedAt, - bodyLength: doc.bodyLength, - ...(doc.body !== undefined && { body: doc.body }), - }, null, 2); -} - -/** - * Format a single DocumentResult as Markdown - */ -export function documentToMarkdown(doc: DocumentResult): string { - let md = `# ${doc.title || doc.displayPath}\n\n`; - if (doc.context) md += `**Context:** ${doc.context}\n\n`; - md += `**File:** ${doc.displayPath}\n`; - md += `**Modified:** ${doc.modifiedAt}\n\n`; - if (doc.body !== undefined) { - md += "---\n\n" + doc.body + "\n"; - } - return md; -} - -/** - * Format a single DocumentResult as XML - */ -export function documentToXml(doc: DocumentResult): string { - let xml = `\n\n`; - xml += ` ${escapeXml(doc.displayPath)}\n`; - xml += ` ${escapeXml(doc.title)}\n`; - if (doc.context) xml += ` ${escapeXml(doc.context)}\n`; - xml += ` ${escapeXml(doc.hash)}\n`; - xml += ` ${escapeXml(doc.modifiedAt)}\n`; - xml += ` ${doc.bodyLength}\n`; - if (doc.body !== undefined) { - xml += ` ${escapeXml(doc.body)}\n`; - } - xml += ``; - return xml; -} - -/** - * Format a single document to the specified format - */ -export function formatDocument(doc: DocumentResult, format: OutputFormat): string { - switch (format) { - case "json": - return documentToJson(doc); - case "md": - return documentToMarkdown(doc); - case "xml": - return documentToXml(doc); - default: - // Default to markdown for CLI and other formats - return documentToMarkdown(doc); - } -} - -// ============================================================================= -// Universal Format Function -// ============================================================================= - -/** - * Format search results to the specified output format - */ -export function formatSearchResults( - results: SearchResult[], - format: OutputFormat, - opts: FormatOptions = {} -): string { - switch (format) { - case "json": - return searchResultsToJson(results, opts); - case "csv": - return searchResultsToCsv(results, opts); - case "files": - return searchResultsToFiles(results); - case "md": - return searchResultsToMarkdown(results, opts); - case "xml": - return searchResultsToXml(results, opts); - case "cli": - // CLI format should be handled separately with colors - // Return a simple text version as fallback - return searchResultsToMarkdown(results, opts); - default: - return searchResultsToJson(results, opts); - } -} - -/** - * Format documents to the specified output format - */ -export function formatDocuments( - results: MultiGetFile[], - format: OutputFormat -): string { - switch (format) { - case "json": - return documentsToJson(results); - case "csv": - return documentsToCsv(results); - case "files": - return documentsToFiles(results); - case "md": - return documentsToMarkdown(results); - case "xml": - return documentsToXml(results); - case "cli": - // CLI format should be handled separately with colors - return documentsToMarkdown(results); - default: - return documentsToJson(results); - } -} diff --git a/tools/qmd/src/llm.test.ts b/tools/qmd/src/llm.test.ts deleted file mode 100644 index 68ce859a..00000000 --- a/tools/qmd/src/llm.test.ts +++ /dev/null @@ -1,559 +0,0 @@ -/** - * llm.test.ts - Unit tests for the LLM abstraction layer (node-llama-cpp) - * - * Run with: bun test src/llm.test.ts - * - * These tests require the actual models to be downloaded. Run the embed or - * rerank functions first to trigger model downloads. - */ - -import { describe, test, expect, beforeAll, afterAll } from "bun:test"; -import { - LlamaCpp, - getDefaultLlamaCpp, - disposeDefaultLlamaCpp, - withLLMSession, - canUnloadLLM, - SessionReleasedError, - type RerankDocument, - type ILLMSession, -} from "./llm.js"; - -// ============================================================================= -// Singleton Tests (no model loading required) -// ============================================================================= - -describe("Default LlamaCpp Singleton", () => { - // Test singleton behavior without resetting to avoid orphan instances - test("getDefaultLlamaCpp returns same instance on subsequent calls", () => { - const llm1 = getDefaultLlamaCpp(); - const llm2 = getDefaultLlamaCpp(); - expect(llm1).toBe(llm2); - expect(llm1).toBeInstanceOf(LlamaCpp); - }); -}); - -// ============================================================================= -// Model Existence Tests -// ============================================================================= - -describe("LlamaCpp.modelExists", () => { - test("returns exists:true for HuggingFace model URIs", async () => { - const llm = getDefaultLlamaCpp(); - const result = await llm.modelExists("hf:org/repo/model.gguf"); - - expect(result.exists).toBe(true); - expect(result.name).toBe("hf:org/repo/model.gguf"); - }); - - test("returns exists:false for non-existent local paths", async () => { - const llm = getDefaultLlamaCpp(); - const result = await llm.modelExists("/nonexistent/path/model.gguf"); - - expect(result.exists).toBe(false); - expect(result.name).toBe("/nonexistent/path/model.gguf"); - }); -}); - -// ============================================================================= -// Integration Tests (require actual models) -// ============================================================================= - -describe("LlamaCpp Integration", () => { - // Use the singleton to avoid multiple Metal contexts - const llm = getDefaultLlamaCpp(); - - afterAll(async () => { - // Ensure native resources are released to avoid ggml-metal asserts on process exit. - await disposeDefaultLlamaCpp(); - }); - - describe("embed", () => { - test("returns embedding with correct dimensions", async () => { - const result = await llm.embed("Hello world"); - - expect(result).not.toBeNull(); - expect(result!.embedding).toBeInstanceOf(Array); - expect(result!.embedding.length).toBeGreaterThan(0); - // embeddinggemma outputs 768 dimensions - expect(result!.embedding.length).toBe(768); - }); - - test("returns consistent embeddings for same input", async () => { - const result1 = await llm.embed("test text"); - const result2 = await llm.embed("test text"); - - expect(result1).not.toBeNull(); - expect(result2).not.toBeNull(); - - // Embeddings should be identical for the same input - for (let i = 0; i < result1!.embedding.length; i++) { - expect(result1!.embedding[i]).toBeCloseTo(result2!.embedding[i]!, 5); - } - }); - - test("returns different embeddings for different inputs", async () => { - const result1 = await llm.embed("cats are great"); - const result2 = await llm.embed("database optimization"); - - expect(result1).not.toBeNull(); - expect(result2).not.toBeNull(); - - // Calculate cosine similarity - should be less than 1.0 (not identical) - let dotProduct = 0; - let norm1 = 0; - let norm2 = 0; - for (let i = 0; i < result1!.embedding.length; i++) { - const v1 = result1!.embedding[i]!; - const v2 = result2!.embedding[i]!; - dotProduct += v1 * v2; - norm1 += v1 ** 2; - norm2 += v2 ** 2; - } - const similarity = dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2)); - - expect(similarity).toBeLessThan(0.95); // Should be meaningfully different - }); - }); - - describe("embedBatch", () => { - test("returns embeddings for multiple texts", async () => { - const texts = ["Hello world", "Test text", "Another document"]; - const results = await llm.embedBatch(texts); - - expect(results).toHaveLength(3); - for (const result of results) { - expect(result).not.toBeNull(); - expect(result!.embedding.length).toBe(768); - } - }); - - test("returns same results as individual embed calls", async () => { - const texts = ["cats are great", "dogs are awesome"]; - - // Get batch embeddings - const batchResults = await llm.embedBatch(texts); - - // Get individual embeddings - const individualResults = await Promise.all(texts.map(t => llm.embed(t))); - - // Compare - should be identical - for (let i = 0; i < texts.length; i++) { - expect(batchResults[i]).not.toBeNull(); - expect(individualResults[i]).not.toBeNull(); - for (let j = 0; j < batchResults[i]!.embedding.length; j++) { - expect(batchResults[i]!.embedding[j]).toBeCloseTo(individualResults[i]!.embedding[j]!, 5); - } - } - }); - - test("handles empty array", async () => { - const results = await llm.embedBatch([]); - expect(results).toHaveLength(0); - }); - - test("batch is faster than sequential", async () => { - const texts = Array(10).fill(null).map((_, i) => `Document number ${i} with content`); - - // Time batch - const batchStart = Date.now(); - await llm.embedBatch(texts); - const batchTime = Date.now() - batchStart; - - // Time sequential - const seqStart = Date.now(); - for (const text of texts) { - await llm.embed(text); - } - const seqTime = Date.now() - seqStart; - - console.log(`Batch: ${batchTime}ms, Sequential: ${seqTime}ms`); - // Performance is machine/load dependent. We only assert batch isn't drastically worse. - expect(batchTime).toBeLessThanOrEqual(seqTime * 3); - }); - - test("handles concurrent embedBatch calls on fresh instance without race condition", async () => { - // This test verifies the fix for a race condition where concurrent calls to - // ensureEmbedContext() could create multiple contexts. Without the promise guard, - // each concurrent embedBatch call sees embedContext === null and creates its own - // context, causing resource leaks and potential "Context is disposed" errors. - // - // See: https://github.com/tobi/qmd/pull/54 - // - // The fix uses a promise guard to ensure only one context creation runs at a time. - // We verify this by instrumenting createEmbeddingContext to count invocations. - - const freshLlm = new LlamaCpp({}); - let contextCreateCount = 0; - - // Instrument the model's createEmbeddingContext to count calls - const originalEnsureEmbedModel = (freshLlm as any).ensureEmbedModel.bind(freshLlm); - let modelInstrumented = false; - (freshLlm as any).ensureEmbedModel = async function() { - const model = await originalEnsureEmbedModel(); - if (!modelInstrumented) { - modelInstrumented = true; - const originalCreate = model.createEmbeddingContext.bind(model); - model.createEmbeddingContext = async function(...args: any[]) { - contextCreateCount++; - return originalCreate(...args); - }; - } - return model; - }; - - const texts = Array(10).fill(null).map((_, i) => `Document ${i}`); - - // Call embedBatch 5 TIMES in parallel on fresh instance. - // Without the promise guard fix, this would create 5 contexts (one per call). - // With the fix, only 1 context should be created. - const batches = await Promise.all([ - freshLlm.embedBatch(texts.slice(0, 2)), - freshLlm.embedBatch(texts.slice(2, 4)), - freshLlm.embedBatch(texts.slice(4, 6)), - freshLlm.embedBatch(texts.slice(6, 8)), - freshLlm.embedBatch(texts.slice(8, 10)), - ]); - - const allResults = batches.flat(); - expect(allResults).toHaveLength(10); - - const successCount = allResults.filter(r => r !== null).length; - expect(successCount).toBe(10); - - // THE KEY ASSERTION: Only 1 context should be created, not 5 - // Without the fix, contextCreateCount would be 5 (one per concurrent embedBatch call) - console.log(`Context creation count: ${contextCreateCount} (expected: 1)`); - expect(contextCreateCount).toBe(1); - - await freshLlm.dispose(); - }, 60000); - }); - - describe("rerank", () => { - test("scores capital of France question correctly", async () => { - const query = "What is the capital of France?"; - const documents: RerankDocument[] = [ - { file: "butterflies.txt", text: "Butterflies indeed fly through the garden." }, - { file: "france.txt", text: "The capital of France is Paris." }, - { file: "canada.txt", text: "The capital of Canada is Ottawa." }, - ]; - - const result = await llm.rerank(query, documents); - - expect(result.results).toHaveLength(3); - - // The France document should score highest - expect(result.results[0]!.file).toBe("france.txt"); - expect(result.results[0]!.score).toBeGreaterThan(0.7); - - // Canada should be somewhat relevant (also about capitals) - expect(result.results[1]!.file).toBe("canada.txt"); - - // Butterflies should score lowest - expect(result.results[2]!.file).toBe("butterflies.txt"); - expect(result.results[2]!.score).toBeLessThan(0.6); - }); - - test("scores authentication query correctly", async () => { - const query = "How do I configure authentication?"; - const documents: RerankDocument[] = [ - { file: "weather.md", text: "The weather today is sunny with mild temperatures." }, - { file: "auth.md", text: "Authentication can be configured by setting the AUTH_SECRET environment variable." }, - { file: "pizza.md", text: "Our restaurant serves the best pizza in town." }, - { file: "jwt.md", text: "JWT authentication requires a secret key and expiration time." }, - ]; - - const result = await llm.rerank(query, documents); - - expect(result.results).toHaveLength(4); - - // Auth documents should score highest - const topTwo = result.results.slice(0, 2).map((r) => r.file); - expect(topTwo).toContain("auth.md"); - expect(topTwo).toContain("jwt.md"); - - // Irrelevant documents should score lowest - const bottomTwo = result.results.slice(2).map((r) => r.file); - expect(bottomTwo).toContain("weather.md"); - expect(bottomTwo).toContain("pizza.md"); - }); - - test("handles programming queries correctly", async () => { - const query = "How do I handle errors in JavaScript?"; - const documents: RerankDocument[] = [ - { file: "cooking.md", text: "To make a good pasta, boil water and add salt." }, - { file: "errors.md", text: "Use try-catch blocks to handle JavaScript errors gracefully." }, - { file: "python.md", text: "Python uses try-except for exception handling." }, - ]; - - const result = await llm.rerank(query, documents); - - // JavaScript errors doc should score highest - expect(result.results[0]!.file).toBe("errors.md"); - expect(result.results[0]!.score).toBeGreaterThan(0.7); - - // Python doc might be somewhat relevant (same concept, different language) - // Cooking should be least relevant - expect(result.results[2]!.file).toBe("cooking.md"); - }); - - test("handles empty document list", async () => { - const result = await llm.rerank("test query", []); - expect(result.results).toHaveLength(0); - }); - - test("handles single document", async () => { - const result = await llm.rerank("test", [{ file: "doc.md", text: "content" }]); - expect(result.results).toHaveLength(1); - expect(result.results[0]!.file).toBe("doc.md"); - }); - - test("preserves original file paths", async () => { - const documents: RerankDocument[] = [ - { file: "path/to/doc1.md", text: "content one" }, - { file: "another/path/doc2.md", text: "content two" }, - ]; - - const result = await llm.rerank("query", documents); - - const files = result.results.map((r) => r.file).sort(); - expect(files).toEqual(["another/path/doc2.md", "path/to/doc1.md"]); - }); - - test("returns scores between 0 and 1", async () => { - const documents: RerankDocument[] = [ - { file: "a.md", text: "The quick brown fox jumps over the lazy dog." }, - { file: "b.md", text: "Machine learning algorithms process data efficiently." }, - { file: "c.md", text: "React components use JSX syntax for rendering." }, - ]; - - const result = await llm.rerank("Tell me about animals", documents); - - for (const doc of result.results) { - expect(doc.score).toBeGreaterThanOrEqual(0); - expect(doc.score).toBeLessThanOrEqual(1); - } - }); - - test("batch reranks multiple documents efficiently", async () => { - // Create 10 documents to verify batch processing works - const documents: RerankDocument[] = Array(10) - .fill(null) - .map((_, i) => ({ - file: `doc${i}.md`, - text: `Document number ${i} with some content about topic ${i % 3}`, - })); - - const start = Date.now(); - const result = await llm.rerank("topic 1", documents); - const elapsed = Date.now() - start; - - expect(result.results).toHaveLength(10); - - // Verify all documents are returned with valid scores - for (const doc of result.results) { - expect(doc.score).toBeGreaterThanOrEqual(0); - expect(doc.score).toBeLessThanOrEqual(1); - } - - // Log timing for monitoring batch performance - console.log(`Batch rerank of 10 docs took ${elapsed}ms`); - }); - }); - - describe("expandQuery", () => { - test("returns query expansions with correct types", async () => { - const result = await llm.expandQuery("test query"); - - // Result is Queryable[] containing lex, vec, and/or hyde entries - expect(result.length).toBeGreaterThanOrEqual(1); - - // Each result should have a valid type - for (const q of result) { - expect(["lex", "vec", "hyde"]).toContain(q.type); - expect(q.text.length).toBeGreaterThan(0); - } - }, 30000); // 30s timeout for model loading - - test("can exclude lexical queries", async () => { - const result = await llm.expandQuery("authentication setup", { includeLexical: false }); - - // Should not contain any 'lex' type entries - const lexEntries = result.filter(q => q.type === "lex"); - expect(lexEntries).toHaveLength(0); - }); - }); -}); - -// ============================================================================= -// Session Management Tests -// ============================================================================= - -describe("LLM Session Management", () => { - describe("withLLMSession", () => { - test("session provides access to LLM operations", async () => { - const result = await withLLMSession(async (session) => { - expect(session.isValid).toBe(true); - const embedding = await session.embed("test text"); - expect(embedding).not.toBeNull(); - expect(embedding!.embedding.length).toBe(768); - return "success"; - }); - expect(result).toBe("success"); - }); - - test("session is invalid after release", async () => { - let capturedSession: ILLMSession | null = null; - - await withLLMSession(async (session) => { - capturedSession = session; - expect(session.isValid).toBe(true); - }); - - // Session should be invalid after withLLMSession returns - expect(capturedSession).not.toBeNull(); - expect(capturedSession!.isValid).toBe(false); - }); - - test("session prevents idle unload during operations", async () => { - await withLLMSession(async (session) => { - // While inside a session, canUnloadLLM should return false - expect(canUnloadLLM()).toBe(false); - - // Perform an operation - await session.embed("test"); - - // Still should not be able to unload - expect(canUnloadLLM()).toBe(false); - }); - - // After session ends, should be able to unload - expect(canUnloadLLM()).toBe(true); - }); - - test("nested sessions increment ref count", async () => { - await withLLMSession(async (outerSession) => { - expect(canUnloadLLM()).toBe(false); - - await withLLMSession(async (innerSession) => { - expect(canUnloadLLM()).toBe(false); - expect(innerSession.isValid).toBe(true); - expect(outerSession.isValid).toBe(true); - }); - - // Inner session released, but outer still active - expect(canUnloadLLM()).toBe(false); - expect(outerSession.isValid).toBe(true); - }); - - // All sessions released - expect(canUnloadLLM()).toBe(true); - }); - - test("session embedBatch works correctly", async () => { - await withLLMSession(async (session) => { - const texts = ["Hello world", "Test text", "Another document"]; - const results = await session.embedBatch(texts); - - expect(results).toHaveLength(3); - for (const result of results) { - expect(result).not.toBeNull(); - expect(result!.embedding.length).toBe(768); - } - }); - }); - - test("session rerank works correctly", async () => { - await withLLMSession(async (session) => { - const documents: RerankDocument[] = [ - { file: "a.txt", text: "The capital of France is Paris." }, - { file: "b.txt", text: "Dogs are great pets." }, - ]; - - const result = await session.rerank("What is the capital of France?", documents); - - expect(result.results).toHaveLength(2); - expect(result.results[0]!.file).toBe("a.txt"); - expect(result.results[0]!.score).toBeGreaterThan(result.results[1]!.score); - }); - }); - - test("max duration aborts session after timeout", async () => { - let aborted = false; - - try { - await withLLMSession(async (session) => { - // Wait longer than max duration - await new Promise(resolve => setTimeout(resolve, 150)); - - // This operation should throw because session was aborted - await session.embed("test"); - }, { maxDuration: 50 }); // 50ms max - } catch (err) { - if (err instanceof SessionReleasedError) { - aborted = true; - } else { - throw err; - } - } - - expect(aborted).toBe(true); - }, 5000); - - test("external abort signal propagates to session", async () => { - const abortController = new AbortController(); - let sessionAborted = false; - - const promise = withLLMSession(async (session) => { - // Wait a bit then check if aborted - await new Promise(resolve => setTimeout(resolve, 100)); - - if (!session.isValid) { - sessionAborted = true; - throw new SessionReleasedError("Session aborted"); - } - - return "should not reach"; - }, { signal: abortController.signal }); - - // Abort after 20ms - setTimeout(() => abortController.abort(), 20); - - try { - await promise; - } catch (err) { - // Expected - } - - expect(sessionAborted).toBe(true); - }, 5000); - - test("session provides abort signal for monitoring", async () => { - await withLLMSession(async (session) => { - expect(session.signal).toBeInstanceOf(AbortSignal); - expect(session.signal.aborted).toBe(false); - }); - }); - - test("returns value from callback", async () => { - const result = await withLLMSession(async (session) => { - await session.embed("test"); - return { status: "complete", count: 42 }; - }); - - expect(result).toEqual({ status: "complete", count: 42 }); - }); - - test("propagates errors from callback", async () => { - const customError = new Error("Custom test error"); - - await expect( - withLLMSession(async () => { - throw customError; - }) - ).rejects.toThrow("Custom test error"); - }); - }); -}); - diff --git a/tools/qmd/src/llm.ts b/tools/qmd/src/llm.ts deleted file mode 100644 index ab39c861..00000000 --- a/tools/qmd/src/llm.ts +++ /dev/null @@ -1,1208 +0,0 @@ -/** - * llm.ts - LLM abstraction layer for QMD using node-llama-cpp - * - * Provides embeddings, text generation, and reranking using local GGUF models. - */ - -import { - getLlama, - resolveModelFile, - LlamaChatSession, - LlamaLogLevel, - type Llama, - type LlamaModel, - type LlamaEmbeddingContext, - type Token as LlamaToken, -} from "node-llama-cpp"; -import { homedir } from "os"; -import { join } from "path"; -import { existsSync, mkdirSync, statSync, unlinkSync, readdirSync, readFileSync, writeFileSync } from "fs"; - -// ============================================================================= -// Embedding Formatting Functions -// ============================================================================= - -/** - * Format a query for embedding. - * Uses nomic-style task prefix format for embeddinggemma. - */ -export function formatQueryForEmbedding(query: string): string { - return `task: search result | query: ${query}`; -} - -/** - * Format a document for embedding. - * Uses nomic-style format with title and text fields. - */ -export function formatDocForEmbedding(text: string, title?: string): string { - return `title: ${title || "none"} | text: ${text}`; -} - -// ============================================================================= -// Types -// ============================================================================= - -/** - * Token with log probability - */ -export type TokenLogProb = { - token: string; - logprob: number; -}; - -/** - * Embedding result - */ -export type EmbeddingResult = { - embedding: number[]; - model: string; -}; - -/** - * Generation result with optional logprobs - */ -export type GenerateResult = { - text: string; - model: string; - logprobs?: TokenLogProb[]; - done: boolean; -}; - -/** - * Rerank result for a single document - */ -export type RerankDocumentResult = { - file: string; - score: number; - index: number; -}; - -/** - * Batch rerank result - */ -export type RerankResult = { - results: RerankDocumentResult[]; - model: string; -}; - -/** - * Model info - */ -export type ModelInfo = { - name: string; - exists: boolean; - path?: string; -}; - -/** - * Options for embedding - */ -export type EmbedOptions = { - model?: string; - isQuery?: boolean; - title?: string; -}; - -/** - * Options for text generation - */ -export type GenerateOptions = { - model?: string; - maxTokens?: number; - temperature?: number; -}; - -/** - * Options for reranking - */ -export type RerankOptions = { - model?: string; -}; - -/** - * Options for LLM sessions - */ -export type LLMSessionOptions = { - /** Max session duration in ms (default: 10 minutes) */ - maxDuration?: number; - /** External abort signal */ - signal?: AbortSignal; - /** Debug name for logging */ - name?: string; -}; - -/** - * Session interface for scoped LLM access with lifecycle guarantees - */ -export interface ILLMSession { - embed(text: string, options?: EmbedOptions): Promise; - embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]>; - expandQuery(query: string, options?: { context?: string; includeLexical?: boolean }): Promise; - rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise; - /** Whether this session is still valid (not released or aborted) */ - readonly isValid: boolean; - /** Abort signal for this session (aborts on release or maxDuration) */ - readonly signal: AbortSignal; -} - -/** - * Supported query types for different search backends - */ -export type QueryType = 'lex' | 'vec' | 'hyde'; - -/** - * A single query and its target backend type - */ -export type Queryable = { - type: QueryType; - text: string; -}; - -/** - * Document to rerank - */ -export type RerankDocument = { - file: string; - text: string; - title?: string; -}; - -// ============================================================================= -// Model Configuration -// ============================================================================= - -// HuggingFace model URIs for node-llama-cpp -// Format: hf:// -const DEFAULT_EMBED_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf"; -const DEFAULT_RERANK_MODEL = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf"; -// const DEFAULT_GENERATE_MODEL = "hf:ggml-org/Qwen3-0.6B-GGUF/Qwen3-0.6B-Q8_0.gguf"; -const DEFAULT_GENERATE_MODEL = "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf"; - -export const DEFAULT_EMBED_MODEL_URI = DEFAULT_EMBED_MODEL; -export const DEFAULT_RERANK_MODEL_URI = DEFAULT_RERANK_MODEL; -export const DEFAULT_GENERATE_MODEL_URI = DEFAULT_GENERATE_MODEL; - -// Local model cache directory -const MODEL_CACHE_DIR = join(homedir(), ".cache", "qmd", "models"); -export const DEFAULT_MODEL_CACHE_DIR = MODEL_CACHE_DIR; - -export type PullResult = { - model: string; - path: string; - sizeBytes: number; - refreshed: boolean; -}; - -type HfRef = { - repo: string; - file: string; -}; - -function parseHfUri(model: string): HfRef | null { - if (!model.startsWith("hf:")) return null; - const without = model.slice(3); - const parts = without.split("/"); - if (parts.length < 3) return null; - const repo = parts.slice(0, 2).join("/"); - const file = parts.slice(2).join("/"); - return { repo, file }; -} - -async function getRemoteEtag(ref: HfRef): Promise { - const url = `https://huggingface.co/${ref.repo}/resolve/main/${ref.file}`; - try { - const resp = await fetch(url, { method: "HEAD" }); - if (!resp.ok) return null; - const etag = resp.headers.get("etag"); - return etag || null; - } catch { - return null; - } -} - -export async function pullModels( - models: string[], - options: { refresh?: boolean; cacheDir?: string } = {} -): Promise { - const cacheDir = options.cacheDir || MODEL_CACHE_DIR; - if (!existsSync(cacheDir)) { - mkdirSync(cacheDir, { recursive: true }); - } - - const results: PullResult[] = []; - for (const model of models) { - let refreshed = false; - const hfRef = parseHfUri(model); - const filename = model.split("/").pop(); - const entries = readdirSync(cacheDir, { withFileTypes: true }); - const cached = filename - ? entries - .filter((entry) => entry.isFile() && entry.name.includes(filename)) - .map((entry) => join(cacheDir, entry.name)) - : []; - - if (hfRef && filename) { - const etagPath = join(cacheDir, `${filename}.etag`); - const remoteEtag = await getRemoteEtag(hfRef); - const localEtag = existsSync(etagPath) - ? readFileSync(etagPath, "utf-8").trim() - : null; - const shouldRefresh = - options.refresh || !remoteEtag || remoteEtag !== localEtag || cached.length === 0; - - if (shouldRefresh) { - for (const candidate of cached) { - if (existsSync(candidate)) unlinkSync(candidate); - } - if (existsSync(etagPath)) unlinkSync(etagPath); - refreshed = cached.length > 0; - } - } else if (options.refresh && filename) { - for (const candidate of cached) { - if (existsSync(candidate)) unlinkSync(candidate); - refreshed = true; - } - } - - const path = await resolveModelFile(model, cacheDir); - const sizeBytes = existsSync(path) ? statSync(path).size : 0; - if (hfRef && filename) { - const remoteEtag = await getRemoteEtag(hfRef); - if (remoteEtag) { - const etagPath = join(cacheDir, `${filename}.etag`); - writeFileSync(etagPath, remoteEtag + "\n", "utf-8"); - } - } - results.push({ model, path, sizeBytes, refreshed }); - } - return results; -} - -// ============================================================================= -// LLM Interface -// ============================================================================= - -/** - * Abstract LLM interface - implement this for different backends - */ -export interface LLM { - /** - * Get embeddings for text - */ - embed(text: string, options?: EmbedOptions): Promise; - - /** - * Generate text completion - */ - generate(prompt: string, options?: GenerateOptions): Promise; - - /** - * Check if a model exists/is available - */ - modelExists(model: string): Promise; - - /** - * Expand a search query into multiple variations for different backends. - * Returns a list of Queryable objects. - */ - expandQuery(query: string, options?: { context?: string, includeLexical?: boolean }): Promise; - - /** - * Rerank documents by relevance to a query - * Returns list of documents with relevance scores (higher = more relevant) - */ - rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise; - - /** - * Dispose of resources - */ - dispose(): Promise; -} - -// ============================================================================= -// node-llama-cpp Implementation -// ============================================================================= - -export type LlamaCppConfig = { - embedModel?: string; - generateModel?: string; - rerankModel?: string; - modelCacheDir?: string; - /** - * Inactivity timeout in ms before unloading contexts (default: 2 minutes, 0 to disable). - * - * Per node-llama-cpp lifecycle guidance, we prefer keeping models loaded and only disposing - * contexts when idle, since contexts (and their sequences) are the heavy per-session objects. - * @see https://node-llama-cpp.withcat.ai/guide/objects-lifecycle - */ - inactivityTimeoutMs?: number; - /** - * Whether to dispose models on inactivity (default: false). - * - * Keeping models loaded avoids repeated VRAM thrash; set to true only if you need aggressive - * memory reclaim. - */ - disposeModelsOnInactivity?: boolean; -}; - -/** - * LLM implementation using node-llama-cpp - */ -// Default inactivity timeout: 5 minutes (keep models warm during typical search sessions) -const DEFAULT_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000; - -export class LlamaCpp implements LLM { - private llama: Llama | null = null; - private embedModel: LlamaModel | null = null; - private embedContext: LlamaEmbeddingContext | null = null; - private generateModel: LlamaModel | null = null; - private rerankModel: LlamaModel | null = null; - private rerankContext: Awaited> | null = null; - - private embedModelUri: string; - private generateModelUri: string; - private rerankModelUri: string; - private modelCacheDir: string; - - // Ensure we don't load the same model/context concurrently (which can allocate duplicate VRAM). - private embedModelLoadPromise: Promise | null = null; - private embedContextCreatePromise: Promise | null = null; - private generateModelLoadPromise: Promise | null = null; - private rerankModelLoadPromise: Promise | null = null; - - // Inactivity timer for auto-unloading models - private inactivityTimer: ReturnType | null = null; - private inactivityTimeoutMs: number; - private disposeModelsOnInactivity: boolean; - - // Track disposal state to prevent double-dispose - private disposed = false; - - - constructor(config: LlamaCppConfig = {}) { - this.embedModelUri = config.embedModel || DEFAULT_EMBED_MODEL; - this.generateModelUri = config.generateModel || DEFAULT_GENERATE_MODEL; - this.rerankModelUri = config.rerankModel || DEFAULT_RERANK_MODEL; - this.modelCacheDir = config.modelCacheDir || MODEL_CACHE_DIR; - this.inactivityTimeoutMs = config.inactivityTimeoutMs ?? DEFAULT_INACTIVITY_TIMEOUT_MS; - this.disposeModelsOnInactivity = config.disposeModelsOnInactivity ?? false; - } - - /** - * Reset the inactivity timer. Called after each model operation. - * When timer fires, models are unloaded to free memory (if no active sessions). - */ - private touchActivity(): void { - // Clear existing timer - if (this.inactivityTimer) { - clearTimeout(this.inactivityTimer); - this.inactivityTimer = null; - } - - // Only set timer if we have disposable contexts and timeout is enabled - if (this.inactivityTimeoutMs > 0 && this.hasLoadedContexts()) { - this.inactivityTimer = setTimeout(() => { - // Check if session manager allows unloading - // canUnloadLLM is defined later in this file - it checks the session manager - // We use dynamic import pattern to avoid circular dependency issues - if (typeof canUnloadLLM === 'function' && !canUnloadLLM()) { - // Active sessions/operations - reschedule timer - this.touchActivity(); - return; - } - this.unloadIdleResources().catch(err => { - console.error("Error unloading idle resources:", err); - }); - }, this.inactivityTimeoutMs); - // Don't keep process alive just for this timer - this.inactivityTimer.unref(); - } - } - - /** - * Check if any contexts are currently loaded (and therefore worth unloading on inactivity). - */ - private hasLoadedContexts(): boolean { - return !!(this.embedContext || this.rerankContext); - } - - /** - * Unload idle resources but keep the instance alive for future use. - * - * By default, this disposes contexts (and their dependent sequences), while keeping models loaded. - * This matches the intended lifecycle: model → context → sequence, where contexts are per-session. - */ - async unloadIdleResources(): Promise { - // Don't unload if already disposed - if (this.disposed) { - return; - } - - // Clear timer - if (this.inactivityTimer) { - clearTimeout(this.inactivityTimer); - this.inactivityTimer = null; - } - - // Dispose contexts first - if (this.embedContext) { - await this.embedContext.dispose(); - this.embedContext = null; - } - if (this.rerankContext) { - await this.rerankContext.dispose(); - this.rerankContext = null; - } - - // Optionally dispose models too (opt-in) - if (this.disposeModelsOnInactivity) { - if (this.embedModel) { - await this.embedModel.dispose(); - this.embedModel = null; - } - if (this.generateModel) { - await this.generateModel.dispose(); - this.generateModel = null; - } - if (this.rerankModel) { - await this.rerankModel.dispose(); - this.rerankModel = null; - } - // Reset load promises so models can be reloaded later - this.embedModelLoadPromise = null; - this.generateModelLoadPromise = null; - this.rerankModelLoadPromise = null; - } - - // Note: We keep llama instance alive - it's lightweight - } - - /** - * Ensure model cache directory exists - */ - private ensureModelCacheDir(): void { - if (!existsSync(this.modelCacheDir)) { - mkdirSync(this.modelCacheDir, { recursive: true }); - } - } - - /** - * Initialize the llama instance (lazy) - */ - private async ensureLlama(): Promise { - if (!this.llama) { - this.llama = await getLlama({ logLevel: LlamaLogLevel.error }); - } - return this.llama; - } - - /** - * Resolve a model URI to a local path, downloading if needed - */ - private async resolveModel(modelUri: string): Promise { - this.ensureModelCacheDir(); - // resolveModelFile handles HF URIs and downloads to the cache dir - return await resolveModelFile(modelUri, this.modelCacheDir); - } - - /** - * Load embedding model (lazy) - */ - private async ensureEmbedModel(): Promise { - if (this.embedModel) { - return this.embedModel; - } - if (this.embedModelLoadPromise) { - return await this.embedModelLoadPromise; - } - - this.embedModelLoadPromise = (async () => { - const llama = await this.ensureLlama(); - const modelPath = await this.resolveModel(this.embedModelUri); - const model = await llama.loadModel({ modelPath }); - this.embedModel = model; - // Model loading counts as activity - ping to keep alive - this.touchActivity(); - return model; - })(); - - try { - return await this.embedModelLoadPromise; - } finally { - // Keep the resolved model cached; clear only the in-flight promise. - this.embedModelLoadPromise = null; - } - } - - /** - * Load embedding context (lazy). Context can be disposed and recreated without reloading the model. - * Uses promise guard to prevent concurrent context creation race condition. - */ - private async ensureEmbedContext(): Promise { - if (!this.embedContext) { - // If context creation is already in progress, wait for it - if (this.embedContextCreatePromise) { - return await this.embedContextCreatePromise; - } - - // Start context creation and store promise so concurrent calls wait - this.embedContextCreatePromise = (async () => { - const model = await this.ensureEmbedModel(); - const context = await model.createEmbeddingContext(); - this.embedContext = context; - return context; - })(); - - try { - const context = await this.embedContextCreatePromise; - this.touchActivity(); - return context; - } finally { - this.embedContextCreatePromise = null; - } - } - this.touchActivity(); - return this.embedContext; - } - - /** - * Load generation model (lazy) - context is created fresh per call - */ - private async ensureGenerateModel(): Promise { - if (!this.generateModel) { - if (this.generateModelLoadPromise) { - return await this.generateModelLoadPromise; - } - - this.generateModelLoadPromise = (async () => { - const llama = await this.ensureLlama(); - const modelPath = await this.resolveModel(this.generateModelUri); - const model = await llama.loadModel({ modelPath }); - this.generateModel = model; - return model; - })(); - - try { - await this.generateModelLoadPromise; - } finally { - this.generateModelLoadPromise = null; - } - } - this.touchActivity(); - if (!this.generateModel) { - throw new Error("Generate model not loaded"); - } - return this.generateModel; - } - - /** - * Load rerank model (lazy) - */ - private async ensureRerankModel(): Promise { - if (this.rerankModel) { - return this.rerankModel; - } - if (this.rerankModelLoadPromise) { - return await this.rerankModelLoadPromise; - } - - this.rerankModelLoadPromise = (async () => { - const llama = await this.ensureLlama(); - const modelPath = await this.resolveModel(this.rerankModelUri); - const model = await llama.loadModel({ modelPath }); - this.rerankModel = model; - // Model loading counts as activity - ping to keep alive - this.touchActivity(); - return model; - })(); - - try { - return await this.rerankModelLoadPromise; - } finally { - this.rerankModelLoadPromise = null; - } - } - - /** - * Load rerank context (lazy). Context can be disposed and recreated without reloading the model. - */ - private async ensureRerankContext(): Promise>> { - if (!this.rerankContext) { - const model = await this.ensureRerankModel(); - this.rerankContext = await model.createRankingContext(); - } - this.touchActivity(); - return this.rerankContext; - } - - // ========================================================================== - // Tokenization - // ========================================================================== - - /** - * Tokenize text using the embedding model's tokenizer - * Returns tokenizer tokens (opaque type from node-llama-cpp) - */ - async tokenize(text: string): Promise { - await this.ensureEmbedContext(); // Ensure model is loaded - if (!this.embedModel) { - throw new Error("Embed model not loaded"); - } - return this.embedModel.tokenize(text); - } - - /** - * Count tokens in text using the embedding model's tokenizer - */ - async countTokens(text: string): Promise { - const tokens = await this.tokenize(text); - return tokens.length; - } - - /** - * Detokenize token IDs back to text - */ - async detokenize(tokens: readonly LlamaToken[]): Promise { - await this.ensureEmbedContext(); - if (!this.embedModel) { - throw new Error("Embed model not loaded"); - } - return this.embedModel.detokenize(tokens); - } - - // ========================================================================== - // Core API methods - // ========================================================================== - - async embed(text: string, options: EmbedOptions = {}): Promise { - // Ping activity at start to keep models alive during this operation - this.touchActivity(); - - try { - const context = await this.ensureEmbedContext(); - const embedding = await context.getEmbeddingFor(text); - - return { - embedding: Array.from(embedding.vector), - model: this.embedModelUri, - }; - } catch (error) { - console.error("Embedding error:", error); - return null; - } - } - - /** - * Batch embed multiple texts efficiently - * Uses Promise.all for parallel embedding - node-llama-cpp handles batching internally - */ - async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> { - // Ping activity at start to keep models alive during this operation - this.touchActivity(); - - if (texts.length === 0) return []; - - try { - const context = await this.ensureEmbedContext(); - - // node-llama-cpp handles batching internally when we make parallel requests - const embeddings = await Promise.all( - texts.map(async (text) => { - try { - const embedding = await context.getEmbeddingFor(text); - this.touchActivity(); // Keep-alive during slow batches - return { - embedding: Array.from(embedding.vector), - model: this.embedModelUri, - }; - } catch (err) { - console.error("Embedding error for text:", err); - return null; - } - }) - ); - - return embeddings; - } catch (error) { - console.error("Batch embedding error:", error); - return texts.map(() => null); - } - } - - async generate(prompt: string, options: GenerateOptions = {}): Promise { - // Ping activity at start to keep models alive during this operation - this.touchActivity(); - - // Ensure model is loaded - await this.ensureGenerateModel(); - - // Create fresh context -> sequence -> session for each call - const context = await this.generateModel!.createContext(); - const sequence = context.getSequence(); - const session = new LlamaChatSession({ contextSequence: sequence }); - - const maxTokens = options.maxTokens ?? 150; - // Qwen3 recommends temp=0.7, topP=0.8, topK=20 for non-thinking mode - // DO NOT use greedy decoding (temp=0) - causes repetition loops - const temperature = options.temperature ?? 0.7; - - let result = ""; - try { - await session.prompt(prompt, { - maxTokens, - temperature, - topK: 20, - topP: 0.8, - onTextChunk: (text) => { - result += text; - }, - }); - - return { - text: result, - model: this.generateModelUri, - done: true, - }; - } finally { - // Dispose context (which disposes dependent sequences/sessions per lifecycle rules) - await context.dispose(); - } - } - - async modelExists(modelUri: string): Promise { - // For HuggingFace URIs, we assume they exist - // For local paths, check if file exists - if (modelUri.startsWith("hf:")) { - return { name: modelUri, exists: true }; - } - - const exists = existsSync(modelUri); - return { - name: modelUri, - exists, - path: exists ? modelUri : undefined, - }; - } - - // ========================================================================== - // High-level abstractions - // ========================================================================== - - async expandQuery(query: string, options: { context?: string, includeLexical?: boolean } = {}): Promise { - // Ping activity at start to keep models alive during this operation - this.touchActivity(); - - const llama = await this.ensureLlama(); - await this.ensureGenerateModel(); - - const includeLexical = options.includeLexical ?? true; - const context = options.context; - - const grammar = await llama.createGrammar({ - grammar: ` - root ::= line+ - line ::= type ": " content "\\n" - type ::= "lex" | "vec" | "hyde" - content ::= [^\\n]+ - ` - }); - - const prompt = `/no_think Expand this search query: ${query}`; - - // Create fresh context for each call - const genContext = await this.generateModel!.createContext(); - const sequence = genContext.getSequence(); - const session = new LlamaChatSession({ contextSequence: sequence }); - - try { - // Qwen3 recommended settings for non-thinking mode: - // temp=0.7, topP=0.8, topK=20, presence_penalty for repetition - // DO NOT use greedy decoding (temp=0) - causes infinite loops - const result = await session.prompt(prompt, { - grammar, - maxTokens: 600, - temperature: 0.7, - topK: 20, - topP: 0.8, - repeatPenalty: { - lastTokens: 64, - presencePenalty: 0.5, - }, - }); - - const lines = result.trim().split("\n"); - const queryLower = query.toLowerCase(); - const queryTerms = queryLower.replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean); - - const hasQueryTerm = (text: string): boolean => { - const lower = text.toLowerCase(); - if (queryTerms.length === 0) return true; - return queryTerms.some(term => lower.includes(term)); - }; - - const queryables: Queryable[] = lines.map(line => { - const colonIdx = line.indexOf(":"); - if (colonIdx === -1) return null; - const type = line.slice(0, colonIdx).trim(); - if (type !== 'lex' && type !== 'vec' && type !== 'hyde') return null; - const text = line.slice(colonIdx + 1).trim(); - if (!hasQueryTerm(text)) return null; - return { type: type as QueryType, text }; - }).filter((q): q is Queryable => q !== null); - - // Filter out lex entries if not requested - const filtered = includeLexical ? queryables : queryables.filter(q => q.type !== 'lex'); - if (filtered.length > 0) return filtered; - - const fallback: Queryable[] = [ - { type: 'hyde', text: `Information about ${query}` }, - { type: 'lex', text: query }, - { type: 'vec', text: query }, - ]; - return includeLexical ? fallback : fallback.filter(q => q.type !== 'lex'); - } catch (error) { - console.error("Structured query expansion failed:", error); - // Fallback to original query - const fallback: Queryable[] = [{ type: 'vec', text: query }]; - if (includeLexical) fallback.unshift({ type: 'lex', text: query }); - return fallback; - } finally { - await genContext.dispose(); - } - } - - async rerank( - query: string, - documents: RerankDocument[], - options: RerankOptions = {} - ): Promise { - // Ping activity at start to keep models alive during this operation - this.touchActivity(); - - const context = await this.ensureRerankContext(); - - // Build a map from document text to original indices (for lookup after sorting) - const textToDoc = new Map(); - documents.forEach((doc, index) => { - textToDoc.set(doc.text, { file: doc.file, index }); - }); - - // Extract just the text for ranking - const texts = documents.map((doc) => doc.text); - - // Use the proper ranking API - returns [{document: string, score: number}] sorted by score - const ranked = await context.rankAndSort(query, texts); - - // Map back to our result format using the text-to-doc map - const results: RerankDocumentResult[] = ranked.map((item) => { - const docInfo = textToDoc.get(item.document)!; - return { - file: docInfo.file, - score: item.score, - index: docInfo.index, - }; - }); - - return { - results, - model: this.rerankModelUri, - }; - } - - async dispose(): Promise { - // Prevent double-dispose - if (this.disposed) { - return; - } - this.disposed = true; - - // Clear inactivity timer - if (this.inactivityTimer) { - clearTimeout(this.inactivityTimer); - this.inactivityTimer = null; - } - - // Disposing llama cascades to models and contexts automatically - // See: https://node-llama-cpp.withcat.ai/guide/objects-lifecycle - // Note: llama.dispose() can hang indefinitely, so we use a timeout - if (this.llama) { - const disposePromise = this.llama.dispose(); - const timeoutPromise = new Promise((resolve) => setTimeout(resolve, 1000)); - await Promise.race([disposePromise, timeoutPromise]); - } - - // Clear references - this.embedContext = null; - this.rerankContext = null; - this.embedModel = null; - this.generateModel = null; - this.rerankModel = null; - this.llama = null; - - // Clear any in-flight load/create promises - this.embedModelLoadPromise = null; - this.embedContextCreatePromise = null; - this.generateModelLoadPromise = null; - this.rerankModelLoadPromise = null; - } -} - -// ============================================================================= -// Session Management Layer -// ============================================================================= - -/** - * Manages LLM session lifecycle with reference counting. - * Coordinates with LlamaCpp idle timeout to prevent disposal during active sessions. - */ -class LLMSessionManager { - private llm: LlamaCpp; - private _activeSessionCount = 0; - private _inFlightOperations = 0; - - constructor(llm: LlamaCpp) { - this.llm = llm; - } - - get activeSessionCount(): number { - return this._activeSessionCount; - } - - get inFlightOperations(): number { - return this._inFlightOperations; - } - - /** - * Returns true only when both session count and in-flight operations are 0. - * Used by LlamaCpp to determine if idle unload is safe. - */ - canUnload(): boolean { - return this._activeSessionCount === 0 && this._inFlightOperations === 0; - } - - acquire(): void { - this._activeSessionCount++; - } - - release(): void { - this._activeSessionCount = Math.max(0, this._activeSessionCount - 1); - } - - operationStart(): void { - this._inFlightOperations++; - } - - operationEnd(): void { - this._inFlightOperations = Math.max(0, this._inFlightOperations - 1); - } - - getLlamaCpp(): LlamaCpp { - return this.llm; - } -} - -/** - * Error thrown when an operation is attempted on a released or aborted session. - */ -export class SessionReleasedError extends Error { - constructor(message = "LLM session has been released or aborted") { - super(message); - this.name = "SessionReleasedError"; - } -} - -/** - * Scoped LLM session with automatic lifecycle management. - * Wraps LlamaCpp methods with operation tracking and abort handling. - */ -class LLMSession implements ILLMSession { - private manager: LLMSessionManager; - private released = false; - private abortController: AbortController; - private maxDurationTimer: ReturnType | null = null; - private name: string; - - constructor(manager: LLMSessionManager, options: LLMSessionOptions = {}) { - this.manager = manager; - this.name = options.name || "unnamed"; - this.abortController = new AbortController(); - - // Link external abort signal if provided - if (options.signal) { - if (options.signal.aborted) { - this.abortController.abort(options.signal.reason); - } else { - options.signal.addEventListener("abort", () => { - this.abortController.abort(options.signal!.reason); - }, { once: true }); - } - } - - // Set up max duration timer - const maxDuration = options.maxDuration ?? 10 * 60 * 1000; // Default 10 minutes - if (maxDuration > 0) { - this.maxDurationTimer = setTimeout(() => { - this.abortController.abort(new Error(`Session "${this.name}" exceeded max duration of ${maxDuration}ms`)); - }, maxDuration); - this.maxDurationTimer.unref(); // Don't keep process alive - } - - // Acquire session lease - this.manager.acquire(); - } - - get isValid(): boolean { - return !this.released && !this.abortController.signal.aborted; - } - - get signal(): AbortSignal { - return this.abortController.signal; - } - - /** - * Release the session and decrement ref count. - * Called automatically by withLLMSession when the callback completes. - */ - release(): void { - if (this.released) return; - this.released = true; - - if (this.maxDurationTimer) { - clearTimeout(this.maxDurationTimer); - this.maxDurationTimer = null; - } - - this.abortController.abort(new Error("Session released")); - this.manager.release(); - } - - /** - * Wrap an operation with tracking and abort checking. - */ - private async withOperation(fn: () => Promise): Promise { - if (!this.isValid) { - throw new SessionReleasedError(); - } - - this.manager.operationStart(); - try { - // Check abort before starting - if (this.abortController.signal.aborted) { - throw new SessionReleasedError( - this.abortController.signal.reason?.message || "Session aborted" - ); - } - return await fn(); - } finally { - this.manager.operationEnd(); - } - } - - async embed(text: string, options?: EmbedOptions): Promise { - return this.withOperation(() => this.manager.getLlamaCpp().embed(text, options)); - } - - async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> { - return this.withOperation(() => this.manager.getLlamaCpp().embedBatch(texts)); - } - - async expandQuery( - query: string, - options?: { context?: string; includeLexical?: boolean } - ): Promise { - return this.withOperation(() => this.manager.getLlamaCpp().expandQuery(query, options)); - } - - async rerank( - query: string, - documents: RerankDocument[], - options?: RerankOptions - ): Promise { - return this.withOperation(() => this.manager.getLlamaCpp().rerank(query, documents, options)); - } -} - -// Session manager for the default LlamaCpp instance -let defaultSessionManager: LLMSessionManager | null = null; - -/** - * Get the session manager for the default LlamaCpp instance. - */ -function getSessionManager(): LLMSessionManager { - const llm = getDefaultLlamaCpp(); - if (!defaultSessionManager || defaultSessionManager.getLlamaCpp() !== llm) { - defaultSessionManager = new LLMSessionManager(llm); - } - return defaultSessionManager; -} - -/** - * Execute a function with a scoped LLM session. - * The session provides lifecycle guarantees - resources won't be disposed mid-operation. - * - * @example - * ```typescript - * await withLLMSession(async (session) => { - * const expanded = await session.expandQuery(query); - * const embeddings = await session.embedBatch(texts); - * const reranked = await session.rerank(query, docs); - * return reranked; - * }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' }); - * ``` - */ -export async function withLLMSession( - fn: (session: ILLMSession) => Promise, - options?: LLMSessionOptions -): Promise { - const manager = getSessionManager(); - const session = new LLMSession(manager, options); - - try { - return await fn(session); - } finally { - session.release(); - } -} - -/** - * Check if idle unload is safe (no active sessions or operations). - * Used internally by LlamaCpp idle timer. - */ -export function canUnloadLLM(): boolean { - if (!defaultSessionManager) return true; - return defaultSessionManager.canUnload(); -} - -// ============================================================================= -// Singleton for default LlamaCpp instance -// ============================================================================= - -let defaultLlamaCpp: LlamaCpp | null = null; - -/** - * Get the default LlamaCpp instance (creates one if needed) - */ -export function getDefaultLlamaCpp(): LlamaCpp { - if (!defaultLlamaCpp) { - defaultLlamaCpp = new LlamaCpp(); - } - return defaultLlamaCpp; -} - -/** - * Set a custom default LlamaCpp instance (useful for testing) - */ -export function setDefaultLlamaCpp(llm: LlamaCpp | null): void { - defaultLlamaCpp = llm; -} - -/** - * Dispose the default LlamaCpp instance if it exists. - * Call this before process exit to prevent NAPI crashes. - */ -export async function disposeDefaultLlamaCpp(): Promise { - if (defaultLlamaCpp) { - await defaultLlamaCpp.dispose(); - defaultLlamaCpp = null; - } -} diff --git a/tools/qmd/src/mcp.test.ts b/tools/qmd/src/mcp.test.ts deleted file mode 100644 index f15b8d48..00000000 --- a/tools/qmd/src/mcp.test.ts +++ /dev/null @@ -1,889 +0,0 @@ -/** - * MCP Server Tests - * - * Tests all MCP tools, resources, and prompts. - * Uses mocked Ollama responses and a test database. - */ - -import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from "bun:test"; -import { Database } from "bun:sqlite"; -import * as sqliteVec from "sqlite-vec"; -import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js"; -import { z } from "zod"; -import { getDefaultLlamaCpp, disposeDefaultLlamaCpp } from "./llm"; -import { mkdtemp, writeFile, readdir, unlink, rmdir } from "node:fs/promises"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import YAML from "yaml"; -import type { CollectionConfig } from "./collections"; - -// ============================================================================= -// Test Database Setup -// ============================================================================= - -let testDb: Database; -let testDbPath: string; -let testConfigDir: string; - -afterAll(async () => { - // Ensure native resources are released to avoid ggml-metal asserts on process exit. - await disposeDefaultLlamaCpp(); -}); - -function initTestDatabase(db: Database): void { - sqliteVec.load(db); - db.exec("PRAGMA journal_mode = WAL"); - - // Content-addressable storage - the source of truth for document content - db.exec(` - CREATE TABLE IF NOT EXISTS content ( - hash TEXT PRIMARY KEY, - doc TEXT NOT NULL, - created_at TEXT NOT NULL - ) - `); - - // Documents table - file system layer mapping virtual paths to content hashes - // Collections are now managed in YAML config - db.exec(` - CREATE TABLE IF NOT EXISTS documents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - collection TEXT NOT NULL, - path TEXT NOT NULL, - title TEXT NOT NULL, - hash TEXT NOT NULL, - created_at TEXT NOT NULL, - modified_at TEXT NOT NULL, - active INTEGER NOT NULL DEFAULT 1, - FOREIGN KEY (hash) REFERENCES content(hash) ON DELETE CASCADE, - UNIQUE(collection, path) - ) - `); - - db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_collection ON documents(collection, active)`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_hash ON documents(hash)`); - - db.exec(` - CREATE TABLE IF NOT EXISTS llm_cache ( - hash TEXT PRIMARY KEY, - result TEXT NOT NULL, - created_at TEXT NOT NULL - ) - `); - - db.exec(` - CREATE TABLE IF NOT EXISTS content_vectors ( - hash TEXT NOT NULL, - seq INTEGER NOT NULL DEFAULT 0, - pos INTEGER NOT NULL DEFAULT 0, - model TEXT NOT NULL, - embedded_at TEXT NOT NULL, - PRIMARY KEY (hash, seq) - ) - `); - - db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( - name, body, - content='documents', - content_rowid='id', - tokenize='porter unicode61' - ) - `); - - db.exec(` - CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN - INSERT INTO documents_fts(rowid, name, body) - SELECT new.id, new.path, content.doc - FROM content - WHERE content.hash = new.hash; - END - `); - - // Create vector table - db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vectors_vec USING vec0(hash_seq TEXT PRIMARY KEY, embedding float[768] distance_metric=cosine)`); -} - -function seedTestData(db: Database): void { - const now = new Date().toISOString(); - - // Note: Collections are now managed in YAML config, not in database - // For tests, we'll use a collection name "docs" - - // Add test documents - const docs = [ - { - path: "readme.md", - title: "Project README", - hash: "hash1", - body: "# Project README\n\nThis is the main readme file for the project.\n\nIt contains important information about setup and usage.", - }, - { - path: "api.md", - title: "API Documentation", - hash: "hash2", - body: "# API Documentation\n\nThis document describes the REST API endpoints.\n\n## Authentication\n\nUse Bearer tokens for auth.", - }, - { - path: "meetings/meeting-2024-01.md", - title: "January Meeting Notes", - hash: "hash3", - body: "# January Meeting Notes\n\nDiscussed Q1 goals and roadmap.\n\n## Action Items\n\n- Review budget\n- Hire new team members", - }, - { - path: "meetings/meeting-2024-02.md", - title: "February Meeting Notes", - hash: "hash4", - body: "# February Meeting Notes\n\nFollowed up on Q1 progress.\n\n## Updates\n\n- Budget approved\n- Two candidates interviewed", - }, - { - path: "large-file.md", - title: "Large Document", - hash: "hash5", - body: "# Large Document\n\n" + "Lorem ipsum ".repeat(2000), // ~24KB - }, - ]; - - for (const doc of docs) { - // Insert content first - db.prepare(` - INSERT OR IGNORE INTO content (hash, doc, created_at) - VALUES (?, ?, ?) - `).run(doc.hash, doc.body, now); - - // Then insert document metadata - db.prepare(` - INSERT INTO documents (collection, path, title, hash, created_at, modified_at, active) - VALUES ('docs', ?, ?, ?, ?, ?, 1) - `).run(doc.path, doc.title, doc.hash, now, now); - } - - // Add embeddings for vector search - const embedding = new Float32Array(768); - for (let i = 0; i < 768; i++) embedding[i] = Math.random(); - - for (const doc of docs.slice(0, 4)) { // Skip large file for embeddings - db.prepare(`INSERT INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, 0, 0, 'embeddinggemma', ?)`).run(doc.hash, now); - db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${doc.hash}_0`, embedding); - } -} - -// ============================================================================= -// MCP Server Test Helpers -// ============================================================================= - -// We need to create a testable version of the MCP handlers -// Since McpServer uses internal routing, we'll test the handler functions directly - -import { - searchFTS, - searchVec, - expandQuery, - rerank, - reciprocalRankFusion, - extractSnippet, - getContextForFile, - findDocument, - getDocumentBody, - findDocuments, - getStatus, - DEFAULT_EMBED_MODEL, - DEFAULT_QUERY_MODEL, - DEFAULT_RERANK_MODEL, - DEFAULT_MULTI_GET_MAX_BYTES, - createStore, -} from "./store"; -import type { RankedResult } from "./store"; -// Note: searchResultsToMcpCsv no longer used in MCP - using structuredContent instead - -// ============================================================================= -// Tests -// ============================================================================= - -describe("MCP Server", () => { - beforeAll(async () => { - // LlamaCpp uses node-llama-cpp for local model inference (no HTTP mocking needed) - // Use shared singleton to avoid creating multiple instances with separate GPU resources - getDefaultLlamaCpp(); - - // Set up test config directory - const configPrefix = join(tmpdir(), `qmd-mcp-config-${Date.now()}-${Math.random().toString(36).slice(2)}`); - testConfigDir = await mkdtemp(configPrefix); - process.env.QMD_CONFIG_DIR = testConfigDir; - - // Create YAML config with test collection - const testConfig: CollectionConfig = { - collections: { - docs: { - path: "/test/docs", - pattern: "**/*.md", - context: { - "/meetings": "Meeting notes and transcripts" - } - } - } - }; - await writeFile(join(testConfigDir, "index.yml"), YAML.stringify(testConfig)); - - testDbPath = `/tmp/qmd-mcp-test-${Date.now()}.sqlite`; - testDb = new Database(testDbPath); - initTestDatabase(testDb); - seedTestData(testDb); - }); - - afterAll(async () => { - testDb.close(); - try { - require("fs").unlinkSync(testDbPath); - } catch {} - - // Clean up test config directory - try { - const files = await readdir(testConfigDir); - for (const file of files) { - await unlink(join(testConfigDir, file)); - } - await rmdir(testConfigDir); - } catch {} - - delete process.env.QMD_CONFIG_DIR; - }); - - // =========================================================================== - // Tool: qmd_search (BM25) - // =========================================================================== - - describe("qmd_search tool", () => { - test("returns results for matching query", () => { - const results = searchFTS(testDb, "readme", 10); - expect(results.length).toBeGreaterThan(0); - expect(results[0]!.displayPath).toBe("docs/readme.md"); - }); - - test("returns empty for non-matching query", () => { - const results = searchFTS(testDb, "xyznonexistent", 10); - expect(results.length).toBe(0); - }); - - test("respects limit parameter", () => { - const results = searchFTS(testDb, "meeting", 1); - expect(results.length).toBe(1); - }); - - // Note: Collection filtering tests removed - collections are now managed in YAML, not DB - - test("formats results as structured content", () => { - const results = searchFTS(testDb, "api", 10); - const filtered = results.map(r => ({ - file: r.displayPath, - title: r.title, - score: Math.round(r.score * 100) / 100, - context: getContextForFile(testDb, r.filepath), - snippet: extractSnippet(r.body || "", "api", 300, r.chunkPos).snippet, - })); - // MCP now returns structuredContent with results array - expect(filtered.length).toBeGreaterThan(0); - expect(filtered[0]).toHaveProperty("file"); - expect(filtered[0]).toHaveProperty("title"); - expect(filtered[0]).toHaveProperty("score"); - expect(filtered[0]).toHaveProperty("snippet"); - }); - }); - - // =========================================================================== - // Tool: qmd_vsearch (Vector) - // =========================================================================== - - describe("qmd_vsearch tool", () => { - test("returns results for semantic query", async () => { - const results = await searchVec(testDb, "project documentation", DEFAULT_EMBED_MODEL, 10); - expect(results.length).toBeGreaterThan(0); - }); - - test("respects limit parameter", async () => { - const results = await searchVec(testDb, "documentation", DEFAULT_EMBED_MODEL, 2); - expect(results.length).toBeLessThanOrEqual(2); - }); - - test("returns empty when no vector table exists", async () => { - const emptyDb = new Database(":memory:"); - initTestDatabase(emptyDb); - emptyDb.exec("DROP TABLE IF EXISTS vectors_vec"); - - const results = await searchVec(emptyDb, "test", DEFAULT_EMBED_MODEL, 10); - expect(results.length).toBe(0); - emptyDb.close(); - }); - }); - - // =========================================================================== - // Tool: qmd_query (Hybrid) - // =========================================================================== - - describe("qmd_query tool", () => { - test("expands query with variations", async () => { - const queries = await expandQuery("api documentation", DEFAULT_QUERY_MODEL, testDb); - // Always returns at least the original query, may have more if generation succeeds - expect(queries.length).toBeGreaterThanOrEqual(1); - expect(queries[0]).toBe("api documentation"); - }, 30000); // 30s timeout for model loading - - test("performs RRF fusion on multiple result lists", () => { - const list1: RankedResult[] = [ - { file: "/a", displayPath: "a.md", title: "A", body: "body", score: 1 }, - { file: "/b", displayPath: "b.md", title: "B", body: "body", score: 0.8 }, - ]; - const list2: RankedResult[] = [ - { file: "/b", displayPath: "b.md", title: "B", body: "body", score: 1 }, - { file: "/c", displayPath: "c.md", title: "C", body: "body", score: 0.9 }, - ]; - - const fused = reciprocalRankFusion([list1, list2]); - expect(fused.length).toBe(3); - // B appears in both lists, should have higher score - const bResult = fused.find(r => r.file === "/b"); - expect(bResult).toBeDefined(); - }); - - test("reranks documents with LLM", async () => { - const docs = [ - { file: "/test/docs/readme.md", text: "Project readme" }, - { file: "/test/docs/api.md", text: "API documentation" }, - ]; - const reranked = await rerank("readme", docs, DEFAULT_RERANK_MODEL, testDb); - expect(reranked.length).toBe(2); - expect(reranked[0]!.score).toBeGreaterThan(0); - }); - - test("full hybrid search pipeline", async () => { - // Simulate full qmd_query flow - const query = "meeting notes"; - const queries = await expandQuery(query, DEFAULT_QUERY_MODEL, testDb); - - const rankedLists: RankedResult[][] = []; - for (const q of queries) { - const ftsResults = searchFTS(testDb, q, 20); - if (ftsResults.length > 0) { - rankedLists.push(ftsResults.map(r => ({ - file: r.filepath, - displayPath: r.displayPath, - title: r.title, - body: r.body || "", - score: r.score, - }))); - } - } - - expect(rankedLists.length).toBeGreaterThan(0); - - const fused = reciprocalRankFusion(rankedLists); - expect(fused.length).toBeGreaterThan(0); - - const candidates = fused.slice(0, 10); - const reranked = await rerank( - query, - candidates.map(c => ({ file: c.file, text: c.body })), - DEFAULT_RERANK_MODEL, - testDb - ); - - expect(reranked.length).toBeGreaterThan(0); - }); - }); - - // =========================================================================== - // Tool: qmd_get (Get Document) - // =========================================================================== - - describe("qmd_get tool", () => { - test("retrieves document by display_path", () => { - const meta = findDocument(testDb, "readme.md", { includeBody: false }); - expect("error" in meta).toBe(false); - if ("error" in meta) return; - const body = getDocumentBody(testDb, meta) ?? ""; - - expect(meta.displayPath).toBe("docs/readme.md"); - expect(body).toContain("Project README"); - }); - - test("retrieves document by filepath", () => { - const meta = findDocument(testDb, "/test/docs/api.md", { includeBody: false }); - expect("error" in meta).toBe(false); - if ("error" in meta) return; - expect(meta.title).toBe("API Documentation"); - }); - - test("retrieves document by partial path", () => { - const result = findDocument(testDb, "api.md", { includeBody: false }); - expect("error" in result).toBe(false); - }); - - test("returns not found for missing document", () => { - const result = findDocument(testDb, "nonexistent.md", { includeBody: false }); - expect("error" in result).toBe(true); - if ("error" in result) { - expect(result.error).toBe("not_found"); - } - }); - - test("suggests similar files when not found", () => { - const result = findDocument(testDb, "readm.md", { includeBody: false }); // typo - expect("error" in result).toBe(true); - if ("error" in result) { - expect(result.similarFiles.length).toBeGreaterThanOrEqual(0); - } - }); - - test("supports line range with :line suffix", () => { - const meta = findDocument(testDb, "readme.md:2", { includeBody: false }); - expect("error" in meta).toBe(false); - if ("error" in meta) return; - const body = getDocumentBody(testDb, meta, 2, 2) ?? ""; - const lines = body.split("\n"); - expect(lines.length).toBeLessThanOrEqual(2); - }); - - test("supports fromLine parameter", () => { - const meta = findDocument(testDb, "readme.md", { includeBody: false }); - expect("error" in meta).toBe(false); - if ("error" in meta) return; - const body = getDocumentBody(testDb, meta, 3) ?? ""; - expect(body).not.toContain("# Project README"); - }); - - test("supports maxLines parameter", () => { - const meta = findDocument(testDb, "api.md", { includeBody: false }); - expect("error" in meta).toBe(false); - if ("error" in meta) return; - const body = getDocumentBody(testDb, meta, 1, 3) ?? ""; - const lines = body.split("\n"); - expect(lines.length).toBeLessThanOrEqual(3); - }); - - test("includes context for documents in context path", () => { - const result = findDocument(testDb, "meetings/meeting-2024-01.md", { includeBody: false }); - expect("error" in result).toBe(false); - if ("error" in result) return; - expect(result.context).toBe("Meeting notes and transcripts"); - }); - }); - - // =========================================================================== - // Tool: qmd_multi_get (Multi Get) - // =========================================================================== - - describe("qmd_multi_get tool", () => { - test("retrieves multiple documents by glob pattern", () => { - const { docs, errors } = findDocuments(testDb, "meetings/*.md", { includeBody: true }); - expect(errors.length).toBe(0); - expect(docs.length).toBe(2); - const paths = docs.map(d => d.doc.displayPath); - expect(paths).toContain("docs/meetings/meeting-2024-01.md"); - expect(paths).toContain("docs/meetings/meeting-2024-02.md"); - }); - - test("retrieves documents by comma-separated list", () => { - const { docs, errors } = findDocuments(testDb, "readme.md, api.md", { includeBody: true }); - expect(errors.length).toBe(0); - expect(docs.length).toBe(2); - }); - - test("returns errors for missing files in comma list", () => { - const { docs, errors } = findDocuments(testDb, "readme.md, nonexistent.md", { includeBody: true }); - expect(docs.length).toBe(1); - expect(errors.length).toBe(1); - expect(errors[0]).toContain("not found"); - }); - - test("skips files larger than maxBytes", () => { - const { docs } = findDocuments(testDb, "*.md", { includeBody: true, maxBytes: 1000 }); // 1KB limit - const large = docs.find(d => d.doc.displayPath === "docs/large-file.md"); - expect(large).toBeDefined(); - expect(large?.skipped).toBe(true); - if (large?.skipped) expect(large.skipReason).toContain("too large"); - }); - - test("respects maxLines parameter", () => { - const { docs } = findDocuments(testDb, "readme.md", { includeBody: true, maxBytes: DEFAULT_MULTI_GET_MAX_BYTES }); - expect(docs.length).toBe(1); - const d = docs[0]!; - expect(d.skipped).toBe(false); - if (d.skipped) return; - if (!("body" in d.doc)) { - throw new Error("Expected body to be included in findDocuments result"); - } - const lines = (d.doc.body || "").split("\n").slice(0, 2); - expect(lines.length).toBeLessThanOrEqual(2); - }); - - test("returns error for non-matching glob", () => { - const { docs, errors } = findDocuments(testDb, "nonexistent/*.md", { includeBody: true }); - expect(docs.length).toBe(0); - expect(errors.length).toBe(1); - expect(errors[0]).toContain("No files matched"); - }); - - test("includes context in results", () => { - const { docs } = findDocuments(testDb, "meetings/meeting-2024-01.md", { includeBody: true }); - expect(docs.length).toBe(1); - const d = docs[0]!; - expect(d.skipped).toBe(false); - if (d.skipped) return; - if (!("context" in d.doc)) { - throw new Error("Expected context to be present on document result"); - } - expect(d.doc.context).toBe("Meeting notes and transcripts"); - }); - }); - - // =========================================================================== - // Tool: qmd_status - // =========================================================================== - - describe("qmd_status tool", () => { - test("returns index status", () => { - const status = getStatus(testDb); - expect(status.totalDocuments).toBe(5); - expect(status.hasVectorIndex).toBe(true); - expect(status.collections.length).toBe(1); - expect(status.collections[0]!.path).toBe("/test/docs"); - }); - - test("shows documents needing embedding", () => { - const status = getStatus(testDb); - // large-file.md doesn't have embeddings - expect(status.needsEmbedding).toBe(1); - }); - }); - - // =========================================================================== - // Resource: qmd://{path} - // =========================================================================== - - describe("qmd:// resource", () => { - test("lists all documents", () => { - const docs = testDb.prepare(` - SELECT path as display_path, title - FROM documents - WHERE active = 1 - ORDER BY modified_at DESC - LIMIT 1000 - `).all() as { display_path: string; title: string }[]; - - expect(docs.length).toBe(5); - expect(docs.map(d => d.display_path)).toContain("readme.md"); - }); - - test("reads document by display_path", () => { - const path = "readme.md"; - const doc = testDb.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.path as display_path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path = ? AND d.active = 1 - `).get(path) as { filepath: string; display_path: string; body: string } | null; - - expect(doc).not.toBeNull(); - expect(doc?.body).toContain("Project README"); - }); - - test("reads document by URL-encoded path", () => { - // Simulate URL encoding that MCP clients may send - const encodedPath = "meetings%2Fmeeting-2024-01.md"; - const decodedPath = decodeURIComponent(encodedPath); - - const doc = testDb.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.path as display_path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path = ? AND d.active = 1 - `).get(decodedPath) as { filepath: string; display_path: string; body: string } | null; - - expect(doc).not.toBeNull(); - expect(doc?.display_path).toBe("meetings/meeting-2024-01.md"); - }); - - test("reads document by suffix match", () => { - const path = "meeting-2024-01.md"; // without meetings/ prefix - let doc = testDb.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.path as display_path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path = ? AND d.active = 1 - `).get(path) as { filepath: string; display_path: string; body: string } | null; - - if (!doc) { - doc = testDb.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.path as display_path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(`%${path}`) as { filepath: string; display_path: string; body: string } | null; - } - - expect(doc).not.toBeNull(); - expect(doc?.display_path).toBe("meetings/meeting-2024-01.md"); - }); - - test("returns not found for missing document", () => { - const path = "nonexistent.md"; - const doc = testDb.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.path as display_path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path = ? AND d.active = 1 - `).get(path) as { filepath: string; display_path: string; body: string } | null; - - expect(doc).toBeNull(); - }); - - test("includes context in document body", () => { - const path = "meetings/meeting-2024-01.md"; - const doc = testDb.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.path as display_path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path = ? AND d.active = 1 - `).get(path) as { filepath: string; display_path: string; body: string } | null; - - expect(doc).not.toBeNull(); - const context = getContextForFile(testDb, doc!.filepath); - expect(context).toBe("Meeting notes and transcripts"); - - // Verify context would be prepended - let text = doc!.body; - if (context) { - text = `\n\n` + text; - } - expect(text).toContain(""); - }); - - test("handles URL-encoded special characters", () => { - // Test various URL encodings - const testCases = [ - { encoded: "readme.md", decoded: "readme.md" }, - { encoded: "meetings%2Fmeeting-2024-01.md", decoded: "meetings/meeting-2024-01.md" }, - { encoded: "api.md%3A10", decoded: "api.md:10" }, // with line number - ]; - - for (const { encoded, decoded } of testCases) { - expect(decodeURIComponent(encoded)).toBe(decoded); - } - }); - - test("handles double-encoded URLs", () => { - // Some clients may double-encode - const doubleEncoded = "meetings%252Fmeeting-2024-01.md"; - const singleDecoded = decodeURIComponent(doubleEncoded); - expect(singleDecoded).toBe("meetings%2Fmeeting-2024-01.md"); - - const fullyDecoded = decodeURIComponent(singleDecoded); - expect(fullyDecoded).toBe("meetings/meeting-2024-01.md"); - }); - - test("handles URL-encoded paths with spaces", () => { - // Add a document with spaces in the path - const now = new Date().toISOString(); - const body = "# Podcast Episode\n\nInterview content here."; - const hash = "hash_spaces"; - const path = "External Podcast/2023 April - Interview.md"; - - // Insert content first - testDb.prepare(` - INSERT OR IGNORE INTO content (hash, doc, created_at) - VALUES (?, ?, ?) - `).run(hash, body, now); - - // Then insert document metadata - testDb.prepare(` - INSERT INTO documents (collection, path, title, hash, created_at, modified_at, active) - VALUES ('docs', ?, ?, ?, ?, ?, 1) - `).run(path, "Podcast Episode", hash, now, now); - - // Simulate URL-encoded path from MCP client - const encodedPath = "External%20Podcast%2F2023%20April%20-%20Interview.md"; - const decodedPath = decodeURIComponent(encodedPath); - - expect(decodedPath).toBe("External Podcast/2023 April - Interview.md"); - - const doc = testDb.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.path as display_path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path = ? AND d.active = 1 - `).get(decodedPath) as { filepath: string; display_path: string; body: string } | null; - - expect(doc).not.toBeNull(); - expect(doc?.display_path).toBe("External Podcast/2023 April - Interview.md"); - expect(doc?.body).toContain("Podcast Episode"); - }); - }); - - // =========================================================================== - // Prompt: query - // =========================================================================== - - describe("query prompt", () => { - test("returns usage guide", () => { - // The prompt content is static, just verify the structure - const promptContent = `# QMD - Quick Markdown Search - -QMD is your on-device search engine for markdown knowledge bases.`; - - expect(promptContent).toContain("QMD"); - expect(promptContent).toContain("search"); - }); - - test("describes all available tools", () => { - const toolNames = [ - "qmd_search", - "qmd_vsearch", - "qmd_query", - "qmd_get", - "qmd_multi_get", - "qmd_status", - ]; - - // Verify these are documented in the prompt - const promptGuide = ` -### 1. qmd_search (Fast keyword search) -### 2. qmd_vsearch (Semantic search) -### 3. qmd_query (Hybrid search - highest quality) -### 4. qmd_get (Retrieve document) -### 5. qmd_multi_get (Retrieve multiple documents) -### 6. qmd_status (Index info) - `; - - for (const tool of toolNames) { - expect(promptGuide).toContain(tool); - } - }); - }); - - // =========================================================================== - // Edge Cases - // =========================================================================== - - describe("edge cases", () => { - test("handles empty query", () => { - const results = searchFTS(testDb, "", 10); - expect(results.length).toBe(0); - }); - - test("handles special characters in query", () => { - const results = searchFTS(testDb, "project's", 10); - // Should not throw - expect(Array.isArray(results)).toBe(true); - }); - - test("handles unicode in query", () => { - const results = searchFTS(testDb, "文档", 10); - expect(Array.isArray(results)).toBe(true); - }); - - test("handles very long query", () => { - const longQuery = "documentation ".repeat(100); - const results = searchFTS(testDb, longQuery, 10); - expect(Array.isArray(results)).toBe(true); - }); - - test("handles query with only stopwords", () => { - const results = searchFTS(testDb, "the and or", 10); - expect(Array.isArray(results)).toBe(true); - }); - - test("extracts snippet around matching text", () => { - const body = "Line 1\nLine 2\nThis is the important line with the keyword\nLine 4\nLine 5"; - const { line, snippet } = extractSnippet(body, "keyword", 200); - expect(snippet).toContain("keyword"); - expect(line).toBe(3); - }); - - test("handles snippet extraction with chunkPos", () => { - const body = "A".repeat(1000) + "KEYWORD" + "B".repeat(1000); - const chunkPos = 1000; // Position of KEYWORD - const { snippet } = extractSnippet(body, "keyword", 200, chunkPos); - expect(snippet).toContain("KEYWORD"); - }); - }); - - // =========================================================================== - // MCP Spec Compliance - // =========================================================================== - - describe("MCP spec compliance", () => { - test("encodeQmdPath preserves slashes but encodes special chars", () => { - // Helper function behavior (tested indirectly through resource URIs) - const path = "External Podcast/2023 April - Interview.md"; - const segments = path.split('/').map(s => encodeURIComponent(s)).join('/'); - expect(segments).toBe("External%20Podcast/2023%20April%20-%20Interview.md"); - expect(segments).toContain("/"); // Slashes preserved - expect(segments).toContain("%20"); // Spaces encoded - }); - - test("search results have correct structure for structuredContent", () => { - const results = searchFTS(testDb, "readme", 5); - const structured = results.map(r => ({ - file: r.displayPath, - title: r.title, - score: Math.round(r.score * 100) / 100, - context: getContextForFile(testDb, r.filepath), - snippet: extractSnippet(r.body || "", "readme", 300, r.chunkPos).snippet, - })); - - expect(structured.length).toBeGreaterThan(0); - const item = structured[0]!; - expect(typeof item.file).toBe("string"); - expect(typeof item.title).toBe("string"); - expect(typeof item.score).toBe("number"); - expect(item.score).toBeGreaterThanOrEqual(0); - expect(item.score).toBeLessThanOrEqual(1); - expect(typeof item.snippet).toBe("string"); - }); - - test("error responses should include isError flag", () => { - // Simulate what MCP server returns for errors - const errorResponse = { - content: [{ type: "text", text: "Collection not found: nonexistent" }], - isError: true, - }; - expect(errorResponse.isError).toBe(true); - expect(errorResponse.content[0]!.type).toBe("text"); - }); - - test("embedded resources include name and title", () => { - // Simulate what qmd_get returns - const meta = findDocument(testDb, "readme.md", { includeBody: false }); - expect("error" in meta).toBe(false); - if ("error" in meta) return; - const body = getDocumentBody(testDb, meta) ?? ""; - const resource = { - uri: `qmd://${meta.displayPath}`, - name: meta.displayPath, - title: meta.title, - mimeType: "text/markdown", - text: body, - }; - expect(resource.name).toBe("docs/readme.md"); - expect(resource.title).toBe("Project README"); - expect(resource.mimeType).toBe("text/markdown"); - }); - - test("status response includes structuredContent", () => { - const status = getStatus(testDb); - // Verify structure matches StatusResult type - expect(typeof status.totalDocuments).toBe("number"); - expect(typeof status.needsEmbedding).toBe("number"); - expect(typeof status.hasVectorIndex).toBe("boolean"); - expect(Array.isArray(status.collections)).toBe(true); - if (status.collections.length > 0) { - const col = status.collections[0]!; - expect(typeof col.name).toBe("string"); // Collections now use names, not IDs - expect(typeof col.path).toBe("string"); - expect(typeof col.pattern).toBe("string"); - expect(typeof col.documents).toBe("number"); - } - }); - }); -}); diff --git a/tools/qmd/src/mcp.ts b/tools/qmd/src/mcp.ts deleted file mode 100644 index aa092048..00000000 --- a/tools/qmd/src/mcp.ts +++ /dev/null @@ -1,626 +0,0 @@ -#!/usr/bin/env bun -/** - * QMD MCP Server - Model Context Protocol server for QMD - * - * Exposes QMD search and document retrieval as MCP tools and resources. - * Documents are accessible via qmd:// URIs. - * - * Follows MCP spec 2025-06-18 for proper response types. - */ - -import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js"; -import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; -import { z } from "zod"; -import { - createStore, - reciprocalRankFusion, - extractSnippet, - DEFAULT_EMBED_MODEL, - DEFAULT_QUERY_MODEL, - DEFAULT_RERANK_MODEL, - DEFAULT_MULTI_GET_MAX_BYTES, -} from "./store.js"; -import type { RankedResult } from "./store.js"; - -// ============================================================================= -// Types for structured content -// ============================================================================= - -type SearchResultItem = { - docid: string; // Short docid (#abc123) for quick reference - file: string; - title: string; - score: number; - context: string | null; - snippet: string; -}; - -type StatusResult = { - totalDocuments: number; - needsEmbedding: number; - hasVectorIndex: boolean; - collections: { - name: string; - path: string; - pattern: string; - documents: number; - lastUpdated: string; - }[]; -}; - -// ============================================================================= -// Helper functions -// ============================================================================= - -/** - * Encode a path for use in qmd:// URIs. - * Encodes special characters but preserves forward slashes for readability. - */ -function encodeQmdPath(path: string): string { - // Encode each path segment separately to preserve slashes - return path.split('/').map(segment => encodeURIComponent(segment)).join('/'); -} - -/** - * Format search results as human-readable text summary - */ -function formatSearchSummary(results: SearchResultItem[], query: string): string { - if (results.length === 0) { - return `No results found for "${query}"`; - } - const lines = [`Found ${results.length} result${results.length === 1 ? '' : 's'} for "${query}":\n`]; - for (const r of results) { - lines.push(`${r.docid} ${Math.round(r.score * 100)}% ${r.file} - ${r.title}`); - } - return lines.join('\n'); -} - -/** - * Add line numbers to text content. - * Each line becomes: "{lineNum}: {content}" - */ -function addLineNumbers(text: string, startLine: number = 1): string { - const lines = text.split('\n'); - return lines.map((line, i) => `${startLine + i}: ${line}`).join('\n'); -} - -// ============================================================================= -// MCP Server -// ============================================================================= - -export async function startMcpServer(): Promise { - // Open database once at startup - keep it open for the lifetime of the server - const store = createStore(); - - const server = new McpServer({ - name: "qmd", - version: "1.0.0", - }); - - // --------------------------------------------------------------------------- - // Resource: qmd://{path} - read-only access to documents by path - // Note: No list() - documents are discovered via search tools - // --------------------------------------------------------------------------- - - server.registerResource( - "document", - new ResourceTemplate("qmd://{+path}", { list: undefined }), - { - title: "QMD Document", - description: "A markdown document from your QMD knowledge base. Use search tools to discover documents.", - mimeType: "text/markdown", - }, - async (uri, { path }) => { - // Decode URL-encoded path (MCP clients send encoded URIs) - const pathStr = Array.isArray(path) ? path.join('/') : (path || ''); - const decodedPath = decodeURIComponent(pathStr); - - // Parse virtual path: collection/relative/path - const parts = decodedPath.split('/'); - const collection = parts[0] || ''; - const relativePath = parts.slice(1).join('/'); - - // Find document by collection and path, join with content table - let doc = store.db.prepare(` - SELECT d.collection, d.path, d.title, c.doc as body - FROM documents d - JOIN content c ON c.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(collection, relativePath) as { collection: string; path: string; title: string; body: string } | null; - - // Try suffix match if exact match fails - if (!doc) { - doc = store.db.prepare(` - SELECT d.collection, d.path, d.title, c.doc as body - FROM documents d - JOIN content c ON c.hash = d.hash - WHERE d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(`%${relativePath}`) as { collection: string; path: string; title: string; body: string } | null; - } - - if (!doc) { - return { contents: [{ uri: uri.href, text: `Document not found: ${decodedPath}` }] }; - } - - // Construct virtual path for context lookup - const virtualPath = `qmd://${doc.collection}/${doc.path}`; - const context = store.getContextForFile(virtualPath); - - let text = addLineNumbers(doc.body); // Default to line numbers - if (context) { - text = `\n\n` + text; - } - - const displayName = `${doc.collection}/${doc.path}`; - return { - contents: [{ - uri: uri.href, - name: displayName, - title: doc.title || doc.path, - mimeType: "text/markdown", - text, - }], - }; - } - ); - - // --------------------------------------------------------------------------- - // Prompt: query guide - // --------------------------------------------------------------------------- - - server.registerPrompt( - "query", - { - title: "QMD Query Guide", - description: "How to effectively search your knowledge base with QMD", - }, - () => ({ - messages: [ - { - role: "user", - content: { - type: "text", - text: `# QMD - Quick Markdown Search - -QMD is your on-device search engine for markdown knowledge bases. Use it to find information across your notes, documents, and meeting transcripts. - -## Available Tools - -### 1. search (Fast keyword search) -Best for: Finding documents with specific keywords or phrases. -- Uses BM25 full-text search -- Fast, no LLM required -- Good for exact matches -- Use \`collection\` parameter to filter to a specific collection - -### 2. vsearch (Semantic search) -Best for: Finding conceptually related content even without exact keyword matches. -- Uses vector embeddings -- Understands meaning and context -- Good for "how do I..." or conceptual queries -- Use \`collection\` parameter to filter to a specific collection - -### 3. query (Hybrid search - highest quality) -Best for: Important searches where you want the best results. -- Combines keyword + semantic search -- Expands your query with variations -- Re-ranks results with LLM -- Slower but most accurate -- Use \`collection\` parameter to filter to a specific collection - -### 4. get (Retrieve document) -Best for: Getting the full content of a single document you found. -- Use the file path from search results -- Supports line ranges: \`file.md:100\` or fromLine/maxLines parameters -- Suggests similar files if not found - -### 5. multi_get (Retrieve multiple documents) -Best for: Getting content from multiple files at once. -- Use glob patterns: \`journals/2025-05*.md\` -- Or comma-separated: \`file1.md, file2.md\` -- Skips files over maxBytes (default 10KB) - use get for large files - -### 6. status (Index info) -Shows collection info, document counts, and embedding status. - -## Resources - -You can also access documents directly via the \`qmd://\` URI scheme: -- List all documents: \`resources/list\` -- Read a document: \`resources/read\` with uri \`qmd://path/to/file.md\` - -## Search Strategy - -1. **Start with search** for quick keyword lookups -2. **Use vsearch** when keywords aren't working or for conceptual queries -3. **Use query** for important searches or when you need high confidence -4. **Use get** to retrieve a single full document -5. **Use multi_get** to batch retrieve multiple related files - -## Tips - -- Use \`minScore: 0.5\` to filter low-relevance results -- Use \`collection: "notes"\` to search only in a specific collection -- Check the "Context" field - it describes what kind of content the file contains -- File paths are relative to their collection (e.g., \`pages/meeting.md\`) -- For glob patterns, match on display_path (e.g., \`journals/2025-*.md\`)`, - }, - }, - ], - }) - ); - - // --------------------------------------------------------------------------- - // Tool: qmd_search (BM25 full-text) - // --------------------------------------------------------------------------- - - server.registerTool( - "search", - { - title: "Search (BM25)", - description: "Fast keyword-based full-text search using BM25. Best for finding documents with specific words or phrases.", - inputSchema: { - query: z.string().describe("Search query - keywords or phrases to find"), - limit: z.number().optional().default(10).describe("Maximum number of results (default: 10)"), - minScore: z.number().optional().default(0).describe("Minimum relevance score 0-1 (default: 0)"), - collection: z.string().optional().describe("Filter to a specific collection by name"), - }, - }, - async ({ query, limit, minScore, collection }) => { - // Note: Collection filtering is now done post-search since collections are managed in YAML - const results = store.searchFTS(query, limit || 10) - .filter(r => !collection || r.collectionName === collection); - const filtered: SearchResultItem[] = results - .filter(r => r.score >= (minScore || 0)) - .map(r => { - const { line, snippet } = extractSnippet(r.body || "", query, 300, r.chunkPos); - return { - docid: `#${r.docid}`, - file: r.displayPath, - title: r.title, - score: Math.round(r.score * 100) / 100, - context: store.getContextForFile(r.filepath), - snippet: addLineNumbers(snippet, line), // Default to line numbers - }; - }); - - return { - content: [{ type: "text", text: formatSearchSummary(filtered, query) }], - structuredContent: { results: filtered }, - }; - } - ); - - // --------------------------------------------------------------------------- - // Tool: qmd_vsearch (Vector semantic search) - // --------------------------------------------------------------------------- - - server.registerTool( - "vsearch", - { - title: "Vector Search (Semantic)", - description: "Semantic similarity search using vector embeddings. Finds conceptually related content even without exact keyword matches. Requires embeddings (run 'qmd embed' first).", - inputSchema: { - query: z.string().describe("Natural language query - describe what you're looking for"), - limit: z.number().optional().default(10).describe("Maximum number of results (default: 10)"), - minScore: z.number().optional().default(0.3).describe("Minimum relevance score 0-1 (default: 0.3)"), - collection: z.string().optional().describe("Filter to a specific collection by name"), - }, - }, - async ({ query, limit, minScore, collection }) => { - const tableExists = store.db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get(); - if (!tableExists) { - return { - content: [{ type: "text", text: "Vector index not found. Run 'qmd embed' first to create embeddings." }], - isError: true, - }; - } - - // Expand query - const queries = await store.expandQuery(query, DEFAULT_QUERY_MODEL); - - // Collect results (filter by collection after search) - const allResults = new Map(); - for (const q of queries) { - const vecResults = await store.searchVec(q, DEFAULT_EMBED_MODEL, limit || 10) - .then(results => results.filter(r => !collection || r.collectionName === collection)); - for (const r of vecResults) { - const existing = allResults.get(r.filepath); - if (!existing || r.score > existing.score) { - allResults.set(r.filepath, { file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score, docid: r.docid }); - } - } - } - - const filtered: SearchResultItem[] = Array.from(allResults.values()) - .sort((a, b) => b.score - a.score) - .slice(0, limit || 10) - .filter(r => r.score >= (minScore || 0.3)) - .map(r => { - const { line, snippet } = extractSnippet(r.body || "", query, 300); - return { - docid: `#${r.docid}`, - file: r.displayPath, - title: r.title, - score: Math.round(r.score * 100) / 100, - context: store.getContextForFile(r.file), - snippet: addLineNumbers(snippet, line), // Default to line numbers - }; - }); - - return { - content: [{ type: "text", text: formatSearchSummary(filtered, query) }], - structuredContent: { results: filtered }, - }; - } - ); - - // --------------------------------------------------------------------------- - // Tool: qmd_query (Hybrid with reranking) - // --------------------------------------------------------------------------- - - server.registerTool( - "query", - { - title: "Hybrid Query (Best Quality)", - description: "Highest quality search combining BM25 + vector + query expansion + LLM reranking. Slower but most accurate. Use for important searches.", - inputSchema: { - query: z.string().describe("Natural language query - describe what you're looking for"), - limit: z.number().optional().default(10).describe("Maximum number of results (default: 10)"), - minScore: z.number().optional().default(0).describe("Minimum relevance score 0-1 (default: 0)"), - collection: z.string().optional().describe("Filter to a specific collection by name"), - }, - }, - async ({ query, limit, minScore, collection }) => { - // Expand query - const queries = await store.expandQuery(query, DEFAULT_QUERY_MODEL); - - // Collect ranked lists (filter by collection after search) - const rankedLists: RankedResult[][] = []; - const docidMap = new Map(); // filepath -> docid - const hasVectors = !!store.db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get(); - - for (const q of queries) { - const ftsResults = store.searchFTS(q, 20) - .filter(r => !collection || r.collectionName === collection); - if (ftsResults.length > 0) { - for (const r of ftsResults) docidMap.set(r.filepath, r.docid); - rankedLists.push(ftsResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score }))); - } - if (hasVectors) { - const vecResults = await store.searchVec(q, DEFAULT_EMBED_MODEL, 20) - .then(results => results.filter(r => !collection || r.collectionName === collection)); - if (vecResults.length > 0) { - for (const r of vecResults) docidMap.set(r.filepath, r.docid); - rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score }))); - } - } - } - - // RRF fusion - const weights = rankedLists.map((_, i) => i < 2 ? 2.0 : 1.0); - const fused = reciprocalRankFusion(rankedLists, weights); - const candidates = fused.slice(0, 30); - - // Rerank - const reranked = await store.rerank( - query, - candidates.map(c => ({ file: c.file, text: c.body })), - DEFAULT_RERANK_MODEL - ); - - // Blend scores - const candidateMap = new Map(candidates.map(c => [c.file, { displayPath: c.displayPath, title: c.title, body: c.body }])); - const rrfRankMap = new Map(candidates.map((c, i) => [c.file, i + 1])); - - const filtered: SearchResultItem[] = reranked.map(r => { - const rrfRank = rrfRankMap.get(r.file) || candidates.length; - let rrfWeight: number; - if (rrfRank <= 3) rrfWeight = 0.75; - else if (rrfRank <= 10) rrfWeight = 0.60; - else rrfWeight = 0.40; - const rrfScore = 1 / rrfRank; - const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score; - const candidate = candidateMap.get(r.file); - const { line, snippet } = extractSnippet(candidate?.body || "", query, 300); - return { - docid: `#${docidMap.get(r.file) || ""}`, - file: candidate?.displayPath || "", - title: candidate?.title || "", - score: Math.round(blendedScore * 100) / 100, - context: store.getContextForFile(r.file), - snippet: addLineNumbers(snippet, line), // Default to line numbers - }; - }).filter(r => r.score >= (minScore || 0)).slice(0, limit || 10); - - return { - content: [{ type: "text", text: formatSearchSummary(filtered, query) }], - structuredContent: { results: filtered }, - }; - } - ); - - // --------------------------------------------------------------------------- - // Tool: qmd_get (Retrieve document) - // --------------------------------------------------------------------------- - - server.registerTool( - "get", - { - title: "Get Document", - description: "Retrieve the full content of a document by its file path or docid. Use paths or docids (#abc123) from search results. Suggests similar files if not found.", - inputSchema: { - file: z.string().describe("File path or docid from search results (e.g., 'pages/meeting.md', '#abc123', or 'pages/meeting.md:100' to start at line 100)"), - fromLine: z.number().optional().describe("Start from this line number (1-indexed)"), - maxLines: z.number().optional().describe("Maximum number of lines to return"), - lineNumbers: z.boolean().optional().default(false).describe("Add line numbers to output (format: 'N: content')"), - }, - }, - async ({ file, fromLine, maxLines, lineNumbers }) => { - // Support :line suffix in `file` (e.g. "foo.md:120") when fromLine isn't provided - let parsedFromLine = fromLine; - let lookup = file; - const colonMatch = lookup.match(/:(\d+)$/); - if (colonMatch && colonMatch[1] && parsedFromLine === undefined) { - parsedFromLine = parseInt(colonMatch[1], 10); - lookup = lookup.slice(0, -colonMatch[0].length); - } - - const result = store.findDocument(lookup, { includeBody: false }); - - if ("error" in result) { - let msg = `Document not found: ${file}`; - if (result.similarFiles.length > 0) { - msg += `\n\nDid you mean one of these?\n${result.similarFiles.map(s => ` - ${s}`).join('\n')}`; - } - return { - content: [{ type: "text", text: msg }], - isError: true, - }; - } - - const body = store.getDocumentBody(result, parsedFromLine, maxLines) ?? ""; - let text = body; - if (lineNumbers) { - const startLine = parsedFromLine || 1; - text = addLineNumbers(text, startLine); - } - if (result.context) { - text = `\n\n` + text; - } - - return { - content: [{ - type: "resource", - resource: { - uri: `qmd://${encodeQmdPath(result.displayPath)}`, - name: result.displayPath, - title: result.title, - mimeType: "text/markdown", - text, - }, - }], - }; - } - ); - - // --------------------------------------------------------------------------- - // Tool: qmd_multi_get (Retrieve multiple documents) - // --------------------------------------------------------------------------- - - server.registerTool( - "multi_get", - { - title: "Multi-Get Documents", - description: "Retrieve multiple documents by glob pattern (e.g., 'journals/2025-05*.md') or comma-separated list. Skips files larger than maxBytes.", - inputSchema: { - pattern: z.string().describe("Glob pattern or comma-separated list of file paths"), - maxLines: z.number().optional().describe("Maximum lines per file"), - maxBytes: z.number().optional().default(10240).describe("Skip files larger than this (default: 10240 = 10KB)"), - lineNumbers: z.boolean().optional().default(false).describe("Add line numbers to output (format: 'N: content')"), - }, - }, - async ({ pattern, maxLines, maxBytes, lineNumbers }) => { - const { docs, errors } = store.findDocuments(pattern, { includeBody: true, maxBytes: maxBytes || DEFAULT_MULTI_GET_MAX_BYTES }); - - if (docs.length === 0 && errors.length === 0) { - return { - content: [{ type: "text", text: `No files matched pattern: ${pattern}` }], - isError: true, - }; - } - - const content: ({ type: "text"; text: string } | { type: "resource"; resource: { uri: string; name: string; title?: string; mimeType: string; text: string } })[] = []; - - if (errors.length > 0) { - content.push({ type: "text", text: `Errors:\n${errors.join('\n')}` }); - } - - for (const result of docs) { - if (result.skipped) { - content.push({ - type: "text", - text: `[SKIPPED: ${result.doc.displayPath} - ${result.skipReason}. Use 'qmd_get' with file="${result.doc.displayPath}" to retrieve.]`, - }); - continue; - } - - let text = result.doc.body || ""; - if (maxLines !== undefined) { - const lines = text.split("\n"); - text = lines.slice(0, maxLines).join("\n"); - if (lines.length > maxLines) { - text += `\n\n[... truncated ${lines.length - maxLines} more lines]`; - } - } - if (lineNumbers) { - text = addLineNumbers(text); - } - if (result.doc.context) { - text = `\n\n` + text; - } - - content.push({ - type: "resource", - resource: { - uri: `qmd://${encodeQmdPath(result.doc.displayPath)}`, - name: result.doc.displayPath, - title: result.doc.title, - mimeType: "text/markdown", - text, - }, - }); - } - - return { content }; - } - ); - - // --------------------------------------------------------------------------- - // Tool: qmd_status (Index status) - // --------------------------------------------------------------------------- - - server.registerTool( - "status", - { - title: "Index Status", - description: "Show the status of the QMD index: collections, document counts, and health information.", - inputSchema: {}, - }, - async () => { - const status: StatusResult = store.getStatus(); - - const summary = [ - `QMD Index Status:`, - ` Total documents: ${status.totalDocuments}`, - ` Needs embedding: ${status.needsEmbedding}`, - ` Vector index: ${status.hasVectorIndex ? 'yes' : 'no'}`, - ` Collections: ${status.collections.length}`, - ]; - - for (const col of status.collections) { - summary.push(` - ${col.path} (${col.documents} docs)`); - } - - return { - content: [{ type: "text", text: summary.join('\n') }], - structuredContent: status, - }; - } - ); - - // --------------------------------------------------------------------------- - // Connect via stdio - // --------------------------------------------------------------------------- - - const transport = new StdioServerTransport(); - await server.connect(transport); - - // Note: Database stays open - it will be closed when the process exits -} - -// Run if this is the main module -if (import.meta.main) { - startMcpServer().catch(console.error); -} diff --git a/tools/qmd/src/qmd.ts b/tools/qmd/src/qmd.ts deleted file mode 100755 index a6b4b312..00000000 --- a/tools/qmd/src/qmd.ts +++ /dev/null @@ -1,2699 +0,0 @@ -#!/usr/bin/env bun -import { Database } from "bun:sqlite"; -import { Glob, $ } from "bun"; -import { parseArgs } from "util"; -import { readFileSync, statSync } from "fs"; -import * as sqliteVec from "sqlite-vec"; -import { - getPwd, - getRealPath, - homedir, - resolve, - enableProductionMode, - searchFTS, - searchVec, - extractSnippet, - getContextForFile, - getContextForPath, - listCollections, - removeCollection, - renameCollection, - findSimilarFiles, - findDocumentByDocid, - isDocid, - matchFilesByGlob, - getHashesNeedingEmbedding, - getHashesForEmbedding, - clearAllEmbeddings, - insertEmbedding, - getStatus, - hashContent, - extractTitle, - formatDocForEmbedding, - formatQueryForEmbedding, - chunkDocument, - chunkDocumentByTokens, - clearCache, - getCacheKey, - getCachedResult, - setCachedResult, - getIndexHealth, - parseVirtualPath, - buildVirtualPath, - isVirtualPath, - resolveVirtualPath, - toVirtualPath, - insertContent, - insertDocument, - findActiveDocument, - updateDocumentTitle, - updateDocument, - deactivateDocument, - getActiveDocumentPaths, - cleanupOrphanedContent, - deleteLLMCache, - deleteInactiveDocuments, - cleanupOrphanedVectors, - vacuumDatabase, - getCollectionsWithoutContext, - getTopLevelPathsWithoutContext, - handelize, - DEFAULT_EMBED_MODEL, - DEFAULT_QUERY_MODEL, - DEFAULT_RERANK_MODEL, - DEFAULT_GLOB, - DEFAULT_MULTI_GET_MAX_BYTES, - createStore, - getDefaultDbPath, -} from "./store.js"; -import { getDefaultLlamaCpp, disposeDefaultLlamaCpp, withLLMSession, pullModels, DEFAULT_EMBED_MODEL_URI, DEFAULT_GENERATE_MODEL_URI, DEFAULT_RERANK_MODEL_URI, DEFAULT_MODEL_CACHE_DIR, type ILLMSession, type RerankDocument, type Queryable, type QueryType } from "./llm.js"; -import type { SearchResult, RankedResult } from "./store.js"; -import { - formatSearchResults, - formatDocuments, - escapeXml, - escapeCSV, - type OutputFormat, -} from "./formatter.js"; -import { - getCollection as getCollectionFromYaml, - listCollections as yamlListCollections, - addContext as yamlAddContext, - removeContext as yamlRemoveContext, - setGlobalContext, - listAllContexts, - setConfigIndexName, -} from "./collections.js"; - -// Enable production mode - allows using default database path -// Tests must set INDEX_PATH or use createStore() with explicit path -enableProductionMode(); - -// ============================================================================= -// Store/DB lifecycle (no legacy singletons in store.ts) -// ============================================================================= - -let store: ReturnType | null = null; -let storeDbPathOverride: string | undefined; - -function getStore(): ReturnType { - if (!store) { - store = createStore(storeDbPathOverride); - } - return store; -} - -function getDb(): Database { - return getStore().db; -} - -function closeDb(): void { - if (store) { - store.close(); - store = null; - } -} - -function getDbPath(): string { - return store?.dbPath ?? storeDbPathOverride ?? getDefaultDbPath(); -} - -function setIndexName(name: string | null): void { - storeDbPathOverride = name ? getDefaultDbPath(name) : undefined; - // Reset open handle so next use opens the new index - closeDb(); -} - -function ensureVecTable(_db: Database, dimensions: number): void { - // Store owns the DB; ignore `_db` and ensure vec table on the active store - getStore().ensureVecTable(dimensions); -} - -// Terminal colors (respects NO_COLOR env) -const useColor = !process.env.NO_COLOR && process.stdout.isTTY; -const c = { - reset: useColor ? "\x1b[0m" : "", - dim: useColor ? "\x1b[2m" : "", - bold: useColor ? "\x1b[1m" : "", - cyan: useColor ? "\x1b[36m" : "", - yellow: useColor ? "\x1b[33m" : "", - green: useColor ? "\x1b[32m" : "", - magenta: useColor ? "\x1b[35m" : "", - blue: useColor ? "\x1b[34m" : "", -}; - -// Terminal cursor control -const cursor = { - hide() { process.stderr.write('\x1b[?25l'); }, - show() { process.stderr.write('\x1b[?25h'); }, -}; - -// Ensure cursor is restored on exit -process.on('SIGINT', () => { cursor.show(); process.exit(130); }); -process.on('SIGTERM', () => { cursor.show(); process.exit(143); }); - -// Terminal progress bar using OSC 9;4 escape sequence -const progress = { - set(percent: number) { - process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`); - }, - clear() { - process.stderr.write(`\x1b]9;4;0\x07`); - }, - indeterminate() { - process.stderr.write(`\x1b]9;4;3\x07`); - }, - error() { - process.stderr.write(`\x1b]9;4;2\x07`); - }, -}; - -// Format seconds into human-readable ETA -function formatETA(seconds: number): string { - if (seconds < 60) return `${Math.round(seconds)}s`; - if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${Math.round(seconds % 60)}s`; - return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`; -} - - -// Check index health and print warnings/tips -function checkIndexHealth(db: Database): void { - const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db); - - // Warn if many docs need embedding - if (needsEmbedding > 0) { - const pct = Math.round((needsEmbedding / totalDocs) * 100); - if (pct >= 10) { - process.stderr.write(`${c.yellow}Warning: ${needsEmbedding} documents (${pct}%) need embeddings. Run 'qmd embed' for better results.${c.reset}\n`); - } else { - process.stderr.write(`${c.dim}Tip: ${needsEmbedding} documents need embeddings. Run 'qmd embed' to index them.${c.reset}\n`); - } - } - - // Check if most recent document update is older than 2 weeks - if (daysStale !== null && daysStale >= 14) { - process.stderr.write(`${c.dim}Tip: Index last updated ${daysStale} days ago. Run 'qmd update' to refresh.${c.reset}\n`); - } -} - -// Compute unique display path for a document -// Always include at least parent folder + filename, add more parent dirs until unique -function computeDisplayPath( - filepath: string, - collectionPath: string, - existingPaths: Set -): string { - // Get path relative to collection (include collection dir name) - const collectionDir = collectionPath.replace(/\/$/, ''); - const collectionName = collectionDir.split('/').pop() || ''; - - let relativePath: string; - if (filepath.startsWith(collectionDir + '/')) { - // filepath is under collection: use collection name + relative path - relativePath = collectionName + filepath.slice(collectionDir.length); - } else { - // Fallback: just use the filepath - relativePath = filepath; - } - - const parts = relativePath.split('/').filter(p => p.length > 0); - - // Always include at least parent folder + filename (minimum 2 parts if available) - // Then add more parent dirs until unique - const minParts = Math.min(2, parts.length); - for (let i = parts.length - minParts; i >= 0; i--) { - const candidate = parts.slice(i).join('/'); - if (!existingPaths.has(candidate)) { - return candidate; - } - } - - // Absolute fallback: use full path (should be unique) - return filepath; -} - -// Rerank documents using node-llama-cpp cross-encoder model -async function rerank(query: string, documents: { file: string; text: string }[], _model: string = DEFAULT_RERANK_MODEL, _db?: Database, session?: ILLMSession): Promise<{ file: string; score: number }[]> { - if (documents.length === 0) return []; - - const total = documents.length; - process.stderr.write(`Reranking ${total} documents...\n`); - progress.indeterminate(); - - const rerankDocs: RerankDocument[] = documents.map((doc) => ({ - file: doc.file, - text: doc.text.slice(0, 4000), // Truncate to context limit - })); - - const result = session - ? await session.rerank(query, rerankDocs) - : await getDefaultLlamaCpp().rerank(query, rerankDocs); - - progress.clear(); - process.stderr.write("\n"); - - return result.results.map((r) => ({ file: r.file, score: r.score })); -} - -function formatTimeAgo(date: Date): string { - const seconds = Math.floor((Date.now() - date.getTime()) / 1000); - if (seconds < 60) return `${seconds}s ago`; - const minutes = Math.floor(seconds / 60); - if (minutes < 60) return `${minutes}m ago`; - const hours = Math.floor(minutes / 60); - if (hours < 24) return `${hours}h ago`; - const days = Math.floor(hours / 24); - return `${days}d ago`; -} - -function formatBytes(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; - return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; -} - -function showStatus(): void { - const dbPath = getDbPath(); - const db = getDb(); - - // Collections are defined in YAML; no duplicate cleanup needed. - // Collections are defined in YAML; no duplicate cleanup needed. - - // Index size - let indexSize = 0; - try { - const stat = statSync(dbPath).size; - indexSize = stat; - } catch { } - - // Collections info (from YAML + database stats) - const collections = listCollections(db); - - // Overall stats - const totalDocs = db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get() as { count: number }; - const vectorCount = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get() as { count: number }; - const needsEmbedding = getHashesNeedingEmbedding(db); - - // Most recent update across all collections - const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get() as { latest: string | null }; - - console.log(`${c.bold}QMD Status${c.reset}\n`); - console.log(`Index: ${dbPath}`); - console.log(`Size: ${formatBytes(indexSize)}\n`); - - console.log(`${c.bold}Documents${c.reset}`); - console.log(` Total: ${totalDocs.count} files indexed`); - console.log(` Vectors: ${vectorCount.count} embedded`); - if (needsEmbedding > 0) { - console.log(` ${c.yellow}Pending: ${needsEmbedding} need embedding${c.reset} (run 'qmd embed')`); - } - if (mostRecent.latest) { - const lastUpdate = new Date(mostRecent.latest); - console.log(` Updated: ${formatTimeAgo(lastUpdate)}`); - } - - // Get all contexts grouped by collection (from YAML) - const allContexts = listAllContexts(); - const contextsByCollection = new Map(); - - for (const ctx of allContexts) { - // Group contexts by collection name - if (!contextsByCollection.has(ctx.collection)) { - contextsByCollection.set(ctx.collection, []); - } - contextsByCollection.get(ctx.collection)!.push({ - path_prefix: ctx.path, - context: ctx.context - }); - } - - if (collections.length > 0) { - console.log(`\n${c.bold}Collections${c.reset}`); - for (const col of collections) { - const lastMod = col.last_modified ? formatTimeAgo(new Date(col.last_modified)) : "never"; - const contexts = contextsByCollection.get(col.name) || []; - - console.log(` ${c.cyan}${col.name}${c.reset} ${c.dim}(qmd://${col.name}/)${c.reset}`); - console.log(` ${c.dim}Pattern:${c.reset} ${col.glob_pattern}`); - console.log(` ${c.dim}Files:${c.reset} ${col.active_count} (updated ${lastMod})`); - - if (contexts.length > 0) { - console.log(` ${c.dim}Contexts:${c.reset} ${contexts.length}`); - for (const ctx of contexts) { - // Handle both empty string and '/' as root context - const pathDisplay = (ctx.path_prefix === '' || ctx.path_prefix === '/') ? '/' : `/${ctx.path_prefix}`; - const contextPreview = ctx.context.length > 60 - ? ctx.context.substring(0, 57) + '...' - : ctx.context; - console.log(` ${c.dim}${pathDisplay}:${c.reset} ${contextPreview}`); - } - } - } - - // Show examples of virtual paths - console.log(`\n${c.bold}Examples${c.reset}`); - console.log(` ${c.dim}# List files in a collection${c.reset}`); - if (collections.length > 0 && collections[0]) { - console.log(` qmd ls ${collections[0].name}`); - } - console.log(` ${c.dim}# Get a document${c.reset}`); - if (collections.length > 0 && collections[0]) { - console.log(` qmd get qmd://${collections[0].name}/path/to/file.md`); - } - console.log(` ${c.dim}# Search within a collection${c.reset}`); - if (collections.length > 0 && collections[0]) { - console.log(` qmd search "query" -c ${collections[0].name}`); - } - } else { - console.log(`\n${c.dim}No collections. Run 'qmd collection add .' to index markdown files.${c.reset}`); - } - - closeDb(); -} - -async function updateCollections(): Promise { - const db = getDb(); - // Collections are defined in YAML; no duplicate cleanup needed. - - // Clear Ollama cache on update - clearCache(db); - - const collections = listCollections(db); - - if (collections.length === 0) { - console.log(`${c.dim}No collections found. Run 'qmd collection add .' to index markdown files.${c.reset}`); - closeDb(); - return; - } - - // Don't close db here - indexFiles will reuse it and close at the end - console.log(`${c.bold}Updating ${collections.length} collection(s)...${c.reset}\n`); - - for (let i = 0; i < collections.length; i++) { - const col = collections[i]; - if (!col) continue; - console.log(`${c.cyan}[${i + 1}/${collections.length}]${c.reset} ${c.bold}${col.name}${c.reset} ${c.dim}(${col.glob_pattern})${c.reset}`); - - // Execute custom update command if specified in YAML - const yamlCol = getCollectionFromYaml(col.name); - if (yamlCol?.update) { - console.log(`${c.dim} Running update command: ${yamlCol.update}${c.reset}`); - try { - const proc = Bun.spawn(["/usr/bin/env", "bash", "-c", yamlCol.update], { - cwd: col.pwd, - stdout: "pipe", - stderr: "pipe", - }); - - const output = await new Response(proc.stdout).text(); - const errorOutput = await new Response(proc.stderr).text(); - const exitCode = await proc.exited; - - if (output.trim()) { - console.log(output.trim().split('\n').map(l => ` ${l}`).join('\n')); - } - if (errorOutput.trim()) { - console.log(errorOutput.trim().split('\n').map(l => ` ${l}`).join('\n')); - } - - if (exitCode !== 0) { - console.log(`${c.yellow}✗ Update command failed with exit code ${exitCode}${c.reset}`); - process.exit(exitCode); - } - } catch (err) { - console.log(`${c.yellow}✗ Update command failed: ${err}${c.reset}`); - process.exit(1); - } - } - - await indexFiles(col.pwd, col.glob_pattern, col.name, true); - console.log(""); - } - - // Check if any documents need embedding (show once at end) - const finalDb = getDb(); - const needsEmbedding = getHashesNeedingEmbedding(finalDb); - closeDb(); - - console.log(`${c.green}✓ All collections updated.${c.reset}`); - if (needsEmbedding > 0) { - console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`); - } -} - -/** - * Detect which collection (if any) contains the given filesystem path. - * Returns { collectionId, collectionName, relativePath } or null if not in any collection. - */ -function detectCollectionFromPath(db: Database, fsPath: string): { collectionName: string; relativePath: string } | null { - const realPath = getRealPath(fsPath); - - // Find collections that this path is under from YAML - const allCollections = yamlListCollections(); - - // Find longest matching path - let bestMatch: { name: string; path: string } | null = null; - for (const coll of allCollections) { - if (realPath.startsWith(coll.path + '/') || realPath === coll.path) { - if (!bestMatch || coll.path.length > bestMatch.path.length) { - bestMatch = { name: coll.name, path: coll.path }; - } - } - } - - if (!bestMatch) return null; - - // Calculate relative path - let relativePath = realPath; - if (relativePath.startsWith(bestMatch.path + '/')) { - relativePath = relativePath.slice(bestMatch.path.length + 1); - } else if (relativePath === bestMatch.path) { - relativePath = ''; - } - - return { - collectionName: bestMatch.name, - relativePath - }; -} - -async function contextAdd(pathArg: string | undefined, contextText: string): Promise { - const db = getDb(); - - // Handle "/" as global context (applies to all collections) - if (pathArg === '/') { - setGlobalContext(contextText); - console.log(`${c.green}✓${c.reset} Set global context`); - console.log(`${c.dim}Context: ${contextText}${c.reset}`); - closeDb(); - return; - } - - // Resolve path - defaults to current directory if not provided - let fsPath = pathArg || '.'; - if (fsPath === '.' || fsPath === './') { - fsPath = getPwd(); - } else if (fsPath.startsWith('~/')) { - fsPath = homedir() + fsPath.slice(1); - } else if (!fsPath.startsWith('/') && !fsPath.startsWith('qmd://')) { - fsPath = resolve(getPwd(), fsPath); - } - - // Handle virtual paths (qmd://collection/path) - if (isVirtualPath(fsPath)) { - const parsed = parseVirtualPath(fsPath); - if (!parsed) { - console.error(`${c.yellow}Invalid virtual path: ${fsPath}${c.reset}`); - process.exit(1); - } - - const coll = getCollectionFromYaml(parsed.collectionName); - if (!coll) { - console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`); - process.exit(1); - } - - yamlAddContext(parsed.collectionName, parsed.path, contextText); - - const displayPath = parsed.path - ? `qmd://${parsed.collectionName}/${parsed.path}` - : `qmd://${parsed.collectionName}/ (collection root)`; - console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`); - console.log(`${c.dim}Context: ${contextText}${c.reset}`); - closeDb(); - return; - } - - // Detect collection from filesystem path - const detected = detectCollectionFromPath(db, fsPath); - if (!detected) { - console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`); - console.error(`${c.dim}Run 'qmd status' to see indexed collections${c.reset}`); - process.exit(1); - } - - yamlAddContext(detected.collectionName, detected.relativePath, contextText); - - const displayPath = detected.relativePath ? `qmd://${detected.collectionName}/${detected.relativePath}` : `qmd://${detected.collectionName}/`; - console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`); - console.log(`${c.dim}Context: ${contextText}${c.reset}`); - closeDb(); -} - -function contextList(): void { - const db = getDb(); - - const allContexts = listAllContexts(); - - if (allContexts.length === 0) { - console.log(`${c.dim}No contexts configured. Use 'qmd context add' to add one.${c.reset}`); - closeDb(); - return; - } - - console.log(`\n${c.bold}Configured Contexts${c.reset}\n`); - - let lastCollection = ''; - for (const ctx of allContexts) { - if (ctx.collection !== lastCollection) { - console.log(`${c.cyan}${ctx.collection}${c.reset}`); - lastCollection = ctx.collection; - } - - const displayPath = ctx.path ? ` ${ctx.path}` : ' / (root)'; - console.log(`${displayPath}`); - console.log(` ${c.dim}${ctx.context}${c.reset}`); - } - - closeDb(); -} - -function contextRemove(pathArg: string): void { - if (pathArg === '/') { - // Remove global context - setGlobalContext(undefined); - console.log(`${c.green}✓${c.reset} Removed global context`); - return; - } - - // Handle virtual paths - if (isVirtualPath(pathArg)) { - const parsed = parseVirtualPath(pathArg); - if (!parsed) { - console.error(`${c.yellow}Invalid virtual path: ${pathArg}${c.reset}`); - process.exit(1); - } - - const coll = getCollectionFromYaml(parsed.collectionName); - if (!coll) { - console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`); - process.exit(1); - } - - const success = yamlRemoveContext(coll.name, parsed.path); - - if (!success) { - console.error(`${c.yellow}No context found for: ${pathArg}${c.reset}`); - process.exit(1); - } - - console.log(`${c.green}✓${c.reset} Removed context for: ${pathArg}`); - return; - } - - // Handle filesystem paths - let fsPath = pathArg; - if (fsPath === '.' || fsPath === './') { - fsPath = getPwd(); - } else if (fsPath.startsWith('~/')) { - fsPath = homedir() + fsPath.slice(1); - } else if (!fsPath.startsWith('/')) { - fsPath = resolve(getPwd(), fsPath); - } - - const db = getDb(); - const detected = detectCollectionFromPath(db, fsPath); - closeDb(); - - if (!detected) { - console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`); - process.exit(1); - } - - const success = yamlRemoveContext(detected.collectionName, detected.relativePath); - - if (!success) { - console.error(`${c.yellow}No context found for: qmd://${detected.collectionName}/${detected.relativePath}${c.reset}`); - process.exit(1); - } - - console.log(`${c.green}✓${c.reset} Removed context for: qmd://${detected.collectionName}/${detected.relativePath}`); -} - -function contextCheck(): void { - const db = getDb(); - - // Get collections without any context - const collectionsWithoutContext = getCollectionsWithoutContext(db); - - // Get all collections to check for missing path contexts - const allCollections = listCollections(db); - - if (collectionsWithoutContext.length === 0 && allCollections.length > 0) { - // Check if all collections have contexts - console.log(`\n${c.green}✓${c.reset} ${c.bold}All collections have context configured${c.reset}\n`); - } - - if (collectionsWithoutContext.length > 0) { - console.log(`\n${c.yellow}Collections without any context:${c.reset}\n`); - - for (const coll of collectionsWithoutContext) { - console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(${coll.doc_count} documents)${c.reset}`); - console.log(` ${c.dim}Suggestion: qmd context add qmd://${coll.name}/ "Description of ${coll.name}"${c.reset}\n`); - } - } - - // Check for top-level paths without context within collections that DO have context - const collectionsWithContext = allCollections.filter(c => - c && !collectionsWithoutContext.some(cwc => cwc.name === c.name) - ); - - let hasPathSuggestions = false; - - for (const coll of collectionsWithContext) { - if (!coll) continue; - const missingPaths = getTopLevelPathsWithoutContext(db, coll.name); - - if (missingPaths.length > 0) { - if (!hasPathSuggestions) { - console.log(`${c.yellow}Top-level directories without context:${c.reset}\n`); - hasPathSuggestions = true; - } - - console.log(`${c.cyan}${coll.name}${c.reset}`); - for (const path of missingPaths) { - console.log(` ${path}`); - console.log(` ${c.dim}Suggestion: qmd context add qmd://${coll.name}/${path} "Description of ${path}"${c.reset}`); - } - console.log(''); - } - } - - if (collectionsWithoutContext.length === 0 && !hasPathSuggestions) { - console.log(`${c.dim}All collections and major paths have context configured.${c.reset}`); - console.log(`${c.dim}Use 'qmd context list' to see all configured contexts.${c.reset}\n`); - } - - closeDb(); -} - -function getDocument(filename: string, fromLine?: number, maxLines?: number, lineNumbers?: boolean): void { - const db = getDb(); - - // Parse :linenum suffix from filename (e.g., "file.md:100") - let inputPath = filename; - const colonMatch = inputPath.match(/:(\d+)$/); - if (colonMatch && !fromLine) { - const matched = colonMatch[1]; - if (matched) { - fromLine = parseInt(matched, 10); - inputPath = inputPath.slice(0, -colonMatch[0].length); - } - } - - // Handle docid lookup (#abc123, abc123, "#abc123", "abc123", etc.) - if (isDocid(inputPath)) { - const docidMatch = findDocumentByDocid(db, inputPath); - if (docidMatch) { - inputPath = docidMatch.filepath; - } else { - console.error(`Document not found: ${filename}`); - closeDb(); - process.exit(1); - } - } - - let doc: { collectionName: string; path: string; body: string } | null = null; - let virtualPath: string; - - // Handle virtual paths (qmd://collection/path) - if (isVirtualPath(inputPath)) { - const parsed = parseVirtualPath(inputPath); - if (!parsed) { - console.error(`Invalid virtual path: ${inputPath}`); - closeDb(); - process.exit(1); - } - - // Try exact match on collection + path - doc = db.prepare(` - SELECT d.collection as collectionName, d.path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(parsed.collectionName, parsed.path) as typeof doc; - - if (!doc) { - // Try fuzzy match by path ending - doc = db.prepare(` - SELECT d.collection as collectionName, d.path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(parsed.collectionName, `%${parsed.path}`) as typeof doc; - } - - virtualPath = inputPath; - } else { - // Try to interpret as collection/path format first (before filesystem path) - // If path is relative (no / or ~ prefix), check if first component is a collection name - if (!inputPath.startsWith('/') && !inputPath.startsWith('~')) { - const parts = inputPath.split('/'); - if (parts.length >= 2) { - const possibleCollection = parts[0]; - const possiblePath = parts.slice(1).join('/'); - - // Check if this collection exists - const collExists = possibleCollection ? db.prepare(` - SELECT 1 FROM documents WHERE collection = ? AND active = 1 LIMIT 1 - `).get(possibleCollection) : null; - - if (collExists) { - // Try exact match on collection + path - doc = db.prepare(` - SELECT d.collection as collectionName, d.path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(possibleCollection || "", possiblePath || "") as { collectionName: string; path: string; body: string } | null; - - if (!doc) { - // Try fuzzy match by path ending - doc = db.prepare(` - SELECT d.collection as collectionName, d.path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(possibleCollection || "", `%${possiblePath}`) as { collectionName: string; path: string; body: string } | null; - } - - if (doc) { - virtualPath = buildVirtualPath(doc.collectionName, doc.path); - // Skip the filesystem path handling below - } - } - } - } - - // If not found as collection/path, handle as filesystem paths - if (!doc) { - let fsPath = inputPath; - - // Expand ~ to home directory - if (fsPath.startsWith('~/')) { - fsPath = homedir() + fsPath.slice(1); - } else if (!fsPath.startsWith('/')) { - // Relative path - resolve from current directory - fsPath = resolve(getPwd(), fsPath); - } - fsPath = getRealPath(fsPath); - - // Try to detect which collection contains this path - const detected = detectCollectionFromPath(db, fsPath); - - if (detected) { - // Found collection - query by collection name + relative path - doc = db.prepare(` - SELECT d.collection as collectionName, d.path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(detected.collectionName, detected.relativePath) as { collectionName: string; path: string; body: string } | null; - } - - // Fuzzy match by filename (last component of path) - if (!doc) { - const filename = inputPath.split('/').pop() || inputPath; - doc = db.prepare(` - SELECT d.collection as collectionName, d.path, content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(`%${filename}`) as { collectionName: string; path: string; body: string } | null; - } - - if (doc) { - virtualPath = buildVirtualPath(doc.collectionName, doc.path); - } else { - virtualPath = inputPath; - } - } - } - - // Ensure doc is not null before proceeding - if (!doc) { - console.error(`Document not found: ${filename}`); - closeDb(); - process.exit(1); - } - - // Get context for this file - const context = getContextForPath(db, doc.collectionName, doc.path); - - let output = doc.body; - const startLine = fromLine || 1; - - // Apply line filtering if specified - if (fromLine !== undefined || maxLines !== undefined) { - const lines = output.split('\n'); - const start = startLine - 1; // Convert to 0-indexed - const end = maxLines !== undefined ? start + maxLines : lines.length; - output = lines.slice(start, end).join('\n'); - } - - // Add line numbers if requested - if (lineNumbers) { - output = addLineNumbers(output, startLine); - } - - // Output context header if exists - if (context) { - console.log(`Folder Context: ${context}\n---\n`); - } - console.log(output); - closeDb(); -} - -// Multi-get: fetch multiple documents by glob pattern or comma-separated list -function multiGet(pattern: string, maxLines?: number, maxBytes: number = DEFAULT_MULTI_GET_MAX_BYTES, format: OutputFormat = "cli"): void { - const db = getDb(); - - // Check if it's a comma-separated list or a glob pattern - const isCommaSeparated = pattern.includes(',') && !pattern.includes('*') && !pattern.includes('?'); - - let files: { filepath: string; displayPath: string; bodyLength: number; collection?: string; path?: string }[]; - - if (isCommaSeparated) { - // Comma-separated list of files (can be virtual paths or relative paths) - const names = pattern.split(',').map(s => s.trim()).filter(Boolean); - files = []; - for (const name of names) { - let doc: { virtual_path: string; body_length: number; collection: string; path: string } | null = null; - - // Handle virtual paths - if (isVirtualPath(name)) { - const parsed = parseVirtualPath(name); - if (parsed) { - // Try exact match on collection + path - doc = db.prepare(` - SELECT - 'qmd://' || d.collection || '/' || d.path as virtual_path, - LENGTH(content.doc) as body_length, - d.collection, - d.path - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(parsed.collectionName, parsed.path) as typeof doc; - } - } else { - // Try exact match on path - doc = db.prepare(` - SELECT - 'qmd://' || d.collection || '/' || d.path as virtual_path, - LENGTH(content.doc) as body_length, - d.collection, - d.path - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path = ? AND d.active = 1 - LIMIT 1 - `).get(name) as { virtual_path: string; body_length: number; collection: string; path: string } | null; - - // Try suffix match - if (!doc) { - doc = db.prepare(` - SELECT - 'qmd://' || d.collection || '/' || d.path as virtual_path, - LENGTH(content.doc) as body_length, - d.collection, - d.path - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(`%${name}`) as { virtual_path: string; body_length: number; collection: string; path: string } | null; - } - } - - if (doc) { - files.push({ - filepath: doc.virtual_path, - displayPath: doc.virtual_path, - bodyLength: doc.body_length, - collection: doc.collection, - path: doc.path - }); - } else { - console.error(`File not found: ${name}`); - } - } - } else { - // Glob pattern - matchFilesByGlob now returns virtual paths - files = matchFilesByGlob(db, pattern).map(f => ({ - ...f, - collection: undefined, // Will be fetched later if needed - path: undefined - })); - if (files.length === 0) { - console.error(`No files matched pattern: ${pattern}`); - closeDb(); - process.exit(1); - } - } - - // Collect results for structured output - const results: { file: string; displayPath: string; title: string; body: string; context: string | null; skipped: boolean; skipReason?: string }[] = []; - - for (const file of files) { - // Parse virtual path to get collection info if not already available - let collection = file.collection; - let path = file.path; - - if (!collection || !path) { - const parsed = parseVirtualPath(file.filepath); - if (parsed) { - collection = parsed.collectionName; - path = parsed.path; - } - } - - // Get context using collection-scoped function - const context = collection && path ? getContextForPath(db, collection, path) : null; - - // Check size limit - if (file.bodyLength > maxBytes) { - results.push({ - file: file.filepath, - displayPath: file.displayPath, - title: file.displayPath.split('/').pop() || file.displayPath, - body: "", - context, - skipped: true, - skipReason: `File too large (${Math.round(file.bodyLength / 1024)}KB > ${Math.round(maxBytes / 1024)}KB). Use 'qmd get ${file.displayPath}' to retrieve.`, - }); - continue; - } - - // Fetch document content using collection and path - if (!collection || !path) continue; - - const doc = db.prepare(` - SELECT content.doc as body, d.title - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(collection, path) as { body: string; title: string } | null; - - if (!doc) continue; - - let body = doc.body; - - // Apply line limit if specified - if (maxLines !== undefined) { - const lines = body.split('\n'); - body = lines.slice(0, maxLines).join('\n'); - if (lines.length > maxLines) { - body += `\n\n[... truncated ${lines.length - maxLines} more lines]`; - } - } - - results.push({ - file: file.filepath, - displayPath: file.displayPath, - title: doc.title || file.displayPath.split('/').pop() || file.displayPath, - body, - context, - skipped: false, - }); - } - - closeDb(); - - // Output based on format - if (format === "json") { - const output = results.map(r => ({ - file: r.displayPath, - title: r.title, - ...(r.context && { context: r.context }), - ...(r.skipped ? { skipped: true, reason: r.skipReason } : { body: r.body }), - })); - console.log(JSON.stringify(output, null, 2)); - } else if (format === "csv") { - const escapeField = (val: string | null | undefined): string => { - if (val === null || val === undefined) return ""; - const str = String(val); - if (str.includes(",") || str.includes('"') || str.includes("\n")) { - return `"${str.replace(/"/g, '""')}"`; - } - return str; - }; - console.log("file,title,context,skipped,body"); - for (const r of results) { - console.log([r.displayPath, r.title, r.context, r.skipped ? "true" : "false", r.skipped ? r.skipReason : r.body].map(escapeField).join(",")); - } - } else if (format === "files") { - for (const r of results) { - const ctx = r.context ? `,"${r.context.replace(/"/g, '""')}"` : ""; - const status = r.skipped ? "[SKIPPED]" : ""; - console.log(`${r.displayPath}${ctx}${status ? `,${status}` : ""}`); - } - } else if (format === "md") { - for (const r of results) { - console.log(`## ${r.displayPath}\n`); - if (r.title && r.title !== r.displayPath) console.log(`**Title:** ${r.title}\n`); - if (r.context) console.log(`**Context:** ${r.context}\n`); - if (r.skipped) { - console.log(`> ${r.skipReason}\n`); - } else { - console.log("```"); - console.log(r.body); - console.log("```\n"); - } - } - } else if (format === "xml") { - console.log(''); - console.log(""); - for (const r of results) { - console.log(" "); - console.log(` ${escapeXml(r.displayPath)}`); - console.log(` ${escapeXml(r.title)}`); - if (r.context) console.log(` ${escapeXml(r.context)}`); - if (r.skipped) { - console.log(` true`); - console.log(` ${escapeXml(r.skipReason || "")}`); - } else { - console.log(` ${escapeXml(r.body)}`); - } - console.log(" "); - } - console.log(""); - } else { - // CLI format (default) - for (const r of results) { - console.log(`\n${'='.repeat(60)}`); - console.log(`File: ${r.displayPath}`); - console.log(`${'='.repeat(60)}\n`); - - if (r.skipped) { - console.log(`[SKIPPED: ${r.skipReason}]`); - continue; - } - - if (r.context) { - console.log(`Folder Context: ${r.context}\n---\n`); - } - console.log(r.body); - } - } -} - -// List files in virtual file tree -function listFiles(pathArg?: string): void { - const db = getDb(); - - if (!pathArg) { - // No argument - list all collections - const yamlCollections = yamlListCollections(); - - if (yamlCollections.length === 0) { - console.log("No collections found. Run 'qmd add .' to index files."); - closeDb(); - return; - } - - // Get file counts from database for each collection - const collections = yamlCollections.map(coll => { - const stats = db.prepare(` - SELECT COUNT(*) as file_count - FROM documents d - WHERE d.collection = ? AND d.active = 1 - `).get(coll.name) as { file_count: number } | null; - - return { - name: coll.name, - file_count: stats?.file_count || 0 - }; - }); - - console.log(`${c.bold}Collections:${c.reset}\n`); - for (const coll of collections) { - console.log(` ${c.dim}qmd://${c.reset}${c.cyan}${coll.name}/${c.reset} ${c.dim}(${coll.file_count} files)${c.reset}`); - } - closeDb(); - return; - } - - // Parse the path argument - let collectionName: string; - let pathPrefix: string | null = null; - - if (pathArg.startsWith('qmd://')) { - // Virtual path format: qmd://collection/path - const parsed = parseVirtualPath(pathArg); - if (!parsed) { - console.error(`Invalid virtual path: ${pathArg}`); - closeDb(); - process.exit(1); - } - collectionName = parsed.collectionName; - pathPrefix = parsed.path; - } else { - // Just collection name or collection/path - const parts = pathArg.split('/'); - collectionName = parts[0] || ''; - if (parts.length > 1) { - pathPrefix = parts.slice(1).join('/'); - } - } - - // Get the collection - const coll = getCollectionFromYaml(collectionName); - if (!coll) { - console.error(`Collection not found: ${collectionName}`); - console.error(`Run 'qmd ls' to see available collections.`); - closeDb(); - process.exit(1); - } - - // List files in the collection with size and modification time - let query: string; - let params: any[]; - - if (pathPrefix) { - // List files under a specific path - query = ` - SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size - FROM documents d - JOIN content ct ON d.hash = ct.hash - WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1 - ORDER BY d.path - `; - params = [coll.name, `${pathPrefix}%`]; - } else { - // List all files in the collection - query = ` - SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size - FROM documents d - JOIN content ct ON d.hash = ct.hash - WHERE d.collection = ? AND d.active = 1 - ORDER BY d.path - `; - params = [coll.name]; - } - - const files = db.prepare(query).all(...params) as { path: string; title: string; modified_at: string; size: number }[]; - - if (files.length === 0) { - if (pathPrefix) { - console.log(`No files found under qmd://${collectionName}/${pathPrefix}`); - } else { - console.log(`No files found in collection: ${collectionName}`); - } - closeDb(); - return; - } - - // Calculate max widths for alignment - const maxSize = Math.max(...files.map(f => formatBytes(f.size).length)); - - // Output in ls -l style - for (const file of files) { - const sizeStr = formatBytes(file.size).padStart(maxSize); - const date = new Date(file.modified_at); - const timeStr = formatLsTime(date); - - // Dim the qmd:// prefix, highlight the filename - console.log(`${sizeStr} ${timeStr} ${c.dim}qmd://${collectionName}/${c.reset}${c.cyan}${file.path}${c.reset}`); - } - - closeDb(); -} - -// Format date/time like ls -l -function formatLsTime(date: Date): string { - const now = new Date(); - const sixMonthsAgo = new Date(now.getTime() - 6 * 30 * 24 * 60 * 60 * 1000); - - const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; - const month = months[date.getMonth()]; - const day = date.getDate().toString().padStart(2, ' '); - - // If file is older than 6 months, show year instead of time - if (date < sixMonthsAgo) { - const year = date.getFullYear(); - return `${month} ${day} ${year}`; - } else { - const hours = date.getHours().toString().padStart(2, '0'); - const minutes = date.getMinutes().toString().padStart(2, '0'); - return `${month} ${day} ${hours}:${minutes}`; - } -} - -// Collection management commands -function collectionList(): void { - const db = getDb(); - const collections = listCollections(db); - - if (collections.length === 0) { - console.log("No collections found. Run 'qmd add .' to create one."); - closeDb(); - return; - } - - console.log(`${c.bold}Collections (${collections.length}):${c.reset}\n`); - - for (const coll of collections) { - const updatedAt = coll.last_modified ? new Date(coll.last_modified) : new Date(); - const timeAgo = formatTimeAgo(updatedAt); - - console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}`); - console.log(` ${c.dim}Pattern:${c.reset} ${coll.glob_pattern}`); - console.log(` ${c.dim}Files:${c.reset} ${coll.active_count}`); - console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`); - console.log(); - } - - closeDb(); -} - -async function collectionAdd(pwd: string, globPattern: string, name?: string): Promise { - // If name not provided, generate from pwd basename - let collName = name; - if (!collName) { - const parts = pwd.split('/').filter(Boolean); - collName = parts[parts.length - 1] || 'root'; - } - - // Check if collection with this name already exists in YAML - const existing = getCollectionFromYaml(collName); - if (existing) { - console.error(`${c.yellow}Collection '${collName}' already exists.${c.reset}`); - console.error(`Use a different name with --name `); - process.exit(1); - } - - // Check if a collection with this pwd+glob already exists in YAML - const allCollections = yamlListCollections(); - const existingPwdGlob = allCollections.find(c => c.path === pwd && c.pattern === globPattern); - - if (existingPwdGlob) { - console.error(`${c.yellow}A collection already exists for this path and pattern:${c.reset}`); - console.error(` Name: ${existingPwdGlob.name} (qmd://${existingPwdGlob.name}/)`); - console.error(` Pattern: ${globPattern}`); - console.error(`\nUse 'qmd update' to re-index it, or remove it first with 'qmd collection remove ${existingPwdGlob.name}'`); - process.exit(1); - } - - // Add to YAML config - const { addCollection } = await import("./collections.js"); - addCollection(collName, pwd, globPattern); - - // Create the collection and index files - console.log(`Creating collection '${collName}'...`); - await indexFiles(pwd, globPattern, collName); - console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`); -} - -function collectionRemove(name: string): void { - // Check if collection exists in YAML - const coll = getCollectionFromYaml(name); - if (!coll) { - console.error(`${c.yellow}Collection not found: ${name}${c.reset}`); - console.error(`Run 'qmd collection list' to see available collections.`); - process.exit(1); - } - - const db = getDb(); - const result = removeCollection(db, name); - closeDb(); - - console.log(`${c.green}✓${c.reset} Removed collection '${name}'`); - console.log(` Deleted ${result.deletedDocs} documents`); - if (result.cleanedHashes > 0) { - console.log(` Cleaned up ${result.cleanedHashes} orphaned content hashes`); - } -} - -function collectionRename(oldName: string, newName: string): void { - // Check if old collection exists in YAML - const coll = getCollectionFromYaml(oldName); - if (!coll) { - console.error(`${c.yellow}Collection not found: ${oldName}${c.reset}`); - console.error(`Run 'qmd collection list' to see available collections.`); - process.exit(1); - } - - // Check if new name already exists in YAML - const existing = getCollectionFromYaml(newName); - if (existing) { - console.error(`${c.yellow}Collection name already exists: ${newName}${c.reset}`); - console.error(`Choose a different name or remove the existing collection first.`); - process.exit(1); - } - - const db = getDb(); - renameCollection(db, oldName, newName); - closeDb(); - - console.log(`${c.green}✓${c.reset} Renamed collection '${oldName}' to '${newName}'`); - console.log(` Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`); -} - -async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, collectionName?: string, suppressEmbedNotice: boolean = false): Promise { - const db = getDb(); - const resolvedPwd = pwd || getPwd(); - const now = new Date().toISOString(); - const excludeDirs = ["node_modules", ".git", ".cache", "vendor", "dist", "build"]; - - // Clear Ollama cache on index - clearCache(db); - - // Collection name must be provided (from YAML) - if (!collectionName) { - throw new Error("Collection name is required. Collections must be defined in ~/.config/qmd/index.yml"); - } - - console.log(`Collection: ${resolvedPwd} (${globPattern})`); - - progress.indeterminate(); - const glob = new Glob(globPattern); - const files: string[] = []; - for await (const file of glob.scan({ cwd: resolvedPwd, onlyFiles: true, followSymlinks: true })) { - // Skip node_modules, hidden folders (.*), and other common excludes - const parts = file.split("/"); - const shouldSkip = parts.some(part => - part === "node_modules" || - part.startsWith(".") || - excludeDirs.includes(part) - ); - if (!shouldSkip) { - files.push(file); - } - } - - const total = files.length; - if (total === 0) { - progress.clear(); - console.log("No files found matching pattern."); - closeDb(); - return; - } - - let indexed = 0, updated = 0, unchanged = 0, processed = 0; - const seenPaths = new Set(); - const startTime = Date.now(); - - for (const relativeFile of files) { - const filepath = getRealPath(resolve(resolvedPwd, relativeFile)); - const path = handelize(relativeFile); // Normalize path for token-friendliness - seenPaths.add(path); - - const content = readFileSync(filepath, "utf-8"); - - // Skip empty files - nothing useful to index - if (!content.trim()) { - processed++; - continue; - } - - const hash = await hashContent(content); - const title = extractTitle(content, relativeFile); - - // Check if document exists in this collection with this path - const existing = findActiveDocument(db, collectionName, path); - - if (existing) { - if (existing.hash === hash) { - // Hash unchanged, but check if title needs updating - if (existing.title !== title) { - updateDocumentTitle(db, existing.id, title, now); - updated++; - } else { - unchanged++; - } - } else { - // Content changed - insert new content hash and update document - insertContent(db, hash, content, now); - const stat = statSync(filepath); - updateDocument(db, existing.id, title, hash, - stat ? new Date(stat.mtime).toISOString() : now); - updated++; - } - } else { - // New document - insert content and document - indexed++; - insertContent(db, hash, content, now); - const stat = statSync(filepath); - insertDocument(db, collectionName, path, title, hash, - stat ? new Date(stat.birthtime).toISOString() : now, - stat ? new Date(stat.mtime).toISOString() : now); - } - - processed++; - progress.set((processed / total) * 100); - const elapsed = (Date.now() - startTime) / 1000; - const rate = processed / elapsed; - const remaining = (total - processed) / rate; - const eta = processed > 2 ? ` ETA: ${formatETA(remaining)}` : ""; - process.stderr.write(`\rIndexing: ${processed}/${total}${eta} `); - } - - // Deactivate documents in this collection that no longer exist - const allActive = getActiveDocumentPaths(db, collectionName); - let removed = 0; - for (const path of allActive) { - if (!seenPaths.has(path)) { - deactivateDocument(db, collectionName, path); - removed++; - } - } - - // Clean up orphaned content hashes (content not referenced by any document) - const orphanedContent = cleanupOrphanedContent(db); - - // Check if vector index needs updating - const needsEmbedding = getHashesNeedingEmbedding(db); - - progress.clear(); - console.log(`\nIndexed: ${indexed} new, ${updated} updated, ${unchanged} unchanged, ${removed} removed`); - if (orphanedContent > 0) { - console.log(`Cleaned up ${orphanedContent} orphaned content hash(es)`); - } - - if (needsEmbedding > 0 && !suppressEmbedNotice) { - console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`); - } - - closeDb(); -} - -function renderProgressBar(percent: number, width: number = 30): string { - const filled = Math.round((percent / 100) * width); - const empty = width - filled; - const bar = "█".repeat(filled) + "░".repeat(empty); - return bar; -} - -async function vectorIndex(model: string = DEFAULT_EMBED_MODEL, force: boolean = false): Promise { - const db = getDb(); - const now = new Date().toISOString(); - - // If force, clear all vectors - if (force) { - console.log(`${c.yellow}Force re-indexing: clearing all vectors...${c.reset}`); - clearAllEmbeddings(db); - } - - // Find unique hashes that need embedding (from active documents) - const hashesToEmbed = getHashesForEmbedding(db); - - if (hashesToEmbed.length === 0) { - console.log(`${c.green}✓ All content hashes already have embeddings.${c.reset}`); - closeDb(); - return; - } - - // Prepare documents with chunks - type ChunkItem = { hash: string; title: string; text: string; seq: number; pos: number; tokens: number; bytes: number; displayName: string }; - const allChunks: ChunkItem[] = []; - let multiChunkDocs = 0; - - // Chunk all documents using actual token counts - process.stderr.write(`Chunking ${hashesToEmbed.length} documents by token count...\n`); - for (const item of hashesToEmbed) { - const encoder = new TextEncoder(); - const bodyBytes = encoder.encode(item.body).length; - if (bodyBytes === 0) continue; // Skip empty - - const title = extractTitle(item.body, item.path); - const displayName = item.path; - const chunks = await chunkDocumentByTokens(item.body); // Uses actual tokenizer - - if (chunks.length > 1) multiChunkDocs++; - - for (let seq = 0; seq < chunks.length; seq++) { - allChunks.push({ - hash: item.hash, - title, - text: chunks[seq]!.text, // Chunk is guaranteed to exist by seq loop - seq, - pos: chunks[seq]!.pos, - tokens: chunks[seq]!.tokens, - bytes: encoder.encode(chunks[seq]!.text).length, - displayName, - }); - } - } - - if (allChunks.length === 0) { - console.log(`${c.green}✓ No non-empty documents to embed.${c.reset}`); - closeDb(); - return; - } - - const totalBytes = allChunks.reduce((sum, chk) => sum + chk.bytes, 0); - const totalChunks = allChunks.length; - const totalDocs = hashesToEmbed.length; - - console.log(`${c.bold}Embedding ${totalDocs} documents${c.reset} ${c.dim}(${totalChunks} chunks, ${formatBytes(totalBytes)})${c.reset}`); - if (multiChunkDocs > 0) { - console.log(`${c.dim}${multiChunkDocs} documents split into multiple chunks${c.reset}`); - } - console.log(`${c.dim}Model: ${model}${c.reset}\n`); - - // Hide cursor during embedding - cursor.hide(); - - // Wrap all LLM embedding operations in a session for lifecycle management - // Use 30 minute timeout for large collections - await withLLMSession(async (session) => { - // Get embedding dimensions from first chunk - progress.indeterminate(); - const firstChunk = allChunks[0]; - if (!firstChunk) { - throw new Error("No chunks available to embed"); - } - const firstText = formatDocForEmbedding(firstChunk.text, firstChunk.title); - const firstResult = await session.embed(firstText); - if (!firstResult) { - throw new Error("Failed to get embedding dimensions from first chunk"); - } - ensureVecTable(db, firstResult.embedding.length); - - let chunksEmbedded = 0, errors = 0, bytesProcessed = 0; - const startTime = Date.now(); - - // Batch embedding for better throughput - // Process in batches of 32 to balance memory usage and efficiency - const BATCH_SIZE = 32; - - for (let batchStart = 0; batchStart < allChunks.length; batchStart += BATCH_SIZE) { - const batchEnd = Math.min(batchStart + BATCH_SIZE, allChunks.length); - const batch = allChunks.slice(batchStart, batchEnd); - - // Format texts for embedding - const texts = batch.map(chunk => formatDocForEmbedding(chunk.text, chunk.title)); - - try { - // Batch embed all texts at once - const embeddings = await session.embedBatch(texts); - - // Insert each embedding - for (let i = 0; i < batch.length; i++) { - const chunk = batch[i]!; - const embedding = embeddings[i]; - - if (embedding) { - insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(embedding.embedding), model, now); - chunksEmbedded++; - } else { - errors++; - console.error(`\n${c.yellow}⚠ Error embedding "${chunk.displayName}" chunk ${chunk.seq}${c.reset}`); - } - bytesProcessed += chunk.bytes; - } - } catch (err) { - // If batch fails, try individual embeddings as fallback - for (const chunk of batch) { - try { - const text = formatDocForEmbedding(chunk.text, chunk.title); - const result = await session.embed(text); - if (result) { - insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(result.embedding), model, now); - chunksEmbedded++; - } else { - errors++; - } - } catch (innerErr) { - errors++; - console.error(`\n${c.yellow}⚠ Error embedding "${chunk.displayName}" chunk ${chunk.seq}: ${innerErr}${c.reset}`); - } - bytesProcessed += chunk.bytes; - } - } - - const percent = (bytesProcessed / totalBytes) * 100; - progress.set(percent); - - const elapsed = (Date.now() - startTime) / 1000; - const bytesPerSec = bytesProcessed / elapsed; - const remainingBytes = totalBytes - bytesProcessed; - const etaSec = remainingBytes / bytesPerSec; - - const bar = renderProgressBar(percent); - const percentStr = percent.toFixed(0).padStart(3); - const throughput = `${formatBytes(bytesPerSec)}/s`; - const eta = elapsed > 2 ? formatETA(etaSec) : "..."; - const errStr = errors > 0 ? ` ${c.yellow}${errors} err${c.reset}` : ""; - - process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${chunksEmbedded}/${totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset} `); - } - - progress.clear(); - cursor.show(); - const totalTimeSec = (Date.now() - startTime) / 1000; - const avgThroughput = formatBytes(totalBytes / totalTimeSec); - - console.log(`\r${c.green}${renderProgressBar(100)}${c.reset} ${c.bold}100%${c.reset} `); - console.log(`\n${c.green}✓ Done!${c.reset} Embedded ${c.bold}${chunksEmbedded}${c.reset} chunks from ${c.bold}${totalDocs}${c.reset} documents in ${c.bold}${formatETA(totalTimeSec)}${c.reset} ${c.dim}(${avgThroughput}/s)${c.reset}`); - if (errors > 0) { - console.log(`${c.yellow}⚠ ${errors} chunks failed${c.reset}`); - } - }, { maxDuration: 30 * 60 * 1000, name: 'embed-command' }); - - closeDb(); -} - -// Sanitize a term for FTS5: remove punctuation except apostrophes -function sanitizeFTS5Term(term: string): string { - // Remove all non-alphanumeric except apostrophes (for contractions like "don't") - return term.replace(/[^\w']/g, '').trim(); -} - -// Build FTS5 query: phrase-aware with fallback to individual terms -function buildFTS5Query(query: string): string { - // Sanitize the full query for phrase matching - const sanitizedQuery = query.replace(/[^\w\s']/g, '').trim(); - - const terms = query - .split(/\s+/) - .map(sanitizeFTS5Term) - .filter(term => term.length >= 2); // Skip single chars and empty - - if (terms.length === 0) return ""; - if (terms.length === 1) return `"${terms[0]!.replace(/"/g, '""')}"`; - - // Strategy: exact phrase OR proximity match OR individual terms - // Exact phrase matches rank highest, then close proximity, then any term - const phrase = `"${sanitizedQuery.replace(/"/g, '""')}"`; - const quotedTerms = terms.map(t => `"${t.replace(/"/g, '""')}"`); - - // FTS5 NEAR syntax: NEAR(term1 term2, distance) - const nearPhrase = `NEAR(${quotedTerms.join(' ')}, 10)`; - const orTerms = quotedTerms.join(' OR '); - - // Exact phrase > proximity > any term - return `(${phrase}) OR (${nearPhrase}) OR (${orTerms})`; -} - -// Normalize BM25 score to 0-1 range using sigmoid -function normalizeBM25(score: number): number { - // BM25 scores are negative in SQLite (lower = better) - // Typical range: -15 (excellent) to -2 (weak match) - // Map to 0-1 where higher is better - const absScore = Math.abs(score); - // Sigmoid-ish normalization: maps ~2-15 range to ~0.1-0.95 - return 1 / (1 + Math.exp(-(absScore - 5) / 3)); -} - -function normalizeScores(results: SearchResult[]): SearchResult[] { - if (results.length === 0) return results; - const maxScore = Math.max(...results.map(r => r.score)); - const minScore = Math.min(...results.map(r => r.score)); - const range = maxScore - minScore || 1; - return results.map(r => ({ ...r, score: (r.score - minScore) / range })); -} - -// Reciprocal Rank Fusion: combines multiple ranked lists -// RRF score = sum(1 / (k + rank)) across all lists where doc appears -// k=60 is standard, provides good balance between top and lower ranks - -function reciprocalRankFusion( - resultLists: RankedResult[][], - weights: number[] = [], // Weight per result list (default 1.0) - k: number = 60 -): RankedResult[] { - const scores = new Map(); - - for (let listIdx = 0; listIdx < resultLists.length; listIdx++) { - const results = resultLists[listIdx]; - if (!results) continue; - const weight = weights[listIdx] ?? 1.0; - for (let rank = 0; rank < results.length; rank++) { - const doc = results[rank]; - if (!doc) continue; // Ensure doc is not undefined - const rrfScore = weight / (k + rank + 1); - const existing = scores.get(doc.file); - if (existing) { - existing.score += rrfScore; - existing.bestRank = Math.min(existing.bestRank, rank); - } else { - scores.set(doc.file, { score: rrfScore, displayPath: doc.displayPath, title: doc.title, body: doc.body, bestRank: rank }); - } - } - } - - // Add bonus for best rank: documents that ranked #1-3 in any list get a boost - // This prevents dilution of exact matches by expansion queries - return Array.from(scores.entries()) - .map(([file, { score, displayPath, title, body, bestRank }]) => { - let bonus = 0; - if (bestRank === 0) bonus = 0.05; // Ranked #1 somewhere - else if (bestRank <= 2) bonus = 0.02; // Ranked top-3 somewhere - return { file, displayPath, title, body, score: score + bonus }; - }) - .sort((a, b) => b.score - a.score); -} - -type OutputOptions = { - format: OutputFormat; - full: boolean; - limit: number; - minScore: number; - all?: boolean; - collection?: string; // Filter by collection name (pwd suffix match) - lineNumbers?: boolean; // Add line numbers to output - context?: string; // Optional context for query expansion -}; - -// Highlight query terms in text (skip short words < 3 chars) -function highlightTerms(text: string, query: string): string { - if (!useColor) return text; - const terms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3); - let result = text; - for (const term of terms) { - const regex = new RegExp(`(${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi'); - result = result.replace(regex, `${c.yellow}${c.bold}$1${c.reset}`); - } - return result; -} - -// Format score with color based on value -function formatScore(score: number): string { - const pct = (score * 100).toFixed(0).padStart(3); - if (!useColor) return `${pct}%`; - if (score >= 0.7) return `${c.green}${pct}%${c.reset}`; - if (score >= 0.4) return `${c.yellow}${pct}%${c.reset}`; - return `${c.dim}${pct}%${c.reset}`; -} - -// Shorten directory path for display - relative to $HOME (used for context paths, not documents) -function shortPath(dirpath: string): string { - const home = homedir(); - if (dirpath.startsWith(home)) { - return '~' + dirpath.slice(home.length); - } - return dirpath; -} - -// Add line numbers to text content -function addLineNumbers(text: string, startLine: number = 1): string { - const lines = text.split('\n'); - return lines.map((line, i) => `${startLine + i}: ${line}`).join('\n'); -} - -function outputResults(results: { file: string; displayPath: string; title: string; body: string; score: number; context?: string | null; chunkPos?: number; hash?: string; docid?: string }[], query: string, opts: OutputOptions): void { - const filtered = results.filter(r => r.score >= opts.minScore).slice(0, opts.limit); - - if (filtered.length === 0) { - console.log("No results found above minimum score threshold."); - return; - } - - // Helper to create qmd:// URI from displayPath - const toQmdPath = (displayPath: string) => `qmd://${displayPath}`; - - if (opts.format === "json") { - // JSON output for LLM consumption - const output = filtered.map(row => { - const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined); - let body = opts.full ? row.body : undefined; - let snippet = !opts.full ? extractSnippet(row.body, query, 300, row.chunkPos).snippet : undefined; - if (opts.lineNumbers) { - if (body) body = addLineNumbers(body); - if (snippet) snippet = addLineNumbers(snippet); - } - return { - ...(docid && { docid: `#${docid}` }), - score: Math.round(row.score * 100) / 100, - file: toQmdPath(row.displayPath), - title: row.title, - ...(row.context && { context: row.context }), - ...(body && { body }), - ...(snippet && { snippet }), - }; - }); - console.log(JSON.stringify(output, null, 2)); - } else if (opts.format === "files") { - // Simple docid,score,filepath,context output - for (const row of filtered) { - const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : ""); - const ctx = row.context ? `,"${row.context.replace(/"/g, '""')}"` : ""; - console.log(`#${docid},${row.score.toFixed(2)},${toQmdPath(row.displayPath)}${ctx}`); - } - } else if (opts.format === "cli") { - for (let i = 0; i < filtered.length; i++) { - const row = filtered[i]; - if (!row) continue; - const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos); - const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined); - - // Line 1: filepath with docid - const path = toQmdPath(row.displayPath); - // Only show :line if we actually found a term match in the snippet body (exclude header line). - const snippetBody = snippet.split("\n").slice(1).join("\n").toLowerCase(); - const hasMatch = query.toLowerCase().split(/\s+/).some(t => t.length > 0 && snippetBody.includes(t)); - const lineInfo = hasMatch ? `:${line}` : ""; - const docidStr = docid ? ` ${c.dim}#${docid}${c.reset}` : ""; - console.log(`${c.cyan}${path}${c.dim}${lineInfo}${c.reset}${docidStr}`); - - // Line 2: Title (if available) - if (row.title) { - console.log(`${c.bold}Title: ${row.title}${c.reset}`); - } - - // Line 3: Context (if available) - if (row.context) { - console.log(`${c.dim}Context: ${row.context}${c.reset}`); - } - - // Line 4: Score - const score = formatScore(row.score); - console.log(`Score: ${c.bold}${score}${c.reset}`); - console.log(); - - // Snippet with highlighting (diff-style header included) - let displaySnippet = opts.lineNumbers ? addLineNumbers(snippet, line) : snippet; - const highlighted = highlightTerms(displaySnippet, query); - console.log(highlighted); - - // Double empty line between results - if (i < filtered.length - 1) console.log('\n'); - } - } else if (opts.format === "md") { - for (let i = 0; i < filtered.length; i++) { - const row = filtered[i]; - if (!row) continue; - const heading = row.title || row.displayPath; - const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined); - let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos).snippet; - if (opts.lineNumbers) { - content = addLineNumbers(content); - } - const docidLine = docid ? `**docid:** \`#${docid}\`\n` : ""; - const contextLine = row.context ? `**context:** ${row.context}\n` : ""; - console.log(`---\n# ${heading}\n${docidLine}${contextLine}\n${content}\n`); - } - } else if (opts.format === "xml") { - for (const row of filtered) { - const titleAttr = row.title ? ` title="${row.title.replace(/"/g, '"')}"` : ""; - const contextAttr = row.context ? ` context="${row.context.replace(/"/g, '"')}"` : ""; - const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : ""); - let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos).snippet; - if (opts.lineNumbers) { - content = addLineNumbers(content); - } - console.log(`\n${content}\n\n`); - } - } else { - // CSV format - console.log("docid,score,file,title,context,line,snippet"); - for (const row of filtered) { - const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos); - let content = opts.full ? row.body : snippet; - if (opts.lineNumbers) { - content = addLineNumbers(content, line); - } - const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : ""); - const snippetText = content || ""; - console.log(`#${docid},${row.score.toFixed(4)},${escapeCSV(toQmdPath(row.displayPath))},${escapeCSV(row.title || "")},${escapeCSV(row.context || "")},${line},${escapeCSV(snippetText)}`); - } - } -} - -function search(query: string, opts: OutputOptions): void { - const db = getDb(); - - // Validate collection filter if specified - let collectionName: string | undefined; - if (opts.collection) { - const coll = getCollectionFromYaml(opts.collection); - if (!coll) { - console.error(`Collection not found: ${opts.collection}`); - closeDb(); - process.exit(1); - } - collectionName = opts.collection; - } - - // Use large limit for --all, otherwise fetch more than needed and let outputResults filter - const fetchLimit = opts.all ? 100000 : Math.max(50, opts.limit * 2); - // searchFTS accepts collection name as number parameter for legacy reasons (will be fixed in store.ts) - const results = searchFTS(db, query, fetchLimit, collectionName as any); - - // Add context to results - const resultsWithContext = results.map(r => ({ - file: r.filepath, - displayPath: r.displayPath, - title: r.title, - body: r.body || "", - score: r.score, - context: getContextForFile(db, r.filepath), - hash: r.hash, - docid: r.docid, - })); - - closeDb(); - - if (resultsWithContext.length === 0) { - console.log("No results found."); - return; - } - outputResults(resultsWithContext, query, opts); -} - -async function vectorSearch(query: string, opts: OutputOptions, model: string = DEFAULT_EMBED_MODEL): Promise { - const db = getDb(); - - // Validate collection filter if specified - let collectionName: string | undefined; - if (opts.collection) { - const coll = getCollectionFromYaml(opts.collection); - if (!coll) { - console.error(`Collection not found: ${opts.collection}`); - closeDb(); - process.exit(1); - } - collectionName = opts.collection; - } - - const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get(); - if (!tableExists) { - console.error("Vector index not found. Run 'qmd embed' first to create embeddings."); - closeDb(); - return; - } - - // Check index health and warn about issues - checkIndexHealth(db); - - // Wrap LLM operations in a session for lifecycle management - await withLLMSession(async (session) => { - // Expand query using structured output (no lexical for vector-only search) - const queryables = await expandQueryStructured(query, false, opts.context, session); - - // Build list of queries for vector search: original, vec, and hyde - const vectorQueries: string[] = [query]; - for (const q of queryables) { - if (q.type === 'vec' || q.type === 'hyde') { - if (q.text && q.text !== query) { - vectorQueries.push(q.text); - } - } - } - - process.stderr.write(`${c.dim}Searching ${vectorQueries.length} vector queries...${c.reset}\n`); - - // Collect results from all query variations - const perQueryLimit = opts.all ? 500 : 20; - const allResults = new Map(); - - // IMPORTANT: Run vector searches sequentially, not with Promise.all. - // node-llama-cpp's embedding context hangs when multiple concurrent embed() calls - // are made. This is a known limitation of the LlamaEmbeddingContext. - // See: https://github.com/tobi/qmd/pull/23 - for (const q of vectorQueries) { - const vecResults = await searchVec(db, q, model, perQueryLimit, collectionName as any, session); - for (const r of vecResults) { - const existing = allResults.get(r.filepath); - if (!existing || r.score > existing.score) { - allResults.set(r.filepath, { file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score, hash: r.hash }); - } - } - } - - // Sort by max score and limit to requested count - const results = Array.from(allResults.values()) - .sort((a, b) => b.score - a.score) - .slice(0, opts.limit) - .map(r => ({ ...r, context: getContextForFile(db, r.file) })); - - closeDb(); - - if (results.length === 0) { - console.log("No results found."); - return; - } - outputResults(results, query, { ...opts, limit: results.length }); // Already limited - }, { maxDuration: 10 * 60 * 1000, name: 'vectorSearch' }); -} - -// Expand query using structured output with GBNF grammar -async function expandQueryStructured(query: string, includeLexical: boolean = true, context?: string, session?: ILLMSession): Promise { - process.stderr.write(`${c.dim}Expanding query...${c.reset}\n`); - - const queryables = session - ? await session.expandQuery(query, { includeLexical, context }) - : await getDefaultLlamaCpp().expandQuery(query, { includeLexical, context }); - - // Log the expansion as a tree - const lines: string[] = []; - const bothLabel = includeLexical ? ' · (lexical+vector)' : ' · (vector)'; - lines.push(`${c.dim}├─ ${query}${bothLabel}${c.reset}`); - - for (let i = 0; i < queryables.length; i++) { - const q = queryables[i]; - if (!q || q.text === query) continue; - - let textPreview = q.text.replace(/\n/g, ' '); - if (textPreview.length > 80) { - textPreview = textPreview.substring(0, 77) + '...'; - } - - const label = q.type === 'lex' ? 'lexical' : (q.type === 'hyde' ? 'hyde' : 'vector'); - lines.push(`${c.dim}├─ ${textPreview} · (${label})${c.reset}`); - } - - // Fix last item to use └─ instead of ├─ - if (lines.length > 0) { - lines[lines.length - 1] = lines[lines.length - 1]!.replace('├─', '└─'); - } - - for (const line of lines) { - process.stderr.write(line + '\n'); - } - - return queryables; -} - -async function expandQuery(query: string, _model: string = DEFAULT_QUERY_MODEL, _db?: Database, session?: ILLMSession): Promise { - const queryables = await expandQueryStructured(query, true, undefined, session); - const queries = new Set([query]); - for (const q of queryables) { - queries.add(q.text); - } - return Array.from(queries); -} - -async function querySearch(query: string, opts: OutputOptions, embedModel: string = DEFAULT_EMBED_MODEL, rerankModel: string = DEFAULT_RERANK_MODEL): Promise { - const db = getDb(); - - // Validate collection filter if specified - let collectionName: string | undefined; - if (opts.collection) { - const coll = getCollectionFromYaml(opts.collection); - if (!coll) { - console.error(`Collection not found: ${opts.collection}`); - closeDb(); - process.exit(1); - } - collectionName = opts.collection; - } - - // Check index health and warn about issues - checkIndexHealth(db); - - // Run initial BM25 search (will be reused for retrieval) - const initialFts = searchFTS(db, query, 20, collectionName as any); - let hasVectors = !!db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get(); - - // Check if initial results have strong signals (skip expansion if so) - // Strong signal = top result is strong AND clearly separated from runner-up. - // This avoids skipping expansion when BM25 has lots of mediocre matches. - const topScore = initialFts[0]?.score ?? 0; - const secondScore = initialFts[1]?.score ?? 0; - const hasStrongSignal = initialFts.length > 0 && topScore >= 0.85 && (topScore - secondScore) >= 0.15; - - // Wrap LLM operations in a session for lifecycle management - await withLLMSession(async (session) => { - let ftsQueries: string[] = [query]; - let vectorQueries: string[] = [query]; - - if (hasStrongSignal) { - // Strong BM25 signal - skip expensive LLM expansion - process.stderr.write(`${c.dim}Strong BM25 signal (${topScore.toFixed(2)}) - skipping expansion${c.reset}\n`); - // Still log the "expansion tree" in the same style as vsearch for consistency. - { - const lines: string[] = []; - lines.push(`${c.dim}├─ ${query} · (lexical+vector)${c.reset}`); - lines[lines.length - 1] = lines[lines.length - 1]!.replace('├─', '└─'); - for (const line of lines) process.stderr.write(line + '\n'); - } - } else { - // Weak signal - expand query for better recall - const queryables = await expandQueryStructured(query, true, opts.context, session); - - for (const q of queryables) { - if (q.type === 'lex') { - if (q.text && q.text !== query) ftsQueries.push(q.text); - } else if (q.type === 'vec' || q.type === 'hyde') { - if (q.text && q.text !== query) vectorQueries.push(q.text); - } - } - } - - process.stderr.write(`${c.dim}Searching ${ftsQueries.length} lexical + ${vectorQueries.length} vector queries...${c.reset}\n`); - - // Collect ranked result lists for RRF fusion - const rankedLists: RankedResult[][] = []; - - // Map to store hash by filepath for final results - const hashMap = new Map(); - - // Run all searches concurrently (FTS + Vector) - const searchPromises: Promise[] = []; - - // FTS searches - for (const q of ftsQueries) { - if (!q) continue; - searchPromises.push((async () => { - const ftsResults = searchFTS(db, q, 20, (collectionName || "") as any); - if (ftsResults.length > 0) { - for (const r of ftsResults) { - // Mutex for hashMap is not strictly needed as it's just adding values - hashMap.set(r.filepath, r.hash); - } - rankedLists.push(ftsResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score }))); - } - })()); - } - - // Vector searches (session ensures contexts stay alive) - if (hasVectors) { - for (const q of vectorQueries) { - if (!q) continue; - searchPromises.push((async () => { - const vecResults = await searchVec(db, q, embedModel, 20, (collectionName || "") as any, session); - if (vecResults.length > 0) { - for (const r of vecResults) hashMap.set(r.filepath, r.hash); - rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score }))); - } - })()); - } - } - - await Promise.all(searchPromises); - - // Apply Reciprocal Rank Fusion to combine all ranked lists - // Give 2x weight to original query results (first 2 lists: FTS + vector) - const weights = rankedLists.map((_, i) => i < 2 ? 2.0 : 1.0); - const fused = reciprocalRankFusion(rankedLists, weights); - // Hard cap reranking for latency/cost. We rerank per-document (best chunk only). - const RERANK_DOC_LIMIT = 40; - const candidates = fused.slice(0, RERANK_DOC_LIMIT); - - if (candidates.length === 0) { - console.log("No results found."); - closeDb(); - return; - } - - // Rerank multiple chunks per document, then aggregate scores - // This improves ranking for long documents where keyword-matched chunk isn't always best - // We only rerank ONE chunk per document (best chunk by a simple keyword heuristic), - // so we never rerank more than RERANK_DOC_LIMIT items. - const chunksToRerank: { file: string; text: string; chunkIdx: number }[] = []; - const docChunkMap = new Map(); - - const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 2); - for (const cand of candidates) { - const chunks = chunkDocument(cand.body); - if (chunks.length === 0) continue; - - // Choose best chunk by keyword matches; fall back to first chunk. - let bestIdx = 0; - let bestScore = -1; - for (let i = 0; i < chunks.length; i++) { - const chunkLower = chunks[i]!.text.toLowerCase(); - const score = queryTerms.reduce((acc, term) => acc + (chunkLower.includes(term) ? 1 : 0), 0); - if (score > bestScore) { - bestScore = score; - bestIdx = i; - } - } - - chunksToRerank.push({ file: cand.file, text: chunks[bestIdx]!.text, chunkIdx: bestIdx }); - docChunkMap.set(cand.file, { chunks, bestIdx }); - } - - // Rerank selected chunks (with caching). One chunk per doc -> one rerank item per doc. - const reranked = await rerank( - query, - chunksToRerank.map(ch => ({ file: ch.file, text: ch.text })), - rerankModel, - db, - session - ); - - const aggregatedScores = new Map(); - for (const r of reranked) { - const chunkInfo = docChunkMap.get(r.file); - aggregatedScores.set(r.file, { score: r.score, bestChunkIdx: chunkInfo?.bestIdx ?? 0 }); - } - - // Blend RRF position score with aggregated reranker score using position-aware weights - // Top retrieval results get more protection from reranker disagreement - const candidateMap = new Map(candidates.map(cand => [cand.file, { displayPath: cand.displayPath, title: cand.title, body: cand.body }])); - const rrfRankMap = new Map(candidates.map((cand, i) => [cand.file, i + 1])); // 1-indexed rank - - const finalResults = Array.from(aggregatedScores.entries()).map(([file, { score: rerankScore, bestChunkIdx }]) => { - const rrfRank = rrfRankMap.get(file) || 30; - // Position-aware blending: top retrieval results preserved more - // Rank 1-3: 75% RRF, 25% reranker (trust retrieval for exact matches) - // Rank 4-10: 60% RRF, 40% reranker - // Rank 11+: 40% RRF, 60% reranker (trust reranker for lower-ranked) - let rrfWeight: number; - if (rrfRank <= 3) { - rrfWeight = 0.75; - } else if (rrfRank <= 10) { - rrfWeight = 0.60; - } else { - rrfWeight = 0.40; - } - const rrfScore = 1 / rrfRank; // Position-based: 1, 0.5, 0.33... - const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * rerankScore; - const candidate = candidateMap.get(file); - // Use the best-scoring chunk's text for the body (better for snippets) - const chunkInfo = docChunkMap.get(file); - const chunkBody = chunkInfo ? (chunkInfo.chunks[bestChunkIdx]?.text || chunkInfo.chunks[0]!.text) : candidate?.body || ""; - const chunkPos = chunkInfo ? (chunkInfo.chunks[bestChunkIdx]?.pos || 0) : 0; - return { - file, - displayPath: candidate?.displayPath || "", - title: candidate?.title || "", - body: chunkBody, - chunkPos, - score: blendedScore, - context: getContextForFile(db, file), - hash: hashMap.get(file) || "", - }; - }).sort((a, b) => b.score - a.score); - - // Deduplicate by file (safety net - shouldn't happen but prevents duplicate output) - const seenFiles = new Set(); - const dedupedResults = finalResults.filter(r => { - if (seenFiles.has(r.file)) return false; - seenFiles.add(r.file); - return true; - }); - - closeDb(); - outputResults(dedupedResults, query, opts); - }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' }); -} - -// Parse CLI arguments using util.parseArgs -function parseCLI() { - const { values, positionals } = parseArgs({ - args: Bun.argv.slice(2), // Skip bun and script path - options: { - // Global options - index: { - type: "string", - }, - context: { - type: "string", - }, - "no-lex": { - type: "boolean", - }, - help: { type: "boolean", short: "h" }, - // Search options - n: { type: "string" }, - "min-score": { type: "string" }, - all: { type: "boolean" }, - full: { type: "boolean" }, - csv: { type: "boolean" }, - md: { type: "boolean" }, - xml: { type: "boolean" }, - files: { type: "boolean" }, - json: { type: "boolean" }, - collection: { type: "string", short: "c" }, // Filter by collection - // Collection options - name: { type: "string" }, // collection name - mask: { type: "string" }, // glob pattern - // Embed options - force: { type: "boolean", short: "f" }, - // Update options - pull: { type: "boolean" }, // git pull before update - refresh: { type: "boolean" }, - // Get options - l: { type: "string" }, // max lines - from: { type: "string" }, // start line - "max-bytes": { type: "string" }, // max bytes for multi-get - "line-numbers": { type: "boolean" }, // add line numbers to output - }, - allowPositionals: true, - strict: false, // Allow unknown options to pass through - }); - - // Select index name (default: "index") - const indexName = values.index as string | undefined; - if (indexName) { - setIndexName(indexName); - setConfigIndexName(indexName); - } - - // Determine output format - let format: OutputFormat = "cli"; - if (values.csv) format = "csv"; - else if (values.md) format = "md"; - else if (values.xml) format = "xml"; - else if (values.files) format = "files"; - else if (values.json) format = "json"; - - // Default limit: 20 for --files/--json, 5 otherwise - // --all means return all results (use very large limit) - const defaultLimit = (format === "files" || format === "json") ? 20 : 5; - const isAll = !!values.all; - - const opts: OutputOptions = { - format, - full: !!values.full, - limit: isAll ? 100000 : (values.n ? parseInt(String(values.n), 10) || defaultLimit : defaultLimit), - minScore: values["min-score"] ? parseFloat(String(values["min-score"])) || 0 : 0, - all: isAll, - collection: values.collection as string | undefined, - lineNumbers: !!values["line-numbers"], - }; - - return { - command: positionals[0] || "", - args: positionals.slice(1), - query: positionals.slice(1).join(" "), - opts, - values, - }; -} - -function showHelp(): void { - console.log("Usage:"); - console.log(" qmd collection add [path] --name --mask - Create/index collection"); - console.log(" qmd collection list - List all collections with details"); - console.log(" qmd collection remove - Remove a collection by name"); - console.log(" qmd collection rename - Rename a collection"); - console.log(" qmd ls [collection[/path]] - List collections or files in a collection"); - console.log(" qmd context add [path] \"text\" - Add context for path (defaults to current dir)"); - console.log(" qmd context list - List all contexts"); - console.log(" qmd context rm - Remove context"); - console.log(" qmd get [:line] [-l N] [--from N] - Get document (optionally from line, max N lines)"); - console.log(" qmd multi-get [-l N] [--max-bytes N] - Get multiple docs by glob or comma-separated list"); - console.log(" qmd status - Show index status and collections"); - console.log(" qmd update [--pull] - Re-index all collections (--pull: git pull first)"); - console.log(" qmd embed [-f] - Create vector embeddings (800 tokens/chunk, 15% overlap)"); - console.log(" qmd cleanup - Remove cache and orphaned data, vacuum DB"); - console.log(" qmd search - Full-text search (BM25)"); - console.log(" qmd vsearch - Vector similarity search"); - console.log(" qmd query - Combined search with query expansion + reranking"); - console.log(" qmd mcp - Start MCP server (for AI agent integration)"); - console.log(""); - console.log("Global options:"); - console.log(" --index - Use custom index name (default: index)"); - console.log(""); - console.log("Search options:"); - console.log(" -n - Number of results (default: 5, or 20 for --files)"); - console.log(" --all - Return all matches (use with --min-score to filter)"); - console.log(" --min-score - Minimum similarity score"); - console.log(" --full - Output full document instead of snippet"); - console.log(" --line-numbers - Add line numbers to output"); - console.log(" --files - Output docid,score,filepath,context (default: 20 results)"); - console.log(" --json - JSON output with snippets (default: 20 results)"); - console.log(" --csv - CSV output with snippets"); - console.log(" --md - Markdown output"); - console.log(" --xml - XML output"); - console.log(" -c, --collection - Filter results to a specific collection"); - console.log(""); - console.log("Multi-get options:"); - console.log(" -l - Maximum lines per file"); - console.log(" --max-bytes - Skip files larger than N bytes (default: 10240)"); - console.log(" --json/--csv/--md/--xml/--files - Output format (same as search)"); - console.log(""); - console.log("Models (auto-downloaded from HuggingFace):"); - console.log(" Embedding: embeddinggemma-300M-Q8_0"); - console.log(" Reranking: qwen3-reranker-0.6b-q8_0"); - console.log(" Generation: Qwen3-0.6B-Q8_0"); - console.log(""); - console.log(`Index: ${getDbPath()}`); -} - -// Main CLI - only run if this is the main module -if (import.meta.main) { - const cli = parseCLI(); - - if (!cli.command || cli.values.help) { - showHelp(); - process.exit(cli.values.help ? 0 : 1); - } - - switch (cli.command) { - case "context": { - const subcommand = cli.args[0]; - if (!subcommand) { - console.error("Usage: qmd context "); - console.error(""); - console.error("Commands:"); - console.error(" qmd context add [path] \"text\" - Add context (defaults to current dir)"); - console.error(" qmd context add / \"text\" - Add global context to all collections"); - console.error(" qmd context list - List all contexts"); - console.error(" qmd context check - Check for missing contexts"); - console.error(" qmd context rm - Remove context"); - process.exit(1); - } - - switch (subcommand) { - case "add": { - if (cli.args.length < 2) { - console.error("Usage: qmd context add [path] \"text\""); - console.error(""); - console.error("Examples:"); - console.error(" qmd context add \"Context for current directory\""); - console.error(" qmd context add . \"Context for current directory\""); - console.error(" qmd context add /subfolder \"Context for subfolder\""); - console.error(" qmd context add / \"Global context for all collections\""); - console.error(""); - console.error(" Using virtual paths:"); - console.error(" qmd context add qmd://journals/ \"Context for entire journals collection\""); - console.error(" qmd context add qmd://journals/2024 \"Context for 2024 journals\""); - process.exit(1); - } - - let pathArg: string | undefined; - let contextText: string; - - // Check if first arg looks like a path or if it's the context text - const firstArg = cli.args[1] || ''; - const secondArg = cli.args[2]; - - if (secondArg) { - // Two args: path + context - pathArg = firstArg; - contextText = cli.args.slice(2).join(" "); - } else { - // One arg: context only (use current directory) - pathArg = undefined; - contextText = firstArg; - } - - await contextAdd(pathArg, contextText); - break; - } - - case "list": { - contextList(); - break; - } - - case "check": { - contextCheck(); - break; - } - - case "rm": - case "remove": { - if (cli.args.length < 2 || !cli.args[1]) { - console.error("Usage: qmd context rm "); - console.error("Examples:"); - console.error(" qmd context rm /"); - console.error(" qmd context rm qmd://journals/2024"); - process.exit(1); - } - contextRemove(cli.args[1]); - break; - } - - default: - console.error(`Unknown subcommand: ${subcommand}`); - console.error("Available: add, list, check, rm"); - process.exit(1); - } - break; - } - - case "get": { - if (!cli.args[0]) { - console.error("Usage: qmd get [:line] [--from ] [-l ] [--line-numbers]"); - process.exit(1); - } - const fromLine = cli.values.from ? parseInt(cli.values.from as string, 10) : undefined; - const maxLines = cli.values.l ? parseInt(cli.values.l as string, 10) : undefined; - getDocument(cli.args[0], fromLine, maxLines, cli.opts.lineNumbers); - break; - } - - case "multi-get": { - if (!cli.args[0]) { - console.error("Usage: qmd multi-get [-l ] [--max-bytes ] [--json|--csv|--md|--xml|--files]"); - console.error(" pattern: glob (e.g., 'journals/2025-05*.md') or comma-separated list"); - process.exit(1); - } - const maxLinesMulti = cli.values.l ? parseInt(cli.values.l as string, 10) : undefined; - const maxBytes = cli.values["max-bytes"] ? parseInt(cli.values["max-bytes"] as string, 10) : DEFAULT_MULTI_GET_MAX_BYTES; - multiGet(cli.args[0], maxLinesMulti, maxBytes, cli.opts.format); - break; - } - - case "ls": { - listFiles(cli.args[0]); - break; - } - - case "collection": { - const subcommand = cli.args[0]; - switch (subcommand) { - case "list": { - collectionList(); - break; - } - - case "add": { - const pwd = cli.args[1] || getPwd(); - const resolvedPwd = pwd === '.' ? getPwd() : getRealPath(resolve(pwd)); - const globPattern = cli.values.mask as string || DEFAULT_GLOB; - const name = cli.values.name as string | undefined; - - await collectionAdd(resolvedPwd, globPattern, name); - break; - } - - case "remove": - case "rm": { - if (!cli.args[1]) { - console.error("Usage: qmd collection remove "); - console.error(" Use 'qmd collection list' to see available collections"); - process.exit(1); - } - collectionRemove(cli.args[1]); - break; - } - - case "rename": - case "mv": { - if (!cli.args[1] || !cli.args[2]) { - console.error("Usage: qmd collection rename "); - console.error(" Use 'qmd collection list' to see available collections"); - process.exit(1); - } - collectionRename(cli.args[1], cli.args[2]); - break; - } - - default: - console.error(`Unknown subcommand: ${subcommand}`); - console.error("Available: list, add, remove, rename"); - process.exit(1); - } - break; - } - - case "status": - showStatus(); - break; - - case "update": - await updateCollections(); - break; - - case "embed": - await vectorIndex(DEFAULT_EMBED_MODEL, !!cli.values.force); - break; - - case "pull": { - const refresh = cli.values.refresh === undefined ? false : Boolean(cli.values.refresh); - const models = [ - DEFAULT_EMBED_MODEL_URI, - DEFAULT_GENERATE_MODEL_URI, - DEFAULT_RERANK_MODEL_URI, - ]; - console.log(`${c.bold}Pulling models${c.reset}`); - const results = await pullModels(models, { - refresh, - cacheDir: DEFAULT_MODEL_CACHE_DIR, - }); - for (const result of results) { - const size = formatBytes(result.sizeBytes); - const note = result.refreshed ? "refreshed" : "cached/checked"; - console.log(`- ${result.model} -> ${result.path} (${size}, ${note})`); - } - break; - } - - case "search": - if (!cli.query) { - console.error("Usage: qmd search [options] "); - process.exit(1); - } - search(cli.query, cli.opts); - break; - - case "vsearch": - if (!cli.query) { - console.error("Usage: qmd vsearch [options] "); - process.exit(1); - } - // Default min-score for vector search is 0.3 - if (!cli.values["min-score"]) { - cli.opts.minScore = 0.3; - } - await vectorSearch(cli.query, cli.opts); - break; - - case "query": - if (!cli.query) { - console.error("Usage: qmd query [options] "); - process.exit(1); - } - await querySearch(cli.query, cli.opts); - break; - - case "mcp": { - const { startMcpServer } = await import("./mcp.js"); - await startMcpServer(); - break; - } - - case "cleanup": { - const db = getDb(); - - // 1. Clear llm_cache - const cacheCount = deleteLLMCache(db); - console.log(`${c.green}✓${c.reset} Cleared ${cacheCount} cached API responses`); - - // 2. Remove orphaned vectors - const orphanedVecs = cleanupOrphanedVectors(db); - if (orphanedVecs > 0) { - console.log(`${c.green}✓${c.reset} Removed ${orphanedVecs} orphaned embedding chunks`); - } else { - console.log(`${c.dim}No orphaned embeddings to remove${c.reset}`); - } - - // 3. Remove inactive documents - const inactiveDocs = deleteInactiveDocuments(db); - if (inactiveDocs > 0) { - console.log(`${c.green}✓${c.reset} Removed ${inactiveDocs} inactive document records`); - } - - // 4. Vacuum to reclaim space - vacuumDatabase(db); - console.log(`${c.green}✓${c.reset} Database vacuumed`); - - closeDb(); - break; - } - - default: - console.error(`Unknown command: ${cli.command}`); - console.error("Run 'qmd --help' for usage."); - process.exit(1); - } - - if (cli.command !== "mcp") { - await disposeDefaultLlamaCpp(); - process.exit(0); - } - -} // end if (import.meta.main) diff --git a/tools/qmd/src/store-paths.test.ts b/tools/qmd/src/store-paths.test.ts deleted file mode 100644 index 2f716781..00000000 --- a/tools/qmd/src/store-paths.test.ts +++ /dev/null @@ -1,395 +0,0 @@ -/** - * store-paths.test.ts - Comprehensive unit tests for Windows path support - * - * Tests all path-related utility functions for cross-platform compatibility: - * - isAbsolutePath() - Unix, Windows (C:\, C:/), and Git Bash (/c/) paths - * - normalizePathSeparators() - backslash to forward slash conversion - * - getRelativePathFromPrefix() - relative path extraction - * - resolve() - path resolution with Unix and Windows paths - * - * Run with: bun test store-paths.test.ts - */ - -import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { - isAbsolutePath, - normalizePathSeparators, - getRelativePathFromPrefix, - resolve, -} from "./store.js"; - -// ============================================================================= -// Test Utilities -// ============================================================================= - -let originalPWD: string | undefined; -let originalProcessCwd: () => string; - -beforeEach(() => { - // Save original environment - originalPWD = Bun.env.PWD; - originalProcessCwd = process.cwd; -}); - -afterEach(() => { - // Restore original environment - if (originalPWD !== undefined) { - Bun.env.PWD = originalPWD; - } else { - delete Bun.env.PWD; - } - process.cwd = originalProcessCwd; -}); - -/** - * Mock the current working directory for testing. - * Sets both Bun.env.PWD and process.cwd() to simulate different environments. - */ -function mockPWD(path: string): void { - Bun.env.PWD = path; - process.cwd = () => path; -} - -// ============================================================================= -// Path Utilities - Cross-platform Support -// ============================================================================= - -describe("Path utilities - Cross-platform support", () => { - - // =========================================================================== - // isAbsolutePath - // =========================================================================== - - describe("isAbsolutePath", () => { - test("Unix absolute paths", () => { - expect(isAbsolutePath("/path/to/file")).toBe(true); - expect(isAbsolutePath("/")).toBe(true); - expect(isAbsolutePath("/home/user/documents")).toBe(true); - expect(isAbsolutePath("/usr/local/bin")).toBe(true); - }); - - test("Unix relative paths", () => { - expect(isAbsolutePath("path/to/file")).toBe(false); - expect(isAbsolutePath("./path/to/file")).toBe(false); - expect(isAbsolutePath("../path/to/file")).toBe(false); - expect(isAbsolutePath("./file")).toBe(false); - expect(isAbsolutePath("../file")).toBe(false); - expect(isAbsolutePath("file.txt")).toBe(false); - }); - - test("Windows absolute paths (native) - forward slash", () => { - expect(isAbsolutePath("C:/path/to/file")).toBe(true); - expect(isAbsolutePath("C:/")).toBe(true); - expect(isAbsolutePath("D:/Users/Documents")).toBe(true); - expect(isAbsolutePath("Z:/")).toBe(true); - expect(isAbsolutePath("c:/lowercase")).toBe(true); - }); - - test("Windows absolute paths (native) - backslash", () => { - expect(isAbsolutePath("C:\\path\\to\\file")).toBe(true); - expect(isAbsolutePath("C:\\")).toBe(true); - expect(isAbsolutePath("D:\\Users\\Documents")).toBe(true); - expect(isAbsolutePath("Z:\\")).toBe(true); - expect(isAbsolutePath("c:\\lowercase")).toBe(true); - }); - - test("Windows relative paths", () => { - expect(isAbsolutePath("path\\to\\file")).toBe(false); - expect(isAbsolutePath(".\\path\\to\\file")).toBe(false); - expect(isAbsolutePath("..\\path\\to\\file")).toBe(false); - expect(isAbsolutePath(".\\file")).toBe(false); - expect(isAbsolutePath("..\\file")).toBe(false); - expect(isAbsolutePath("file.txt")).toBe(false); - }); - - test("Git Bash style paths", () => { - expect(isAbsolutePath("/c/Users/name/file")).toBe(true); - expect(isAbsolutePath("/C/Users/name/file")).toBe(true); - expect(isAbsolutePath("/d/Projects")).toBe(true); - expect(isAbsolutePath("/D/Projects")).toBe(true); - expect(isAbsolutePath("/z/")).toBe(true); - }); - - test("Edge cases", () => { - expect(isAbsolutePath("")).toBe(false); - expect(isAbsolutePath("C:")).toBe(true); // Drive letter only - expect(isAbsolutePath("C")).toBe(false); // Just a letter - expect(isAbsolutePath(":")).toBe(false); - expect(isAbsolutePath("/a")).toBe(true); // Short Unix path - expect(isAbsolutePath("/1/")).toBe(true); // Number after slash (not Git Bash) - }); - }); - - // =========================================================================== - // normalizePathSeparators - // =========================================================================== - - describe("normalizePathSeparators", () => { - test("Windows paths with backslashes", () => { - expect(normalizePathSeparators("C:\\Users\\name\\file.txt")) - .toBe("C:/Users/name/file.txt"); - expect(normalizePathSeparators("D:\\Projects\\qmd\\src")) - .toBe("D:/Projects/qmd/src"); - expect(normalizePathSeparators("\\path\\to\\file")) - .toBe("/path/to/file"); - }); - - test("Mixed separators", () => { - expect(normalizePathSeparators("C:\\Users/name\\file.txt")) - .toBe("C:/Users/name/file.txt"); - expect(normalizePathSeparators("path\\to/file/here")) - .toBe("path/to/file/here"); - }); - - test("Unix paths (should remain unchanged)", () => { - expect(normalizePathSeparators("/path/to/file")) - .toBe("/path/to/file"); - expect(normalizePathSeparators("/usr/local/bin")) - .toBe("/usr/local/bin"); - expect(normalizePathSeparators("relative/path")) - .toBe("relative/path"); - }); - - test("Multiple consecutive backslashes", () => { - expect(normalizePathSeparators("path\\\\to\\\\file")) - .toBe("path//to//file"); - expect(normalizePathSeparators("C:\\\\Users\\\\name")) - .toBe("C://Users//name"); - }); - - test("Edge cases", () => { - expect(normalizePathSeparators("")).toBe(""); - expect(normalizePathSeparators("\\")).toBe("/"); - expect(normalizePathSeparators("\\\\")).toBe("//"); - expect(normalizePathSeparators("file.txt")).toBe("file.txt"); - }); - }); - - // =========================================================================== - // getRelativePathFromPrefix - // =========================================================================== - - describe("getRelativePathFromPrefix", () => { - test("Exact match (path equals prefix)", () => { - expect(getRelativePathFromPrefix("/home/user", "/home/user")).toBe(""); - expect(getRelativePathFromPrefix("C:/Users/name", "C:/Users/name")).toBe(""); - expect(getRelativePathFromPrefix("/path", "/path")).toBe(""); - }); - - test("Path under prefix", () => { - expect(getRelativePathFromPrefix("/home/user/documents", "/home/user")) - .toBe("documents"); - expect(getRelativePathFromPrefix("/home/user/documents/file.txt", "/home/user")) - .toBe("documents/file.txt"); - expect(getRelativePathFromPrefix("C:/Users/name/Documents/file.txt", "C:/Users/name")) - .toBe("Documents/file.txt"); - }); - - test("Path not under prefix", () => { - expect(getRelativePathFromPrefix("/home/other", "/home/user")).toBeNull(); - expect(getRelativePathFromPrefix("/usr/local", "/home/user")).toBeNull(); - expect(getRelativePathFromPrefix("C:/Users/other", "D:/Users")).toBeNull(); - }); - - test("Windows paths with normalized separators", () => { - // Backslashes should be normalized - expect(getRelativePathFromPrefix("C:\\Users\\name\\Documents", "C:\\Users\\name")) - .toBe("Documents"); - expect(getRelativePathFromPrefix("C:\\Users\\name\\Documents\\file.txt", "C:/Users/name")) - .toBe("Documents/file.txt"); - }); - - test("Prefix with trailing slash", () => { - expect(getRelativePathFromPrefix("/home/user/documents", "/home/user/")) - .toBe("documents"); - expect(getRelativePathFromPrefix("C:/Users/name/Documents", "C:/Users/name/")) - .toBe("Documents"); - }); - - test("Prefix without trailing slash", () => { - expect(getRelativePathFromPrefix("/home/user/documents", "/home/user")) - .toBe("documents"); - expect(getRelativePathFromPrefix("C:/Users/name/Documents", "C:/Users/name")) - .toBe("Documents"); - }); - - test("Edge cases", () => { - // Empty prefix - expect(getRelativePathFromPrefix("/path/to/file", "")).toBeNull(); - - // Path is prefix substring but not in hierarchy - expect(getRelativePathFromPrefix("/home/username", "/home/user")).toBeNull(); - - // Root prefix - expect(getRelativePathFromPrefix("/home/user", "/")).toBe("home/user"); - }); - }); - - // =========================================================================== - // resolve - Unix environment - // =========================================================================== - - describe("resolve - Unix environment", () => { - beforeEach(() => { - mockPWD("/home/user"); - }); - - test("Unix relative paths", () => { - expect(resolve("/base", "relative")).toBe("/base/relative"); - expect(resolve("/base", "a/b/c")).toBe("/base/a/b/c"); - expect(resolve("/home", "user/documents")).toBe("/home/user/documents"); - }); - - test("Unix absolute paths", () => { - expect(resolve("/base", "/absolute")).toBe("/absolute"); - expect(resolve("/home/user", "/usr/local")).toBe("/usr/local"); - expect(resolve("/any", "/")).toBe("/"); - }); - - test("Path with .. and .", () => { - expect(resolve("/base", "../other")).toBe("/other"); - expect(resolve("/base/sub", "..")).toBe("/base"); - expect(resolve("/base", "./file")).toBe("/base/file"); - expect(resolve("/base/a/b", "../../c")).toBe("/base/c"); - }); - - test("Multiple path segments", () => { - expect(resolve("/a", "b", "c")).toBe("/a/b/c"); - expect(resolve("/a", "b", "../c")).toBe("/a/c"); - expect(resolve("/a", "b", "/c")).toBe("/c"); - }); - - test("Relative path without base (uses PWD)", () => { - expect(resolve("relative")).toBe("/home/user/relative"); - expect(resolve("a/b/c")).toBe("/home/user/a/b/c"); - expect(resolve("./file")).toBe("/home/user/file"); - }); - - test("Absolute path alone", () => { - expect(resolve("/absolute/path")).toBe("/absolute/path"); - expect(resolve("/")).toBe("/"); - }); - }); - - // =========================================================================== - // resolve - Windows environment - // =========================================================================== - - describe("resolve - Windows environment", () => { - beforeEach(() => { - mockPWD("C:/Users/name"); - }); - - test("Windows relative paths", () => { - expect(resolve("C:/base", "relative")).toBe("C:/base/relative"); - expect(resolve("C:/base", "a/b/c")).toBe("C:/base/a/b/c"); - expect(resolve("D:/Projects", "qmd/src")).toBe("D:/Projects/qmd/src"); - }); - - test("Windows absolute paths", () => { - expect(resolve("C:/base", "D:/other")).toBe("D:/other"); - expect(resolve("C:/Users", "C:/Program Files")).toBe("C:/Program Files"); - expect(resolve("D:/any", "E:/other")).toBe("E:/other"); - }); - - test("Windows with backslashes", () => { - expect(resolve("C:\\base", "relative")).toBe("C:/base/relative"); - expect(resolve("C:\\Users\\name", "Documents")).toBe("C:/Users/name/Documents"); - expect(resolve("C:\\base", "a\\b\\c")).toBe("C:/base/a/b/c"); - }); - - test("Path with .. and .", () => { - expect(resolve("C:/base", "../other")).toBe("C:/other"); - expect(resolve("C:/base/sub", "..")).toBe("C:/base"); - expect(resolve("C:/base", "./file")).toBe("C:/base/file"); - expect(resolve("C:/base/a/b", "../../c")).toBe("C:/base/c"); - }); - - test("Multiple path segments", () => { - expect(resolve("C:/a", "b", "c")).toBe("C:/a/b/c"); - expect(resolve("C:/a", "b", "../c")).toBe("C:/a/c"); - expect(resolve("C:/a", "b", "D:/c")).toBe("D:/c"); - }); - - test("Relative path without base (uses PWD)", () => { - expect(resolve("relative")).toBe("C:/Users/name/relative"); - expect(resolve("a/b/c")).toBe("C:/Users/name/a/b/c"); - expect(resolve(".\\file")).toBe("C:/Users/name/file"); - }); - - test("Drive letter only", () => { - expect(resolve("C:")).toBe("C:/"); - expect(resolve("D:")).toBe("D:/"); - }); - }); - - // =========================================================================== - // resolve - Git Bash style paths - // =========================================================================== - - describe("resolve - Git Bash style paths", () => { - test("Git Bash to Windows conversion", () => { - expect(resolve("/c/Users/name")).toBe("C:/Users/name"); - expect(resolve("/C/Users/name")).toBe("C:/Users/name"); - expect(resolve("/d/Projects")).toBe("D:/Projects"); - expect(resolve("/D/Projects")).toBe("D:/Projects"); - }); - - test("Git Bash with relative paths", () => { - expect(resolve("/c/base", "relative")).toBe("C:/base/relative"); - expect(resolve("/d/Projects", "qmd/src")).toBe("D:/Projects/qmd/src"); - }); - - test("Git Bash with .. and .", () => { - expect(resolve("/c/base", "../other")).toBe("C:/other"); - expect(resolve("/c/base/sub", "..")).toBe("C:/base"); - expect(resolve("/c/base", "./file")).toBe("C:/base/file"); - }); - - test("Multiple Git Bash segments", () => { - expect(resolve("/c/a", "b", "c")).toBe("C:/a/b/c"); - expect(resolve("/c/a", "b", "/d/c")).toBe("D:/c"); - }); - }); - - // =========================================================================== - // resolve - Edge cases and mixed scenarios - // =========================================================================== - - describe("resolve - Edge cases", () => { - test("Empty path segments are filtered", () => { - expect(resolve("/base", "", "file")).toBe("/base/file"); - expect(resolve("C:/base", "", "file")).toBe("C:/base/file"); - }); - - test("Multiple consecutive slashes", () => { - expect(resolve("/base//path///file")).toBe("/base/path/file"); - expect(resolve("C:/base//path///file")).toBe("C:/base/path/file"); - }); - - test("Trailing slashes", () => { - expect(resolve("/base/", "file")).toBe("/base/file"); - expect(resolve("C:/base/", "file")).toBe("C:/base/file"); - }); - - test("Complex .. navigation", () => { - expect(resolve("/a/b/c/d", "../../../e")).toBe("/a/e"); - expect(resolve("C:/a/b/c/d", "../../../e")).toBe("C:/a/e"); - }); - - test("Too many .. (should not go above root)", () => { - expect(resolve("/base", "../../../../other")).toBe("/other"); - expect(resolve("C:/base", "../../../../other")).toBe("C:/other"); - }); - - test("Mixed Unix and Windows (normalized)", () => { - mockPWD("C:/Users/name"); - expect(resolve("/unix/path")).toBe("/unix/path"); - expect(resolve("relative")).toBe("C:/Users/name/relative"); - }); - - test("Error on no arguments", () => { - expect(() => resolve()).toThrow("resolve: at least one path segment is required"); - }); - }); -}); diff --git a/tools/qmd/src/store.test.ts b/tools/qmd/src/store.test.ts deleted file mode 100644 index af34dd62..00000000 --- a/tools/qmd/src/store.test.ts +++ /dev/null @@ -1,2483 +0,0 @@ -/** - * store.test.ts - Comprehensive unit tests for the QMD store module - * - * Run with: bun test store.test.ts - * - * LLM operations use LlamaCpp with local GGUF models (node-llama-cpp). - */ - -import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, mock, spyOn } from "bun:test"; -import { Database } from "bun:sqlite"; -import { unlink, mkdtemp, rmdir, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import YAML from "yaml"; -import { disposeDefaultLlamaCpp } from "./llm.js"; -import { - createStore, - getDefaultDbPath, - homedir, - resolve, - getPwd, - getRealPath, - hashContent, - extractTitle, - formatQueryForEmbedding, - formatDocForEmbedding, - chunkDocument, - chunkDocumentByTokens, - reciprocalRankFusion, - extractSnippet, - getCacheKey, - handelize, - normalizeVirtualPath, - isVirtualPath, - parseVirtualPath, - normalizeDocid, - isDocid, - type Store, - type DocumentResult, - type SearchResult, - type RankedResult, -} from "./store.js"; -import type { CollectionConfig } from "./collections.js"; - -// ============================================================================= -// LlamaCpp Setup -// ============================================================================= - -// Note: LlamaCpp uses node-llama-cpp for local GGUF model inference. -// No HTTP mocking needed - tests use real LlamaCpp calls for integration tests. - -// ============================================================================= -// Test Utilities -// ============================================================================= - -let testDir: string; -let testDbPath: string; -let testConfigDir: string; - -async function createTestStore(): Promise { - testDbPath = join(testDir, `test-${Date.now()}-${Math.random().toString(36).slice(2)}.sqlite`); - - // Set up test config directory - const configPrefix = join(testDir, `config-${Date.now()}-${Math.random().toString(36).slice(2)}`); - testConfigDir = await mkdtemp(configPrefix); - - // Set environment variable to use test config - process.env.QMD_CONFIG_DIR = testConfigDir; - - // Create empty YAML config - const emptyConfig: CollectionConfig = { collections: {} }; - await writeFile( - join(testConfigDir, "index.yml"), - YAML.stringify(emptyConfig) - ); - - return createStore(testDbPath); -} - -async function cleanupTestDb(store: Store): Promise { - store.close(); - try { - await unlink(store.dbPath); - } catch { - // Ignore if file doesn't exist - } - - // Clean up test config directory - try { - const { readdir, unlink: unlinkFile, rmdir: rmdirAsync } = await import("node:fs/promises"); - const files = await readdir(testConfigDir); - for (const file of files) { - await unlinkFile(join(testConfigDir, file)); - } - await rmdirAsync(testConfigDir); - } catch { - // Ignore cleanup errors - } - - // Clear environment variable - delete process.env.QMD_CONFIG_DIR; -} - -// Helper to insert a test document directly into the database -async function insertTestDocument( - db: Database, - collectionName: string, - opts: { - name?: string; - title?: string; - hash?: string; - displayPath?: string; - filepath?: string; - body?: string; - active?: number; - } -): Promise { - const now = new Date().toISOString(); - const name = opts.name || "test-doc"; - const title = opts.title || "Test Document"; - - // Use displayPath if provided, otherwise filepath's basename, otherwise default - let path: string; - if (opts.displayPath) { - path = opts.displayPath; - } else if (opts.filepath) { - // Extract relative path from filepath by removing collection path - // For tests, assume filepath is either relative or we want the whole path as the document path - path = opts.filepath.startsWith('/') ? opts.filepath : opts.filepath; - } else { - path = `test/${name}.md`; - } - - const body = opts.body || "# Test Document\n\nThis is test content."; - const active = opts.active ?? 1; - - // Generate hash from body if not provided - const hash = opts.hash || await hashContent(body); - - // Insert content (with OR IGNORE for deduplication) - db.prepare(` - INSERT OR IGNORE INTO content (hash, doc, created_at) - VALUES (?, ?, ?) - `).run(hash, body, now); - - // Insert document - const result = db.prepare(` - INSERT INTO documents (collection, path, title, hash, created_at, modified_at, active) - VALUES (?, ?, ?, ?, ?, ?, ?) - `).run(collectionName, path, title, hash, now, now, active); - - return Number(result.lastInsertRowid); -} - -// Helper to create a test collection in YAML config -async function createTestCollection( - options: { pwd?: string; glob?: string; name?: string } = {} -): Promise { - const pwd = options.pwd || "/test/collection"; - const glob = options.glob || "**/*.md"; - const name = options.name || pwd.split('/').filter(Boolean).pop() || 'test'; - - // Read current config - const configPath = join(testConfigDir, "index.yml"); - const { readFile } = await import("node:fs/promises"); - const content = await readFile(configPath, "utf-8"); - const config = YAML.parse(content) as CollectionConfig; - - // Add collection - config.collections[name] = { - path: pwd, - pattern: glob, - }; - - // Write back - await writeFile(configPath, YAML.stringify(config)); - return name; -} - -// Helper to add path context in YAML config -async function addPathContext(collectionName: string, pathPrefix: string, contextText: string): Promise { - // Read current config - const configPath = join(testConfigDir, "index.yml"); - const { readFile } = await import("node:fs/promises"); - const content = await readFile(configPath, "utf-8"); - const config = YAML.parse(content) as CollectionConfig; - - // Add context to collection - if (!config.collections[collectionName]) { - throw new Error(`Collection ${collectionName} not found`); - } - - if (!config.collections[collectionName].context) { - config.collections[collectionName].context = {}; - } - - config.collections[collectionName].context![pathPrefix] = contextText; - - // Write back - await writeFile(configPath, YAML.stringify(config)); -} - -// Helper to add global context in YAML config -async function addGlobalContext(contextText: string): Promise { - const configPath = join(testConfigDir, "index.yml"); - const { readFile } = await import("node:fs/promises"); - const content = await readFile(configPath, "utf-8"); - const config = YAML.parse(content) as CollectionConfig; - - config.global_context = contextText; - - await writeFile(configPath, YAML.stringify(config)); -} - -// ============================================================================= -// Test Setup -// ============================================================================= - -beforeAll(async () => { - testDir = await mkdtemp(join(tmpdir(), "qmd-test-")); -}); - -afterAll(async () => { - // Ensure native resources are released to avoid ggml-metal asserts on process exit. - await disposeDefaultLlamaCpp(); - - try { - // Clean up test directory - const { readdir, unlink } = await import("node:fs/promises"); - const files = await readdir(testDir); - for (const file of files) { - await unlink(join(testDir, file)); - } - await rmdir(testDir); - } catch { - // Ignore cleanup errors - } -}); - -// ============================================================================= -// Path Utilities Tests -// ============================================================================= - -describe("Path Utilities", () => { - test("homedir returns HOME environment variable", () => { - const result = homedir(); - expect(result).toBe(Bun.env.HOME || "/tmp"); - }); - - test("resolve handles absolute paths", () => { - expect(resolve("/foo/bar")).toBe("/foo/bar"); - expect(resolve("/foo", "/bar")).toBe("/bar"); - }); - - test("resolve handles relative paths", () => { - const pwd = Bun.env.PWD || process.cwd(); - expect(resolve("foo")).toBe(`${pwd}/foo`); - expect(resolve("foo", "bar")).toBe(`${pwd}/foo/bar`); - }); - - test("resolve normalizes . and ..", () => { - expect(resolve("/foo/bar/./baz")).toBe("/foo/bar/baz"); - expect(resolve("/foo/bar/../baz")).toBe("/foo/baz"); - expect(resolve("/foo/bar/../../baz")).toBe("/baz"); - }); - - test("getDefaultDbPath throws in test mode without INDEX_PATH", () => { - // In test mode, getDefaultDbPath should throw to prevent accidental writes to global index - // This is intentional safety behavior - const originalIndexPath = process.env.INDEX_PATH; - delete process.env.INDEX_PATH; - - expect(() => getDefaultDbPath()).toThrow("Database path not set"); - - // Restore - if (originalIndexPath) process.env.INDEX_PATH = originalIndexPath; - }); - - test("getDefaultDbPath uses INDEX_PATH when set", () => { - const originalIndexPath = process.env.INDEX_PATH; - process.env.INDEX_PATH = "/tmp/test-index.sqlite"; - - expect(getDefaultDbPath()).toBe("/tmp/test-index.sqlite"); - expect(getDefaultDbPath("custom")).toBe("/tmp/test-index.sqlite"); // INDEX_PATH overrides name - - // Restore - if (originalIndexPath) { - process.env.INDEX_PATH = originalIndexPath; - } else { - delete process.env.INDEX_PATH; - } - }); - - test("getPwd returns current working directory", () => { - const pwd = getPwd(); - expect(pwd).toBeTruthy(); - expect(typeof pwd).toBe("string"); - }); - - test("getRealPath resolves symlinks", () => { - const result = getRealPath("/tmp"); - expect(result).toBeTruthy(); - // On macOS, /tmp is a symlink to /private/tmp - expect(result === "/tmp" || result === "/private/tmp").toBe(true); - }); -}); - -// ============================================================================= -// Handelize Tests - path normalization for token-friendly filenames -// ============================================================================= - -describe("handelize", () => { - test("converts to lowercase", () => { - expect(handelize("README.md")).toBe("readme.md"); - expect(handelize("MyFile.MD")).toBe("myfile.md"); - }); - - test("preserves folder structure", () => { - expect(handelize("a/b/c/d.md")).toBe("a/b/c/d.md"); - expect(handelize("docs/api/README.md")).toBe("docs/api/readme.md"); - }); - - test("replaces non-word characters with dash", () => { - expect(handelize("hello world.md")).toBe("hello-world.md"); - expect(handelize("file (1).md")).toBe("file-1.md"); - expect(handelize("foo@bar#baz.md")).toBe("foo-bar-baz.md"); - }); - - test("collapses multiple special chars into single dash", () => { - expect(handelize("hello world.md")).toBe("hello-world.md"); - expect(handelize("foo---bar.md")).toBe("foo-bar.md"); - expect(handelize("a - b.md")).toBe("a-b.md"); - }); - - test("removes leading and trailing dashes from segments", () => { - expect(handelize("-hello-.md")).toBe("hello.md"); - expect(handelize("--test--.md")).toBe("test.md"); - expect(handelize("a/-b-/c.md")).toBe("a/b/c.md"); - }); - - test("converts triple underscore to folder separator", () => { - expect(handelize("foo___bar.md")).toBe("foo/bar.md"); - expect(handelize("notes___2025___january.md")).toBe("notes/2025/january.md"); - expect(handelize("a/b___c/d.md")).toBe("a/b/c/d.md"); - }); - - test("handles complex real-world meeting notes", () => { - // Example: "Money Movement Licensing Review - 2025／11／19 10:25 EST - Notes by Gemini.md" - const complexName = "Money Movement Licensing Review - 2025／11／19 10:25 EST - Notes by Gemini.md"; - const result = handelize(complexName); - expect(result).toBe("money-movement-licensing-review-2025-11-19-10-25-est-notes-by-gemini.md"); - expect(result).not.toContain(" "); - expect(result).not.toContain("／"); - expect(result).not.toContain(":"); - }); - - test("handles unicode characters", () => { - // Pure unicode filenames are now supported (fixes GitHub issue #10) - expect(handelize("日本語.md")).toBe("日本語.md"); - expect(handelize("Зоны и проекты.md")).toBe("зоны-и-проекты.md"); - // Mixed unicode/ascii preserves both - expect(handelize("café-notes.md")).toBe("café-notes.md"); - expect(handelize("naïve.md")).toBe("naïve.md"); - expect(handelize("日本語-notes.md")).toBe("日本語-notes.md"); - }); - - test("handles dates and times in filenames", () => { - expect(handelize("meeting-2025-01-15.md")).toBe("meeting-2025-01-15.md"); - expect(handelize("notes 2025/01/15.md")).toBe("notes-2025/01/15.md"); - expect(handelize("call_10:30_AM.md")).toBe("call-10-30-am.md"); - }); - - test("handles special project naming patterns", () => { - expect(handelize("PROJECT_ABC_v2.0.md")).toBe("project-abc-v2-0.md"); - expect(handelize("[WIP] Feature Request.md")).toBe("wip-feature-request.md"); - expect(handelize("(DRAFT) Proposal v1.md")).toBe("draft-proposal-v1.md"); - }); - - test("filters out empty segments", () => { - expect(handelize("a//b/c.md")).toBe("a/b/c.md"); - expect(handelize("/a/b/")).toBe("a/b"); - expect(handelize("///test///")).toBe("test"); - }); - - test("throws error for invalid inputs", () => { - expect(() => handelize("")).toThrow("path cannot be empty"); - expect(() => handelize(" ")).toThrow("path cannot be empty"); - expect(() => handelize(".md")).toThrow("no valid filename content"); - expect(() => handelize("...")).toThrow("no valid filename content"); - expect(() => handelize("___")).toThrow("no valid filename content"); - }); - - test("handles minimal valid inputs", () => { - expect(handelize("a")).toBe("a"); - expect(handelize("1")).toBe("1"); - expect(handelize("a.md")).toBe("a.md"); - }); -}); - -// ============================================================================= -// Store Creation Tests -// ============================================================================= - -describe("Store Creation", () => { - test("createStore throws without explicit path in test mode", () => { - // In test mode, createStore without path should throw to prevent accidental writes - const originalIndexPath = process.env.INDEX_PATH; - delete process.env.INDEX_PATH; - - expect(() => createStore()).toThrow("Database path not set"); - - // Restore - if (originalIndexPath) process.env.INDEX_PATH = originalIndexPath; - }); - - test("createStore creates a new store with custom path", async () => { - const store = await createTestStore(); - expect(store.dbPath).toBe(testDbPath); - expect(store.db).toBeInstanceOf(Database); - await cleanupTestDb(store); - }); - - test("createStore initializes database schema", async () => { - const store = await createTestStore(); - - // Check tables exist - const tables = store.db.prepare(` - SELECT name FROM sqlite_master WHERE type='table' ORDER BY name - `).all() as { name: string }[]; - - const tableNames = tables.map(t => t.name); - expect(tableNames).toContain("documents"); - expect(tableNames).toContain("documents_fts"); - expect(tableNames).toContain("content_vectors"); - expect(tableNames).toContain("llm_cache"); - // Note: path_contexts table removed in favor of YAML-based context storage - - await cleanupTestDb(store); - }); - - test("createStore sets WAL journal mode", async () => { - const store = await createTestStore(); - const result = store.db.prepare("PRAGMA journal_mode").get() as { journal_mode: string }; - expect(result.journal_mode).toBe("wal"); - await cleanupTestDb(store); - }); - - test("store.close closes the database connection", async () => { - const store = await createTestStore(); - store.close(); - // Attempting to use db after close should throw - expect(() => store.db.prepare("SELECT 1").get()).toThrow(); - try { - await unlink(testDbPath); - } catch {} - }); -}); - -// ============================================================================= -// Document Hashing & Title Extraction Tests -// ============================================================================= - -describe("Document Helpers", () => { - test("hashContent produces consistent SHA256 hashes", async () => { - const content = "Hello, World!"; - const hash1 = await hashContent(content); - const hash2 = await hashContent(content); - expect(hash1).toBe(hash2); - expect(hash1).toMatch(/^[a-f0-9]{64}$/); - }); - - test("hashContent produces different hashes for different content", async () => { - const hash1 = await hashContent("Hello"); - const hash2 = await hashContent("World"); - expect(hash1).not.toBe(hash2); - }); - - test("extractTitle extracts H1 heading", () => { - const content = "# My Title\n\nSome content here."; - expect(extractTitle(content, "file.md")).toBe("My Title"); - }); - - test("extractTitle extracts H2 heading if no H1", () => { - const content = "## My Subtitle\n\nSome content here."; - expect(extractTitle(content, "file.md")).toBe("My Subtitle"); - }); - - test("extractTitle falls back to filename", () => { - const content = "Just some plain text without headings."; - expect(extractTitle(content, "my-document.md")).toBe("my-document"); - }); - - test("extractTitle skips generic 'Notes' heading", () => { - const content = "# Notes\n\n## Actual Title\n\nContent"; - expect(extractTitle(content, "file.md")).toBe("Actual Title"); - }); - - test("extractTitle handles 📝 Notes heading", () => { - const content = "# 📝 Notes\n\n## Meeting Summary\n\nContent"; - expect(extractTitle(content, "file.md")).toBe("Meeting Summary"); - }); -}); - -// ============================================================================= -// Embedding Format Tests -// ============================================================================= - -describe("Embedding Formatting", () => { - test("formatQueryForEmbedding adds search task prefix", () => { - const formatted = formatQueryForEmbedding("how to deploy"); - expect(formatted).toBe("task: search result | query: how to deploy"); - }); - - test("formatDocForEmbedding adds title and text prefix", () => { - const formatted = formatDocForEmbedding("Some content", "My Title"); - expect(formatted).toBe("title: My Title | text: Some content"); - }); - - test("formatDocForEmbedding handles missing title", () => { - const formatted = formatDocForEmbedding("Some content"); - expect(formatted).toBe("title: none | text: Some content"); - }); -}); - -// ============================================================================= -// Document Chunking Tests -// ============================================================================= - -describe("Document Chunking", () => { - test("chunkDocument returns single chunk for small documents", () => { - const content = "Small document content"; - const chunks = chunkDocument(content, 1000, 0); - expect(chunks).toHaveLength(1); - expect(chunks[0]!.text).toBe(content); - expect(chunks[0]!.pos).toBe(0); - }); - - test("chunkDocument splits large documents", () => { - const content = "A".repeat(10000); - const chunks = chunkDocument(content, 1000, 0); - expect(chunks.length).toBeGreaterThan(1); - - // All chunks should have correct positions - for (let i = 0; i < chunks.length; i++) { - expect(chunks[i]!.pos).toBeGreaterThanOrEqual(0); - if (i > 0) { - expect(chunks[i]!.pos).toBeGreaterThan(chunks[i - 1]!.pos); - } - } - }); - - test("chunkDocument with overlap creates overlapping chunks", () => { - const content = "A".repeat(3000); - const chunks = chunkDocument(content, 1000, 150); // 15% overlap - expect(chunks.length).toBeGreaterThan(1); - - // With overlap, positions should be closer together than without - // Each new chunk starts 150 chars before where the previous one ended - for (let i = 1; i < chunks.length; i++) { - const prevEnd = chunks[i - 1]!.pos + chunks[i - 1]!.text.length; - const currentStart = chunks[i]!.pos; - // Current chunk should start before the previous chunk ended (overlap) - expect(currentStart).toBeLessThan(prevEnd); - // But should still make forward progress - expect(currentStart).toBeGreaterThan(chunks[i - 1]!.pos); - } - }); - - test("chunkDocument prefers paragraph breaks", () => { - const content = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph.".repeat(50); - const chunks = chunkDocument(content, 500, 0); - - // Chunks should end at paragraph breaks when possible - for (const chunk of chunks.slice(0, -1)) { - // Most chunks should end near a paragraph break - const endsNearParagraph = chunk.text.endsWith("\n\n") || - chunk.text.endsWith(".") || - chunk.text.endsWith("\n"); - // This is a soft check - not all chunks can end at breaks - } - expect(chunks.length).toBeGreaterThan(1); - }); - - test("chunkDocument handles UTF-8 characters correctly", () => { - const content = "こんにちは世界".repeat(500); // Japanese text - const chunks = chunkDocument(content, 1000, 0); - - // Should not split in the middle of a multi-byte character - for (const chunk of chunks) { - expect(() => new TextEncoder().encode(chunk.text)).not.toThrow(); - } - }); - - test("chunkDocument with default params uses 800-token chunks", () => { - // Default is CHUNK_SIZE_CHARS (3200 chars) with CHUNK_OVERLAP_CHARS (480 chars) - const content = "Word ".repeat(2000); // ~10000 chars - const chunks = chunkDocument(content); - expect(chunks.length).toBeGreaterThan(1); - // Each chunk should be around 3200 chars (except last) - expect(chunks[0]!.text.length).toBeGreaterThan(2500); - expect(chunks[0]!.text.length).toBeLessThanOrEqual(3200); - }); -}); - -describe("Token-based Chunking", () => { - test("chunkDocumentByTokens returns single chunk for small documents", async () => { - const content = "This is a small document."; - const chunks = await chunkDocumentByTokens(content, 800, 120); - expect(chunks).toHaveLength(1); - expect(chunks[0]!.text).toBe(content); - expect(chunks[0]!.pos).toBe(0); - expect(chunks[0]!.tokens).toBeGreaterThan(0); - expect(chunks[0]!.tokens).toBeLessThan(800); - }); - - test("chunkDocumentByTokens splits large documents", async () => { - // Create a document that's definitely more than 800 tokens - const content = "The quick brown fox jumps over the lazy dog. ".repeat(200); - const chunks = await chunkDocumentByTokens(content, 800, 120); - - expect(chunks.length).toBeGreaterThan(1); - - // Each chunk should have ~800 tokens or less - for (const chunk of chunks) { - expect(chunk.tokens).toBeLessThanOrEqual(850); // Allow slight overage - expect(chunk.tokens).toBeGreaterThan(0); - } - - // Chunks should have correct positions - for (let i = 0; i < chunks.length; i++) { - expect(chunks[i]!.pos).toBeGreaterThanOrEqual(0); - if (i > 0) { - expect(chunks[i]!.pos).toBeGreaterThan(chunks[i - 1]!.pos); - } - } - }); - - test("chunkDocumentByTokens creates overlapping chunks", async () => { - const content = "Word ".repeat(500); // ~500 tokens - const chunks = await chunkDocumentByTokens(content, 200, 30); // 15% overlap - - expect(chunks.length).toBeGreaterThan(1); - - // With overlap, consecutive chunks should have overlapping positions - for (let i = 1; i < chunks.length; i++) { - const prevEnd = chunks[i - 1]!.pos + chunks[i - 1]!.text.length; - const currentStart = chunks[i]!.pos; - // Current chunk should start before the previous chunk ended (overlap) - expect(currentStart).toBeLessThan(prevEnd); - } - }); - - test("chunkDocumentByTokens returns actual token counts", async () => { - const content = "Hello world, this is a test."; - const chunks = await chunkDocumentByTokens(content); - - expect(chunks).toHaveLength(1); - // The token count should be reasonable (not 0, not equal to char count) - expect(chunks[0]!.tokens).toBeGreaterThan(0); - expect(chunks[0]!.tokens).toBeLessThan(content.length); // Tokens < chars for English - }); -}); - -// ============================================================================= -// Caching Tests -// ============================================================================= - -describe("Caching", () => { - test("getCacheKey generates consistent keys", () => { - const key1 = getCacheKey("http://example.com", { query: "test" }); - const key2 = getCacheKey("http://example.com", { query: "test" }); - expect(key1).toBe(key2); - expect(key1).toMatch(/^[a-f0-9]{64}$/); - }); - - test("getCacheKey generates different keys for different inputs", () => { - const key1 = getCacheKey("http://example.com", { query: "test1" }); - const key2 = getCacheKey("http://example.com", { query: "test2" }); - expect(key1).not.toBe(key2); - }); - - test("store cache operations work correctly", async () => { - const store = await createTestStore(); - - const key = "test-cache-key"; - const value = "cached result"; - - // Initially empty - expect(store.getCachedResult(key)).toBeNull(); - - // Set cache - store.setCachedResult(key, value); - - // Retrieve cache - expect(store.getCachedResult(key)).toBe(value); - - // Clear cache - store.clearCache(); - expect(store.getCachedResult(key)).toBeNull(); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Context Tests -// ============================================================================= - -describe("Path Context", () => { - test("getContextForFile returns null when no context set", async () => { - const store = await createTestStore(); - const context = store.getContextForFile("/some/random/path.md"); - expect(context).toBeNull(); - await cleanupTestDb(store); - }); - - test("getContextForFile returns matching context", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/test/collection", glob: "**/*.md" }); - await addPathContext(collectionName, "/docs", "Documentation files"); - - // Insert a document so getContextForFile can find it - await insertTestDocument(store.db, collectionName, { - name: "readme", - displayPath: "docs/readme.md", - }); - - const context = store.getContextForFile("/test/collection/docs/readme.md"); - expect(context).toBe("Documentation files"); - - await cleanupTestDb(store); - }); - - test("getContextForFile returns all matching contexts", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/test/collection", glob: "**/*.md" }); - await addPathContext(collectionName, "/", "General test files"); - await addPathContext(collectionName, "/docs", "Documentation files"); - await addPathContext(collectionName, "/docs/api", "API documentation"); - - // Insert documents so getContextForFile can find them - await insertTestDocument(store.db, collectionName, { - name: "readme", - displayPath: "readme.md", - }); - await insertTestDocument(store.db, collectionName, { - name: "guide", - displayPath: "docs/guide.md", - }); - await insertTestDocument(store.db, collectionName, { - name: "reference", - displayPath: "docs/api/reference.md", - }); - - // Context now returns ALL matching contexts joined with \n\n - expect(store.getContextForFile("/test/collection/readme.md")).toBe("General test files"); - expect(store.getContextForFile("/test/collection/docs/guide.md")).toBe("General test files\n\nDocumentation files"); - expect(store.getContextForFile("/test/collection/docs/api/reference.md")).toBe("General test files\n\nDocumentation files\n\nAPI documentation"); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Collection Tests -// ============================================================================= - -describe("Collections", () => { - test("collections are managed via YAML config", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/home/user/projects/myapp", glob: "**/*.md" }); - - // Collections are now in YAML, not in the database - expect(collectionName).toBe("myapp"); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// FTS Search Tests -// ============================================================================= - -describe("FTS Search", () => { - test("searchFTS returns empty array for no matches", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - await insertTestDocument(store.db, collectionName, { - name: "doc1", - body: "The quick brown fox jumps over the lazy dog", - }); - - const results = store.searchFTS("nonexistent-term-xyz", 10); - expect(results).toHaveLength(0); - - await cleanupTestDb(store); - }); - - test("searchFTS finds documents by keyword", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - await insertTestDocument(store.db, collectionName, { - name: "doc1", - title: "Fox Document", - body: "The quick brown fox jumps over the lazy dog", - displayPath: "test/doc1.md", - }); - - const results = store.searchFTS("fox", 10); - expect(results.length).toBeGreaterThan(0); - expect(results[0]!.displayPath).toBe(`${collectionName}/test/doc1.md`); - expect(results[0]!.filepath).toBe(`qmd://${collectionName}/test/doc1.md`); - expect(results[0]!.source).toBe("fts"); - - await cleanupTestDb(store); - }); - - test("searchFTS ranks title matches higher", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - // Document with "fox" in body only - await insertTestDocument(store.db, collectionName, { - name: "body-match", - title: "Some Other Title", - body: "The fox is here in the body", - displayPath: "test/body.md", - }); - - // Document with "fox" in title (via name field which is indexed) - await insertTestDocument(store.db, collectionName, { - name: "fox", - title: "Fox Title", - body: "Different content without the animal fox", - displayPath: "test/title.md", - }); - - const results = store.searchFTS("fox", 10); - // Both documents contain "fox" in the body now, so we should get 2 results - expect(results.length).toBe(2); - // Title/name match should rank higher due to BM25 weights - expect(results[0]!.displayPath).toBe(`${collectionName}/test/title.md`); - - await cleanupTestDb(store); - }); - - test("searchFTS respects limit parameter", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - // Insert 10 documents - for (let i = 0; i < 10; i++) { - await insertTestDocument(store.db, collectionName, { - name: `doc${i}`, - body: "common keyword appears here", - displayPath: `test/doc${i}.md`, - }); - } - - const results = store.searchFTS("common keyword", 3); - expect(results).toHaveLength(3); - - await cleanupTestDb(store); - }); - - test("searchFTS filters by collection name", async () => { - const store = await createTestStore(); - const collection1 = await createTestCollection({ pwd: "/path/one", glob: "**/*.md", name: "one" }); - const collection2 = await createTestCollection({ pwd: "/path/two", glob: "**/*.md", name: "two" }); - - await insertTestDocument(store.db, collection1, { - name: "doc1", - body: "searchable content", - displayPath: "doc1.md", - }); - - await insertTestDocument(store.db, collection2, { - name: "doc2", - body: "searchable content", - displayPath: "doc2.md", - }); - - const allResults = store.searchFTS("searchable", 10); - expect(allResults).toHaveLength(2); - - // Filter by collection name (collectionId is now treated as collection name string) - const filtered = store.searchFTS("searchable", 10, collection1 as unknown as number); - expect(filtered).toHaveLength(1); - expect(filtered[0]!.displayPath).toBe(`${collection1}/doc1.md`); - - await cleanupTestDb(store); - }); - - test("searchFTS handles special characters in query", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - await insertTestDocument(store.db, collectionName, { - name: "doc1", - body: "Function with params: foo(bar, baz)", - displayPath: "test/doc1.md", - }); - - // Should not throw on special characters - const results = store.searchFTS("foo(bar)", 10); - // Results may vary based on FTS5 handling - expect(Array.isArray(results)).toBe(true); - - await cleanupTestDb(store); - }); - - test("searchFTS ignores inactive documents", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "active", - body: "findme content", - displayPath: "test/active.md", - active: 1, - }); - - await insertTestDocument(store.db, collectionName, { - name: "inactive", - body: "findme content", - displayPath: "test/inactive.md", - active: 0, - }); - - const results = store.searchFTS("findme", 10); - expect(results).toHaveLength(1); - expect(results[0]!.displayPath).toBe(`${collectionName}/test/active.md`); - expect(results[0]!.filepath).toBe(`qmd://${collectionName}/test/active.md`); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Document Retrieval Tests -// ============================================================================= - -describe("Document Retrieval", () => { - describe("findDocument", () => { - test("findDocument finds by exact filepath", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/exact/path", glob: "**/*.md" }); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - title: "My Document", - displayPath: "mydoc.md", - body: "Document content here", - }); - - const result = store.findDocument("/exact/path/mydoc.md"); - expect("error" in result).toBe(false); - if (!("error" in result)) { - expect(result.title).toBe("My Document"); - expect(result.displayPath).toBe(`${collectionName}/mydoc.md`); - expect(result.filepath).toBe(`qmd://${collectionName}/mydoc.md`); - expect(result.body).toBeUndefined(); // body not included by default - } - - await cleanupTestDb(store); - }); - - test("findDocument finds by display_path", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/some/path", glob: "**/*.md" }); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - displayPath: "docs/mydoc.md", - }); - - const result = store.findDocument("docs/mydoc.md"); - expect("error" in result).toBe(false); - - await cleanupTestDb(store); - }); - - test("findDocument finds by partial path match", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/very/long/path/to", glob: "**/*.md" }); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - displayPath: "mydoc.md", - }); - - const result = store.findDocument("mydoc.md"); - expect("error" in result).toBe(false); - - await cleanupTestDb(store); - }); - - test("findDocument includes body when requested", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/path", glob: "**/*.md" }); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - displayPath: "mydoc.md", - body: "The actual body content", - }); - - const result = store.findDocument("/path/mydoc.md", { includeBody: true }); - expect("error" in result).toBe(false); - if (!("error" in result)) { - expect(result.body).toBe("The actual body content"); - } - - await cleanupTestDb(store); - }); - - test("findDocument returns error with suggestions for not found", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - await insertTestDocument(store.db, collectionName, { - name: "similar", - filepath: "/path/similar.md", - displayPath: "similar.md", - }); - - const result = store.findDocument("simlar.md"); // typo - 1 char diff - expect("error" in result).toBe(true); - if ("error" in result) { - expect(result.error).toBe("not_found"); - // Levenshtein distance of 1 should be found with maxDistance 3 - expect(result.similarFiles.length).toBeGreaterThanOrEqual(0); // May or may not find depending on distance calc - } - - await cleanupTestDb(store); - }); - - test("findDocument handles :line suffix", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - filepath: "/path/mydoc.md", - displayPath: "mydoc.md", - }); - - const result = store.findDocument("mydoc.md:100"); - expect("error" in result).toBe(false); - - await cleanupTestDb(store); - }); - - test("findDocument expands ~ to home directory", async () => { - const store = await createTestStore(); - const home = homedir(); - const collectionName = await createTestCollection({ pwd: home, name: "home" }); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - filepath: `${home}/docs/mydoc.md`, - displayPath: "docs/mydoc.md", - }); - - const result = store.findDocument("~/docs/mydoc.md"); - expect("error" in result).toBe(false); - - await cleanupTestDb(store); - }); - - test("findDocument includes context from path_contexts", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/path" }); - await addPathContext(collectionName, "docs", "Documentation"); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - displayPath: "docs/mydoc.md", - }); - - const result = store.findDocument("/path/docs/mydoc.md"); - expect("error" in result).toBe(false); - if (!("error" in result)) { - expect(result.context).toBe("Documentation"); - } - - await cleanupTestDb(store); - }); - - test("findDocument includes hierarchical contexts (global + collection + path)", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/archive", name: "archive" }); - - // Add global context - await addGlobalContext("Global context for all documents"); - - // Add collection root context - await addPathContext(collectionName, "/", "Archive collection context"); - - // Add path-specific contexts at different levels - await addPathContext(collectionName, "/podcasts", "Podcast episodes"); - await addPathContext(collectionName, "/podcasts/external", "External podcast interviews"); - - // Insert document in nested path - await insertTestDocument(store.db, collectionName, { - name: "interview", - displayPath: "podcasts/external/2024-jan-interview.md", - }); - - const result = store.findDocument("/archive/podcasts/external/2024-jan-interview.md"); - expect("error" in result).toBe(false); - if (!("error" in result)) { - // Should have all contexts joined with double newlines - expect(result.context).toBe( - "Global context for all documents\n\n" + - "Archive collection context\n\n" + - "Podcast episodes\n\n" + - "External podcast interviews" - ); - } - - await cleanupTestDb(store); - }); - }); - - describe("getDocumentBody", () => { - test("getDocumentBody returns full body", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/path" }); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - displayPath: "mydoc.md", - body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5", - }); - - const body = store.getDocumentBody({ filepath: "/path/mydoc.md" }); - expect(body).toBe("Line 1\nLine 2\nLine 3\nLine 4\nLine 5"); - - await cleanupTestDb(store); - }); - - test("getDocumentBody supports line range", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/path" }); - await insertTestDocument(store.db, collectionName, { - name: "mydoc", - displayPath: "mydoc.md", - body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5", - }); - - const body = store.getDocumentBody({ filepath: "/path/mydoc.md" }, 2, 2); - expect(body).toBe("Line 2\nLine 3"); - - await cleanupTestDb(store); - }); - - test("getDocumentBody returns null for non-existent document", async () => { - const store = await createTestStore(); - const body = store.getDocumentBody({ filepath: "/nonexistent.md" }); - expect(body).toBeNull(); - await cleanupTestDb(store); - }); - }); - - describe("findDocuments (multi-get)", () => { - test("findDocuments finds by glob pattern", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "doc1", - filepath: "/path/journals/2024-01.md", - displayPath: "journals/2024-01.md", - }); - await insertTestDocument(store.db, collectionName, { - name: "doc2", - filepath: "/path/journals/2024-02.md", - displayPath: "journals/2024-02.md", - }); - await insertTestDocument(store.db, collectionName, { - name: "doc3", - filepath: "/path/other/file.md", - displayPath: "other/file.md", - }); - - const { docs, errors } = store.findDocuments("journals/2024-*.md"); - expect(errors).toHaveLength(0); - expect(docs).toHaveLength(2); - - await cleanupTestDb(store); - }); - - test("findDocuments finds by comma-separated list", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "doc1", - filepath: "/path/doc1.md", - displayPath: "doc1.md", - }); - await insertTestDocument(store.db, collectionName, { - name: "doc2", - filepath: "/path/doc2.md", - displayPath: "doc2.md", - }); - - const { docs, errors } = store.findDocuments("doc1.md, doc2.md"); - expect(errors).toHaveLength(0); - expect(docs).toHaveLength(2); - - await cleanupTestDb(store); - }); - - test("findDocuments reports errors for not found files", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "doc1", - filepath: "/path/doc1.md", - displayPath: "doc1.md", - }); - - const { docs, errors } = store.findDocuments("doc1.md, nonexistent.md"); - expect(docs).toHaveLength(1); - expect(errors).toHaveLength(1); - expect(errors[0]).toContain("not found"); - - await cleanupTestDb(store); - }); - - test("findDocuments skips large files", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "large", - filepath: "/path/large.md", - displayPath: "large.md", - body: "x".repeat(20000), // 20KB - }); - - const { docs } = store.findDocuments("large.md", { maxBytes: 10000 }); - expect(docs).toHaveLength(1); - expect(docs[0]!.skipped).toBe(true); - if (docs[0]!.skipped) { - expect((docs[0] as { skipped: true; skipReason: string }).skipReason).toContain("too large"); - } - - await cleanupTestDb(store); - }); - - test("findDocuments includes body when requested", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "doc1", - filepath: "/path/doc1.md", - displayPath: "doc1.md", - body: "The content", - }); - - const { docs } = store.findDocuments("doc1.md", { includeBody: true }); - expect(docs[0]!.skipped).toBe(false); - if (!docs[0]!.skipped) { - expect((docs[0] as { doc: { body: string }; skipped: false }).doc.body).toBe("The content"); - } - - await cleanupTestDb(store); - }); - }); - -}); - -// ============================================================================= -// Snippet Extraction Tests -// ============================================================================= - -describe("Snippet Extraction", () => { - test("extractSnippet finds query terms", () => { - const body = "First line.\nSecond line with keyword.\nThird line.\nFourth line."; - const { line, snippet } = extractSnippet(body, "keyword", 500); - - expect(line).toBe(2); // Line 2 contains "keyword" - expect(snippet).toContain("keyword"); - }); - - test("extractSnippet includes context lines", () => { - const body = "Line 1\nLine 2\nLine 3 has keyword\nLine 4\nLine 5"; - const { snippet } = extractSnippet(body, "keyword", 500); - - expect(snippet).toContain("Line 2"); // Context before - expect(snippet).toContain("Line 3 has keyword"); - expect(snippet).toContain("Line 4"); // Context after - }); - - test("extractSnippet respects maxLen for content", () => { - const body = "A".repeat(1000); - const result = extractSnippet(body, "query", 100); - - // Snippet includes header + content, content should be truncated - expect(result.snippet).toContain("@@"); // Has diff header - expect(result.snippet).toContain("..."); // Content was truncated - }); - - test("extractSnippet uses chunkPos hint", () => { - const body = "First section...\n".repeat(50) + "Target keyword here\n" + "More content...".repeat(50); - const chunkPos = body.indexOf("Target keyword"); - - const { snippet } = extractSnippet(body, "Target", 200, chunkPos); - expect(snippet).toContain("Target keyword"); - }); - - test("extractSnippet returns beginning when no match", () => { - const body = "First line\nSecond line\nThird line"; - const { line, snippet } = extractSnippet(body, "nonexistent", 500); - - expect(line).toBe(1); - expect(snippet).toContain("First line"); - }); - - test("extractSnippet includes diff-style header", () => { - const body = "Line 1\nLine 2\nLine 3 has keyword\nLine 4\nLine 5"; - const { snippet, linesBefore, linesAfter, snippetLines } = extractSnippet(body, "keyword", 500); - - // Header should show line position and context info - expect(snippet).toMatch(/^@@ -\d+,\d+ @@ \(\d+ before, \d+ after\)/); - expect(linesBefore).toBe(1); // Line 1 comes before - expect(linesAfter).toBe(0); // Snippet includes to end (lines 2-5) - expect(snippetLines).toBe(4); // Lines 2, 3, 4, 5 - }); - - test("extractSnippet calculates linesBefore and linesAfter correctly", () => { - const body = "L1\nL2\nL3\nL4 match\nL5\nL6\nL7\nL8\nL9\nL10"; - const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "match", 500); - - expect(line).toBe(4); // "L4 match" is line 4 - expect(linesBefore).toBe(2); // L1, L2 before snippet (snippet starts at L3) - expect(snippetLines).toBe(4); // L3, L4, L5, L6 - expect(linesAfter).toBe(4); // L7, L8, L9, L10 after snippet - }); - - test("extractSnippet header format matches diff style", () => { - const body = "A\nB\nC keyword\nD\nE\nF\nG\nH"; - const { snippet } = extractSnippet(body, "keyword", 500); - - // Should start with @@ -line,count @@ (N before, M after) - const headerMatch = snippet.match(/^@@ -(\d+),(\d+) @@ \((\d+) before, (\d+) after\)/); - expect(headerMatch).not.toBeNull(); - - const [, startLine, count, before, after] = headerMatch!; - expect(parseInt(startLine!)).toBe(2); // Snippet starts at line 2 (B) - expect(parseInt(count!)).toBe(4); // 4 lines: B, C keyword, D, E - expect(parseInt(before!)).toBe(1); // A is before - expect(parseInt(after!)).toBe(3); // F, G, H are after - }); - - test("extractSnippet at document start shows 0 before", () => { - const body = "First line keyword\nSecond\nThird\nFourth\nFifth"; - const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "keyword", 500); - - expect(line).toBe(1); // Keyword on first line - expect(linesBefore).toBe(0); // Nothing before - expect(snippetLines).toBe(3); // First, Second, Third (bestLine-1 to bestLine+3, clamped) - expect(linesAfter).toBe(2); // Fourth, Fifth - }); - - test("extractSnippet at document end shows 0 after", () => { - const body = "First\nSecond\nThird\nFourth\nFifth keyword"; - const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "keyword", 500); - - expect(line).toBe(5); // Keyword on last line - expect(linesBefore).toBe(3); // First, Second, Third before snippet - expect(snippetLines).toBe(2); // Fourth, Fifth keyword (bestLine-1 to bestLine+3, clamped) - expect(linesAfter).toBe(0); // Nothing after - }); - - test("extractSnippet with single line document", () => { - const body = "Single line with keyword"; - const { linesBefore, linesAfter, snippetLines, snippet } = extractSnippet(body, "keyword", 500); - - expect(linesBefore).toBe(0); - expect(linesAfter).toBe(0); - expect(snippetLines).toBe(1); - expect(snippet).toContain("@@ -1,1 @@ (0 before, 0 after)"); - expect(snippet).toContain("Single line with keyword"); - }); - - test("extractSnippet with chunkPos adjusts line numbers correctly", () => { - // 50 lines of padding, then keyword, then more content - const padding = "Padding line\n".repeat(50); - const body = padding + "Target keyword here\nMore content\nEven more"; - const chunkPos = padding.length; // Position of "Target keyword" - - const { line, linesBefore, linesAfter } = extractSnippet(body, "keyword", 200, chunkPos); - - expect(line).toBe(51); // "Target keyword" is line 51 - expect(linesBefore).toBeGreaterThan(40); // Many lines before - }); -}); - -// ============================================================================= -// Reciprocal Rank Fusion Tests -// ============================================================================= - -describe("Reciprocal Rank Fusion", () => { - const makeResult = (file: string, score: number): RankedResult => ({ - file, - displayPath: file, - title: file, - body: "body", - score, - }); - - test("RRF combines single list correctly", () => { - const list1 = [ - makeResult("doc1", 0.9), - makeResult("doc2", 0.8), - makeResult("doc3", 0.7), - ]; - - const fused = reciprocalRankFusion([list1]); - - // Order should be preserved - expect(fused[0]!.file).toBe("doc1"); - expect(fused[1]!.file).toBe("doc2"); - expect(fused[2]!.file).toBe("doc3"); - }); - - test("RRF merges documents from multiple lists", () => { - const list1 = [makeResult("doc1", 0.9), makeResult("doc2", 0.8)]; - const list2 = [makeResult("doc2", 0.95), makeResult("doc3", 0.85)]; - - const fused = reciprocalRankFusion([list1, list2]); - - // doc2 appears in both lists, should have higher combined score - expect(fused.find(r => r.file === "doc2")).toBeDefined(); - expect(fused.find(r => r.file === "doc1")).toBeDefined(); - expect(fused.find(r => r.file === "doc3")).toBeDefined(); - }); - - test("RRF respects weights", () => { - const list1 = [makeResult("doc1", 0.9)]; - const list2 = [makeResult("doc2", 0.9)]; - - // Give double weight to list1 - const fused = reciprocalRankFusion([list1, list2], [2.0, 1.0]); - - // doc1 should rank higher due to weight - expect(fused[0]!.file).toBe("doc1"); - }); - - test("RRF adds top-rank bonus", () => { - // doc1 is #1 in list1, doc2 is #2 in list1 - const list1 = [makeResult("doc1", 0.9), makeResult("doc2", 0.8)]; - const list2 = [makeResult("doc3", 0.85)]; - - const fused = reciprocalRankFusion([list1, list2]); - - // doc1 should get +0.05 bonus for being #1 - // doc2 should get +0.02 bonus for being #2-3 - const doc1 = fused.find(r => r.file === "doc1"); - const doc2 = fused.find(r => r.file === "doc2"); - - expect(doc1!.score).toBeGreaterThan(doc2!.score); - }); - - test("RRF handles empty lists", () => { - const fused = reciprocalRankFusion([[], []]); - expect(fused).toHaveLength(0); - }); - - test("RRF uses k parameter correctly", () => { - const list = [makeResult("doc1", 0.9)]; - - // With different k values, scores should differ - const fused60 = reciprocalRankFusion([list], [], 60); - const fused30 = reciprocalRankFusion([list], [], 30); - - // Lower k = higher scores for top ranks - expect(fused30[0]!.score).toBeGreaterThan(fused60[0]!.score); - }); -}); - -// ============================================================================= -// Index Status Tests -// ============================================================================= - -describe("Index Status", () => { - test("getStatus returns correct structure", async () => { - const store = await createTestStore(); - const status = store.getStatus(); - expect(status).toHaveProperty("totalDocuments"); - expect(status).toHaveProperty("needsEmbedding"); - expect(status).toHaveProperty("hasVectorIndex"); - expect(status).toHaveProperty("collections"); - expect(Array.isArray(status.collections)).toBe(true); - - await cleanupTestDb(store); - }); - - test("getStatus counts documents correctly", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { name: "doc1", active: 1 }); - await insertTestDocument(store.db, collectionName, { name: "doc2", active: 1 }); - await insertTestDocument(store.db, collectionName, { name: "doc3", active: 0 }); // inactive - - const status = store.getStatus(); - expect(status.totalDocuments).toBe(2); // Only active docs - - await cleanupTestDb(store); - }); - - test("getStatus reports collection info", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/test/path", glob: "**/*.md" }); - await insertTestDocument(store.db, collectionName, { name: "doc1" }); - - const status = store.getStatus(); - expect(status.collections.length).toBeGreaterThanOrEqual(1); - const col = status.collections.find(c => c.name === collectionName); - expect(col).toBeDefined(); - expect(col?.path).toBe("/test/path"); - expect(col?.pattern).toBe("**/*.md"); - expect(col?.documents).toBe(1); - - await cleanupTestDb(store); - }); - - test("getHashesNeedingEmbedding counts correctly", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - // Add documents with different hashes - await insertTestDocument(store.db, collectionName, { name: "doc1", hash: "hash1" }); - await insertTestDocument(store.db, collectionName, { name: "doc2", hash: "hash2" }); - await insertTestDocument(store.db, collectionName, { name: "doc3", hash: "hash1" }); // same hash as doc1 - - const needsEmbedding = store.getHashesNeedingEmbedding(); - expect(needsEmbedding).toBe(2); // hash1 and hash2 - - await cleanupTestDb(store); - }); - - test("getIndexHealth returns health info", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - await insertTestDocument(store.db, collectionName, { name: "doc1" }); - - const health = store.getIndexHealth(); - expect(health).toHaveProperty("needsEmbedding"); - expect(health).toHaveProperty("totalDocs"); - expect(health).toHaveProperty("daysStale"); - expect(health.totalDocs).toBe(1); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Fuzzy Matching Tests -// ============================================================================= - -describe("Fuzzy Matching", () => { - test("findSimilarFiles finds similar paths", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "readme", - displayPath: "docs/readme.md", - }); - await insertTestDocument(store.db, collectionName, { - name: "readmi", - displayPath: "docs/readmi.md", // typo - }); - - const similar = store.findSimilarFiles("docs/readme.md", 3, 5); - expect(similar).toContain("docs/readme.md"); - - await cleanupTestDb(store); - }); - - test("findSimilarFiles respects maxDistance", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "abc", - displayPath: "abc.md", - }); - await insertTestDocument(store.db, collectionName, { - name: "xyz", - displayPath: "xyz.md", // very different - }); - - const similar = store.findSimilarFiles("abc.md", 1, 5); // max distance 1 - expect(similar).toContain("abc.md"); - expect(similar).not.toContain("xyz.md"); - - await cleanupTestDb(store); - }); - - test("matchFilesByGlob matches patterns", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - filepath: "/p/journals/2024-01.md", - displayPath: "journals/2024-01.md", - }); - await insertTestDocument(store.db, collectionName, { - filepath: "/p/journals/2024-02.md", - displayPath: "journals/2024-02.md", - }); - await insertTestDocument(store.db, collectionName, { - filepath: "/p/docs/readme.md", - displayPath: "docs/readme.md", - }); - - const matches = store.matchFilesByGlob("journals/*.md"); - expect(matches).toHaveLength(2); - expect(matches.every(m => m.displayPath.startsWith("journals/"))).toBe(true); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Vector Table Tests -// ============================================================================= - -describe("Vector Table", () => { - test("ensureVecTable creates vector table", async () => { - const store = await createTestStore(); - - // Initially no vector table - let exists = store.db.prepare(` - SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec' - `).get(); - expect(exists).toBeFalsy(); // null or undefined - - // Create vector table - store.ensureVecTable(768); - - exists = store.db.prepare(` - SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec' - `).get(); - expect(exists).toBeTruthy(); - - await cleanupTestDb(store); - }); - - test("ensureVecTable recreates table if dimensions change", async () => { - const store = await createTestStore(); - - // Create with 768 dimensions - store.ensureVecTable(768); - - // Check dimensions - let tableInfo = store.db.prepare(` - SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec' - `).get() as { sql: string }; - expect(tableInfo.sql).toContain("float[768]"); - - // Recreate with different dimensions - store.ensureVecTable(1024); - - tableInfo = store.db.prepare(` - SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec' - `).get() as { sql: string }; - expect(tableInfo.sql).toContain("float[1024]"); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Integration Tests -// ============================================================================= - -describe("Integration", () => { - test("full document lifecycle: create, search, retrieve", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection({ pwd: "/test/notes", glob: "**/*.md" }); - - // Add context - use "/" for collection root - await addPathContext(collectionName, "/", "Personal notes"); - - // Insert documents - await insertTestDocument(store.db, collectionName, { - name: "meeting", - title: "Team Meeting Notes", - filepath: "/test/notes/meeting.md", - displayPath: "notes/meeting.md", - body: "# Team Meeting Notes\n\nDiscussed project timeline and deliverables.", - }); - - await insertTestDocument(store.db, collectionName, { - name: "ideas", - title: "Project Ideas", - filepath: "/test/notes/ideas.md", - displayPath: "notes/ideas.md", - body: "# Project Ideas\n\nBrainstorming new features for the product.", - }); - - // Search - const searchResults = store.searchFTS("project", 10); - expect(searchResults.length).toBe(2); - - // Status - SKIPPED: getStatus() has bug (queries non-existent collections table) - // const status = store.getStatus(); - // expect(status.totalDocuments).toBe(2); - // expect(status.collections).toHaveLength(1); - - // Retrieve single document - const doc = store.findDocument("notes/meeting.md", { includeBody: true }); - expect("error" in doc).toBe(false); - if (!("error" in doc)) { - expect(doc.title).toBe("Team Meeting Notes"); - expect(doc.context).toBe("Personal notes"); - expect(doc.body).toContain("Team Meeting"); - } - - // Multi-get - const { docs, errors } = store.findDocuments("notes/*.md", { includeBody: true }); - expect(errors).toHaveLength(0); - expect(docs).toHaveLength(2); - - await cleanupTestDb(store); - }); - - test("multiple stores can operate independently", async () => { - const store1 = await createTestStore(); - const store2 = await createTestStore(); - - const col1 = await createTestCollection({ pwd: "/store1", glob: "**/*.md", name: "store1" }); - const col2 = await createTestCollection({ pwd: "/store2", glob: "**/*.md", name: "store2" }); - - await insertTestDocument(store1.db, col1, { - name: "doc1", - body: "unique content for store1", - displayPath: "doc.md", - }); - - await insertTestDocument(store2.db, col2, { - name: "doc2", - body: "different content for store2", - displayPath: "doc.md", - }); - - // Each store should only see its own documents - const results1 = store1.searchFTS("unique", 10); - const results2 = store2.searchFTS("different", 10); - - expect(results1).toHaveLength(1); - expect(results1[0]!.displayPath).toBe("store1/doc.md"); - expect(results1[0]!.filepath).toBe("qmd://store1/doc.md"); - - expect(results2).toHaveLength(1); - expect(results2[0]!.displayPath).toBe("store2/doc.md"); - expect(results2[0]!.filepath).toBe("qmd://store2/doc.md"); - - // Cross-check: store1 shouldn't find store2's content - const cross1 = store1.searchFTS("different", 10); - const cross2 = store2.searchFTS("unique", 10); - - expect(cross1).toHaveLength(0); - expect(cross2).toHaveLength(0); - - await cleanupTestDb(store1); - await cleanupTestDb(store2); - }); -}); - -// ============================================================================= -// LlamaCpp Integration Tests (using real local models) -// ============================================================================= - -describe("LlamaCpp Integration", () => { - test("searchVec returns empty when no vector index", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - await insertTestDocument(store.db, collectionName, { - name: "doc1", - body: "Some content", - }); - - // No vectors_vec table exists, should return empty - const results = await store.searchVec("query", "embeddinggemma", 10); - expect(results).toHaveLength(0); - - await cleanupTestDb(store); - }); - - test("searchVec returns results when vector index exists", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - const hash = "testhash123"; - await insertTestDocument(store.db, collectionName, { - name: "doc1", - hash, - body: "Some content about testing", - filepath: "/test/doc1.md", - displayPath: "doc1.md", - }); - - // Create vector table and insert a vector - store.ensureVecTable(768); - const embedding = Array(768).fill(0).map(() => Math.random()); - store.db.prepare(`INSERT INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, 0, 0, 'test', ?)`).run(hash, new Date().toISOString()); - store.db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${hash}_0`, new Float32Array(embedding)); - - const results = await store.searchVec("test query", "embeddinggemma", 10); - expect(results).toHaveLength(1); - expect(results[0]!.displayPath).toBe(`${collectionName}/doc1.md`); - expect(results[0]!.filepath).toBe(`qmd://${collectionName}/doc1.md`); - expect(results[0]!.source).toBe("vec"); - - await cleanupTestDb(store); - }); - - test("searchVec filters by collection name", async () => { - const store = await createTestStore(); - const collection1 = await createTestCollection({ name: "coll1", pwd: "/test/coll1" }); - const collection2 = await createTestCollection({ name: "coll2", pwd: "/test/coll2" }); - - const hash1 = "hash1abc"; - const hash2 = "hash2xyz"; - - await insertTestDocument(store.db, collection1, { - name: "doc1", - hash: hash1, - body: "Content in collection one", - }); - - await insertTestDocument(store.db, collection2, { - name: "doc2", - hash: hash2, - body: "Content in collection two", - }); - - // Create vectors_vec table with correct dimensions (768 for embeddinggemma) - store.ensureVecTable(768); - const embedding1 = Array(768).fill(0).map(() => Math.random()); - const embedding2 = Array(768).fill(0).map(() => Math.random()); - store.db.prepare(`INSERT INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, 0, 0, 'test', ?)`).run(hash1, new Date().toISOString()); - store.db.prepare(`INSERT INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, 0, 0, 'test', ?)`).run(hash2, new Date().toISOString()); - store.db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${hash1}_0`, new Float32Array(embedding1)); - store.db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${hash2}_0`, new Float32Array(embedding2)); - - // Search without filter - should return both - const allResults = await store.searchVec("content", "embeddinggemma", 10); - expect(allResults).toHaveLength(2); - - // Search with collection filter - should return only from collection1 - const filtered = await store.searchVec("content", "embeddinggemma", 10, collection1); - expect(filtered).toHaveLength(1); - expect(filtered[0]!.collectionName).toBe(collection1); - - await cleanupTestDb(store); - }); - - // Regression test for https://github.com/tobi/qmd/pull/23 - // sqlite-vec virtual tables hang when combined with JOINs in the same query. - // The fix uses a two-step approach: vector query first, then separate JOINs. - test("searchVec uses two-step query to avoid sqlite-vec JOIN hang", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - const hash = "regression_test_hash"; - await insertTestDocument(store.db, collectionName, { - name: "regression-doc", - hash, - body: "Test content for vector search regression", - filepath: "/test/regression.md", - displayPath: "regression.md", - }); - - // Create vector table and insert a test vector - store.ensureVecTable(768); - const embedding = Array(768).fill(0).map(() => Math.random()); - store.db.prepare(`INSERT INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, 0, 0, 'test', ?)`).run(hash, new Date().toISOString()); - store.db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${hash}_0`, new Float32Array(embedding)); - - // This should complete quickly (not hang) due to the two-step fix - // The old code with JOINs in the sqlite-vec query would hang indefinitely - const startTime = Date.now(); - const results = await store.searchVec("test content", "embeddinggemma", 5); - const elapsed = Date.now() - startTime; - - // If the query took more than 5 seconds, something is wrong - // (the hang bug would cause it to never return at all) - expect(elapsed).toBeLessThan(5000); - expect(results.length).toBeGreaterThan(0); - - await cleanupTestDb(store); - }); - - test("expandQuery returns original plus expanded queries", async () => { - const store = await createTestStore(); - - const queries = await store.expandQuery("test query"); - expect(queries).toContain("test query"); - expect(queries[0]).toBe("test query"); - // LlamaCpp returns original + variations - expect(queries.length).toBeGreaterThanOrEqual(1); - - await cleanupTestDb(store); - }, 30000); - - test("expandQuery caches results", async () => { - const store = await createTestStore(); - - // First call - const queries1 = await store.expandQuery("cached query test"); - // Second call - should hit cache - const queries2 = await store.expandQuery("cached query test"); - - expect(queries1[0]).toBe(queries2[0]); - - await cleanupTestDb(store); - }, 30000); - - test("rerank scores documents", async () => { - const store = await createTestStore(); - - const docs = [ - { file: "doc1.md", text: "Relevant content about the topic" }, - { file: "doc2.md", text: "Other content" }, - ]; - - const results = await store.rerank("topic", docs); - expect(results).toHaveLength(2); - // LlamaCpp reranker returns relevance scores - expect(results[0]!.score).toBeGreaterThan(0); - - await cleanupTestDb(store); - }); - - test("rerank caches results", async () => { - const store = await createTestStore(); - - const docs = [{ file: "doc1.md", text: "Content for caching test" }]; - - // First call - await store.rerank("cache test query", docs); - // Second call - should hit cache - const results = await store.rerank("cache test query", docs); - - expect(results).toHaveLength(1); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Edge Cases & Error Handling -// ============================================================================= - -describe("Edge Cases", () => { - test("handles empty database gracefully", async () => { - const store = await createTestStore(); - - const searchResults = store.searchFTS("anything", 10); - expect(searchResults).toHaveLength(0); - - // SKIPPED: getStatus() has bug (queries non-existent collections table) - // const status = store.getStatus(); - // expect(status.totalDocuments).toBe(0); - // expect(status.collections).toHaveLength(0); - - const doc = store.findDocument("nonexistent.md"); - expect("error" in doc).toBe(true); - - await cleanupTestDb(store); - }); - - test("handles very long document bodies", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - const longBody = "word ".repeat(100000); // ~600KB - await insertTestDocument(store.db, collectionName, { - name: "long", - body: longBody, - displayPath: "long.md", - }); - - const results = store.searchFTS("word", 10); - expect(results).toHaveLength(1); - - await cleanupTestDb(store); - }); - - test("handles unicode content correctly", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "unicode", - title: "日本語タイトル", - body: "# 日本語\n\n内容は日本語で書かれています。\n\nEmoji: 🎉🚀✨", - displayPath: "unicode.md", - }); - - // Should be searchable - const results = store.searchFTS("日本語", 10); - expect(results.length).toBeGreaterThan(0); - - // Should retrieve correctly - const doc = store.findDocument("unicode.md", { includeBody: true }); - expect("error" in doc).toBe(false); - if (!("error" in doc)) { - expect(doc.title).toBe("日本語タイトル"); - expect(doc.body).toContain("🎉"); - } - - await cleanupTestDb(store); - }); - - test("handles documents with special characters in paths", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - await insertTestDocument(store.db, collectionName, { - name: "special", - filepath: "/path/file with spaces.md", - displayPath: "file with spaces.md", - body: "Content", - }); - - const doc = store.findDocument("file with spaces.md"); - expect("error" in doc).toBe(false); - - await cleanupTestDb(store); - }); - - test("handles concurrent operations", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - // Insert multiple documents concurrently - const inserts = Array.from({ length: 10 }, (_, i) => - insertTestDocument(store.db, collectionName, { - name: `concurrent${i}`, - body: `Content ${i} searchterm`, - displayPath: `concurrent${i}.md`, - }) - ); - - await Promise.all(inserts); - - // All should be searchable - const results = store.searchFTS("searchterm", 20); - expect(results).toHaveLength(10); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Content-Addressable Storage Tests -// ============================================================================= - -describe("Content-Addressable Storage", () => { - test("same content gets same hash from multiple collections", async () => { - const store = await createTestStore(); - - // Create two collections - const collection1 = await createTestCollection({ pwd: "/path/collection1", name: "collection1" }); - const collection2 = await createTestCollection({ pwd: "/path/collection2", name: "collection2" }); - - // Add same content to both collections - const content = "# Same Content\n\nThis is the same content in two places."; - const hash1 = await hashContent(content); - - const doc1 = await insertTestDocument(store.db, collection1, { - name: "doc1", - body: content, - displayPath: "doc1.md", - }); - - const doc2 = await insertTestDocument(store.db, collection2, { - name: "doc2", - body: content, - displayPath: "doc2.md", - }); - - // Both should have the same hash - const hash1Db = store.db.prepare(`SELECT hash FROM documents WHERE id = ?`).get(doc1) as { hash: string }; - const hash2Db = store.db.prepare(`SELECT hash FROM documents WHERE id = ?`).get(doc2) as { hash: string }; - - expect(hash1Db.hash).toBe(hash2Db.hash); - expect(hash1Db.hash).toBe(hash1); - - // There should only be one entry in the content table - const contentCount = store.db.prepare(`SELECT COUNT(*) as count FROM content WHERE hash = ?`).get(hash1) as { count: number }; - expect(contentCount.count).toBe(1); - - await cleanupTestDb(store); - }); - - test("removing one collection preserves content used by another", async () => { - const store = await createTestStore(); - - // Create two collections - const collection1 = await createTestCollection({ pwd: "/path/collection1", name: "collection1" }); - const collection2 = await createTestCollection({ pwd: "/path/collection2", name: "collection2" }); - - // Add same content to both collections - const sharedContent = "# Shared Content\n\nThis is shared."; - const sharedHash = await hashContent(sharedContent); - - await insertTestDocument(store.db, collection1, { - name: "shared1", - body: sharedContent, - displayPath: "shared1.md", - }); - - await insertTestDocument(store.db, collection2, { - name: "shared2", - body: sharedContent, - displayPath: "shared2.md", - }); - - // Add unique content to collection1 - const uniqueContent = "# Unique Content\n\nThis is unique to collection1."; - const uniqueHash = await hashContent(uniqueContent); - - await insertTestDocument(store.db, collection1, { - name: "unique", - body: uniqueContent, - displayPath: "unique.md", - }); - - // Verify both hashes exist in content table - const sharedExists1 = store.db.prepare(`SELECT hash FROM content WHERE hash = ?`).get(sharedHash); - const uniqueExists1 = store.db.prepare(`SELECT hash FROM content WHERE hash = ?`).get(uniqueHash); - expect(sharedExists1).toBeTruthy(); - expect(uniqueExists1).toBeTruthy(); - - // Remove collection1 documents (collections are in YAML now) - store.db.prepare(`DELETE FROM documents WHERE collection = ?`).run(collection1); - - // Clean up orphaned content (mimics what the CLI does) - store.db.prepare(` - DELETE FROM content - WHERE hash NOT IN (SELECT DISTINCT hash FROM documents WHERE active = 1) - `).run(); - - // Shared content should still exist (used by collection2) - const sharedExists2 = store.db.prepare(`SELECT hash FROM content WHERE hash = ?`).get(sharedHash); - expect(sharedExists2).toBeTruthy(); - - // Unique content should be removed (only used by collection1) - const uniqueExists2 = store.db.prepare(`SELECT hash FROM content WHERE hash = ?`).get(uniqueHash); - expect(uniqueExists2).toBeFalsy(); - - await cleanupTestDb(store); - }); - - test("deduplicates content across many collections", async () => { - const store = await createTestStore(); - - const sharedContent = "# Common Header\n\nThis appears everywhere."; - const sharedHash = await hashContent(sharedContent); - - // Create 5 collections with the same content - const collectionNames = []; - for (let i = 0; i < 5; i++) { - const collName = await createTestCollection({ pwd: `/path/collection${i}`, name: `collection${i}` }); - collectionNames.push(collName); - - await insertTestDocument(store.db, collName, { - name: `doc${i}`, - body: sharedContent, - displayPath: `doc${i}.md`, - }); - } - - // Should have 5 documents - const docCount = store.db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get() as { count: number }; - expect(docCount.count).toBe(5); - - // But only 1 content entry - const contentCount = store.db.prepare(`SELECT COUNT(*) as count FROM content WHERE hash = ?`).get(sharedHash) as { count: number }; - expect(contentCount.count).toBe(1); - - // All documents should point to the same hash - const hashes = store.db.prepare(`SELECT DISTINCT hash FROM documents WHERE active = 1`).all() as { hash: string }[]; - expect(hashes).toHaveLength(1); - expect(hashes[0]!.hash).toBe(sharedHash); - - await cleanupTestDb(store); - }); - - test("different content gets different hashes", async () => { - const store = await createTestStore(); - const collectionName = await createTestCollection(); - - const content1 = "# Content One"; - const content2 = "# Content Two"; - const hash1 = await hashContent(content1); - const hash2 = await hashContent(content2); - - // Hashes should be different - expect(hash1).not.toBe(hash2); - - const doc1 = await insertTestDocument(store.db, collectionName, { - name: "doc1", - body: content1, - displayPath: "doc1.md", - }); - - const doc2 = await insertTestDocument(store.db, collectionName, { - name: "doc2", - body: content2, - displayPath: "doc2.md", - }); - - // Both hashes should exist in content table - const hash1Db = store.db.prepare(`SELECT hash FROM documents WHERE id = ?`).get(doc1) as { hash: string }; - const hash2Db = store.db.prepare(`SELECT hash FROM documents WHERE id = ?`).get(doc2) as { hash: string }; - - expect(hash1Db.hash).toBe(hash1); - expect(hash2Db.hash).toBe(hash2); - expect(hash1Db.hash).not.toBe(hash2Db.hash); - - // Should have 2 entries in content table - const contentCount = store.db.prepare(`SELECT COUNT(*) as count FROM content`).get() as { count: number }; - expect(contentCount.count).toBe(2); - - await cleanupTestDb(store); - }); -}); - -// ============================================================================= -// Virtual Path Normalization Tests -// ============================================================================= - -describe("normalizeVirtualPath", () => { - test("already normalized qmd:// path passes through", () => { - expect(normalizeVirtualPath("qmd://collection/path.md")).toBe("qmd://collection/path.md"); - expect(normalizeVirtualPath("qmd://journals/2025-01-01.md")).toBe("qmd://journals/2025-01-01.md"); - }); - - test("handles //collection/path format (missing qmd: prefix)", () => { - expect(normalizeVirtualPath("//collection/path.md")).toBe("qmd://collection/path.md"); - expect(normalizeVirtualPath("//journals/2025-01-01.md")).toBe("qmd://journals/2025-01-01.md"); - }); - - test("handles qmd:// with extra slashes", () => { - expect(normalizeVirtualPath("qmd:////collection/path.md")).toBe("qmd://collection/path.md"); - expect(normalizeVirtualPath("qmd:///journals/2025-01-01.md")).toBe("qmd://journals/2025-01-01.md"); - expect(normalizeVirtualPath("qmd:///////archive/file.md")).toBe("qmd://archive/file.md"); - }); - - test("handles collection root paths", () => { - expect(normalizeVirtualPath("qmd://collection/")).toBe("qmd://collection/"); - expect(normalizeVirtualPath("qmd://collection")).toBe("qmd://collection"); - expect(normalizeVirtualPath("//collection/")).toBe("qmd://collection/"); - }); - - test("preserves bare collection/path format (not auto-converted)", () => { - // Bare paths without qmd:// or // prefix are NOT converted - // (could be relative filesystem paths) - expect(normalizeVirtualPath("collection/path.md")).toBe("collection/path.md"); - expect(normalizeVirtualPath("journals/2025-01-01.md")).toBe("journals/2025-01-01.md"); - }); - - test("preserves absolute filesystem paths", () => { - expect(normalizeVirtualPath("/Users/test/file.md")).toBe("/Users/test/file.md"); - expect(normalizeVirtualPath("/absolute/path/file.md")).toBe("/absolute/path/file.md"); - }); - - test("preserves home-relative paths", () => { - expect(normalizeVirtualPath("~/Documents/file.md")).toBe("~/Documents/file.md"); - }); - - test("preserves docid format", () => { - expect(normalizeVirtualPath("#abc123")).toBe("#abc123"); - expect(normalizeVirtualPath("#def456")).toBe("#def456"); - }); - - test("handles whitespace trimming", () => { - expect(normalizeVirtualPath(" qmd://collection/path.md ")).toBe("qmd://collection/path.md"); - expect(normalizeVirtualPath(" //collection/path.md ")).toBe("qmd://collection/path.md"); - }); -}); - -describe("isVirtualPath", () => { - test("recognizes qmd:// paths", () => { - expect(isVirtualPath("qmd://collection/path.md")).toBe(true); - expect(isVirtualPath("qmd://journals/2025-01-01.md")).toBe(true); - expect(isVirtualPath("qmd://collection")).toBe(true); - }); - - test("recognizes //collection/path format", () => { - expect(isVirtualPath("//collection/path.md")).toBe(true); - expect(isVirtualPath("//journals/2025-01-01.md")).toBe(true); - }); - - test("does not auto-recognize bare collection/path format", () => { - // Bare paths could be relative filesystem paths, so not auto-detected as virtual - expect(isVirtualPath("collection/path.md")).toBe(false); - expect(isVirtualPath("journals/2025-01-01.md")).toBe(false); - expect(isVirtualPath("archive/subfolder/file.md")).toBe(false); - }); - - test("rejects docid format", () => { - expect(isVirtualPath("#abc123")).toBe(false); - expect(isVirtualPath("#def456")).toBe(false); - }); - - test("rejects absolute filesystem paths", () => { - expect(isVirtualPath("/Users/test/file.md")).toBe(false); - expect(isVirtualPath("/absolute/path/file.md")).toBe(false); - }); - - test("rejects home-relative paths", () => { - expect(isVirtualPath("~/Documents/file.md")).toBe(false); - expect(isVirtualPath("~/notes/journal.md")).toBe(false); - }); - - test("rejects paths without slashes", () => { - expect(isVirtualPath("file.md")).toBe(false); - expect(isVirtualPath("document")).toBe(false); - }); -}); - -describe("parseVirtualPath", () => { - test("parses standard qmd:// paths", () => { - expect(parseVirtualPath("qmd://collection/path.md")).toEqual({ - collectionName: "collection", - path: "path.md", - }); - expect(parseVirtualPath("qmd://journals/2025-01-01.md")).toEqual({ - collectionName: "journals", - path: "2025-01-01.md", - }); - }); - - test("parses paths with nested directories", () => { - expect(parseVirtualPath("qmd://archive/subfolder/file.md")).toEqual({ - collectionName: "archive", - path: "subfolder/file.md", - }); - }); - - test("parses collection root paths", () => { - expect(parseVirtualPath("qmd://collection/")).toEqual({ - collectionName: "collection", - path: "", - }); - expect(parseVirtualPath("qmd://collection")).toEqual({ - collectionName: "collection", - path: "", - }); - }); - - test("parses //collection/path format (normalizes first)", () => { - expect(parseVirtualPath("//collection/path.md")).toEqual({ - collectionName: "collection", - path: "path.md", - }); - }); - - test("parses qmd:// with extra slashes (normalizes first)", () => { - expect(parseVirtualPath("qmd:////collection/path.md")).toEqual({ - collectionName: "collection", - path: "path.md", - }); - }); - - test("returns null for non-virtual paths", () => { - expect(parseVirtualPath("/absolute/path.md")).toBe(null); - expect(parseVirtualPath("~/home/path.md")).toBe(null); - expect(parseVirtualPath("#docid")).toBe(null); - expect(parseVirtualPath("file.md")).toBe(null); - // Bare collection/path is not recognized as virtual - expect(parseVirtualPath("collection/path.md")).toBe(null); - }); -}); - -// ============================================================================= -// Docid Functions -// ============================================================================= - -describe("normalizeDocid", () => { - test("strips leading # from docid", () => { - expect(normalizeDocid("#abc123")).toBe("abc123"); - expect(normalizeDocid("#def456")).toBe("def456"); - }); - - test("returns bare hex unchanged", () => { - expect(normalizeDocid("abc123")).toBe("abc123"); - expect(normalizeDocid("def456")).toBe("def456"); - }); - - test("strips surrounding double quotes", () => { - expect(normalizeDocid('"#abc123"')).toBe("abc123"); - expect(normalizeDocid('"abc123"')).toBe("abc123"); - }); - - test("strips surrounding single quotes", () => { - expect(normalizeDocid("'#abc123'")).toBe("abc123"); - expect(normalizeDocid("'abc123'")).toBe("abc123"); - }); - - test("handles quoted docid without #", () => { - expect(normalizeDocid('"def456"')).toBe("def456"); - expect(normalizeDocid("'def456'")).toBe("def456"); - }); - - test("handles whitespace", () => { - expect(normalizeDocid(" #abc123 ")).toBe("abc123"); - expect(normalizeDocid(" abc123 ")).toBe("abc123"); - }); - - test("handles uppercase hex", () => { - expect(normalizeDocid("#ABC123")).toBe("ABC123"); - expect(normalizeDocid('"ABC123"')).toBe("ABC123"); - }); - - test("does not strip mismatched quotes", () => { - expect(normalizeDocid('"abc123\'')).toBe('"abc123\''); - expect(normalizeDocid("'abc123\"")).toBe("'abc123\""); - }); -}); - -describe("isDocid", () => { - test("accepts #hash format", () => { - expect(isDocid("#abc123")).toBe(true); - expect(isDocid("#def456")).toBe(true); - expect(isDocid("#ABCDEF")).toBe(true); - }); - - test("accepts bare 6-char hex", () => { - expect(isDocid("abc123")).toBe(true); - expect(isDocid("def456")).toBe(true); - expect(isDocid("ABCDEF")).toBe(true); - }); - - test("accepts longer hex strings", () => { - expect(isDocid("abc123def456")).toBe(true); - expect(isDocid("#abc123def456")).toBe(true); - }); - - test("accepts double-quoted docids", () => { - expect(isDocid('"#abc123"')).toBe(true); - expect(isDocid('"abc123"')).toBe(true); - }); - - test("accepts single-quoted docids", () => { - expect(isDocid("'#abc123'")).toBe(true); - expect(isDocid("'abc123'")).toBe(true); - }); - - test("rejects non-hex strings", () => { - expect(isDocid("ghijkl")).toBe(false); - expect(isDocid("#ghijkl")).toBe(false); - expect(isDocid("abc12g")).toBe(false); - }); - - test("rejects strings shorter than 6 chars", () => { - expect(isDocid("abc12")).toBe(false); - expect(isDocid("#abc1")).toBe(false); - expect(isDocid("'abc'")).toBe(false); - }); - - test("rejects empty strings", () => { - expect(isDocid("")).toBe(false); - expect(isDocid("#")).toBe(false); - expect(isDocid('""')).toBe(false); - }); - - test("rejects file paths", () => { - expect(isDocid("/path/to/file.md")).toBe(false); - expect(isDocid("path/to/file.md")).toBe(false); - expect(isDocid("qmd://collection/file.md")).toBe(false); - }); - - test("rejects paths that look like hex with extensions", () => { - expect(isDocid("abc123.md")).toBe(false); - }); -}); diff --git a/tools/qmd/src/store.ts b/tools/qmd/src/store.ts deleted file mode 100644 index fb7dd70d..00000000 --- a/tools/qmd/src/store.ts +++ /dev/null @@ -1,2571 +0,0 @@ -/** - * QMD Store - Core data access and retrieval functions - * - * This module provides all database operations, search functions, and document - * retrieval for QMD. It returns raw data structures that can be formatted by - * CLI or MCP consumers. - * - * Usage: - * const store = createStore("/path/to/db.sqlite"); - * // or use default path: - * const store = createStore(); - */ - -import { Database } from "bun:sqlite"; -import { Glob } from "bun"; -import { realpathSync, statSync } from "node:fs"; -import * as sqliteVec from "sqlite-vec"; -import { - LlamaCpp, - getDefaultLlamaCpp, - formatQueryForEmbedding, - formatDocForEmbedding, - type RerankDocument, - type ILLMSession, -} from "./llm"; -import { - findContextForPath as collectionsFindContextForPath, - addContext as collectionsAddContext, - removeContext as collectionsRemoveContext, - listAllContexts as collectionsListAllContexts, - getCollection, - listCollections as collectionsListCollections, - addCollection as collectionsAddCollection, - removeCollection as collectionsRemoveCollection, - renameCollection as collectionsRenameCollection, - setGlobalContext, - loadConfig as collectionsLoadConfig, - type NamedCollection, -} from "./collections"; - -// ============================================================================= -// Configuration -// ============================================================================= - -const HOME = Bun.env.HOME || "/tmp"; -export const DEFAULT_EMBED_MODEL = "embeddinggemma"; -export const DEFAULT_RERANK_MODEL = "ExpedientFalcon/qwen3-reranker:0.6b-q8_0"; -export const DEFAULT_QUERY_MODEL = "Qwen/Qwen3-1.7B"; -export const DEFAULT_GLOB = "**/*.md"; -export const DEFAULT_MULTI_GET_MAX_BYTES = 10 * 1024; // 10KB - -// Chunking: 800 tokens per chunk with 15% overlap -export const CHUNK_SIZE_TOKENS = 800; -export const CHUNK_OVERLAP_TOKENS = Math.floor(CHUNK_SIZE_TOKENS * 0.15); // 120 tokens (15% overlap) -// Fallback char-based approximation for sync chunking (~4 chars per token) -export const CHUNK_SIZE_CHARS = CHUNK_SIZE_TOKENS * 4; // 3200 chars -export const CHUNK_OVERLAP_CHARS = CHUNK_OVERLAP_TOKENS * 4; // 480 chars - -// ============================================================================= -// Path utilities -// ============================================================================= - -export function homedir(): string { - return HOME; -} - -/** - * Check if a path is absolute. - * Supports: - * - Unix paths: /path/to/file - * - Windows native: C:\path or C:/path - * - Git Bash: /c/path or /C/path (C-Z drives, excluding A/B floppy drives) - * - * Note: /c without trailing slash is treated as Unix path (directory named "c"), - * while /c/ or /c/path are treated as Git Bash paths (C: drive). - */ -export function isAbsolutePath(path: string): boolean { - if (!path) return false; - - // Unix absolute path - if (path.startsWith('/')) { - // Check if it's a Git Bash style path like /c/ or /c/Users (C-Z only, not A or B) - // Requires path[2] === '/' to distinguish from Unix paths like /c or /cache - if (path.length >= 3 && path[2] === '/') { - const driveLetter = path[1]; - if (driveLetter && /[c-zC-Z]/.test(driveLetter)) { - return true; - } - } - // Any other path starting with / is Unix absolute - return true; - } - - // Windows native path: C:\ or C:/ (any letter A-Z) - if (path.length >= 2 && /[a-zA-Z]/.test(path[0]!) && path[1] === ':') { - return true; - } - - return false; -} - -/** - * Normalize path separators to forward slashes. - * Converts Windows backslashes to forward slashes. - */ -export function normalizePathSeparators(path: string): string { - return path.replace(/\\/g, '/'); -} - -/** - * Get the relative path from a prefix. - * Returns null if path is not under prefix. - * Returns empty string if path equals prefix. - */ -export function getRelativePathFromPrefix(path: string, prefix: string): string | null { - // Empty prefix is invalid - if (!prefix) { - return null; - } - - const normalizedPath = normalizePathSeparators(path); - const normalizedPrefix = normalizePathSeparators(prefix); - - // Ensure prefix ends with / for proper matching - const prefixWithSlash = !normalizedPrefix.endsWith('/') - ? normalizedPrefix + '/' - : normalizedPrefix; - - // Exact match - if (normalizedPath === normalizedPrefix) { - return ''; - } - - // Check if path starts with prefix - if (normalizedPath.startsWith(prefixWithSlash)) { - return normalizedPath.slice(prefixWithSlash.length); - } - - return null; -} - -export function resolve(...paths: string[]): string { - if (paths.length === 0) { - throw new Error("resolve: at least one path segment is required"); - } - - // Normalize all paths to use forward slashes - const normalizedPaths = paths.map(normalizePathSeparators); - - let result = ''; - let windowsDrive = ''; - - // Check if first path is absolute - const firstPath = normalizedPaths[0]!; - if (isAbsolutePath(firstPath)) { - result = firstPath; - - // Extract Windows drive letter if present - if (firstPath.length >= 2 && /[a-zA-Z]/.test(firstPath[0]!) && firstPath[1] === ':') { - windowsDrive = firstPath.slice(0, 2); - result = firstPath.slice(2); - } else if (firstPath.startsWith('/') && firstPath.length >= 3 && firstPath[2] === '/') { - // Git Bash style: /c/ -> C: (C-Z drives only, not A or B) - const driveLetter = firstPath[1]; - if (driveLetter && /[c-zC-Z]/.test(driveLetter)) { - windowsDrive = driveLetter.toUpperCase() + ':'; - result = firstPath.slice(2); - } - } - } else { - // Start with PWD or cwd, then append the first relative path - const pwd = normalizePathSeparators(Bun.env.PWD || process.cwd()); - - // Extract Windows drive from PWD if present - if (pwd.length >= 2 && /[a-zA-Z]/.test(pwd[0]!) && pwd[1] === ':') { - windowsDrive = pwd.slice(0, 2); - result = pwd.slice(2) + '/' + firstPath; - } else { - result = pwd + '/' + firstPath; - } - } - - // Process remaining paths - for (let i = 1; i < normalizedPaths.length; i++) { - const p = normalizedPaths[i]!; - if (isAbsolutePath(p)) { - // Absolute path replaces everything - result = p; - - // Update Windows drive if present - if (p.length >= 2 && /[a-zA-Z]/.test(p[0]!) && p[1] === ':') { - windowsDrive = p.slice(0, 2); - result = p.slice(2); - } else if (p.startsWith('/') && p.length >= 3 && p[2] === '/') { - // Git Bash style (C-Z drives only, not A or B) - const driveLetter = p[1]; - if (driveLetter && /[c-zC-Z]/.test(driveLetter)) { - windowsDrive = driveLetter.toUpperCase() + ':'; - result = p.slice(2); - } else { - windowsDrive = ''; - } - } else { - windowsDrive = ''; - } - } else { - // Relative path - append - result = result + '/' + p; - } - } - - // Normalize . and .. components - const parts = result.split('/').filter(Boolean); - const normalized: string[] = []; - for (const part of parts) { - if (part === '..') { - normalized.pop(); - } else if (part !== '.') { - normalized.push(part); - } - } - - // Build final path - const finalPath = '/' + normalized.join('/'); - - // Prepend Windows drive if present - if (windowsDrive) { - return windowsDrive + finalPath; - } - - return finalPath; -} - -// Flag to indicate production mode (set by qmd.ts at startup) -let _productionMode = false; - -export function enableProductionMode(): void { - _productionMode = true; -} - -export function getDefaultDbPath(indexName: string = "index"): string { - // Always allow override via INDEX_PATH (for testing) - if (Bun.env.INDEX_PATH) { - return Bun.env.INDEX_PATH; - } - - // In non-production mode (tests), require explicit path - if (!_productionMode) { - throw new Error( - "Database path not set. Tests must set INDEX_PATH env var or use createStore() with explicit path. " + - "This prevents tests from accidentally writing to the global index." - ); - } - - const cacheDir = Bun.env.XDG_CACHE_HOME || resolve(homedir(), ".cache"); - const qmdCacheDir = resolve(cacheDir, "qmd"); - try { Bun.spawnSync(["mkdir", "-p", qmdCacheDir]); } catch { } - return resolve(qmdCacheDir, `${indexName}.sqlite`); -} - -export function getPwd(): string { - return process.env.PWD || process.cwd(); -} - -export function getRealPath(path: string): string { - try { - return realpathSync(path); - } catch { - return resolve(path); - } -} - -// ============================================================================= -// Virtual Path Utilities (qmd://) -// ============================================================================= - -export type VirtualPath = { - collectionName: string; - path: string; // relative path within collection -}; - -/** - * Normalize explicit virtual path formats to standard qmd:// format. - * Only handles paths that are already explicitly virtual: - * - qmd://collection/path.md (already normalized) - * - qmd:////collection/path.md (extra slashes - normalize) - * - //collection/path.md (missing qmd: prefix - add it) - * - * Does NOT handle: - * - collection/path.md (bare paths - could be filesystem relative) - * - :linenum suffix (should be parsed separately before calling this) - */ -export function normalizeVirtualPath(input: string): string { - let path = input.trim(); - - // Handle qmd:// with extra slashes: qmd:////collection/path -> qmd://collection/path - if (path.startsWith('qmd:')) { - // Remove qmd: prefix and normalize slashes - path = path.slice(4); - // Remove leading slashes and re-add exactly two - path = path.replace(/^\/+/, ''); - return `qmd://${path}`; - } - - // Handle //collection/path (missing qmd: prefix) - if (path.startsWith('//')) { - path = path.replace(/^\/+/, ''); - return `qmd://${path}`; - } - - // Return as-is for other cases (filesystem paths, docids, bare collection/path, etc.) - return path; -} - -/** - * Parse a virtual path like "qmd://collection-name/path/to/file.md" - * into its components. - * Also supports collection root: "qmd://collection-name/" or "qmd://collection-name" - */ -export function parseVirtualPath(virtualPath: string): VirtualPath | null { - // Normalize the path first - const normalized = normalizeVirtualPath(virtualPath); - - // Match: qmd://collection-name[/optional-path] - // Allows: qmd://name, qmd://name/, qmd://name/path - const match = normalized.match(/^qmd:\/\/([^\/]+)\/?(.*)$/); - if (!match?.[1]) return null; - return { - collectionName: match[1], - path: match[2] ?? '', // Empty string for collection root - }; -} - -/** - * Build a virtual path from collection name and relative path. - */ -export function buildVirtualPath(collectionName: string, path: string): string { - return `qmd://${collectionName}/${path}`; -} - -/** - * Check if a path is explicitly a virtual path. - * Only recognizes explicit virtual path formats: - * - qmd://collection/path.md - * - //collection/path.md - * - * Does NOT consider bare collection/path.md as virtual - that should be - * handled separately by checking if the first component is a collection name. - */ -export function isVirtualPath(path: string): boolean { - const trimmed = path.trim(); - - // Explicit qmd:// prefix (with any number of slashes) - if (trimmed.startsWith('qmd:')) return true; - - // //collection/path format (missing qmd: prefix) - if (trimmed.startsWith('//')) return true; - - return false; -} - -/** - * Resolve a virtual path to absolute filesystem path. - */ -export function resolveVirtualPath(db: Database, virtualPath: string): string | null { - const parsed = parseVirtualPath(virtualPath); - if (!parsed) return null; - - const coll = getCollectionByName(db, parsed.collectionName); - if (!coll) return null; - - return resolve(coll.pwd, parsed.path); -} - -/** - * Convert an absolute filesystem path to a virtual path. - * Returns null if the file is not in any indexed collection. - */ -export function toVirtualPath(db: Database, absolutePath: string): string | null { - // Get all collections from YAML config - const collections = collectionsListCollections(); - - // Find which collection this absolute path belongs to - for (const coll of collections) { - if (absolutePath.startsWith(coll.path + '/') || absolutePath === coll.path) { - // Extract relative path - const relativePath = absolutePath.startsWith(coll.path + '/') - ? absolutePath.slice(coll.path.length + 1) - : ''; - - // Verify this document exists in the database - const doc = db.prepare(` - SELECT d.path - FROM documents d - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - LIMIT 1 - `).get(coll.name, relativePath) as { path: string } | null; - - if (doc) { - return buildVirtualPath(coll.name, relativePath); - } - } - } - - return null; -} - -// ============================================================================= -// Database initialization -// ============================================================================= - -function setSQLiteFromBrewPrefixEnv(): void { - const candidates: string[] = []; - - if (process.platform === "darwin") { - // Use BREW_PREFIX for non-standard Homebrew installs (common on corporate Macs). - const brewPrefix = Bun.env.BREW_PREFIX || Bun.env.HOMEBREW_PREFIX; - if (brewPrefix) { - // Homebrew can place SQLite in opt/sqlite (keg-only) or directly under the prefix. - candidates.push(`${brewPrefix}/opt/sqlite/lib/libsqlite3.dylib`); - candidates.push(`${brewPrefix}/lib/libsqlite3.dylib`); - } else { - candidates.push("/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib"); - candidates.push("/usr/local/opt/sqlite/lib/libsqlite3.dylib"); - } - } - - for (const candidate of candidates) { - try { - if (statSync(candidate).size > 0) { - Database.setCustomSQLite(candidate); - return; - } - } catch { } - } -} - -setSQLiteFromBrewPrefixEnv(); - -function initializeDatabase(db: Database): void { - try { - sqliteVec.load(db); - } catch (err) { - if (err instanceof Error && err.message.includes("does not support dynamic extension loading")) { - throw new Error( - "SQLite build does not support dynamic extension loading. " + - "Install Homebrew SQLite so the sqlite-vec extension can be loaded, " + - "and set BREW_PREFIX if Homebrew is installed in a non-standard location." - ); - } - throw err; - } - db.exec("PRAGMA journal_mode = WAL"); - db.exec("PRAGMA foreign_keys = ON"); - - // Drop legacy tables that are now managed in YAML - db.exec(`DROP TABLE IF EXISTS path_contexts`); - db.exec(`DROP TABLE IF EXISTS collections`); - - // Content-addressable storage - the source of truth for document content - db.exec(` - CREATE TABLE IF NOT EXISTS content ( - hash TEXT PRIMARY KEY, - doc TEXT NOT NULL, - created_at TEXT NOT NULL - ) - `); - - // Documents table - file system layer mapping virtual paths to content hashes - // Collections are now managed in ~/.config/qmd/index.yml - db.exec(` - CREATE TABLE IF NOT EXISTS documents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - collection TEXT NOT NULL, - path TEXT NOT NULL, - title TEXT NOT NULL, - hash TEXT NOT NULL, - created_at TEXT NOT NULL, - modified_at TEXT NOT NULL, - active INTEGER NOT NULL DEFAULT 1, - FOREIGN KEY (hash) REFERENCES content(hash) ON DELETE CASCADE, - UNIQUE(collection, path) - ) - `); - - db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_collection ON documents(collection, active)`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_hash ON documents(hash)`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_path ON documents(path, active)`); - - // Cache table for LLM API calls - db.exec(` - CREATE TABLE IF NOT EXISTS llm_cache ( - hash TEXT PRIMARY KEY, - result TEXT NOT NULL, - created_at TEXT NOT NULL - ) - `); - - // Content vectors - const cvInfo = db.prepare(`PRAGMA table_info(content_vectors)`).all() as { name: string }[]; - const hasSeqColumn = cvInfo.some(col => col.name === 'seq'); - if (cvInfo.length > 0 && !hasSeqColumn) { - db.exec(`DROP TABLE IF EXISTS content_vectors`); - db.exec(`DROP TABLE IF EXISTS vectors_vec`); - } - db.exec(` - CREATE TABLE IF NOT EXISTS content_vectors ( - hash TEXT NOT NULL, - seq INTEGER NOT NULL DEFAULT 0, - pos INTEGER NOT NULL DEFAULT 0, - model TEXT NOT NULL, - embedded_at TEXT NOT NULL, - PRIMARY KEY (hash, seq) - ) - `); - - // FTS - index filepath (collection/path), title, and content - db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( - filepath, title, body, - tokenize='porter unicode61' - ) - `); - - // Triggers to keep FTS in sync - db.exec(` - CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents - WHEN new.active = 1 - BEGIN - INSERT INTO documents_fts(rowid, filepath, title, body) - SELECT - new.id, - new.collection || '/' || new.path, - new.title, - (SELECT doc FROM content WHERE hash = new.hash) - WHERE new.active = 1; - END - `); - - db.exec(` - CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN - DELETE FROM documents_fts WHERE rowid = old.id; - END - `); - - db.exec(` - CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents - BEGIN - -- Delete from FTS if no longer active - DELETE FROM documents_fts WHERE rowid = old.id AND new.active = 0; - - -- Update FTS if still/newly active - INSERT OR REPLACE INTO documents_fts(rowid, filepath, title, body) - SELECT - new.id, - new.collection || '/' || new.path, - new.title, - (SELECT doc FROM content WHERE hash = new.hash) - WHERE new.active = 1; - END - `); -} - - -function ensureVecTableInternal(db: Database, dimensions: number): void { - const tableInfo = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get() as { sql: string } | null; - if (tableInfo) { - const match = tableInfo.sql.match(/float\[(\d+)\]/); - const hasHashSeq = tableInfo.sql.includes('hash_seq'); - const hasCosine = tableInfo.sql.includes('distance_metric=cosine'); - const existingDims = match?.[1] ? parseInt(match[1], 10) : null; - if (existingDims === dimensions && hasHashSeq && hasCosine) return; - // Table exists but wrong schema - need to rebuild - db.exec("DROP TABLE IF EXISTS vectors_vec"); - } - db.exec(`CREATE VIRTUAL TABLE vectors_vec USING vec0(hash_seq TEXT PRIMARY KEY, embedding float[${dimensions}] distance_metric=cosine)`); -} - -// ============================================================================= -// Store Factory -// ============================================================================= - -export type Store = { - db: Database; - dbPath: string; - close: () => void; - ensureVecTable: (dimensions: number) => void; - - // Index health - getHashesNeedingEmbedding: () => number; - getIndexHealth: () => IndexHealthInfo; - getStatus: () => IndexStatus; - - // Caching - getCacheKey: typeof getCacheKey; - getCachedResult: (cacheKey: string) => string | null; - setCachedResult: (cacheKey: string, result: string) => void; - clearCache: () => void; - - // Cleanup and maintenance - deleteLLMCache: () => number; - deleteInactiveDocuments: () => number; - cleanupOrphanedContent: () => number; - cleanupOrphanedVectors: () => number; - vacuumDatabase: () => void; - - // Context - getContextForFile: (filepath: string) => string | null; - getContextForPath: (collectionName: string, path: string) => string | null; - getCollectionByName: (name: string) => { name: string; pwd: string; glob_pattern: string } | null; - getCollectionsWithoutContext: () => { name: string; pwd: string; doc_count: number }[]; - getTopLevelPathsWithoutContext: (collectionName: string) => string[]; - - // Virtual paths - parseVirtualPath: typeof parseVirtualPath; - buildVirtualPath: typeof buildVirtualPath; - isVirtualPath: typeof isVirtualPath; - resolveVirtualPath: (virtualPath: string) => string | null; - toVirtualPath: (absolutePath: string) => string | null; - - // Search - searchFTS: (query: string, limit?: number, collectionId?: number) => SearchResult[]; - searchVec: (query: string, model: string, limit?: number, collectionName?: string) => Promise; - - // Query expansion & reranking - expandQuery: (query: string, model?: string) => Promise; - rerank: (query: string, documents: { file: string; text: string }[], model?: string) => Promise<{ file: string; score: number }[]>; - - // Document retrieval - findDocument: (filename: string, options?: { includeBody?: boolean }) => DocumentResult | DocumentNotFound; - getDocumentBody: (doc: DocumentResult | { filepath: string }, fromLine?: number, maxLines?: number) => string | null; - findDocuments: (pattern: string, options?: { includeBody?: boolean; maxBytes?: number }) => { docs: MultiGetResult[]; errors: string[] }; - - // Fuzzy matching and docid lookup - findSimilarFiles: (query: string, maxDistance?: number, limit?: number) => string[]; - matchFilesByGlob: (pattern: string) => { filepath: string; displayPath: string; bodyLength: number }[]; - findDocumentByDocid: (docid: string) => { filepath: string; hash: string } | null; - - // Document indexing operations - insertContent: (hash: string, content: string, createdAt: string) => void; - insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => void; - findActiveDocument: (collectionName: string, path: string) => { id: number; hash: string; title: string } | null; - updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => void; - updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => void; - deactivateDocument: (collectionName: string, path: string) => void; - getActiveDocumentPaths: (collectionName: string) => string[]; - - // Vector/embedding operations - getHashesForEmbedding: () => { hash: string; body: string; path: string }[]; - clearAllEmbeddings: () => void; - insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string) => void; -}; - -/** - * Create a new store instance with the given database path. - * If no path is provided, uses the default path (~/.cache/qmd/index.sqlite). - * - * @param dbPath - Path to the SQLite database file - * @returns Store instance with all methods bound to the database - */ -export function createStore(dbPath?: string): Store { - const resolvedPath = dbPath || getDefaultDbPath(); - const db = new Database(resolvedPath); - initializeDatabase(db); - - return { - db, - dbPath: resolvedPath, - close: () => db.close(), - ensureVecTable: (dimensions: number) => ensureVecTableInternal(db, dimensions), - - // Index health - getHashesNeedingEmbedding: () => getHashesNeedingEmbedding(db), - getIndexHealth: () => getIndexHealth(db), - getStatus: () => getStatus(db), - - // Caching - getCacheKey, - getCachedResult: (cacheKey: string) => getCachedResult(db, cacheKey), - setCachedResult: (cacheKey: string, result: string) => setCachedResult(db, cacheKey, result), - clearCache: () => clearCache(db), - - // Cleanup and maintenance - deleteLLMCache: () => deleteLLMCache(db), - deleteInactiveDocuments: () => deleteInactiveDocuments(db), - cleanupOrphanedContent: () => cleanupOrphanedContent(db), - cleanupOrphanedVectors: () => cleanupOrphanedVectors(db), - vacuumDatabase: () => vacuumDatabase(db), - - // Context - getContextForFile: (filepath: string) => getContextForFile(db, filepath), - getContextForPath: (collectionName: string, path: string) => getContextForPath(db, collectionName, path), - getCollectionByName: (name: string) => getCollectionByName(db, name), - getCollectionsWithoutContext: () => getCollectionsWithoutContext(db), - getTopLevelPathsWithoutContext: (collectionName: string) => getTopLevelPathsWithoutContext(db, collectionName), - - // Virtual paths - parseVirtualPath, - buildVirtualPath, - isVirtualPath, - resolveVirtualPath: (virtualPath: string) => resolveVirtualPath(db, virtualPath), - toVirtualPath: (absolutePath: string) => toVirtualPath(db, absolutePath), - - // Search - searchFTS: (query: string, limit?: number, collectionId?: number) => searchFTS(db, query, limit, collectionId), - searchVec: (query: string, model: string, limit?: number, collectionName?: string) => searchVec(db, query, model, limit, collectionName), - - // Query expansion & reranking - expandQuery: (query: string, model?: string) => expandQuery(query, model, db), - rerank: (query: string, documents: { file: string; text: string }[], model?: string) => rerank(query, documents, model, db), - - // Document retrieval - findDocument: (filename: string, options?: { includeBody?: boolean }) => findDocument(db, filename, options), - getDocumentBody: (doc: DocumentResult | { filepath: string }, fromLine?: number, maxLines?: number) => getDocumentBody(db, doc, fromLine, maxLines), - findDocuments: (pattern: string, options?: { includeBody?: boolean; maxBytes?: number }) => findDocuments(db, pattern, options), - - // Fuzzy matching and docid lookup - findSimilarFiles: (query: string, maxDistance?: number, limit?: number) => findSimilarFiles(db, query, maxDistance, limit), - matchFilesByGlob: (pattern: string) => matchFilesByGlob(db, pattern), - findDocumentByDocid: (docid: string) => findDocumentByDocid(db, docid), - - // Document indexing operations - insertContent: (hash: string, content: string, createdAt: string) => insertContent(db, hash, content, createdAt), - insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => insertDocument(db, collectionName, path, title, hash, createdAt, modifiedAt), - findActiveDocument: (collectionName: string, path: string) => findActiveDocument(db, collectionName, path), - updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => updateDocumentTitle(db, documentId, title, modifiedAt), - updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => updateDocument(db, documentId, title, hash, modifiedAt), - deactivateDocument: (collectionName: string, path: string) => deactivateDocument(db, collectionName, path), - getActiveDocumentPaths: (collectionName: string) => getActiveDocumentPaths(db, collectionName), - - // Vector/embedding operations - getHashesForEmbedding: () => getHashesForEmbedding(db), - clearAllEmbeddings: () => clearAllEmbeddings(db), - insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string) => insertEmbedding(db, hash, seq, pos, embedding, model, embeddedAt), - }; -} - -// ============================================================================= -// Core Document Type -// ============================================================================= - -/** - * Unified document result type with all metadata. - * Body is optional - use getDocumentBody() to load it separately if needed. - */ -export type DocumentResult = { - filepath: string; // Full filesystem path - displayPath: string; // Short display path (e.g., "docs/readme.md") - title: string; // Document title (from first heading or filename) - context: string | null; // Folder context description if configured - hash: string; // Content hash for caching/change detection - docid: string; // Short docid (first 6 chars of hash) for quick reference - collectionName: string; // Parent collection name - modifiedAt: string; // Last modification timestamp - bodyLength: number; // Body length in bytes (useful before loading) - body?: string; // Document body (optional, load with getDocumentBody) -}; - -/** - * Extract short docid from a full hash (first 6 characters). - */ -export function getDocid(hash: string): string { - return hash.slice(0, 6); -} - -/** - * Handelize a filename to be more token-friendly. - * - Convert triple underscore `___` to `/` (folder separator) - * - Convert to lowercase - * - Replace sequences of non-word chars (except /) with single dash - * - Remove leading/trailing dashes from path segments - * - Preserve folder structure (a/b/c/d.md stays structured) - * - Preserve file extension - */ -export function handelize(path: string): string { - if (!path || path.trim() === '') { - throw new Error('handelize: path cannot be empty'); - } - - // Check for paths that are just extensions or only dots/special chars - // A valid path must have at least one letter or digit (including Unicode) - const segments = path.split('/').filter(Boolean); - const lastSegment = segments[segments.length - 1] || ''; - const filenameWithoutExt = lastSegment.replace(/\.[^.]+$/, ''); - const hasValidContent = /[\p{L}\p{N}]/u.test(filenameWithoutExt); - if (!hasValidContent) { - throw new Error(`handelize: path "${path}" has no valid filename content`); - } - - const result = path - .replace(/___/g, '/') // Triple underscore becomes folder separator - .toLowerCase() - .split('/') - .map((segment, idx, arr) => { - const isLastSegment = idx === arr.length - 1; - - if (isLastSegment) { - // For the filename (last segment), preserve the extension - const extMatch = segment.match(/(\.[a-z0-9]+)$/i); - const ext = extMatch ? extMatch[1] : ''; - const nameWithoutExt = ext ? segment.slice(0, -ext.length) : segment; - - const cleanedName = nameWithoutExt - .replace(/[^\p{L}\p{N}]+/gu, '-') // Replace non-letter/digit chars with dash - .replace(/^-+|-+$/g, ''); // Remove leading/trailing dashes - - return cleanedName + ext; - } else { - // For directories, just clean normally - return segment - .replace(/[^\p{L}\p{N}]+/gu, '-') - .replace(/^-+|-+$/g, ''); - } - }) - .filter(Boolean) - .join('/'); - - if (!result) { - throw new Error(`handelize: path "${path}" resulted in empty string after processing`); - } - - return result; -} - -/** - * Search result extends DocumentResult with score and source info - */ -export type SearchResult = DocumentResult & { - score: number; // Relevance score (0-1) - source: "fts" | "vec"; // Search source (full-text or vector) - chunkPos?: number; // Character position of matching chunk (for vector search) -}; - -/** - * Ranked result for RRF fusion (simplified, used internally) - */ -export type RankedResult = { - file: string; - displayPath: string; - title: string; - body: string; - score: number; -}; - -/** - * Error result when document is not found - */ -export type DocumentNotFound = { - error: "not_found"; - query: string; - similarFiles: string[]; -}; - -/** - * Result from multi-get operations - */ -export type MultiGetResult = { - doc: DocumentResult; - skipped: false; -} | { - doc: Pick; - skipped: true; - skipReason: string; -}; - -export type CollectionInfo = { - name: string; - path: string; - pattern: string; - documents: number; - lastUpdated: string; -}; - -export type IndexStatus = { - totalDocuments: number; - needsEmbedding: number; - hasVectorIndex: boolean; - collections: CollectionInfo[]; -}; - -// ============================================================================= -// Index health -// ============================================================================= - -export function getHashesNeedingEmbedding(db: Database): number { - const result = db.prepare(` - SELECT COUNT(DISTINCT d.hash) as count - FROM documents d - LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0 - WHERE d.active = 1 AND v.hash IS NULL - `).get() as { count: number }; - return result.count; -} - -export type IndexHealthInfo = { - needsEmbedding: number; - totalDocs: number; - daysStale: number | null; -}; - -export function getIndexHealth(db: Database): IndexHealthInfo { - const needsEmbedding = getHashesNeedingEmbedding(db); - const totalDocs = (db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get() as { count: number }).count; - - const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get() as { latest: string | null }; - let daysStale: number | null = null; - if (mostRecent?.latest) { - const lastUpdate = new Date(mostRecent.latest); - daysStale = Math.floor((Date.now() - lastUpdate.getTime()) / (24 * 60 * 60 * 1000)); - } - - return { needsEmbedding, totalDocs, daysStale }; -} - -// ============================================================================= -// Caching -// ============================================================================= - -export function getCacheKey(url: string, body: object): string { - const hash = new Bun.CryptoHasher("sha256"); - hash.update(url); - hash.update(JSON.stringify(body)); - return hash.digest("hex"); -} - -export function getCachedResult(db: Database, cacheKey: string): string | null { - const row = db.prepare(`SELECT result FROM llm_cache WHERE hash = ?`).get(cacheKey) as { result: string } | null; - return row?.result || null; -} - -export function setCachedResult(db: Database, cacheKey: string, result: string): void { - const now = new Date().toISOString(); - db.prepare(`INSERT OR REPLACE INTO llm_cache (hash, result, created_at) VALUES (?, ?, ?)`).run(cacheKey, result, now); - if (Math.random() < 0.01) { - db.exec(`DELETE FROM llm_cache WHERE hash NOT IN (SELECT hash FROM llm_cache ORDER BY created_at DESC LIMIT 1000)`); - } -} - -export function clearCache(db: Database): void { - db.exec(`DELETE FROM llm_cache`); -} - -// ============================================================================= -// Cleanup and maintenance operations -// ============================================================================= - -/** - * Delete cached LLM API responses. - * Returns the number of cached responses deleted. - */ -export function deleteLLMCache(db: Database): number { - const result = db.prepare(`DELETE FROM llm_cache`).run(); - return result.changes; -} - -/** - * Remove inactive document records (active = 0). - * Returns the number of inactive documents deleted. - */ -export function deleteInactiveDocuments(db: Database): number { - const result = db.prepare(`DELETE FROM documents WHERE active = 0`).run(); - return result.changes; -} - -/** - * Remove orphaned content hashes that are not referenced by any active document. - * Returns the number of orphaned content hashes deleted. - */ -export function cleanupOrphanedContent(db: Database): number { - const result = db.prepare(` - DELETE FROM content - WHERE hash NOT IN (SELECT DISTINCT hash FROM documents WHERE active = 1) - `).run(); - return result.changes; -} - -/** - * Remove orphaned vector embeddings that are not referenced by any active document. - * Returns the number of orphaned embedding chunks deleted. - */ -export function cleanupOrphanedVectors(db: Database): number { - // Check if vectors_vec table exists - const tableExists = db.prepare(` - SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec' - `).get(); - - if (!tableExists) { - return 0; - } - - // Count orphaned vectors first - const countResult = db.prepare(` - SELECT COUNT(*) as c FROM content_vectors cv - WHERE NOT EXISTS ( - SELECT 1 FROM documents d WHERE d.hash = cv.hash AND d.active = 1 - ) - `).get() as { c: number }; - - if (countResult.c === 0) { - return 0; - } - - // Delete from vectors_vec first - db.exec(` - DELETE FROM vectors_vec WHERE hash_seq IN ( - SELECT cv.hash || '_' || cv.seq FROM content_vectors cv - WHERE NOT EXISTS ( - SELECT 1 FROM documents d WHERE d.hash = cv.hash AND d.active = 1 - ) - ) - `); - - // Delete from content_vectors - db.exec(` - DELETE FROM content_vectors WHERE hash NOT IN ( - SELECT hash FROM documents WHERE active = 1 - ) - `); - - return countResult.c; -} - -/** - * Run VACUUM to reclaim unused space in the database. - * This operation rebuilds the database file to eliminate fragmentation. - */ -export function vacuumDatabase(db: Database): void { - db.exec(`VACUUM`); -} - -// ============================================================================= -// Document helpers -// ============================================================================= - -export async function hashContent(content: string): Promise { - const hash = new Bun.CryptoHasher("sha256"); - hash.update(content); - return hash.digest("hex"); -} - -const titleExtractors: Record string | null> = { - '.md': (content) => { - const match = content.match(/^##?\s+(.+)$/m); - if (match) { - const title = (match[1] ?? "").trim(); - if (title === "📝 Notes" || title === "Notes") { - const nextMatch = content.match(/^##\s+(.+)$/m); - if (nextMatch?.[1]) return nextMatch[1].trim(); - } - return title; - } - return null; - }, - '.org': (content) => { - const titleProp = content.match(/^#\+TITLE:\s*(.+)$/im); - if (titleProp?.[1]) return titleProp[1].trim(); - const heading = content.match(/^\*+\s+(.+)$/m); - if (heading?.[1]) return heading[1].trim(); - return null; - }, -}; - -export function extractTitle(content: string, filename: string): string { - const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase(); - const extractor = titleExtractors[ext]; - if (extractor) { - const title = extractor(content); - if (title) return title; - } - return filename.replace(/\.[^.]+$/, "").split("/").pop() || filename; -} - -// ============================================================================= -// Document indexing operations -// ============================================================================= - -/** - * Insert content into the content table (content-addressable storage). - * Uses INSERT OR IGNORE so duplicate hashes are skipped. - */ -export function insertContent(db: Database, hash: string, content: string, createdAt: string): void { - db.prepare(`INSERT OR IGNORE INTO content (hash, doc, created_at) VALUES (?, ?, ?)`) - .run(hash, content, createdAt); -} - -/** - * Insert a new document into the documents table. - */ -export function insertDocument( - db: Database, - collectionName: string, - path: string, - title: string, - hash: string, - createdAt: string, - modifiedAt: string -): void { - db.prepare(` - INSERT INTO documents (collection, path, title, hash, created_at, modified_at, active) - VALUES (?, ?, ?, ?, ?, ?, 1) - `).run(collectionName, path, title, hash, createdAt, modifiedAt); -} - -/** - * Find an active document by collection name and path. - */ -export function findActiveDocument( - db: Database, - collectionName: string, - path: string -): { id: number; hash: string; title: string } | null { - return db.prepare(` - SELECT id, hash, title FROM documents - WHERE collection = ? AND path = ? AND active = 1 - `).get(collectionName, path) as { id: number; hash: string; title: string } | null; -} - -/** - * Update the title and modified_at timestamp for a document. - */ -export function updateDocumentTitle( - db: Database, - documentId: number, - title: string, - modifiedAt: string -): void { - db.prepare(`UPDATE documents SET title = ?, modified_at = ? WHERE id = ?`) - .run(title, modifiedAt, documentId); -} - -/** - * Update an existing document's hash, title, and modified_at timestamp. - * Used when content changes but the file path stays the same. - */ -export function updateDocument( - db: Database, - documentId: number, - title: string, - hash: string, - modifiedAt: string -): void { - db.prepare(`UPDATE documents SET title = ?, hash = ?, modified_at = ? WHERE id = ?`) - .run(title, hash, modifiedAt, documentId); -} - -/** - * Deactivate a document (mark as inactive but don't delete). - */ -export function deactivateDocument(db: Database, collectionName: string, path: string): void { - db.prepare(`UPDATE documents SET active = 0 WHERE collection = ? AND path = ? AND active = 1`) - .run(collectionName, path); -} - -/** - * Get all active document paths for a collection. - */ -export function getActiveDocumentPaths(db: Database, collectionName: string): string[] { - const rows = db.prepare(` - SELECT path FROM documents WHERE collection = ? AND active = 1 - `).all(collectionName) as { path: string }[]; - return rows.map(r => r.path); -} - -export { formatQueryForEmbedding, formatDocForEmbedding }; - -export function chunkDocument(content: string, maxChars: number = CHUNK_SIZE_CHARS, overlapChars: number = CHUNK_OVERLAP_CHARS): { text: string; pos: number }[] { - if (content.length <= maxChars) { - return [{ text: content, pos: 0 }]; - } - - const chunks: { text: string; pos: number }[] = []; - let charPos = 0; - - while (charPos < content.length) { - // Calculate end position for this chunk - let endPos = Math.min(charPos + maxChars, content.length); - - // If not at the end, try to find a good break point - if (endPos < content.length) { - const slice = content.slice(charPos, endPos); - - // Look for break points in the last 30% of the chunk - const searchStart = Math.floor(slice.length * 0.7); - const searchSlice = slice.slice(searchStart); - - // Priority: paragraph > sentence > line > word - let breakOffset = -1; - const paragraphBreak = searchSlice.lastIndexOf('\n\n'); - if (paragraphBreak >= 0) { - breakOffset = searchStart + paragraphBreak + 2; - } else { - const sentenceEnd = Math.max( - searchSlice.lastIndexOf('. '), - searchSlice.lastIndexOf('.\n'), - searchSlice.lastIndexOf('? '), - searchSlice.lastIndexOf('?\n'), - searchSlice.lastIndexOf('! '), - searchSlice.lastIndexOf('!\n') - ); - if (sentenceEnd >= 0) { - breakOffset = searchStart + sentenceEnd + 2; - } else { - const lineBreak = searchSlice.lastIndexOf('\n'); - if (lineBreak >= 0) { - breakOffset = searchStart + lineBreak + 1; - } else { - const spaceBreak = searchSlice.lastIndexOf(' '); - if (spaceBreak >= 0) { - breakOffset = searchStart + spaceBreak + 1; - } - } - } - } - - if (breakOffset > 0) { - endPos = charPos + breakOffset; - } - } - - // Ensure we make progress - if (endPos <= charPos) { - endPos = Math.min(charPos + maxChars, content.length); - } - - chunks.push({ text: content.slice(charPos, endPos), pos: charPos }); - - // Move forward, but overlap with previous chunk - // For last chunk, don't overlap (just go to the end) - if (endPos >= content.length) { - break; - } - charPos = endPos - overlapChars; - const lastChunkPos = chunks.at(-1)!.pos; - if (charPos <= lastChunkPos) { - // Prevent infinite loop - move forward at least a bit - charPos = endPos; - } - } - - return chunks; -} - -/** - * Chunk a document by actual token count using the LLM tokenizer. - * More accurate than character-based chunking but requires async. - */ -export async function chunkDocumentByTokens( - content: string, - maxTokens: number = CHUNK_SIZE_TOKENS, - overlapTokens: number = CHUNK_OVERLAP_TOKENS -): Promise<{ text: string; pos: number; tokens: number }[]> { - const llm = getDefaultLlamaCpp(); - - // Tokenize once upfront - const allTokens = await llm.tokenize(content); - const totalTokens = allTokens.length; - - if (totalTokens <= maxTokens) { - return [{ text: content, pos: 0, tokens: totalTokens }]; - } - - const chunks: { text: string; pos: number; tokens: number }[] = []; - const step = maxTokens - overlapTokens; - const avgCharsPerToken = content.length / totalTokens; - let tokenPos = 0; - - while (tokenPos < totalTokens) { - const chunkEnd = Math.min(tokenPos + maxTokens, totalTokens); - const chunkTokens = allTokens.slice(tokenPos, chunkEnd); - let chunkText = await llm.detokenize(chunkTokens); - - // Find a good break point if not at end of document - if (chunkEnd < totalTokens) { - const searchStart = Math.floor(chunkText.length * 0.7); - const searchSlice = chunkText.slice(searchStart); - - let breakOffset = -1; - const paragraphBreak = searchSlice.lastIndexOf('\n\n'); - if (paragraphBreak >= 0) { - breakOffset = paragraphBreak + 2; - } else { - const sentenceEnd = Math.max( - searchSlice.lastIndexOf('. '), - searchSlice.lastIndexOf('.\n'), - searchSlice.lastIndexOf('? '), - searchSlice.lastIndexOf('?\n'), - searchSlice.lastIndexOf('! '), - searchSlice.lastIndexOf('!\n') - ); - if (sentenceEnd >= 0) { - breakOffset = sentenceEnd + 2; - } else { - const lineBreak = searchSlice.lastIndexOf('\n'); - if (lineBreak >= 0) { - breakOffset = lineBreak + 1; - } - } - } - - if (breakOffset >= 0) { - chunkText = chunkText.slice(0, searchStart + breakOffset); - } - } - - // Approximate character position based on token position - const charPos = Math.floor(tokenPos * avgCharsPerToken); - chunks.push({ text: chunkText, pos: charPos, tokens: chunkTokens.length }); - - // Move forward - if (chunkEnd >= totalTokens) break; - - // Advance by step tokens (maxTokens - overlap) - tokenPos += step; - } - - return chunks; -} - -// ============================================================================= -// Fuzzy matching -// ============================================================================= - -function levenshtein(a: string, b: string): number { - const m = a.length, n = b.length; - if (m === 0) return n; - if (n === 0) return m; - const dp: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0)); - for (let i = 0; i <= m; i++) dp[i]![0] = i; - for (let j = 0; j <= n; j++) dp[0]![j] = j; - for (let i = 1; i <= m; i++) { - for (let j = 1; j <= n; j++) { - const cost = a[i - 1] === b[j - 1] ? 0 : 1; - dp[i]![j] = Math.min( - dp[i - 1]![j]! + 1, - dp[i]![j - 1]! + 1, - dp[i - 1]![j - 1]! + cost - ); - } - } - return dp[m]![n]!; -} - -/** - * Normalize a docid input by stripping surrounding quotes and leading #. - * Handles: "#abc123", 'abc123', "abc123", #abc123, abc123 - * Returns the bare hex string. - */ -export function normalizeDocid(docid: string): string { - let normalized = docid.trim(); - - // Strip surrounding quotes (single or double) - if ((normalized.startsWith('"') && normalized.endsWith('"')) || - (normalized.startsWith("'") && normalized.endsWith("'"))) { - normalized = normalized.slice(1, -1); - } - - // Strip leading # if present - if (normalized.startsWith('#')) { - normalized = normalized.slice(1); - } - - return normalized; -} - -/** - * Check if a string looks like a docid reference. - * Accepts: #abc123, abc123, "#abc123", "abc123", '#abc123', 'abc123' - * Returns true if the normalized form is a valid hex string of 6+ chars. - */ -export function isDocid(input: string): boolean { - const normalized = normalizeDocid(input); - // Must be at least 6 hex characters - return normalized.length >= 6 && /^[a-f0-9]+$/i.test(normalized); -} - -/** - * Find a document by its short docid (first 6 characters of hash). - * Returns the document's virtual path if found, null otherwise. - * If multiple documents match the same short hash (collision), returns the first one. - * - * Accepts lenient input: #abc123, abc123, "#abc123", "abc123" - */ -export function findDocumentByDocid(db: Database, docid: string): { filepath: string; hash: string } | null { - const shortHash = normalizeDocid(docid); - - if (shortHash.length < 1) return null; - - // Look up documents where hash starts with the short hash - const doc = db.prepare(` - SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.hash - FROM documents d - WHERE d.hash LIKE ? AND d.active = 1 - LIMIT 1 - `).get(`${shortHash}%`) as { filepath: string; hash: string } | null; - - return doc; -} - -export function findSimilarFiles(db: Database, query: string, maxDistance: number = 3, limit: number = 5): string[] { - const allFiles = db.prepare(` - SELECT d.path - FROM documents d - WHERE d.active = 1 - `).all() as { path: string }[]; - const queryLower = query.toLowerCase(); - const scored = allFiles - .map(f => ({ path: f.path, dist: levenshtein(f.path.toLowerCase(), queryLower) })) - .filter(f => f.dist <= maxDistance) - .sort((a, b) => a.dist - b.dist) - .slice(0, limit); - return scored.map(f => f.path); -} - -export function matchFilesByGlob(db: Database, pattern: string): { filepath: string; displayPath: string; bodyLength: number }[] { - const allFiles = db.prepare(` - SELECT - 'qmd://' || d.collection || '/' || d.path as virtual_path, - LENGTH(content.doc) as body_length, - d.path, - d.collection - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.active = 1 - `).all() as { virtual_path: string; body_length: number; path: string; collection: string }[]; - - const glob = new Glob(pattern); - return allFiles - .filter(f => glob.match(f.virtual_path) || glob.match(f.path)) - .map(f => ({ - filepath: f.virtual_path, // Virtual path for precise lookup - displayPath: f.path, // Relative path for display - bodyLength: f.body_length - })); -} - -// ============================================================================= -// Context -// ============================================================================= - -/** - * Get context for a file path using hierarchical inheritance. - * Contexts are collection-scoped and inherit from parent directories. - * For example, context at "/talks" applies to "/talks/2024/keynote.md". - * - * @param db Database instance (unused - kept for compatibility) - * @param collectionName Collection name - * @param path Relative path within the collection - * @returns Context string or null if no context is defined - */ -export function getContextForPath(db: Database, collectionName: string, path: string): string | null { - const config = collectionsLoadConfig(); - const coll = getCollection(collectionName); - - if (!coll) return null; - - // Collect ALL matching contexts (global + all path prefixes) - const contexts: string[] = []; - - // Add global context if present - if (config.global_context) { - contexts.push(config.global_context); - } - - // Add all matching path contexts (from most general to most specific) - if (coll.context) { - const normalizedPath = path.startsWith("/") ? path : `/${path}`; - - // Collect all matching prefixes - const matchingContexts: { prefix: string; context: string }[] = []; - for (const [prefix, context] of Object.entries(coll.context)) { - const normalizedPrefix = prefix.startsWith("/") ? prefix : `/${prefix}`; - if (normalizedPath.startsWith(normalizedPrefix)) { - matchingContexts.push({ prefix: normalizedPrefix, context }); - } - } - - // Sort by prefix length (shortest/most general first) - matchingContexts.sort((a, b) => a.prefix.length - b.prefix.length); - - // Add all matching contexts - for (const match of matchingContexts) { - contexts.push(match.context); - } - } - - // Join all contexts with double newline - return contexts.length > 0 ? contexts.join('\n\n') : null; -} - -/** - * Get context for a file path (virtual or filesystem). - * Resolves the collection and relative path using the YAML collections config. - */ -export function getContextForFile(db: Database, filepath: string): string | null { - // Handle undefined or null filepath - if (!filepath) return null; - - // Get all collections from YAML config - const collections = collectionsListCollections(); - const config = collectionsLoadConfig(); - - // Parse virtual path format: qmd://collection/path - let collectionName: string | null = null; - let relativePath: string | null = null; - - const parsedVirtual = filepath.startsWith('qmd://') ? parseVirtualPath(filepath) : null; - if (parsedVirtual) { - collectionName = parsedVirtual.collectionName; - relativePath = parsedVirtual.path; - } else { - // Filesystem path: find which collection this absolute path belongs to - for (const coll of collections) { - // Skip collections with missing paths - if (!coll || !coll.path) continue; - - if (filepath.startsWith(coll.path + '/') || filepath === coll.path) { - collectionName = coll.name; - // Extract relative path - relativePath = filepath.startsWith(coll.path + '/') - ? filepath.slice(coll.path.length + 1) - : ''; - break; - } - } - - if (!collectionName || relativePath === null) return null; - } - - // Get the collection from config - const coll = getCollection(collectionName); - if (!coll) return null; - - // Verify this document exists in the database - const doc = db.prepare(` - SELECT d.path - FROM documents d - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - LIMIT 1 - `).get(collectionName, relativePath) as { path: string } | null; - - if (!doc) return null; - - // Collect ALL matching contexts (global + all path prefixes) - const contexts: string[] = []; - - // Add global context if present - if (config.global_context) { - contexts.push(config.global_context); - } - - // Add all matching path contexts (from most general to most specific) - if (coll.context) { - const normalizedPath = relativePath.startsWith("/") ? relativePath : `/${relativePath}`; - - // Collect all matching prefixes - const matchingContexts: { prefix: string; context: string }[] = []; - for (const [prefix, context] of Object.entries(coll.context)) { - const normalizedPrefix = prefix.startsWith("/") ? prefix : `/${prefix}`; - if (normalizedPath.startsWith(normalizedPrefix)) { - matchingContexts.push({ prefix: normalizedPrefix, context }); - } - } - - // Sort by prefix length (shortest/most general first) - matchingContexts.sort((a, b) => a.prefix.length - b.prefix.length); - - // Add all matching contexts - for (const match of matchingContexts) { - contexts.push(match.context); - } - } - - // Join all contexts with double newline - return contexts.length > 0 ? contexts.join('\n\n') : null; -} - -/** - * Get collection by name from YAML config. - * Returns collection metadata from ~/.config/qmd/index.yml - */ -export function getCollectionByName(db: Database, name: string): { name: string; pwd: string; glob_pattern: string } | null { - const collection = getCollection(name); - if (!collection) return null; - - return { - name: collection.name, - pwd: collection.path, - glob_pattern: collection.pattern, - }; -} - -/** - * List all collections with document counts from database. - * Merges YAML config with database statistics. - */ -export function listCollections(db: Database): { name: string; pwd: string; glob_pattern: string; doc_count: number; active_count: number; last_modified: string | null }[] { - const collections = collectionsListCollections(); - - // Get document counts from database for each collection - const result = collections.map(coll => { - const stats = db.prepare(` - SELECT - COUNT(d.id) as doc_count, - SUM(CASE WHEN d.active = 1 THEN 1 ELSE 0 END) as active_count, - MAX(d.modified_at) as last_modified - FROM documents d - WHERE d.collection = ? - `).get(coll.name) as { doc_count: number; active_count: number; last_modified: string | null } | null; - - return { - name: coll.name, - pwd: coll.path, - glob_pattern: coll.pattern, - doc_count: stats?.doc_count || 0, - active_count: stats?.active_count || 0, - last_modified: stats?.last_modified || null, - }; - }); - - return result; -} - -/** - * Remove a collection and clean up its documents. - * Uses collections.ts to remove from YAML config and cleans up database. - */ -export function removeCollection(db: Database, collectionName: string): { deletedDocs: number; cleanedHashes: number } { - // Delete documents from database - const docResult = db.prepare(`DELETE FROM documents WHERE collection = ?`).run(collectionName); - - // Clean up orphaned content hashes - const cleanupResult = db.prepare(` - DELETE FROM content - WHERE hash NOT IN (SELECT DISTINCT hash FROM documents WHERE active = 1) - `).run(); - - // Remove from YAML config (returns true if found and removed) - collectionsRemoveCollection(collectionName); - - return { - deletedDocs: docResult.changes, - cleanedHashes: cleanupResult.changes - }; -} - -/** - * Rename a collection. - * Updates both YAML config and database documents table. - */ -export function renameCollection(db: Database, oldName: string, newName: string): void { - // Update all documents with the new collection name in database - db.prepare(`UPDATE documents SET collection = ? WHERE collection = ?`) - .run(newName, oldName); - - // Rename in YAML config - collectionsRenameCollection(oldName, newName); -} - -// ============================================================================= -// Context Management Operations -// ============================================================================= - -/** - * Insert or update a context for a specific collection and path prefix. - */ -export function insertContext(db: Database, collectionId: number, pathPrefix: string, context: string): void { - // Get collection name from ID - const coll = db.prepare(`SELECT name FROM collections WHERE id = ?`).get(collectionId) as { name: string } | null; - if (!coll) { - throw new Error(`Collection with id ${collectionId} not found`); - } - - // Use collections.ts to add context - collectionsAddContext(coll.name, pathPrefix, context); -} - -/** - * Delete a context for a specific collection and path prefix. - * Returns the number of contexts deleted. - */ -export function deleteContext(db: Database, collectionName: string, pathPrefix: string): number { - // Use collections.ts to remove context - const success = collectionsRemoveContext(collectionName, pathPrefix); - return success ? 1 : 0; -} - -/** - * Delete all global contexts (contexts with empty path_prefix). - * Returns the number of contexts deleted. - */ -export function deleteGlobalContexts(db: Database): number { - let deletedCount = 0; - - // Remove global context - setGlobalContext(undefined); - deletedCount++; - - // Remove root context (empty string) from all collections - const collections = collectionsListCollections(); - for (const coll of collections) { - const success = collectionsRemoveContext(coll.name, ''); - if (success) { - deletedCount++; - } - } - - return deletedCount; -} - -/** - * List all contexts, grouped by collection. - * Returns contexts ordered by collection name, then by path prefix length (longest first). - */ -export function listPathContexts(db: Database): { collection_name: string; path_prefix: string; context: string }[] { - const allContexts = collectionsListAllContexts(); - - // Convert to expected format and sort - return allContexts.map(ctx => ({ - collection_name: ctx.collection, - path_prefix: ctx.path, - context: ctx.context, - })).sort((a, b) => { - // Sort by collection name first - if (a.collection_name !== b.collection_name) { - return a.collection_name.localeCompare(b.collection_name); - } - // Then by path prefix length (longest first) - if (a.path_prefix.length !== b.path_prefix.length) { - return b.path_prefix.length - a.path_prefix.length; - } - // Then alphabetically - return a.path_prefix.localeCompare(b.path_prefix); - }); -} - -/** - * Get all collections (name only - from YAML config). - */ -export function getAllCollections(db: Database): { name: string }[] { - const collections = collectionsListCollections(); - return collections.map(c => ({ name: c.name })); -} - -/** - * Check which collections don't have any context defined. - * Returns collections that have no context entries at all (not even root context). - */ -export function getCollectionsWithoutContext(db: Database): { name: string; pwd: string; doc_count: number }[] { - // Get all collections from YAML config - const yamlCollections = collectionsListCollections(); - - // Filter to those without context - const collectionsWithoutContext: { name: string; pwd: string; doc_count: number }[] = []; - - for (const coll of yamlCollections) { - // Check if collection has any context - if (!coll.context || Object.keys(coll.context).length === 0) { - // Get doc count from database - const stats = db.prepare(` - SELECT COUNT(d.id) as doc_count - FROM documents d - WHERE d.collection = ? AND d.active = 1 - `).get(coll.name) as { doc_count: number } | null; - - collectionsWithoutContext.push({ - name: coll.name, - pwd: coll.path, - doc_count: stats?.doc_count || 0, - }); - } - } - - return collectionsWithoutContext.sort((a, b) => a.name.localeCompare(b.name)); -} - -/** - * Get top-level directories in a collection that don't have context. - * Useful for suggesting where context might be needed. - */ -export function getTopLevelPathsWithoutContext(db: Database, collectionName: string): string[] { - // Get all paths in the collection from database - const paths = db.prepare(` - SELECT DISTINCT path FROM documents - WHERE collection = ? AND active = 1 - `).all(collectionName) as { path: string }[]; - - // Get existing contexts for this collection from YAML - const yamlColl = getCollection(collectionName); - if (!yamlColl) return []; - - const contextPrefixes = new Set(); - if (yamlColl.context) { - for (const prefix of Object.keys(yamlColl.context)) { - contextPrefixes.add(prefix); - } - } - - // Extract top-level directories (first path component) - const topLevelDirs = new Set(); - for (const { path } of paths) { - const parts = path.split('/').filter(Boolean); - if (parts.length > 1) { - const dir = parts[0]; - if (dir) topLevelDirs.add(dir); - } - } - - // Filter out directories that already have context (exact or parent) - const missing: string[] = []; - for (const dir of topLevelDirs) { - let hasContext = false; - - // Check if this dir or any parent has context - for (const prefix of contextPrefixes) { - if (prefix === '' || prefix === dir || dir.startsWith(prefix + '/')) { - hasContext = true; - break; - } - } - - if (!hasContext) { - missing.push(dir); - } - } - - return missing.sort(); -} - -// ============================================================================= -// FTS Search -// ============================================================================= - -function sanitizeFTS5Term(term: string): string { - return term.replace(/[^\p{L}\p{N}']/gu, '').toLowerCase(); -} - -function buildFTS5Query(query: string): string | null { - const terms = query.split(/\s+/) - .map(t => sanitizeFTS5Term(t)) - .filter(t => t.length > 0); - if (terms.length === 0) return null; - if (terms.length === 1) return `"${terms[0]}"*`; - return terms.map(t => `"${t}"*`).join(' AND '); -} - -export function searchFTS(db: Database, query: string, limit: number = 20, collectionId?: number): SearchResult[] { - const ftsQuery = buildFTS5Query(query); - if (!ftsQuery) return []; - - let sql = ` - SELECT - 'qmd://' || d.collection || '/' || d.path as filepath, - d.collection || '/' || d.path as display_path, - d.title, - content.doc as body, - d.hash, - bm25(documents_fts, 10.0, 1.0) as bm25_score - FROM documents_fts f - JOIN documents d ON d.id = f.rowid - JOIN content ON content.hash = d.hash - WHERE documents_fts MATCH ? AND d.active = 1 - `; - const params: (string | number)[] = [ftsQuery]; - - if (collectionId) { - // Note: collectionId is a legacy parameter that should be phased out - // Collections are now managed in YAML. For now, we interpret it as a collection name filter. - // This code path is likely unused as collection filtering should be done at CLI level. - sql += ` AND d.collection = ?`; - params.push(String(collectionId)); - } - - // bm25 lower is better; sort ascending. - sql += ` ORDER BY bm25_score ASC LIMIT ?`; - params.push(limit); - - const rows = db.prepare(sql).all(...params) as { filepath: string; display_path: string; title: string; body: string; hash: string; bm25_score: number }[]; - return rows.map(row => { - const collectionName = row.filepath.split('//')[1]?.split('/')[0] || ""; - // Convert bm25 (negative, lower is better) into a stable (0..1] score where higher is better. - // BM25 scores in SQLite FTS5 are negative (e.g., -10 is strong, -2 is weak). - // Avoid per-query normalization so "strong signal" heuristics can work. - const score = 1 / (1 + Math.abs(row.bm25_score)); - return { - filepath: row.filepath, - displayPath: row.display_path, - title: row.title, - hash: row.hash, - docid: getDocid(row.hash), - collectionName, - modifiedAt: "", // Not available in FTS query - bodyLength: row.body.length, - body: row.body, - context: getContextForFile(db, row.filepath), - score, - source: "fts" as const, - }; - }); -} - -// ============================================================================= -// Vector Search -// ============================================================================= - -export async function searchVec(db: Database, query: string, model: string, limit: number = 20, collectionName?: string, session?: ILLMSession): Promise { - const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get(); - if (!tableExists) return []; - - const embedding = await getEmbedding(query, model, true, session); - if (!embedding) return []; - - // IMPORTANT: We use a two-step query approach here because sqlite-vec virtual tables - // hang indefinitely when combined with JOINs in the same query. Do NOT try to - // "optimize" this by combining into a single query with JOINs - it will break. - // See: https://github.com/tobi/qmd/pull/23 - - // Step 1: Get vector matches from sqlite-vec (no JOINs allowed) - const vecResults = db.prepare(` - SELECT hash_seq, distance - FROM vectors_vec - WHERE embedding MATCH ? AND k = ? - `).all(new Float32Array(embedding), limit * 3) as { hash_seq: string; distance: number }[]; - - if (vecResults.length === 0) return []; - - // Step 2: Get chunk info and document data - const hashSeqs = vecResults.map(r => r.hash_seq); - const distanceMap = new Map(vecResults.map(r => [r.hash_seq, r.distance])); - - // Build query for document lookup - const placeholders = hashSeqs.map(() => '?').join(','); - let docSql = ` - SELECT - cv.hash || '_' || cv.seq as hash_seq, - cv.hash, - cv.pos, - 'qmd://' || d.collection || '/' || d.path as filepath, - d.collection || '/' || d.path as display_path, - d.title, - content.doc as body - FROM content_vectors cv - JOIN documents d ON d.hash = cv.hash AND d.active = 1 - JOIN content ON content.hash = d.hash - WHERE cv.hash || '_' || cv.seq IN (${placeholders}) - `; - const params: string[] = [...hashSeqs]; - - if (collectionName) { - docSql += ` AND d.collection = ?`; - params.push(collectionName); - } - - const docRows = db.prepare(docSql).all(...params) as { - hash_seq: string; hash: string; pos: number; filepath: string; - display_path: string; title: string; body: string; - }[]; - - // Combine with distances and dedupe by filepath - const seen = new Map(); - for (const row of docRows) { - const distance = distanceMap.get(row.hash_seq) ?? 1; - const existing = seen.get(row.filepath); - if (!existing || distance < existing.bestDist) { - seen.set(row.filepath, { row, bestDist: distance }); - } - } - - return Array.from(seen.values()) - .sort((a, b) => a.bestDist - b.bestDist) - .slice(0, limit) - .map(({ row, bestDist }) => { - const collectionName = row.filepath.split('//')[1]?.split('/')[0] || ""; - return { - filepath: row.filepath, - displayPath: row.display_path, - title: row.title, - hash: row.hash, - docid: getDocid(row.hash), - collectionName, - modifiedAt: "", // Not available in vec query - bodyLength: row.body.length, - body: row.body, - context: getContextForFile(db, row.filepath), - score: 1 - bestDist, // Cosine similarity = 1 - cosine distance - source: "vec" as const, - chunkPos: row.pos, - }; - }); -} - -// ============================================================================= -// Embeddings -// ============================================================================= - -async function getEmbedding(text: string, model: string, isQuery: boolean, session?: ILLMSession): Promise { - // Format text using the appropriate prompt template - const formattedText = isQuery ? formatQueryForEmbedding(text) : formatDocForEmbedding(text); - const result = session - ? await session.embed(formattedText, { model, isQuery }) - : await getDefaultLlamaCpp().embed(formattedText, { model, isQuery }); - return result?.embedding || null; -} - -/** - * Get all unique content hashes that need embeddings (from active documents). - * Returns hash, document body, and a sample path for display purposes. - */ -export function getHashesForEmbedding(db: Database): { hash: string; body: string; path: string }[] { - return db.prepare(` - SELECT d.hash, c.doc as body, MIN(d.path) as path - FROM documents d - JOIN content c ON d.hash = c.hash - LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0 - WHERE d.active = 1 AND v.hash IS NULL - GROUP BY d.hash - `).all() as { hash: string; body: string; path: string }[]; -} - -/** - * Clear all embeddings from the database (force re-index). - * Deletes all rows from content_vectors and drops the vectors_vec table. - */ -export function clearAllEmbeddings(db: Database): void { - db.exec(`DELETE FROM content_vectors`); - db.exec(`DROP TABLE IF EXISTS vectors_vec`); -} - -/** - * Insert a single embedding into both content_vectors and vectors_vec tables. - * The hash_seq key is formatted as "hash_seq" for the vectors_vec table. - */ -export function insertEmbedding( - db: Database, - hash: string, - seq: number, - pos: number, - embedding: Float32Array, - model: string, - embeddedAt: string -): void { - const hashSeq = `${hash}_${seq}`; - const insertVecStmt = db.prepare(`INSERT OR REPLACE INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`); - const insertContentVectorStmt = db.prepare(`INSERT OR REPLACE INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, ?, ?, ?, ?)`); - - insertVecStmt.run(hashSeq, embedding); - insertContentVectorStmt.run(hash, seq, pos, model, embeddedAt); -} - -// ============================================================================= -// Query expansion -// ============================================================================= - -export async function expandQuery(query: string, model: string = DEFAULT_QUERY_MODEL, db: Database): Promise { - // Check cache first - const cacheKey = getCacheKey("expandQuery", { query, model }); - const cached = getCachedResult(db, cacheKey); - if (cached) { - const lines = cached.split('\n').map(l => l.trim()).filter(l => l.length > 0); - return [query, ...lines.slice(0, 2)]; - } - - const llm = getDefaultLlamaCpp(); - // Note: LlamaCpp uses hardcoded model, model parameter is ignored - const results = await llm.expandQuery(query); - const queryTexts = results.map(r => r.text); - - // Cache the expanded queries (excluding original) - const expandedOnly = queryTexts.filter(t => t !== query); - if (expandedOnly.length > 0) { - setCachedResult(db, cacheKey, expandedOnly.join('\n')); - } - - return Array.from(new Set([query, ...queryTexts])); -} - -// ============================================================================= -// Reranking -// ============================================================================= - -export async function rerank(query: string, documents: { file: string; text: string }[], model: string = DEFAULT_RERANK_MODEL, db: Database): Promise<{ file: string; score: number }[]> { - const cachedResults: Map = new Map(); - const uncachedDocs: RerankDocument[] = []; - - // Check cache for each document - for (const doc of documents) { - const cacheKey = getCacheKey("rerank", { query, file: doc.file, model }); - const cached = getCachedResult(db, cacheKey); - if (cached !== null) { - cachedResults.set(doc.file, parseFloat(cached)); - } else { - uncachedDocs.push({ file: doc.file, text: doc.text }); - } - } - - // Rerank uncached documents using LlamaCpp - if (uncachedDocs.length > 0) { - const llm = getDefaultLlamaCpp(); - const rerankResult = await llm.rerank(query, uncachedDocs, { model }); - - // Cache results - for (const result of rerankResult.results) { - const cacheKey = getCacheKey("rerank", { query, file: result.file, model }); - setCachedResult(db, cacheKey, result.score.toString()); - cachedResults.set(result.file, result.score); - } - } - - // Return all results sorted by score - return documents - .map(doc => ({ file: doc.file, score: cachedResults.get(doc.file) || 0 })) - .sort((a, b) => b.score - a.score); -} - -// ============================================================================= -// Reciprocal Rank Fusion -// ============================================================================= - -export function reciprocalRankFusion( - resultLists: RankedResult[][], - weights: number[] = [], - k: number = 60 -): RankedResult[] { - const scores = new Map(); - - for (let listIdx = 0; listIdx < resultLists.length; listIdx++) { - const list = resultLists[listIdx]; - if (!list) continue; - const weight = weights[listIdx] ?? 1.0; - - for (let rank = 0; rank < list.length; rank++) { - const result = list[rank]; - if (!result) continue; - const rrfContribution = weight / (k + rank + 1); - const existing = scores.get(result.file); - - if (existing) { - existing.rrfScore += rrfContribution; - existing.topRank = Math.min(existing.topRank, rank); - } else { - scores.set(result.file, { - result, - rrfScore: rrfContribution, - topRank: rank, - }); - } - } - } - - // Top-rank bonus - for (const entry of scores.values()) { - if (entry.topRank === 0) { - entry.rrfScore += 0.05; - } else if (entry.topRank <= 2) { - entry.rrfScore += 0.02; - } - } - - return Array.from(scores.values()) - .sort((a, b) => b.rrfScore - a.rrfScore) - .map(e => ({ ...e.result, score: e.rrfScore })); -} - -// ============================================================================= -// Document retrieval -// ============================================================================= - -type DbDocRow = { - virtual_path: string; - display_path: string; - title: string; - hash: string; - collection: string; - path: string; - modified_at: string; - body_length: number; - body?: string; -}; - -/** - * Find a document by filename/path, docid (#hash), or with fuzzy matching. - * Returns document metadata without body by default. - * - * Supports: - * - Virtual paths: qmd://collection/path/to/file.md - * - Absolute paths: /path/to/file.md - * - Relative paths: path/to/file.md - * - Short docid: #abc123 (first 6 chars of hash) - */ -export function findDocument(db: Database, filename: string, options: { includeBody?: boolean } = {}): DocumentResult | DocumentNotFound { - let filepath = filename; - const colonMatch = filepath.match(/:(\d+)$/); - if (colonMatch) { - filepath = filepath.slice(0, -colonMatch[0].length); - } - - // Check if this is a docid lookup (#abc123, abc123, "#abc123", "abc123", etc.) - if (isDocid(filepath)) { - const docidMatch = findDocumentByDocid(db, filepath); - if (docidMatch) { - filepath = docidMatch.filepath; - } else { - return { error: "not_found", query: filename, similarFiles: [] }; - } - } - - if (filepath.startsWith('~/')) { - filepath = homedir() + filepath.slice(1); - } - - const bodyCol = options.includeBody ? `, content.doc as body` : ``; - - // Build computed columns - // Note: absoluteFilepath is computed from YAML collections after query - const selectCols = ` - 'qmd://' || d.collection || '/' || d.path as virtual_path, - d.collection || '/' || d.path as display_path, - d.title, - d.hash, - d.collection, - d.modified_at, - LENGTH(content.doc) as body_length - ${bodyCol} - `; - - // Try to match by virtual path first - let doc = db.prepare(` - SELECT ${selectCols} - FROM documents d - JOIN content ON content.hash = d.hash - WHERE 'qmd://' || d.collection || '/' || d.path = ? AND d.active = 1 - `).get(filepath) as DbDocRow | null; - - // Try fuzzy match by virtual path - if (!doc) { - doc = db.prepare(` - SELECT ${selectCols} - FROM documents d - JOIN content ON content.hash = d.hash - WHERE 'qmd://' || d.collection || '/' || d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(`%${filepath}`) as DbDocRow | null; - } - - // Try to match by absolute path (requires looking up collection paths from YAML) - if (!doc && !filepath.startsWith('qmd://')) { - const collections = collectionsListCollections(); - for (const coll of collections) { - let relativePath: string | null = null; - - // If filepath is absolute and starts with collection path, extract relative part - if (filepath.startsWith(coll.path + '/')) { - relativePath = filepath.slice(coll.path.length + 1); - } - // Otherwise treat filepath as relative to collection - else if (!filepath.startsWith('/')) { - relativePath = filepath; - } - - if (relativePath) { - doc = db.prepare(` - SELECT ${selectCols} - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(coll.name, relativePath) as DbDocRow | null; - if (doc) break; - } - } - } - - if (!doc) { - const similar = findSimilarFiles(db, filepath, 5, 5); - return { error: "not_found", query: filename, similarFiles: similar }; - } - - // Get context using virtual path - const virtualPath = doc.virtual_path || `qmd://${doc.collection}/${doc.display_path}`; - const context = getContextForFile(db, virtualPath); - - return { - filepath: virtualPath, - displayPath: doc.display_path, - title: doc.title, - context, - hash: doc.hash, - docid: getDocid(doc.hash), - collectionName: doc.collection, - modifiedAt: doc.modified_at, - bodyLength: doc.body_length, - ...(options.includeBody && doc.body !== undefined && { body: doc.body }), - }; -} - -/** - * Get the body content for a document - * Optionally slice by line range - */ -export function getDocumentBody(db: Database, doc: DocumentResult | { filepath: string }, fromLine?: number, maxLines?: number): string | null { - const filepath = doc.filepath; - - // Try to resolve document by filepath (absolute or virtual) - let row: { body: string } | null = null; - - // Try virtual path first - if (filepath.startsWith('qmd://')) { - row = db.prepare(` - SELECT content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE 'qmd://' || d.collection || '/' || d.path = ? AND d.active = 1 - `).get(filepath) as { body: string } | null; - } - - // Try absolute path by looking up in YAML collections - if (!row) { - const collections = collectionsListCollections(); - for (const coll of collections) { - if (filepath.startsWith(coll.path + '/')) { - const relativePath = filepath.slice(coll.path.length + 1); - row = db.prepare(` - SELECT content.doc as body - FROM documents d - JOIN content ON content.hash = d.hash - WHERE d.collection = ? AND d.path = ? AND d.active = 1 - `).get(coll.name, relativePath) as { body: string } | null; - if (row) break; - } - } - } - - if (!row) return null; - - let body = row.body; - if (fromLine !== undefined || maxLines !== undefined) { - const lines = body.split('\n'); - const start = (fromLine || 1) - 1; - const end = maxLines !== undefined ? start + maxLines : lines.length; - body = lines.slice(start, end).join('\n'); - } - - return body; -} - -/** - * Find multiple documents by glob pattern or comma-separated list - * Returns documents without body by default (use getDocumentBody to load) - */ -export function findDocuments( - db: Database, - pattern: string, - options: { includeBody?: boolean; maxBytes?: number } = {} -): { docs: MultiGetResult[]; errors: string[] } { - const isCommaSeparated = pattern.includes(',') && !pattern.includes('*') && !pattern.includes('?'); - const errors: string[] = []; - const maxBytes = options.maxBytes ?? DEFAULT_MULTI_GET_MAX_BYTES; - - const bodyCol = options.includeBody ? `, content.doc as body` : ``; - const selectCols = ` - 'qmd://' || d.collection || '/' || d.path as virtual_path, - d.collection || '/' || d.path as display_path, - d.title, - d.hash, - d.collection, - d.modified_at, - LENGTH(content.doc) as body_length - ${bodyCol} - `; - - let fileRows: DbDocRow[]; - - if (isCommaSeparated) { - const names = pattern.split(',').map(s => s.trim()).filter(Boolean); - fileRows = []; - for (const name of names) { - let doc = db.prepare(` - SELECT ${selectCols} - FROM documents d - JOIN content ON content.hash = d.hash - WHERE 'qmd://' || d.collection || '/' || d.path = ? AND d.active = 1 - `).get(name) as DbDocRow | null; - if (!doc) { - doc = db.prepare(` - SELECT ${selectCols} - FROM documents d - JOIN content ON content.hash = d.hash - WHERE 'qmd://' || d.collection || '/' || d.path LIKE ? AND d.active = 1 - LIMIT 1 - `).get(`%${name}`) as DbDocRow | null; - } - if (doc) { - fileRows.push(doc); - } else { - const similar = findSimilarFiles(db, name, 5, 3); - let msg = `File not found: ${name}`; - if (similar.length > 0) { - msg += ` (did you mean: ${similar.join(', ')}?)`; - } - errors.push(msg); - } - } - } else { - // Glob pattern match - const matched = matchFilesByGlob(db, pattern); - if (matched.length === 0) { - errors.push(`No files matched pattern: ${pattern}`); - return { docs: [], errors }; - } - const virtualPaths = matched.map(m => m.filepath); - const placeholders = virtualPaths.map(() => '?').join(','); - fileRows = db.prepare(` - SELECT ${selectCols} - FROM documents d - JOIN content ON content.hash = d.hash - WHERE 'qmd://' || d.collection || '/' || d.path IN (${placeholders}) AND d.active = 1 - `).all(...virtualPaths) as DbDocRow[]; - } - - const results: MultiGetResult[] = []; - - for (const row of fileRows) { - // Get context using virtual path - const virtualPath = row.virtual_path || `qmd://${row.collection}/${row.display_path}`; - const context = getContextForFile(db, virtualPath); - - if (row.body_length > maxBytes) { - results.push({ - doc: { filepath: virtualPath, displayPath: row.display_path }, - skipped: true, - skipReason: `File too large (${Math.round(row.body_length / 1024)}KB > ${Math.round(maxBytes / 1024)}KB)`, - }); - continue; - } - - results.push({ - doc: { - filepath: virtualPath, - displayPath: row.display_path, - title: row.title || row.display_path.split('/').pop() || row.display_path, - context, - hash: row.hash, - docid: getDocid(row.hash), - collectionName: row.collection, - modifiedAt: row.modified_at, - bodyLength: row.body_length, - ...(options.includeBody && row.body !== undefined && { body: row.body }), - }, - skipped: false, - }); - } - - return { docs: results, errors }; -} - -// ============================================================================= -// Status -// ============================================================================= - -export function getStatus(db: Database): IndexStatus { - // Load collections from YAML - const yamlCollections = collectionsListCollections(); - - // Get document counts and last update times for each collection - const collections = yamlCollections.map(col => { - const stats = db.prepare(` - SELECT - COUNT(*) as active_count, - MAX(modified_at) as last_doc_update - FROM documents - WHERE collection = ? AND active = 1 - `).get(col.name) as { active_count: number; last_doc_update: string | null }; - - return { - name: col.name, - path: col.path, - pattern: col.pattern, - documents: stats.active_count, - lastUpdated: stats.last_doc_update || new Date().toISOString(), - }; - }); - - // Sort by last update time (most recent first) - collections.sort((a, b) => { - if (!a.lastUpdated) return 1; - if (!b.lastUpdated) return -1; - return new Date(b.lastUpdated).getTime() - new Date(a.lastUpdated).getTime(); - }); - - const totalDocs = (db.prepare(`SELECT COUNT(*) as c FROM documents WHERE active = 1`).get() as { c: number }).c; - const needsEmbedding = getHashesNeedingEmbedding(db); - const hasVectors = !!db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get(); - - return { - totalDocuments: totalDocs, - needsEmbedding, - hasVectorIndex: hasVectors, - collections, - }; -} - -// ============================================================================= -// Snippet extraction -// ============================================================================= - -export type SnippetResult = { - line: number; // 1-indexed line number of best match - snippet: string; // The snippet text with diff-style header - linesBefore: number; // Lines in document before snippet - linesAfter: number; // Lines in document after snippet - snippetLines: number; // Number of lines in snippet -}; - -export function extractSnippet(body: string, query: string, maxLen = 500, chunkPos?: number): SnippetResult { - const totalLines = body.split('\n').length; - let searchBody = body; - let lineOffset = 0; - - if (chunkPos && chunkPos > 0) { - const contextStart = Math.max(0, chunkPos - 100); - const contextEnd = Math.min(body.length, chunkPos + maxLen + 100); - searchBody = body.slice(contextStart, contextEnd); - if (contextStart > 0) { - lineOffset = body.slice(0, contextStart).split('\n').length - 1; - } - } - - const lines = searchBody.split('\n'); - const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 0); - let bestLine = 0, bestScore = -1; - - for (let i = 0; i < lines.length; i++) { - const lineLower = (lines[i] ?? "").toLowerCase(); - let score = 0; - for (const term of queryTerms) { - if (lineLower.includes(term)) score++; - } - if (score > bestScore) { - bestScore = score; - bestLine = i; - } - } - - const start = Math.max(0, bestLine - 1); - const end = Math.min(lines.length, bestLine + 3); - const snippetLines = lines.slice(start, end); - let snippetText = snippetLines.join('\n'); - - // If we focused on a chunk window and it produced an empty/whitespace-only snippet, - // fall back to a full-document snippet so we always show something useful. - if (chunkPos && chunkPos > 0 && snippetText.trim().length === 0) { - return extractSnippet(body, query, maxLen, undefined); - } - - if (snippetText.length > maxLen) snippetText = snippetText.substring(0, maxLen - 3) + "..."; - - const absoluteStart = lineOffset + start + 1; // 1-indexed - const snippetLineCount = snippetLines.length; - const linesBefore = absoluteStart - 1; - const linesAfter = totalLines - (absoluteStart + snippetLineCount - 1); - - // Format with diff-style header: @@ -start,count @@ (linesBefore before, linesAfter after) - const header = `@@ -${absoluteStart},${snippetLineCount} @@ (${linesBefore} before, ${linesAfter} after)`; - const snippet = `${header}\n${snippetText}`; - - return { - line: lineOffset + bestLine + 1, - snippet, - linesBefore, - linesAfter, - snippetLines: snippetLineCount, - }; -} diff --git a/tools/qmd/tsconfig.json b/tools/qmd/tsconfig.json deleted file mode 100644 index bfa0fead..00000000 --- a/tools/qmd/tsconfig.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "compilerOptions": { - // Environment setup & latest features - "lib": ["ESNext"], - "target": "ESNext", - "module": "Preserve", - "moduleDetection": "force", - "jsx": "react-jsx", - "allowJs": true, - - // Bundler mode - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "noEmit": true, - - // Best practices - "strict": true, - "skipLibCheck": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedIndexedAccess": true, - "noImplicitOverride": true, - - // Some stricter flags (disabled by default) - "noUnusedLocals": false, - "noUnusedParameters": false, - "noPropertyAccessFromIndexSignature": false - } -}