fix: make search snippets unicode-safe

This commit is contained in:
lucaronin
2026-04-30 10:23:54 +02:00
parent 8754b1bc77
commit 0d6e3853f2
5 changed files with 490 additions and 39 deletions

View File

@@ -46,3 +46,90 @@ pub fn batch_archive_notes(
Ok(count)
})
}
#[cfg(test)]
mod tests {
use super::*;
fn note_path(dir: &tempfile::TempDir, name: &str) -> String {
dir.path().join(name).to_string_lossy().into_owned()
}
fn write_note(path: &str, content: &str) {
std::fs::write(path, content).unwrap();
}
#[test]
fn update_frontmatter_command_validates_and_updates_note() {
let dir = tempfile::TempDir::new().unwrap();
let path = note_path(&dir, "note.md");
write_note(&path, "---\nStatus: Draft\n---\n# Note\n");
let updated = update_frontmatter(
path.clone(),
"Status".to_string(),
FrontmatterValue::String("Done".to_string()),
Some(dir.path().to_string_lossy().into_owned()),
)
.unwrap();
assert!(updated.contains("Status: Done"));
assert_eq!(std::fs::read_to_string(path).unwrap(), updated);
}
#[test]
fn delete_frontmatter_property_command_removes_existing_key() {
let dir = tempfile::TempDir::new().unwrap();
let path = note_path(&dir, "note.md");
write_note(&path, "---\nStatus: Draft\nOwner: Ada\n---\n# Note\n");
let updated = delete_frontmatter_property(
path,
"Owner".to_string(),
Some(dir.path().to_string_lossy().into_owned()),
)
.unwrap();
assert!(!updated.contains("Owner:"));
assert!(updated.contains("Status: Draft"));
}
#[test]
fn batch_archive_notes_command_marks_each_note_archived() {
let dir = tempfile::TempDir::new().unwrap();
let first = note_path(&dir, "first.md");
let second = note_path(&dir, "second.md");
write_note(&first, "---\nStatus: Draft\n---\n# First\n");
write_note(&second, "# Second\n");
let count = batch_archive_notes(
vec![first.clone(), second.clone()],
Some(dir.path().to_string_lossy().into_owned()),
)
.unwrap();
assert_eq!(count, 2);
assert!(std::fs::read_to_string(first)
.unwrap()
.contains("_archived: true"));
assert!(std::fs::read_to_string(second)
.unwrap()
.contains("_archived: true"));
}
#[test]
fn batch_archive_notes_command_rejects_notes_outside_vault() {
let vault = tempfile::TempDir::new().unwrap();
let outside = tempfile::TempDir::new().unwrap();
let outside_note = note_path(&outside, "outside.md");
write_note(&outside_note, "# Outside\n");
let error = batch_archive_notes(
vec![outside_note],
Some(vault.path().to_string_lossy().into_owned()),
)
.unwrap_err();
assert!(error.contains("Path must stay inside the active vault"));
}
}

View File

@@ -112,8 +112,18 @@ pub async fn search_vault(
#[cfg(test)]
mod tests {
use super::{collect_registered_vault_roots, find_registered_vault_root};
use super::{
collect_registered_vault_roots, find_registered_vault_root, reload_vault_entry,
resolve_reload_vault_path, search_vault,
};
use crate::vault_list::{VaultEntry as VaultListEntry, VaultList};
use std::path::{Path, PathBuf};
fn write_note(root: &Path, name: &str, content: &str) -> std::path::PathBuf {
let path = root.join(name);
std::fs::write(&path, content).unwrap();
path
}
#[test]
fn finds_registered_vault_root_for_an_absolute_note_path() {
@@ -173,4 +183,109 @@ mod tests {
Some(nested_root),
);
}
#[test]
fn find_registered_vault_root_ignores_missing_registered_roots() {
let dir = tempfile::TempDir::new().unwrap();
let vault_root = dir.path().join("vault");
std::fs::create_dir_all(&vault_root).unwrap();
let note_path = write_note(&vault_root, "note.md", "# Note\n");
let registered_roots = vec![dir.path().join("missing"), vault_root.clone()];
let canonical_note_path = note_path.canonicalize().unwrap();
assert_eq!(
find_registered_vault_root(canonical_note_path.as_path(), &registered_roots),
Some(vault_root),
);
}
#[test]
fn collect_registered_vault_roots_includes_active_vault() {
let vault_list = VaultList {
vaults: vec![VaultListEntry {
label: "Listed".to_string(),
path: "/listed".to_string(),
}],
active_vault: Some("/active".to_string()),
hidden_defaults: vec![],
};
let roots = collect_registered_vault_roots(&vault_list);
assert_eq!(
roots,
vec![PathBuf::from("/listed"), PathBuf::from("/active")]
);
}
#[test]
fn resolve_reload_vault_path_uses_explicit_vault_path() {
let explicit = Path::new("/tmp/vault");
assert_eq!(
resolve_reload_vault_path(Path::new("note.md"), Some(explicit)).unwrap(),
Some(explicit.to_path_buf()),
);
}
#[test]
fn resolve_reload_vault_path_skips_relative_note_paths() {
assert_eq!(
resolve_reload_vault_path(Path::new("note.md"), None).unwrap(),
None,
);
}
#[test]
fn reload_vault_entry_command_reads_note_inside_vault() {
let dir = tempfile::TempDir::new().unwrap();
let note_path = write_note(dir.path(), "note.md", "# Reloaded Title\n\nBody");
let entry = reload_vault_entry(note_path, Some(dir.path().to_path_buf())).unwrap();
assert_eq!(entry.title, "Reloaded Title");
}
#[tokio::test]
async fn search_vault_command_uses_default_limit_and_returns_results() {
let dir = tempfile::Builder::new()
.prefix("scan-search-")
.tempdir_in(std::env::current_dir().unwrap())
.unwrap();
write_note(dir.path(), "search.md", "# Searchable\n\nneedle");
let response = search_vault(
dir.path().to_string_lossy().into_owned(),
"needle".to_string(),
"keyword".to_string(),
None,
)
.await
.unwrap();
assert_eq!(response.results.len(), 1);
assert_eq!(response.results[0].title, "Searchable");
assert_eq!(response.mode, "keyword");
}
#[tokio::test]
async fn search_vault_command_honors_explicit_limit() {
let dir = tempfile::Builder::new()
.prefix("scan-search-limit-")
.tempdir_in(std::env::current_dir().unwrap())
.unwrap();
write_note(dir.path(), "first.md", "# First\n\nneedle");
write_note(dir.path(), "second.md", "# Second\n\nneedle");
let response = search_vault(
dir.path().to_string_lossy().into_owned(),
"needle".to_string(),
"keyword".to_string(),
Some(1),
)
.await
.unwrap();
assert_eq!(response.results.len(), 1);
}
}

View File

@@ -28,41 +28,90 @@ pub struct SearchOptions<'a> {
pub hide_gitignored_files: bool,
}
fn extract_snippet(content: &str, query_lower: &str) -> String {
let content_lower = content.to_lowercase();
let pos = match content_lower.find(query_lower) {
Some(p) => p,
None => return String::new(),
};
let start = content[..pos]
.rfind('\n')
.map(|i| i + 1)
.unwrap_or(pos.saturating_sub(60));
let end = content[pos..]
.find('\n')
.map(|i| pos + i)
.unwrap_or_else(|| (pos + 120).min(content.len()));
let snippet = &content[start..end];
if snippet.len() > 200 {
format!("{}", &snippet[..200])
} else {
snippet.to_string()
struct Utf8Boundary<'a> {
text: &'a str,
}
struct SnippetRequest<'a> {
content: &'a str,
query_lower: &'a str,
}
struct MatchScoreRequest<'a> {
title_lower: &'a str,
content_lower: &'a str,
query_lower: &'a str,
}
impl Utf8Boundary<'_> {
fn floor(&self, index: usize) -> usize {
let mut boundary = index.min(self.text.len());
while boundary > 0 && !self.text.is_char_boundary(boundary) {
boundary -= 1;
}
boundary
}
fn lower_to_source(&self, lower_index: usize) -> usize {
let mut lowered_len = 0;
for (source_index, ch) in self.text.char_indices() {
if lowered_len >= lower_index {
return source_index;
}
lowered_len += ch.to_lowercase().map(|c| c.len_utf8()).sum::<usize>();
if lowered_len > lower_index {
return source_index;
}
}
self.text.len()
}
}
fn score_match(title_lower: &str, content_lower: &str, query_lower: &str) -> f64 {
let title_exact = title_lower.contains(query_lower);
let title_word = title_lower.split_whitespace().any(|w| w == query_lower);
let content_count = content_lower.matches(query_lower).count();
let mut score = 0.0;
if title_word {
score += 10.0;
} else if title_exact {
score += 5.0;
impl SnippetRequest<'_> {
fn extract(&self) -> String {
let content_lower = self.content.to_lowercase();
let lower_pos = match content_lower.find(self.query_lower) {
Some(p) => p,
None => return String::new(),
};
let content_boundary = Utf8Boundary { text: self.content };
let pos = content_boundary.lower_to_source(lower_pos);
let start = self.content[..pos]
.rfind('\n')
.map(|i| i + 1)
.unwrap_or_else(|| content_boundary.floor(pos.saturating_sub(60)));
let end = self.content[pos..]
.find('\n')
.map(|i| pos + i)
.unwrap_or_else(|| content_boundary.floor((pos + 120).min(self.content.len())));
let snippet = &self.content[start..end];
if snippet.len() > 200 {
let end = Utf8Boundary { text: snippet }.floor(200);
format!("{}", &snippet[..end])
} else {
snippet.to_string()
}
}
}
impl MatchScoreRequest<'_> {
fn score(&self) -> f64 {
let title_exact = self.title_lower.contains(self.query_lower);
let title_word = self
.title_lower
.split_whitespace()
.any(|word| word == self.query_lower);
let content_count = self.content_lower.matches(self.query_lower).count();
let mut score = 0.0;
if title_word {
score += 10.0;
} else if title_exact {
score += 5.0;
}
score += (content_count as f64).min(20.0) * 0.5;
score
}
score += (content_count as f64).min(20.0) * 0.5;
score
}
pub fn search_vault(
@@ -126,8 +175,17 @@ pub fn search_vault_with_options(options: SearchOptions<'_>) -> Result<SearchRes
continue;
}
let score = score_match(&title_lower, &content_lower, &query_lower);
let snippet = extract_snippet(&content, &query_lower);
let score = MatchScoreRequest {
title_lower: &title_lower,
content_lower: &content_lower,
query_lower: &query_lower,
}
.score();
let snippet = SnippetRequest {
content: &content,
query_lower: &query_lower,
}
.extract();
let full_path = path.to_string_lossy().to_string();
results.push(SearchResult {
@@ -170,28 +228,49 @@ mod tests {
.unwrap();
}
macro_rules! snippet {
($content:expr, $query_lower:expr) => {
SnippetRequest {
content: $content,
query_lower: $query_lower,
}
.extract()
};
}
macro_rules! match_score {
($title_lower:expr, $content_lower:expr, $query_lower:expr) => {
MatchScoreRequest {
title_lower: $title_lower,
content_lower: $content_lower,
query_lower: $query_lower,
}
.score()
};
}
#[test]
fn test_extract_snippet_basic() {
let content = "line one\nline with keyword here\nline three";
let snippet = extract_snippet(content, "keyword");
let snippet = snippet!(content, "keyword");
assert!(snippet.contains("keyword"));
}
#[test]
fn test_extract_snippet_no_match() {
let snippet = extract_snippet("nothing here", "missing");
let snippet = snippet!("nothing here", "missing");
assert!(snippet.is_empty());
}
#[test]
fn test_score_match_title_word() {
let score = score_match("my keyword", "", "keyword");
let score = match_score!("my keyword", "", "keyword");
assert!(score >= 10.0);
}
#[test]
fn test_score_match_content_only() {
let score = score_match("unrelated", "some keyword text keyword", "keyword");
let score = match_score!("unrelated", "some keyword text keyword", "keyword");
assert!(score > 0.0);
assert!(score < 10.0);
}
@@ -200,10 +279,52 @@ mod tests {
fn test_extract_snippet_long() {
let long_line = "a".repeat(300);
let content = format!("start\n{}keyword{}\nend", long_line, long_line);
let snippet = extract_snippet(&content, "keyword");
let snippet = snippet!(&content, "keyword");
assert!(snippet.len() <= 203); // 200 + "…" (3 bytes UTF-8)
}
#[test]
fn test_extract_snippet_multibyte_context_start() {
let prefix = format!("{}a", "".repeat(21));
let content = format!("{prefix}needle after multibyte prefix");
let snippet = snippet!(&content, "needle");
assert!(snippet.contains("needle"));
assert!(snippet.is_char_boundary(snippet.len()));
}
#[test]
fn test_extract_snippet_multibyte_context_end() {
let content = format!("x{}", "".repeat(50));
let snippet = snippet!(&content, "x");
assert!(snippet.starts_with('x'));
assert!(snippet.is_char_boundary(snippet.len()));
}
#[test]
fn test_extract_snippet_multibyte_truncation() {
let content = format!("key {}\n", "".repeat(100));
let snippet = snippet!(&content, "key");
assert!(snippet.starts_with("key"));
assert!(snippet.ends_with('…'));
assert!(snippet.is_char_boundary(snippet.len()));
}
#[test]
fn test_extract_snippet_maps_expanded_lowercase_to_source_boundary() {
let content = "İstanbul needle";
let snippet = snippet!(content, "i");
assert!(snippet.starts_with("İstanbul"));
assert!(snippet.is_char_boundary(snippet.len()));
}
#[test]
fn test_search_vault_uses_h1_for_result_title() {
let dir = Builder::new()

View File

@@ -162,6 +162,82 @@ mod desktop {
*active = None;
Ok(())
}
#[cfg(test)]
mod tests {
use notify::event::{AccessKind, CreateKind, EventAttributes};
use notify::{Event, EventKind};
use super::*;
fn event(kind: EventKind, paths: &[&str]) -> Event {
Event {
kind,
paths: paths.iter().map(PathBuf::from).collect(),
attrs: EventAttributes::default(),
}
}
#[test]
fn validate_vault_path_accepts_existing_directories_only() {
let dir = tempfile::TempDir::new().unwrap();
assert_eq!(
validate_vault_path(dir.path().to_path_buf()).unwrap(),
dir.path()
);
assert_eq!(
validate_vault_path(PathBuf::new()).unwrap_err(),
"Vault path is required"
);
assert!(validate_vault_path(dir.path().join("missing"))
.unwrap_err()
.contains("Vault path is not a directory"));
}
#[test]
fn changed_paths_ignores_access_events() {
let paths = changed_paths(event(
EventKind::Access(AccessKind::Read),
&["notes/today.md"],
));
assert!(paths.is_empty());
}
#[test]
fn changed_paths_filters_unwatchable_paths() {
let paths = changed_paths(event(
EventKind::Create(CreateKind::File),
&[
".git/index.lock",
"node_modules/pkg/index.js",
"notes/today.md",
],
));
assert_eq!(paths, vec!["notes/today.md"]);
}
#[test]
fn changed_paths_filters_editor_temporary_files() {
let paths = changed_paths(event(
EventKind::Create(CreateKind::File),
&[
".DS_Store",
".tolaria-rename-txn",
".#draft.md",
"draft.md~",
"draft.tmp",
"draft.swp",
"draft.swx",
"notes/keep.md",
],
));
assert_eq!(paths, vec!["notes/keep.md"]);
}
}
}
#[cfg(not(desktop))]

View File

@@ -0,0 +1,52 @@
import { expect, test } from '@playwright/test'
import fs from 'fs'
import path from 'path'
import { createFixtureVaultCopy, openFixtureVaultDesktopHarness, removeFixtureVaultCopy } from '../helpers/fixtureVault'
let tempVaultDir: string
async function showNoteListSearch(page: import('@playwright/test').Page) {
await page.getByTitle('Search notes').click()
await expect(page.getByPlaceholder('Search notes...')).toBeVisible()
}
test.describe('Multibyte note-list search', () => {
test.beforeEach(async ({ page }) => {
tempVaultDir = createFixtureVaultCopy()
const noteDir = path.join(tempVaultDir, 'note')
fs.mkdirSync(noteDir, { recursive: true })
fs.writeFileSync(
path.join(noteDir, 'multibyte-search-boundary.md'),
[
'---',
'Is A: Note',
'---',
'# Multibyte Search Boundary',
'',
`${'한'.repeat(21)}aneedle after multibyte prefix`,
].join('\n'),
'utf-8',
)
await openFixtureVaultDesktopHarness(page, tempVaultDir)
})
test.afterEach(() => {
removeFixtureVaultCopy(tempVaultDir)
})
test('searching content near Korean text keeps the result openable @smoke', async ({ page }) => {
await showNoteListSearch(page)
const noteList = page.getByTestId('note-list-container')
const searchInput = page.getByPlaceholder('Search notes...')
await searchInput.fill('needle')
await expect(page.getByTestId('note-list-search-loading')).toHaveCount(0)
await expect(noteList.getByText('Multibyte Search Boundary', { exact: true })).toBeVisible()
await noteList.getByText('Multibyte Search Boundary', { exact: true }).click()
await expect(page.getByTestId('breadcrumb-filename-trigger')).toContainText(
'multibyte-search-boundary',
)
})
})