fix: vault scanner fallback parser for malformed YAML frontmatter

Files with malformed YAML (e.g. Notion exports containing unescaped
double quotes inside double-quoted strings) caused gray_matter to return
Pod::Null instead of a parsed Hash. This silently reset all frontmatter
fields to defaults, making trashed/archived notes appear in Inbox.

Add a line-by-line fallback parser that extracts simple key:value pairs
when gray_matter fails, so critical fields (Trashed, Archived, type)
are never lost. Bump cache version to 9 to force rescan.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Test
2026-04-01 22:40:25 +02:00
parent ee69e30b6b
commit f84faac7c3
4 changed files with 236 additions and 7 deletions

View File

@@ -8,7 +8,7 @@ use super::{parse_md_file, scan_vault, VaultEntry};
// --- Vault Cache ---
/// Bump this when VaultEntry fields change to force a full rescan.
const CACHE_VERSION: u32 = 8;
const CACHE_VERSION: u32 = 9;
#[derive(Debug, Serialize, Deserialize)]
struct VaultCache {

View File

@@ -284,20 +284,113 @@ fn pod_to_json(pod: gray_matter::Pod) -> serde_json::Value {
}
}
/// Strip matching outer quotes (single or double) from a YAML scalar.
fn unquote(s: &str) -> &str {
s.strip_prefix('"')
.and_then(|rest| rest.strip_suffix('"'))
.or_else(|| s.strip_prefix('\'').and_then(|rest| rest.strip_suffix('\'')))
.unwrap_or(s)
}
/// Parse a scalar YAML value into a JSON value.
fn parse_scalar(s: &str) -> serde_json::Value {
let trimmed = unquote(s);
match trimmed.to_lowercase().as_str() {
"true" | "yes" => serde_json::Value::Bool(true),
"false" | "no" => serde_json::Value::Bool(false),
_ => trimmed
.parse::<i64>()
.map(|n| serde_json::json!(n))
.unwrap_or_else(|_| serde_json::Value::String(trimmed.to_string())),
}
}
/// Return the key from a top-level `key:` or `"key":` YAML line.
/// Returns `None` for indented, blank, or non-key lines.
fn extract_yaml_key(line: &str) -> Option<&str> {
if line.is_empty() || line.starts_with(' ') || line.starts_with('\t') {
return None;
}
let (k, _) = line.split_once(':')?;
Some(k.trim().trim_matches('"'))
}
/// Flush a pending list accumulator into the map.
fn flush_list(
map: &mut HashMap<String, serde_json::Value>,
key: &mut Option<String>,
items: &mut Vec<serde_json::Value>,
) {
if let Some(k) = key.take() {
if !items.is_empty() {
map.insert(k, serde_json::Value::Array(std::mem::take(items)));
}
}
}
/// Fallback parser for when gray_matter fails to parse YAML (returns raw string).
/// Extracts simple `key: value` lines, handling booleans, numbers, quoted strings,
/// and YAML lists.
fn fallback_parse_yaml_string(raw: &str) -> HashMap<String, serde_json::Value> {
let mut map = HashMap::new();
let mut list_key: Option<String> = None;
let mut list_items: Vec<serde_json::Value> = Vec::new();
for line in raw.lines() {
// Accumulate list items under the current key
if list_key.is_some() {
if let Some(item) = line.strip_prefix(" - ") {
list_items.push(parse_scalar(item.trim()));
continue;
}
flush_list(&mut map, &mut list_key, &mut list_items);
}
let Some(key) = extract_yaml_key(line) else { continue };
let value_part = line.split_once(':').map(|(_, v)| v.trim()).unwrap_or("");
if value_part.is_empty() {
list_key = Some(key.to_string());
} else {
map.insert(key.to_string(), parse_scalar(value_part));
}
}
flush_list(&mut map, &mut list_key, &mut list_items);
map
}
/// Extract the raw YAML frontmatter string from between `---` delimiters.
fn extract_raw_frontmatter(content: &str) -> Option<&str> {
let rest = content.strip_prefix("---")?;
let rest = rest.strip_prefix('\n').or_else(|| rest.strip_prefix("\r\n"))?;
let end = rest.find("\n---")?;
Some(&rest[..end])
}
/// Extract frontmatter, relationships, and custom properties from parsed gray_matter data.
/// When gray_matter fails to parse YAML (e.g. malformed quotes from Notion exports),
/// `raw_content` is used as a fallback: simple key:value pairs are extracted line-by-line
/// so that critical fields like Trashed, Archived, type are not silently lost.
pub(crate) fn extract_fm_and_rels(
data: Option<gray_matter::Pod>,
raw_content: &str,
) -> (
Frontmatter,
HashMap<String, Vec<String>>,
HashMap<String, serde_json::Value>,
) {
let hash = match data {
Some(gray_matter::Pod::Hash(map)) => map,
_ => return (Frontmatter::default(), HashMap::new(), HashMap::new()),
let json_map = match data {
Some(gray_matter::Pod::Hash(map)) => {
map.into_iter().map(|(k, v)| (k, pod_to_json(v))).collect()
}
_ => {
// gray_matter returned Null, String, or None — YAML parse failed.
// Fall back to line-by-line extraction from the raw frontmatter block.
match extract_raw_frontmatter(raw_content) {
Some(raw) => fallback_parse_yaml_string(raw),
None => return (Frontmatter::default(), HashMap::new(), HashMap::new()),
}
}
};
let json_map: HashMap<String, serde_json::Value> =
hash.into_iter().map(|(k, v)| (k, pod_to_json(v))).collect();
(
parse_frontmatter(&json_map),
extract_relationships(&json_map),

View File

@@ -45,7 +45,7 @@ pub fn parse_md_file(path: &Path) -> Result<VaultEntry, String> {
let matter = Matter::<YAML>::new();
let parsed = matter.parse(&content);
let (frontmatter, mut relationships, properties) = extract_fm_and_rels(parsed.data);
let (frontmatter, mut relationships, properties) = extract_fm_and_rels(parsed.data, &content);
let title = extract_title(frontmatter.title.as_deref(), &content, &filename);
let snippet = extract_snippet(&content);

View File

@@ -1103,6 +1103,142 @@ fn test_parse_visible_missing_defaults_to_none() {
assert_eq!(entry.visible, None);
}
// --- Regression: trashed/archived must survive unquoted date in "Trashed at" ---
#[test]
fn test_trashed_true_with_unquoted_date_in_trashed_at() {
let dir = TempDir::new().unwrap();
// Reproduces the engineering-management.md scenario: Trashed at has an
// unquoted YAML date (2026-03-11) which gray_matter may parse as a non-string.
// The entire Frontmatter deserialization must NOT fail because of this.
let content = "---\ntype: Topic\nTrashed: true\n\"Trashed at\": 2026-03-11\n---\n# Engineering Management\n";
let entry = parse_test_entry(&dir, "engineering-management.md", content);
assert!(
entry.trashed,
"Trashed must be true even when 'Trashed at' contains an unquoted date"
);
}
#[test]
fn test_archived_true_with_extra_non_string_fields() {
let dir = TempDir::new().unwrap();
// If any StringOrList field gets a non-string value, archived must still parse.
let content = "---\nArchived: true\norder: 5\n---\n# Archived Note\n";
let entry = parse_test_entry(&dir, "archived-extra.md", content);
assert!(
entry.archived,
"Archived must be true even with other fields present"
);
}
#[test]
fn test_trashed_with_reviewed_false_field() {
let dir = TempDir::new().unwrap();
let content = "---\ntype: Topic\nReviewed: False\nTrashed: true\n---\n# Test\n";
let entry = parse_test_entry(&dir, "reviewed-test.md", content);
assert!(
entry.trashed,
"Trashed must be true even when frontmatter contains Reviewed: False"
);
assert_eq!(entry.is_a, Some("Topic".to_string()));
}
/// Regression: wikilinks containing curly braces + nested quotes in YAML arrays
/// cause gray_matter to produce Hash values instead of strings for array elements.
/// This must NOT make parse_frontmatter fall back to default (losing trashed/archived).
#[test]
fn test_trashed_survives_malformed_wikilinks_in_yaml() {
let dir = TempDir::new().unwrap();
// This YAML has curly braces inside a double-quoted string, producing nested
// Hash values in some YAML parsers. The Frontmatter serde must not fail.
let content = "---\ntype: Topic\nNotes:\n - \"[[foo|bar]]\"\n - \"[[slug|{'Title': 'Subtitle'}]]\"\nTrashed: true\n---\n# Test\n";
let entry = parse_test_entry(&dir, "malformed-links.md", content);
assert!(
entry.trashed,
"Trashed must be true even with curly-brace wikilinks in frontmatter arrays"
);
}
/// Regression: files with malformed YAML (e.g. Notion exports with unescaped quotes
/// in wikilinks) cause gray_matter to return Null instead of a Hash. The fallback
/// parser must still extract Trashed, type, and other simple key:value fields.
#[test]
fn test_parse_real_engineering_management_file() {
let path = std::path::Path::new("/Users/luca/Laputa/engineering-management.md");
if !path.exists() {
return; // Skip when the Laputa vault is not available
}
let entry = parse_md_file(path).unwrap();
assert!(
entry.trashed,
"engineering-management.md must be trashed (has Trashed: true in frontmatter)"
);
assert_eq!(entry.is_a, Some("Topic".to_string()));
}
#[test]
fn test_fallback_parser_extracts_trashed_from_malformed_yaml() {
let dir = TempDir::new().unwrap();
// Simulate malformed YAML that gray_matter can't parse: unescaped double
// quotes inside a double-quoted YAML string cause the YAML parser to fail.
// The fallback line-by-line parser must still extract simple key:value pairs.
//
// Write the file manually with literal unescaped quotes (can't use Rust string
// escaping for this since the YAML itself is the malformed part).
let fm = [
"---",
"type: Topic",
"Status: Draft",
"Belongs to:",
" - \"[[engineering|Engineering]]\"",
"aliases:",
" - Engineering Management",
"Notes:",
// This line has unescaped " inside a "-quoted YAML string — malformed YAML
" - \"[[slug|{\"Title\": 'Subtitle'}]]\"",
"Trashed: true",
"\"Trashed at\": 2026-03-11",
"---",
"",
"# Engineering Management",
];
let content = fm.join("\n");
create_test_file(dir.path(), "eng-mgmt.md", &content);
let entry = parse_md_file(&dir.path().join("eng-mgmt.md")).unwrap();
assert!(
entry.trashed,
"Trashed must be true even when YAML is malformed (fallback parser)"
);
assert_eq!(
entry.is_a,
Some("Topic".to_string()),
"isA must be extracted by fallback parser"
);
}
#[test]
fn test_fallback_parser_extracts_archived_from_malformed_yaml() {
let dir = TempDir::new().unwrap();
let fm = [
"---",
"type: Essay",
"Notes:",
" - \"[[slug|{\"Broken\": 'quotes'}]]\"",
"Archived: true",
"---",
"",
"# Archived Essay",
];
let content = fm.join("\n");
create_test_file(dir.path(), "archived-essay.md", &content);
let entry = parse_md_file(&dir.path().join("archived-essay.md")).unwrap();
assert!(
entry.archived,
"Archived must be true even when YAML is malformed"
);
assert_eq!(entry.is_a, Some("Essay".to_string()));
}
#[test]
fn test_visible_not_in_relationships() {
let dir = TempDir::new().unwrap();