fix: vault scanner fallback parser for malformed YAML frontmatter
Files with malformed YAML (e.g. Notion exports containing unescaped double quotes inside double-quoted strings) caused gray_matter to return Pod::Null instead of a parsed Hash. This silently reset all frontmatter fields to defaults, making trashed/archived notes appear in Inbox. Add a line-by-line fallback parser that extracts simple key:value pairs when gray_matter fails, so critical fields (Trashed, Archived, type) are never lost. Bump cache version to 9 to force rescan. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,7 @@ use super::{parse_md_file, scan_vault, VaultEntry};
|
||||
// --- Vault Cache ---
|
||||
|
||||
/// Bump this when VaultEntry fields change to force a full rescan.
|
||||
const CACHE_VERSION: u32 = 8;
|
||||
const CACHE_VERSION: u32 = 9;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct VaultCache {
|
||||
|
||||
@@ -284,20 +284,113 @@ fn pod_to_json(pod: gray_matter::Pod) -> serde_json::Value {
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip matching outer quotes (single or double) from a YAML scalar.
|
||||
fn unquote(s: &str) -> &str {
|
||||
s.strip_prefix('"')
|
||||
.and_then(|rest| rest.strip_suffix('"'))
|
||||
.or_else(|| s.strip_prefix('\'').and_then(|rest| rest.strip_suffix('\'')))
|
||||
.unwrap_or(s)
|
||||
}
|
||||
|
||||
/// Parse a scalar YAML value into a JSON value.
|
||||
fn parse_scalar(s: &str) -> serde_json::Value {
|
||||
let trimmed = unquote(s);
|
||||
match trimmed.to_lowercase().as_str() {
|
||||
"true" | "yes" => serde_json::Value::Bool(true),
|
||||
"false" | "no" => serde_json::Value::Bool(false),
|
||||
_ => trimmed
|
||||
.parse::<i64>()
|
||||
.map(|n| serde_json::json!(n))
|
||||
.unwrap_or_else(|_| serde_json::Value::String(trimmed.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the key from a top-level `key:` or `"key":` YAML line.
|
||||
/// Returns `None` for indented, blank, or non-key lines.
|
||||
fn extract_yaml_key(line: &str) -> Option<&str> {
|
||||
if line.is_empty() || line.starts_with(' ') || line.starts_with('\t') {
|
||||
return None;
|
||||
}
|
||||
let (k, _) = line.split_once(':')?;
|
||||
Some(k.trim().trim_matches('"'))
|
||||
}
|
||||
|
||||
/// Flush a pending list accumulator into the map.
|
||||
fn flush_list(
|
||||
map: &mut HashMap<String, serde_json::Value>,
|
||||
key: &mut Option<String>,
|
||||
items: &mut Vec<serde_json::Value>,
|
||||
) {
|
||||
if let Some(k) = key.take() {
|
||||
if !items.is_empty() {
|
||||
map.insert(k, serde_json::Value::Array(std::mem::take(items)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fallback parser for when gray_matter fails to parse YAML (returns raw string).
|
||||
/// Extracts simple `key: value` lines, handling booleans, numbers, quoted strings,
|
||||
/// and YAML lists.
|
||||
fn fallback_parse_yaml_string(raw: &str) -> HashMap<String, serde_json::Value> {
|
||||
let mut map = HashMap::new();
|
||||
let mut list_key: Option<String> = None;
|
||||
let mut list_items: Vec<serde_json::Value> = Vec::new();
|
||||
|
||||
for line in raw.lines() {
|
||||
// Accumulate list items under the current key
|
||||
if list_key.is_some() {
|
||||
if let Some(item) = line.strip_prefix(" - ") {
|
||||
list_items.push(parse_scalar(item.trim()));
|
||||
continue;
|
||||
}
|
||||
flush_list(&mut map, &mut list_key, &mut list_items);
|
||||
}
|
||||
|
||||
let Some(key) = extract_yaml_key(line) else { continue };
|
||||
let value_part = line.split_once(':').map(|(_, v)| v.trim()).unwrap_or("");
|
||||
if value_part.is_empty() {
|
||||
list_key = Some(key.to_string());
|
||||
} else {
|
||||
map.insert(key.to_string(), parse_scalar(value_part));
|
||||
}
|
||||
}
|
||||
flush_list(&mut map, &mut list_key, &mut list_items);
|
||||
map
|
||||
}
|
||||
|
||||
/// Extract the raw YAML frontmatter string from between `---` delimiters.
|
||||
fn extract_raw_frontmatter(content: &str) -> Option<&str> {
|
||||
let rest = content.strip_prefix("---")?;
|
||||
let rest = rest.strip_prefix('\n').or_else(|| rest.strip_prefix("\r\n"))?;
|
||||
let end = rest.find("\n---")?;
|
||||
Some(&rest[..end])
|
||||
}
|
||||
|
||||
/// Extract frontmatter, relationships, and custom properties from parsed gray_matter data.
|
||||
/// When gray_matter fails to parse YAML (e.g. malformed quotes from Notion exports),
|
||||
/// `raw_content` is used as a fallback: simple key:value pairs are extracted line-by-line
|
||||
/// so that critical fields like Trashed, Archived, type are not silently lost.
|
||||
pub(crate) fn extract_fm_and_rels(
|
||||
data: Option<gray_matter::Pod>,
|
||||
raw_content: &str,
|
||||
) -> (
|
||||
Frontmatter,
|
||||
HashMap<String, Vec<String>>,
|
||||
HashMap<String, serde_json::Value>,
|
||||
) {
|
||||
let hash = match data {
|
||||
Some(gray_matter::Pod::Hash(map)) => map,
|
||||
_ => return (Frontmatter::default(), HashMap::new(), HashMap::new()),
|
||||
let json_map = match data {
|
||||
Some(gray_matter::Pod::Hash(map)) => {
|
||||
map.into_iter().map(|(k, v)| (k, pod_to_json(v))).collect()
|
||||
}
|
||||
_ => {
|
||||
// gray_matter returned Null, String, or None — YAML parse failed.
|
||||
// Fall back to line-by-line extraction from the raw frontmatter block.
|
||||
match extract_raw_frontmatter(raw_content) {
|
||||
Some(raw) => fallback_parse_yaml_string(raw),
|
||||
None => return (Frontmatter::default(), HashMap::new(), HashMap::new()),
|
||||
}
|
||||
}
|
||||
};
|
||||
let json_map: HashMap<String, serde_json::Value> =
|
||||
hash.into_iter().map(|(k, v)| (k, pod_to_json(v))).collect();
|
||||
(
|
||||
parse_frontmatter(&json_map),
|
||||
extract_relationships(&json_map),
|
||||
|
||||
@@ -45,7 +45,7 @@ pub fn parse_md_file(path: &Path) -> Result<VaultEntry, String> {
|
||||
|
||||
let matter = Matter::<YAML>::new();
|
||||
let parsed = matter.parse(&content);
|
||||
let (frontmatter, mut relationships, properties) = extract_fm_and_rels(parsed.data);
|
||||
let (frontmatter, mut relationships, properties) = extract_fm_and_rels(parsed.data, &content);
|
||||
|
||||
let title = extract_title(frontmatter.title.as_deref(), &content, &filename);
|
||||
let snippet = extract_snippet(&content);
|
||||
|
||||
@@ -1103,6 +1103,142 @@ fn test_parse_visible_missing_defaults_to_none() {
|
||||
assert_eq!(entry.visible, None);
|
||||
}
|
||||
|
||||
// --- Regression: trashed/archived must survive unquoted date in "Trashed at" ---
|
||||
|
||||
#[test]
|
||||
fn test_trashed_true_with_unquoted_date_in_trashed_at() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Reproduces the engineering-management.md scenario: Trashed at has an
|
||||
// unquoted YAML date (2026-03-11) which gray_matter may parse as a non-string.
|
||||
// The entire Frontmatter deserialization must NOT fail because of this.
|
||||
let content = "---\ntype: Topic\nTrashed: true\n\"Trashed at\": 2026-03-11\n---\n# Engineering Management\n";
|
||||
let entry = parse_test_entry(&dir, "engineering-management.md", content);
|
||||
assert!(
|
||||
entry.trashed,
|
||||
"Trashed must be true even when 'Trashed at' contains an unquoted date"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archived_true_with_extra_non_string_fields() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// If any StringOrList field gets a non-string value, archived must still parse.
|
||||
let content = "---\nArchived: true\norder: 5\n---\n# Archived Note\n";
|
||||
let entry = parse_test_entry(&dir, "archived-extra.md", content);
|
||||
assert!(
|
||||
entry.archived,
|
||||
"Archived must be true even with other fields present"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_trashed_with_reviewed_false_field() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let content = "---\ntype: Topic\nReviewed: False\nTrashed: true\n---\n# Test\n";
|
||||
let entry = parse_test_entry(&dir, "reviewed-test.md", content);
|
||||
assert!(
|
||||
entry.trashed,
|
||||
"Trashed must be true even when frontmatter contains Reviewed: False"
|
||||
);
|
||||
assert_eq!(entry.is_a, Some("Topic".to_string()));
|
||||
}
|
||||
|
||||
/// Regression: wikilinks containing curly braces + nested quotes in YAML arrays
|
||||
/// cause gray_matter to produce Hash values instead of strings for array elements.
|
||||
/// This must NOT make parse_frontmatter fall back to default (losing trashed/archived).
|
||||
#[test]
|
||||
fn test_trashed_survives_malformed_wikilinks_in_yaml() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// This YAML has curly braces inside a double-quoted string, producing nested
|
||||
// Hash values in some YAML parsers. The Frontmatter serde must not fail.
|
||||
let content = "---\ntype: Topic\nNotes:\n - \"[[foo|bar]]\"\n - \"[[slug|{'Title': 'Subtitle'}]]\"\nTrashed: true\n---\n# Test\n";
|
||||
let entry = parse_test_entry(&dir, "malformed-links.md", content);
|
||||
assert!(
|
||||
entry.trashed,
|
||||
"Trashed must be true even with curly-brace wikilinks in frontmatter arrays"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression: files with malformed YAML (e.g. Notion exports with unescaped quotes
|
||||
/// in wikilinks) cause gray_matter to return Null instead of a Hash. The fallback
|
||||
/// parser must still extract Trashed, type, and other simple key:value fields.
|
||||
#[test]
|
||||
fn test_parse_real_engineering_management_file() {
|
||||
let path = std::path::Path::new("/Users/luca/Laputa/engineering-management.md");
|
||||
if !path.exists() {
|
||||
return; // Skip when the Laputa vault is not available
|
||||
}
|
||||
let entry = parse_md_file(path).unwrap();
|
||||
assert!(
|
||||
entry.trashed,
|
||||
"engineering-management.md must be trashed (has Trashed: true in frontmatter)"
|
||||
);
|
||||
assert_eq!(entry.is_a, Some("Topic".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fallback_parser_extracts_trashed_from_malformed_yaml() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Simulate malformed YAML that gray_matter can't parse: unescaped double
|
||||
// quotes inside a double-quoted YAML string cause the YAML parser to fail.
|
||||
// The fallback line-by-line parser must still extract simple key:value pairs.
|
||||
//
|
||||
// Write the file manually with literal unescaped quotes (can't use Rust string
|
||||
// escaping for this since the YAML itself is the malformed part).
|
||||
let fm = [
|
||||
"---",
|
||||
"type: Topic",
|
||||
"Status: Draft",
|
||||
"Belongs to:",
|
||||
" - \"[[engineering|Engineering]]\"",
|
||||
"aliases:",
|
||||
" - Engineering Management",
|
||||
"Notes:",
|
||||
// This line has unescaped " inside a "-quoted YAML string — malformed YAML
|
||||
" - \"[[slug|{\"Title\": 'Subtitle'}]]\"",
|
||||
"Trashed: true",
|
||||
"\"Trashed at\": 2026-03-11",
|
||||
"---",
|
||||
"",
|
||||
"# Engineering Management",
|
||||
];
|
||||
let content = fm.join("\n");
|
||||
create_test_file(dir.path(), "eng-mgmt.md", &content);
|
||||
let entry = parse_md_file(&dir.path().join("eng-mgmt.md")).unwrap();
|
||||
assert!(
|
||||
entry.trashed,
|
||||
"Trashed must be true even when YAML is malformed (fallback parser)"
|
||||
);
|
||||
assert_eq!(
|
||||
entry.is_a,
|
||||
Some("Topic".to_string()),
|
||||
"isA must be extracted by fallback parser"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fallback_parser_extracts_archived_from_malformed_yaml() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let fm = [
|
||||
"---",
|
||||
"type: Essay",
|
||||
"Notes:",
|
||||
" - \"[[slug|{\"Broken\": 'quotes'}]]\"",
|
||||
"Archived: true",
|
||||
"---",
|
||||
"",
|
||||
"# Archived Essay",
|
||||
];
|
||||
let content = fm.join("\n");
|
||||
create_test_file(dir.path(), "archived-essay.md", &content);
|
||||
let entry = parse_md_file(&dir.path().join("archived-essay.md")).unwrap();
|
||||
assert!(
|
||||
entry.archived,
|
||||
"Archived must be true even when YAML is malformed"
|
||||
);
|
||||
assert_eq!(entry.is_a, Some("Essay".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_visible_not_in_relationships() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user