Merge branch 'main' into fix/view_filters

docs/ABSTRACTIONS.md

@@ -632,7 +632,7 @@ Defined in `src/components/tolariaEditorFormatting.tsx` and `src/components/tola
 ```mermaid
 flowchart LR
     A["📄 Raw markdown\n(from disk)"] --> B["splitFrontmatter()\n→ yaml + body"]
-    B --> C["preProcessDurableEditorMarkdown(body)\nmermaid/tldraw fences → tokens"]
+    B --> C["preProcessDurableEditorMarkdown(body)\nmermaid/tldraw fences + file links → tokens"]
     C --> D["preProcessWikilinks(body)\n[[target]] → ‹token›"]
     D --> E["preProcessMathMarkdown(body)\n$...$ / $$...$$ → tokens"]
     E --> F["tryParseMarkdownToBlocks()\n→ BlockNote block tree"]
@@ -643,7 +643,7 @@ flowchart LR
     style H fill:#d4edda,stroke:#28a745,color:#000
 ```
 
-> Wikilink placeholder tokens use `\u2039` and `\u203A`; math, Mermaid, and tldraw placeholder tokens use ASCII sentinels with URI-encoded payloads.
+> Wikilink placeholder tokens use `\u2039` and `\u203A`; math, Mermaid, tldraw, and standalone file-attachment link placeholders use ASCII sentinels with URI-encoded payloads.
 
 ### BlockNote-to-Markdown Pipeline (Save)
 
@@ -658,7 +658,7 @@ flowchart LR
     style E fill:#d4edda,stroke:#28a745,color:#000
 ```
 
-Rich-editor change events are coalesced before this serialization runs. `useEditorTabSwap` keeps the latest BlockNote state in the editor, schedules one Markdown serialization for a short idle window, and exposes an explicit flush hook for save, note switch, raw-mode entry, and destructive note actions. `src/utils/richEditorMarkdown.ts` is the shared BlockNote-to-Markdown owner for autosave/tab-swap and raw-mode entry, so wikilink restoration, durable schema-node serialization, frontmatter preservation, and portable attachment paths cannot drift between editor modes. This keeps long notes from paying full-document Markdown serialization on every keystroke while preserving the disk-first save path.
+Rich-editor change events are coalesced before this serialization runs. `useEditorTabSwap` keeps the latest BlockNote state in the editor, schedules one Markdown serialization for a short idle window, and exposes an explicit flush hook for save, note switch, raw-mode entry, and destructive note actions. `src/utils/richEditorMarkdown.ts` is the shared BlockNote-to-Markdown owner for autosave/tab-swap and raw-mode entry, so wikilink restoration, durable schema-node serialization, frontmatter preservation, file-attachment block round-tripping, and portable attachment paths cannot drift between editor modes. This keeps long notes from paying full-document Markdown serialization on every keystroke while preserving the disk-first save path.
 
 Autosave then waits for a 1.5s idle window before invoking `save_note_content`. If an older save resolves after the user has already typed newer content, the older save is treated as stale and cannot clear the newer pending buffer or repaint tab state over it; the latest pending content remains scheduled for its own save.
 

src/utils/editorDurableMarkdown.ts

@@ -5,6 +5,12 @@ import {
   serializeDurableMarkdownBlocks,
   type MarkdownSerializer,
 } from './durableMarkdownBlocks'
+import {
+  hasFileAttachmentBlocks,
+  injectFileAttachmentBlocks,
+  preProcessFileAttachmentMarkdown,
+  serializeFileAttachmentBlocks,
+} from './fileAttachmentMarkdown'
 import { serializeMathAwareBlocks } from './mathMarkdown'
 import { mermaidMarkdownCodec } from './mermaidMarkdown'
 import { tldrawMarkdownCodec } from './tldrawMarkdown'
@@ -15,29 +21,34 @@ const EDITOR_DURABLE_MARKDOWN_CODECS = [
 ] as const
 
 export function preProcessDurableEditorMarkdown({ markdown }: { markdown: string }): string {
-  return preProcessDurableMarkdownBlocks({
+  const withDurableBlocks = preProcessDurableMarkdownBlocks({
     markdown,
     codecs: EDITOR_DURABLE_MARKDOWN_CODECS,
   })
+  return preProcessFileAttachmentMarkdown({ markdown: withDurableBlocks })
 }
 
 export function injectDurableEditorMarkdownBlocks(blocks: unknown[]): unknown[] {
-  return injectDurableMarkdownBlocks({
+  const withDurableBlocks = injectDurableMarkdownBlocks({
     blocks,
     codecs: EDITOR_DURABLE_MARKDOWN_CODECS,
   })
+  return injectFileAttachmentBlocks(withDurableBlocks)
 }
 
 export function serializeDurableEditorBlocks(editor: MarkdownSerializer, blocks: unknown[]): string {
-  return serializeDurableMarkdownBlocks({
+  return serializeFileAttachmentBlocks({
     blocks,
-    codecs: EDITOR_DURABLE_MARKDOWN_CODECS,
-    serializeOrdinaryBlocks: ordinaryBlocks => serializeMathAwareBlocks(editor, ordinaryBlocks),
+    serializeOrdinaryBlocks: ordinaryBlocks => serializeDurableMarkdownBlocks({
+      blocks: ordinaryBlocks,
+      codecs: EDITOR_DURABLE_MARKDOWN_CODECS,
+      serializeOrdinaryBlocks: durableOrdinaryBlocks => serializeMathAwareBlocks(editor, durableOrdinaryBlocks),
+    }),
   })
 }
 
 export function hasDurableEditorBlocks(blocks: unknown[]): boolean {
-  return hasDurableMarkdownBlocks({
+  return hasFileAttachmentBlocks(blocks) || hasDurableMarkdownBlocks({
     blocks,
     codecs: EDITOR_DURABLE_MARKDOWN_CODECS,
   })

src/utils/fileAttachmentMarkdown.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+  injectDurableEditorMarkdownBlocks,
+  preProcessDurableEditorMarkdown,
+} from './editorDurableMarkdown'
+import { serializeRichEditorDocumentToMarkdown } from './richEditorMarkdown'
+
+function makeEditor(document: unknown[]) {
+  return {
+    document,
+    blocksToMarkdownLossy: vi.fn(() => ''),
+  }
+}
+
+function parsePreprocessedParagraph(markdown: string) {
+  return [{
+    type: 'paragraph',
+    content: [{ type: 'text', text: markdown.trim(), styles: {} }],
+    children: [],
+  }]
+}
+
+describe('file attachment Markdown roundtrip', () => {
+  it('serializes file blocks as portable attachment links', () => {
+    const editor = makeEditor([{
+      type: 'file',
+      props: {
+        name: 'report.pdf',
+        url: 'asset://localhost/%2Fvault%2Fattachments%2Freport.pdf',
+      },
+      children: [],
+    }])
+
+    expect(serializeRichEditorDocumentToMarkdown(
+      editor as never,
+      '---\ntitle: Note A\n---\n',
+      '/vault',
+      'a.md',
+    )).toBe('---\ntitle: Note A\n---\n[report.pdf](attachments/report.pdf)\n')
+  })
+
+  it('rebuilds file blocks from standalone attachment links', () => {
+    const preprocessed = preProcessDurableEditorMarkdown({
+      markdown: '[report.pdf](attachments/report.pdf)\n',
+    })
+
+    expect(injectDurableEditorMarkdownBlocks(parsePreprocessedParagraph(preprocessed))).toEqual([
+      expect.objectContaining({
+        type: 'file',
+        props: expect.objectContaining({
+          name: 'report.pdf',
+          url: 'attachments/report.pdf',
+        }),
+      }),
+    ])
+  })
+})

src/utils/fileAttachmentMarkdown.ts

@@ -0,0 +1,368 @@
+import {
+  lineEnding,
+  lineText,
+  type BlockLike,
+} from './durableMarkdownBlocks'
+import {
+  isVaultAttachmentUrl,
+  portableAttachmentPathFromCurrentVaultAssetUrl,
+} from './vaultAttachments'
+
+interface FileAttachmentPayload {
+  name: string
+  url: string
+  caption?: string
+}
+
+interface SerializeFileAttachmentBlocksOptions {
+  blocks: unknown[]
+  serializeOrdinaryBlocks: (blocks: unknown[]) => string
+}
+
+type FileAttachmentLineTransform = (payload: FileAttachmentPayload) => string | null
+type MarkdownFence = { character: string; length: number }
+type AttachmentUrl = string
+type Markdown = string
+type MarkdownLine = string
+type MarkdownText = string
+type VaultPath = string
+
+interface PreProcessFileAttachmentMarkdownOptions {
+  markdown: Markdown
+}
+
+interface TransformFileAttachmentLinksOptions {
+  markdown: Markdown
+  transform: FileAttachmentLineTransform
+}
+
+interface TransformLineOptions {
+  fence: MarkdownFence | null
+  line: MarkdownLine
+  transform: FileAttachmentLineTransform
+}
+
+interface TransformedLine {
+  fence: MarkdownFence | null
+  line: MarkdownLine
+}
+
+const FILE_ATTACHMENT_TOKEN_PREFIX = '@@TOLARIA_FILE_ATTACHMENT:'
+const FILE_ATTACHMENT_TOKEN_SUFFIX = '@@'
+const STANDALONE_ATTACHMENT_LINK_PATTERN = /^( {0,3})\[((?:\\.|[^\]\\\n])*)\]\((<[^>\n]+>|(?:\\.|[^)\s\n])+)(?:[ \t]+"((?:\\.|[^"\\\n])*)")?\)[ \t]*$/u
+
+function splitMarkdownLines(markdown: Markdown): MarkdownLine[] {
+  const lines = markdown.match(/[^\n]*(?:\n|$)/g) ?? []
+  return lines.filter((line, index) => line !== '' || index < lines.length - 1)
+}
+
+function safeDecode(value: MarkdownText): MarkdownText {
+  try {
+    return decodeURIComponent(value)
+  } catch {
+    return value
+  }
+}
+
+function escapeMarkdownLabel(value: MarkdownText): MarkdownText {
+  return value.replace(/\\/g, '\\\\').replace(/\]/g, '\\]')
+}
+
+function escapeMarkdownTitle(value: MarkdownText): MarkdownText {
+  return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
+}
+
+function escapeMarkdownDestination(value: AttachmentUrl): MarkdownText {
+  if (/[\s<>]/u.test(value)) return `<${value.replace(/>/g, '%3E')}>`
+  return value.replace(/\\/g, '\\\\').replace(/\)/g, '\\)')
+}
+
+function unescapeMarkdownText(value: MarkdownText): MarkdownText {
+  return value.replace(/\\([\\\]"'])/g, '$1')
+}
+
+function unescapeMarkdownDestination(value: MarkdownText): AttachmentUrl {
+  const withoutAngles = value.startsWith('<') && value.endsWith('>')
+    ? value.slice(1, -1)
+    : value
+  return withoutAngles.replace(/\\([\\()<>])/g, '$1')
+}
+
+function fileNameFromUrl(url: AttachmentUrl): MarkdownText {
+  const path = safeDecode(url.split(/[?#]/u)[0] ?? url)
+  return path.split(/[\\/]/u).filter(Boolean).at(-1) ?? url
+}
+
+function serializeMarkdownFileLink(payload: FileAttachmentPayload): Markdown {
+  const title = payload.caption ? ` "${escapeMarkdownTitle(payload.caption)}"` : ''
+  return `[${escapeMarkdownLabel(payload.name)}](${escapeMarkdownDestination(payload.url)}${title})`
+}
+
+function encodePayload(payload: FileAttachmentPayload): MarkdownText {
+  return encodeURIComponent(JSON.stringify(payload))
+}
+
+function fileAttachmentToken(payload: FileAttachmentPayload): MarkdownText {
+  return `${FILE_ATTACHMENT_TOKEN_PREFIX}${encodePayload(payload)}${FILE_ATTACHMENT_TOKEN_SUFFIX}`
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function readNonEmptyText(value: unknown): MarkdownText | null {
+  return typeof value === 'string' && value.trim() ? value : null
+}
+
+function readAttachmentUrl(value: unknown): AttachmentUrl | null {
+  const url = readNonEmptyText(value)
+  return url && isVaultAttachmentUrl({ url }) ? url : null
+}
+
+function readPayloadName(record: Record<string, unknown>, url: AttachmentUrl): MarkdownText {
+  return readNonEmptyText(record.name) ?? fileNameFromUrl(url)
+}
+
+function readOptionalCaption(value: unknown): MarkdownText | undefined {
+  return typeof value === 'string' && value ? value : undefined
+}
+
+function fileAttachmentPayload(payload: FileAttachmentPayload): FileAttachmentPayload {
+  return payload.caption
+    ? payload
+    : { name: payload.name, url: payload.url }
+}
+
+function normalizePayload(value: unknown): FileAttachmentPayload | null {
+  if (!isRecord(value)) return null
+
+  const url = readAttachmentUrl(value.url)
+  if (!url) return null
+
+  return fileAttachmentPayload({
+    name: readPayloadName(value, url),
+    url,
+    caption: readOptionalCaption(value.caption),
+  })
+}
+
+function readFileAttachmentToken(text: MarkdownText): FileAttachmentPayload | null {
+  const trimmed = text.trim()
+  if (!trimmed.startsWith(FILE_ATTACHMENT_TOKEN_PREFIX)) return null
+  if (!trimmed.endsWith(FILE_ATTACHMENT_TOKEN_SUFFIX)) return null
+
+  const encoded = trimmed.slice(
+    FILE_ATTACHMENT_TOKEN_PREFIX.length,
+    -FILE_ATTACHMENT_TOKEN_SUFFIX.length,
+  )
+  try {
+    return normalizePayload(JSON.parse(decodeURIComponent(encoded)))
+  } catch {
+    return null
+  }
+}
+
+function readStandaloneFileAttachmentLink(text: MarkdownText): FileAttachmentPayload | null {
+  const match = STANDALONE_ATTACHMENT_LINK_PATTERN.exec(text)
+  if (!match) return null
+
+  const url = readAttachmentUrl(unescapeMarkdownDestination(match[3] ?? ''))
+  if (!url) return null
+
+  const name = unescapeMarkdownText(match[2] ?? '') || fileNameFromUrl(url)
+  const caption = match[4] ? unescapeMarkdownText(match[4]) : undefined
+  return fileAttachmentPayload({ name, url, caption })
+}
+
+function readFenceOpening(text: MarkdownText): MarkdownFence | null {
+  const match = /^( {0,3})(`{3,}|~{3,})/u.exec(text)
+  const fence = match?.[2]
+  return fence ? { character: fence.charAt(0), length: fence.length } : null
+}
+
+function isFenceClosing(text: MarkdownText, fence: MarkdownFence): boolean {
+  const match = /^( {0,3})(`{3,}|~{3,})[ \t]*$/u.exec(text)
+  const closing = match?.[2]
+  return !!closing && closing.charAt(0) === fence.character && closing.length >= fence.length
+}
+
+function transformAttachmentLine(
+  line: MarkdownLine,
+  transform: FileAttachmentLineTransform,
+): MarkdownLine {
+  const payload = readStandaloneFileAttachmentLink(lineText({ line }))
+  if (!payload) return line
+
+  const transformed = transform(payload)
+  return transformed ? `${transformed}${lineEnding({ line })}` : line
+}
+
+function transformFencedLine(line: MarkdownLine, text: MarkdownText, fence: MarkdownFence): TransformedLine {
+  return {
+    line,
+    fence: isFenceClosing(text, fence) ? null : fence,
+  }
+}
+
+function transformUnfencedLine(
+  line: MarkdownLine,
+  text: MarkdownText,
+  transform: FileAttachmentLineTransform,
+): TransformedLine {
+  const opening = readFenceOpening(text)
+  if (opening) {
+    return {
+      line,
+      fence: opening,
+    }
+  }
+
+  const transformedLine = transformAttachmentLine(line, transform)
+  return {
+    line: transformedLine,
+    fence: null,
+  }
+}
+
+function transformLine(options: TransformLineOptions): TransformedLine {
+  const text = lineText({ line: options.line })
+  if (options.fence) {
+    return transformFencedLine(options.line, text, options.fence)
+  }
+
+  return transformUnfencedLine(options.line, text, options.transform)
+}
+
+function transformStandaloneFileAttachmentLinks(options: TransformFileAttachmentLinksOptions): Markdown {
+  const lines: MarkdownLine[] = []
+  let fence: MarkdownFence | null = null
+
+  for (const line of splitMarkdownLines(options.markdown)) {
+    const transformed = transformLine({ line, fence, transform: options.transform })
+    lines.push(transformed.line)
+    fence = transformed.fence
+  }
+
+  return lines.join('')
+}
+
+function readTokenPayload(block: BlockLike): FileAttachmentPayload | null {
+  const content = block.content
+  if (!Array.isArray(content) || content.length !== 1) return null
+
+  const item = content[0]
+  if (item?.type !== 'text' || typeof item.text !== 'string') return null
+  return readFileAttachmentToken(item.text)
+}
+
+function buildFileAttachmentBlock(block: BlockLike, payload: FileAttachmentPayload): BlockLike {
+  return {
+    ...block,
+    type: 'file',
+    props: {
+      ...block.props,
+      backgroundColor: block.props?.backgroundColor ?? 'default',
+      name: payload.name,
+      url: payload.url,
+      caption: payload.caption ?? '',
+    },
+    content: undefined,
+    children: [],
+  }
+}
+
+function injectFileAttachmentBlock(block: BlockLike): BlockLike {
+  const payload = readTokenPayload(block)
+  if (payload) return buildFileAttachmentBlock(block, payload)
+
+  const children = Array.isArray(block.children)
+    ? block.children.map(injectFileAttachmentBlock)
+    : block.children
+  return { ...block, children }
+}
+
+export function injectFileAttachmentBlocks(blocks: unknown[]): unknown[] {
+  return (blocks as BlockLike[]).map(injectFileAttachmentBlock)
+}
+
+function readBlockAttachmentUrl(block: BlockLike): AttachmentUrl | null {
+  return block.type === 'file' ? readAttachmentUrl(block.props?.url) : null
+}
+
+function readBlockAttachmentName(block: BlockLike, url: AttachmentUrl): MarkdownText {
+  return block.props?.name?.trim() || fileNameFromUrl(url)
+}
+
+function readFileAttachmentBlockPayload(block: BlockLike): FileAttachmentPayload | null {
+  const url = readBlockAttachmentUrl(block)
+  if (!url) return null
+
+  return fileAttachmentPayload({
+    name: readBlockAttachmentName(block, url),
+    url,
+    caption: block.props?.caption?.trim(),
+  })
+}
+
+export function serializeFileAttachmentBlocks({
+  blocks,
+  serializeOrdinaryBlocks,
+}: SerializeFileAttachmentBlocksOptions): string {
+  const chunks: string[] = []
+  let pending: unknown[] = []
+
+  const flushPending = () => {
+    if (pending.length === 0) return
+
+    const markdown = serializeOrdinaryBlocks(pending).trimEnd()
+    if (markdown) chunks.push(markdown)
+    pending = []
+  }
+
+  for (const block of blocks as BlockLike[]) {
+    const payload = readFileAttachmentBlockPayload(block)
+    if (!payload) {
+      pending.push(block)
+      continue
+    }
+
+    flushPending()
+    chunks.push(serializeMarkdownFileLink(payload))
+  }
+
+  flushPending()
+  return chunks.join('\n\n')
+}
+
+function hasFileAttachmentBlock(block: BlockLike): boolean {
+  if (readFileAttachmentBlockPayload(block)) return true
+  return Array.isArray(block.children)
+    ? block.children.some(hasFileAttachmentBlock)
+    : false
+}
+
+export function hasFileAttachmentBlocks(blocks: unknown[]): boolean {
+  return (blocks as BlockLike[]).some(hasFileAttachmentBlock)
+}
+
+function portableAttachmentUrl(payload: FileAttachmentPayload, vaultPath: VaultPath): AttachmentUrl | null {
+  return portableAttachmentPathFromCurrentVaultAssetUrl({ url: payload.url, vaultPath })
+}
+
+export function portableFileAttachmentUrls(markdown: Markdown, vaultPath: VaultPath): Markdown {
+  if (!vaultPath) return markdown
+
+  return transformStandaloneFileAttachmentLinks({ markdown, transform: (payload) => {
+    const url = portableAttachmentUrl(payload, vaultPath)
+    return url ? serializeMarkdownFileLink({ ...payload, url }) : null
+  } })
+}
+
+export function preProcessFileAttachmentMarkdown(
+  options: PreProcessFileAttachmentMarkdownOptions,
+): Markdown {
+  return transformStandaloneFileAttachmentLinks({
+    markdown: options.markdown,
+    transform: fileAttachmentToken,
+  })
+}

src/utils/richEditorMarkdown.ts

@@ -1,6 +1,7 @@
 import type { useCreateBlockNote } from '@blocknote/react'
 import { compactMarkdown } from './compact-markdown'
 import { serializeDurableEditorBlocks } from './editorDurableMarkdown'
+import { portableFileAttachmentUrls } from './fileAttachmentMarkdown'
 import { portableImageUrls } from './vaultImages'
 import { restoreWikilinksInBlocks, splitFrontmatter } from './wikilinks'
 
@@ -19,7 +20,10 @@ export function serializeRichEditorDocumentToMarkdown(
 ): string {
   const rawBodyMarkdown = serializeRichEditorBodyToMarkdown(editor)
   const bodyMarkdown = vaultPath
-    ? portableImageUrls(rawBodyMarkdown, vaultPath, notePath)
+    ? portableFileAttachmentUrls(
+      portableImageUrls(rawBodyMarkdown, vaultPath, notePath),
+      vaultPath,
+    )
     : rawBodyMarkdown
   const [frontmatter] = splitFrontmatter(tabContent)
   return `${frontmatter}${bodyMarkdown}`