diff --git a/docs/ABSTRACTIONS.md b/docs/ABSTRACTIONS.md index 0d31b2df..d50126a8 100644 --- a/docs/ABSTRACTIONS.md +++ b/docs/ABSTRACTIONS.md @@ -590,6 +590,7 @@ Defined in `src/components/editorSchema.tsx` and styled in `src/components/Edito - The schema overrides BlockNote's default `codeBlock` spec with `createCodeBlockSpec({ ...codeBlockOptions, defaultLanguage: "text" })` from `@blocknote/code-block`. - Fenced code blocks now use BlockNote's supported Shiki-backed highlighter path, which renders `.shiki` token spans directly inside the editor DOM. +- Missing common grammars live in `src/utils/codeBlockLanguageCatalog.ts` and are registered lazily from direct `@shikijs/langs` imports by `src/components/codeBlockOptions.ts`; known aliases such as `ps1` and `vb` normalize to canonical picker values during Markdown import. - Tolaria keeps `defaultLanguage: "text"` so unlabeled code blocks do not silently become JavaScript at creation time. Parsed unlabeled code blocks then run through Tolaria's lightweight language inference, while explicit fence languages and user dropdown choices still win. - Inline-code chip styling remains scoped to `.bn-inline-content code`, so fenced `pre > code` nodes keep the dedicated code-block shell instead of inheriting the muted inline surface. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 94aefe7b..35e960d3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -134,6 +134,7 @@ The note list opportunistically preloads visible and adjacent markdown/text entr | Frontend | React + TypeScript | React 19, TS 5.9 | | Editor | BlockNote | 0.46.2 | | Code block highlighting | @blocknote/code-block | 0.46.2 | +| Additional code grammars | @shikijs/langs | 3.23.0 | | Diagram rendering | Mermaid | 11.14.0 | | Whiteboard rendering | tldraw | 4.5.10 | | Raw editor | CodeMirror 6 | - | @@ -228,7 +229,7 @@ flowchart TD - **Sidebar** (220-400px, resizable): Top-level filters (All Notes, Changes, Pulse), saved Views, collapsible type-based section groups, and a dedicated folder tree. The folder tree starts with a vault-root row labeled from the opened vault path, shows root-level files when selected, and nests user-created folders plus default vault folders such as `attachments/` and `views/` underneath it; only the dedicated `type/` directory stays hidden because note types already have their own sidebar section. Saved Views persist a top-level YAML `order` field in each view file and use the same ordered-list mental model as Types for single-vault lists: pointer users can drag the existing view row, double-click to rename it, or right-click for edit/rename/appearance/delete actions, while keyboard users can use the row context key for the same menu and command-palette move actions for ordering. In multiple-vault mode, saved View rows are keyed by source vault plus filename so duplicate filenames do not collide, and edits/deletes route to the owning vault. The folder tree supports inline folder creation and rename, exposes a right-click menu for rename/delete plus filesystem reveal/copy-path actions on mutable folders, and auto-expands ancestor folders when the current selection or rename target is nested. Folder creation sends the selected folder's vault-relative path and mounted root to `create_vault_folder`, so a new folder is created under the focused parent instead of defaulting to the active vault root. Type sections and folder rows also act as note drop targets: dropping a note on a type updates its `type:` frontmatter, while dropping it on a folder runs the same crash-safe move path as the command palette flow. Each type can have a custom icon, color, sort, and visibility set via its `type: Type` document; new type documents created by Tolaria are written at the vault root. In mounted multi-vault graphs, duplicate type names still render as one sidebar section, but the visibility picker becomes a workspace matrix and writes visibility to the specific vault's Type document, so hidden type definitions suppress only notes of that type from the same workspace. - **Note List / Pulse View** (220-500px, resizable): When a section group, filter, or saved view is selected, shows filtered notes with snippets, modified dates, status indicators, and per-context note-list controls. When `selection.kind === 'entity'`, the same pane enters **Neighborhood** mode: the source note is pinned at the top as a normal active row, outgoing relationship groups render first, inverse/backlink groups follow, empty groups stay visible with `0`, and duplicates across groups are allowed when multiple relationships are true. Plain click / `Enter` open the focused note without replacing the current Neighborhood, while Cmd/Ctrl-click and Cmd/Ctrl-`Enter` pivot the pane into the clicked note's Neighborhood. Inbox organization auto-advance is coordinated by `useInboxOrganizeAdvance`, which only opens the next visible Inbox note when the organized note is still the active requested tab after the write finishes. Folder-backed lists also show non-Markdown files: previewable media and PDF binaries get file indicators and open in the editor pane, while unsupported binaries remain muted instead of auto-launching an external app. Saved views reuse the same sort and visible-column controls as the built-in lists, and those changes persist back into the view `.yml` definition (`sort`, `listPropertiesDisplay`). When Pulse filter is active, shows `PulseView` — a chronological git activity feed grouped by day. -- **Editor** (flex, fills remaining space): Single note open at a time (no tabs — see ADR-0003). Breadcrumb bar with filename controls, read-only legacy display-title context when a no-H1 note's title differs from its filename, word count, rich-editor width toggle, and the secondary-overflow Table of Contents action, BlockNote rich text editor with wikilink support, Markdown-compatible inline/display math rendering, first-class Mermaid diagram blocks, markdown-safe formatting controls, and schema-backed fenced code block highlighting via `@blocknote/code-block`. Can toggle to diff view (modified files), raw CodeMirror view, or a wide rich-editor reading surface with preserved side margins; raw CodeMirror remains full-width and unaffected by note width mode. Inline rich-editor images open in a localized shadcn lightbox on double-click while normal single-click BlockNote selection remains untouched, and tiny tracking-style images are ignored. Binary image, audio, video, and PDF files render through `FilePreview` as ordinary vault files using Tauri asset URLs; editor-embedded audio and video use the same scoped asset sources through the CSP `media-src` allow-list. Linux AppImage builds ask the native runtime whether audio/video should fall back to external-open controls before mounting webview media elements. External-open actions call `open_vault_file_external` so the target is validated against the active vault before the native default app opens it. Unsupported/broken binaries show explicit fallback states and keyboard focus returns to the note list on `Escape`. Decomposed into `Editor` (orchestrator), `EditorContent`, `FilePreview`, `EditorRightPanel`, `TableOfContentsPanel`, `SingleEditorView`, with hooks `useDiffMode`, `useEditorFocus`, and `useEditorSave`, plus the `useRawMode`/`RawEditorView` pair for markdown source editing. Rich BlockNote input and raw CodeMirror input both route typed `->`, `<-`, and `<->` through the shared `src/utils/arrowLigatures.ts` resolver so arrow ligatures stay consistent across mode switches while escaped ASCII sequences remain literal. Navigation history (Cmd+[/]) replaces tabs. +- **Editor** (flex, fills remaining space): Single note open at a time (no tabs — see ADR-0003). Breadcrumb bar with filename controls, read-only legacy display-title context when a no-H1 note's title differs from its filename, word count, rich-editor width toggle, and the secondary-overflow Table of Contents action, BlockNote rich text editor with wikilink support, Markdown-compatible inline/display math rendering, first-class Mermaid diagram blocks, markdown-safe formatting controls, and schema-backed fenced code block highlighting via `@blocknote/code-block` plus lazy direct `@shikijs/langs` registrations for missing common grammars. Can toggle to diff view (modified files), raw CodeMirror view, or a wide rich-editor reading surface with preserved side margins; raw CodeMirror remains full-width and unaffected by note width mode. Inline rich-editor images open in a localized shadcn lightbox on double-click while normal single-click BlockNote selection remains untouched, and tiny tracking-style images are ignored. Binary image, audio, video, and PDF files render through `FilePreview` as ordinary vault files using Tauri asset URLs; editor-embedded audio and video use the same scoped asset sources through the CSP `media-src` allow-list. Linux AppImage builds ask the native runtime whether audio/video should fall back to external-open controls before mounting webview media elements. External-open actions call `open_vault_file_external` so the target is validated against the active vault before the native default app opens it. Unsupported/broken binaries show explicit fallback states and keyboard focus returns to the note list on `Escape`. Decomposed into `Editor` (orchestrator), `EditorContent`, `FilePreview`, `EditorRightPanel`, `TableOfContentsPanel`, `SingleEditorView`, with hooks `useDiffMode`, `useEditorFocus`, and `useEditorSave`, plus the `useRawMode`/`RawEditorView` pair for markdown source editing. Rich BlockNote input and raw CodeMirror input both route typed `->`, `<-`, and `<->` through the shared `src/utils/arrowLigatures.ts` resolver so arrow ligatures stay consistent across mode switches while escaped ASCII sequences remain literal. Navigation history (Cmd+[/]) replaces tabs. Note PDF export stays renderer-owned for layout: `useEditorPdfExport` exits diff/raw views, applies a print-only stylesheet to the rendered note root, and opens the platform print/save-to-PDF dialog through the Tauri `print_current_webview` command, with `window.print()` as the browser fallback. The export reuses rendered BlockNote output so math, images, Mermaid diagrams, tldraw blocks, code, tables, and links degrade through their existing DOM rather than a second Markdown-to-PDF renderer, and the source Markdown is never modified. - **Right side panels** (200-500px or hidden): Properties and Table of Contents are mutually exclusive panels mounted by `EditorRightPanel` and coordinated by `useRightPanelExclusion`. Properties shows frontmatter, relationships, instances, backlinks, and git history; Table of Contents is lazy-mounted only while open, derives a title-rooted H1/H2/H3 hierarchy through a debounced Web Worker per ADR-0109, and reuses folder-tree indentation/guide geometry with heading icons while resolving live BlockNote block IDs at click time for navigation. The breadcrumb bar toggles Table of Contents and Properties actions. Per-note `icon` is a suggested Properties field and the command palette's "Set Note Icon" action opens that field directly. When viewing a Type note, Properties shows an **Instances** section listing all notes of that type (sorted by modified_at desc, capped at 50). diff --git a/docs/adr/0134-direct-shiki-language-registrations.md b/docs/adr/0134-direct-shiki-language-registrations.md new file mode 100644 index 00000000..5eeb8d1b --- /dev/null +++ b/docs/adr/0134-direct-shiki-language-registrations.md @@ -0,0 +1,27 @@ +--- +type: ADR +id: "0134" +title: "Direct Shiki language registrations for code blocks" +status: active +date: 2026-05-29 +--- + +## Context + +Tolaria uses `@blocknote/code-block` for rich-editor fenced-code highlighting. The bundled BlockNote highlighter covers the common web and systems languages already in the editor menu, but it does not include several common Shiki grammars such as PowerShell, VBScript, Dart, Dockerfile, Terraform/HCL, and TOML. Users still expect imported fences like `powershell`, `ps1`, `vb`, and `vbscript` to highlight, show a valid language picker state, and serialize back to a stable Markdown fence. + +## Decision + +**Tolaria keeps BlockNote's code-block integration and adds direct, lazy `@shikijs/langs` registrations for missing common languages and aliases.** + +## Options considered + +- **Keep only the BlockNote bundle**: simplest, but leaves PowerShell/VBScript and other common fences unsupported. +- **Register selected `@shikijs/langs` grammars lazily** (chosen): preserves BlockNote's schema and parser path while adding only the extra grammars users need. +- **Replace BlockNote's highlighter with a full custom Shiki bundle**: more control, but a larger structural change than the current requirement needs. + +## Consequences + +`src/components/codeBlockOptions.ts` remains the owner of the BlockNote highlighter configuration, but extra grammar modules are now imported directly from `@shikijs/langs` only when a matching fence or picker value needs highlighting. `src/utils/codeBlockLanguageCatalog.ts` owns the supported extra language labels and aliases, and `src/utils/codeBlockLanguage.ts` normalizes known imported aliases such as `ps1` and `vb` to the canonical picker language. + +The language menu grows, but unsupported aliases still fail safely by staying as plain explicit fence names. If Tolaria later needs a generated language bundle, export-time highlighting, or a substantially smaller menu, this ADR should be superseded by a custom Shiki packaging decision. diff --git a/docs/adr/README.md b/docs/adr/README.md index f8344437..76badb05 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -185,3 +185,4 @@ proposed → active → superseded | [0131](0131-reusable-release-artifact-build-workflow.md) | Reusable release artifact build workflow | active | | [0132](0132-alpha-authenticode-soft-gate.md) | Alpha Authenticode soft gate | active | | [0133](0133-request-scoped-ai-stream-events.md) | Request-scoped AI stream event channels | active | +| [0134](0134-direct-shiki-language-registrations.md) | Direct Shiki language registrations for code blocks | active | diff --git a/package.json b/package.json index 0ad4647a..2295d40f 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "@radix-ui/react-tabs": "^1.1.13", "@radix-ui/react-tooltip": "^1.2.8", "@sentry/react": "^10.47.0", + "@shikijs/langs": "3.23.0", "@tailwindcss/vite": "^4.1.18", "@tauri-apps/api": "^2.10.1", "@tauri-apps/plugin-deep-link": "2.4.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6621258e..4f278b50 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -122,6 +122,9 @@ importers: '@sentry/react': specifier: ^10.47.0 version: 10.47.0(react@19.2.4) + '@shikijs/langs': + specifier: 3.23.0 + version: 3.23.0 '@tailwindcss/vite': specifier: ^4.1.18 version: 4.1.18(vite@7.3.2(@types/node@24.10.13)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.3)) diff --git a/src/components/codeBlockOptions.ts b/src/components/codeBlockOptions.ts index 9b6510bf..38e6bb0e 100644 --- a/src/components/codeBlockOptions.ts +++ b/src/components/codeBlockOptions.ts @@ -1,10 +1,15 @@ import { codeBlockOptions } from '@blocknote/code-block' import type { CodeBlockOptions } from '@blocknote/core' +import { + canonicalKnownCodeBlockLanguage, + codeBlockLanguageOptions, + EXTRA_CODE_BLOCK_LANGUAGES, + GO_CODE_BLOCK_LANGUAGE, +} from '../utils/codeBlockLanguageCatalog' import { supportsModernRegexFeatures } from '../utils/regexCapabilities' const LIGHT_CODE_THEME = 'github-light' const DARK_CODE_THEME = 'github-dark' -const GO_LANGUAGE = { name: 'Go', aliases: ['go', 'golang'] } const GO_LANGUAGE_REGISTRATION = { name: 'go', displayName: 'Go', @@ -48,6 +53,15 @@ const GO_LANGUAGE_REGISTRATION = { type TolariaCodeHighlighter = Awaited>> type TolariaLoadLanguage = TolariaCodeHighlighter['loadLanguage'] type TolariaLanguageInput = Parameters[number] +type TolariaLanguageLoader = () => Promise +type TolariaNamedLanguageRegistration = Record & { + name: string + displayName?: string + aliases?: string[] +} + +const GO_LANGUAGE = codeBlockLanguageOptions([GO_CODE_BLOCK_LANGUAGE]).go +const EXTRA_SUPPORTED_LANGUAGES = codeBlockLanguageOptions(EXTRA_CODE_BLOCK_LANGUAGES) function currentCodeBlockTheme() { if (typeof document === 'undefined') return LIGHT_CODE_THEME @@ -62,12 +76,76 @@ function prioritizeTheme(themes: string[], theme: string) { return [theme, ...themes.filter((candidate) => candidate !== theme)] } -function expandGoLanguage(language: TolariaLanguageInput): TolariaLanguageInput[] { - if (typeof language !== 'string') return [language] - const languageName: string = language - return languageName === 'go' || languageName === 'golang' +function languageInputs(languages: readonly TolariaLanguageInput[]): TolariaLanguageInput[] { + return [...languages] +} + +function namedLanguageRegistration(value: TolariaLanguageInput): TolariaNamedLanguageRegistration | null { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return null + const record = value as Record + return typeof record.name === 'string' + ? record as TolariaNamedLanguageRegistration + : null +} + +function renameLanguageRegistration( + languages: readonly TolariaLanguageInput[], + sourceName: string, + nextLanguage: { name: string; displayName: string; aliases: string[] }, +): TolariaLanguageInput[] { + return languages.map((language) => { + const registration = namedLanguageRegistration(language) + if (!registration || registration.name !== sourceName) return language + return { ...registration, ...nextLanguage } as TolariaLanguageInput + }) +} + +async function loadVbScriptLanguage(): Promise { + const language = await import('@shikijs/langs/vb') + return renameLanguageRegistration(language.default, 'vb', { + name: 'vbscript', + displayName: 'VBScript', + aliases: ['vb', 'vbs', 'vba', 'visual-basic', 'visualbasic'], + }) +} + +const EXTRA_LANGUAGE_LOADERS = new Map([ + ['powershell', async () => languageInputs((await import('@shikijs/langs/powershell')).default)], + ['vbscript', loadVbScriptLanguage], + ['dart', async () => languageInputs((await import('@shikijs/langs/dart')).default)], + ['groovy', async () => languageInputs((await import('@shikijs/langs/groovy')).default)], + ['matlab', async () => languageInputs((await import('@shikijs/langs/matlab')).default)], + ['perl', async () => languageInputs((await import('@shikijs/langs/perl')).default)], + ['elixir', async () => languageInputs((await import('@shikijs/langs/elixir')).default)], + ['erlang', async () => languageInputs((await import('@shikijs/langs/erlang')).default)], + ['fsharp', async () => languageInputs((await import('@shikijs/langs/fsharp')).default)], + ['clojure', async () => languageInputs((await import('@shikijs/langs/clojure')).default)], + ['asm', async () => languageInputs((await import('@shikijs/langs/asm')).default)], + ['zig', async () => languageInputs((await import('@shikijs/langs/zig')).default)], + ['hcl', async () => languageInputs((await import('@shikijs/langs/hcl')).default)], + ['terraform', async () => languageInputs((await import('@shikijs/langs/terraform')).default)], + ['dockerfile', async () => languageInputs((await import('@shikijs/langs/dockerfile')).default)], + ['batch', async () => languageInputs((await import('@shikijs/langs/bat')).default)], + ['diff', async () => languageInputs((await import('@shikijs/langs/diff')).default)], + ['ini', async () => languageInputs((await import('@shikijs/langs/ini')).default)], + ['toml', async () => languageInputs((await import('@shikijs/langs/toml')).default)], +]) + +function expandGoLanguage(language: string): TolariaLanguageInput[] | null { + return canonicalKnownCodeBlockLanguage(language) === 'go' ? [GO_LANGUAGE_REGISTRATION as TolariaLanguageInput] - : [language] + : null +} + +async function expandExternalLanguage(language: string): Promise { + const canonicalLanguage = canonicalKnownCodeBlockLanguage(language) ?? language.trim().toLowerCase() + const loadLanguage = EXTRA_LANGUAGE_LOADERS.get(canonicalLanguage) + return loadLanguage ? loadLanguage() : null +} + +async function expandLanguage(language: TolariaLanguageInput): Promise { + if (typeof language !== 'string') return [language] + return expandGoLanguage(language) ?? await expandExternalLanguage(language) ?? [language] } async function createTolariaCodeHighlighter(): Promise { @@ -75,7 +153,10 @@ async function createTolariaCodeHighlighter(): Promise { return { ...highlighter, getLoadedThemes: () => prioritizeTheme(highlighter.getLoadedThemes(), currentCodeBlockTheme()), - loadLanguage: (...languages) => highlighter.loadLanguage(...languages.flatMap(expandGoLanguage)), + loadLanguage: async (...languages) => { + const expandedLanguages = await Promise.all(languages.map(expandLanguage)) + return highlighter.loadLanguage(...expandedLanguages.flat()) + }, } } @@ -87,6 +168,7 @@ export function createTolariaCodeBlockOptions(): Partial { supportedLanguages: { ...codeBlockOptions.supportedLanguages, go: GO_LANGUAGE, + ...EXTRA_SUPPORTED_LANGUAGES, }, } diff --git a/src/components/editorSchema.codeBlockLanguages.test.ts b/src/components/editorSchema.codeBlockLanguages.test.ts new file mode 100644 index 00000000..939b88b9 --- /dev/null +++ b/src/components/editorSchema.codeBlockLanguages.test.ts @@ -0,0 +1,28 @@ +import { BlockNoteEditor } from '@blocknote/core' +import { describe, expect, it } from 'vitest' +import { inferCodeBlockLanguages } from '../utils/codeBlockLanguage' +import { schema } from './editorSchema' + +describe('editor schema code block languages', () => { + it.each([ + ['powershell', 'powershell'], + ['ps1', 'powershell'], + ['vbscript', 'vbscript'], + ['vb', 'vbscript'], + ['php', 'php'], + ])('imports %s fences as %s code blocks and exports the canonical fence', async (fence, language) => { + const editor = BlockNoteEditor.create({ schema }) + const blocks = await editor.tryParseMarkdownToBlocks([ + `\`\`\`${fence}`, + 'example', + '```', + ].join('\n')) + const repairedBlocks = inferCodeBlockLanguages(blocks) as typeof blocks + + expect(repairedBlocks[0]).toMatchObject({ + type: 'codeBlock', + props: { language }, + }) + expect(editor.blocksToMarkdownLossy(repairedBlocks)).toContain(`\`\`\`${language}`) + }) +}) diff --git a/src/components/editorSchema.webkit.test.ts b/src/components/editorSchema.webkit.test.ts index fe4e9fdd..d796c98c 100644 --- a/src/components/editorSchema.webkit.test.ts +++ b/src/components/editorSchema.webkit.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' +import { EXTRA_CODE_BLOCK_LANGUAGES } from '../utils/codeBlockLanguageCatalog' const nativeRegExpDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'RegExp') const NativeRegExp = RegExp @@ -82,6 +83,42 @@ describe('editor schema code block highlighting', () => { }) }) + it('registers additional common Shiki code block languages', async () => { + vi.resetModules() + + const { createTolariaCodeBlockOptions } = await import('./codeBlockOptions') + const options = createTolariaCodeBlockOptions() + + expect(options.supportedLanguages?.powershell).toMatchObject({ + name: 'PowerShell', + aliases: ['powershell', 'ps', 'ps1'], + }) + expect(options.supportedLanguages?.vbscript).toMatchObject({ + name: 'VBScript', + aliases: ['vbscript', 'vbs', 'vb', 'vba', 'visual-basic', 'visualbasic'], + }) + expect(options.supportedLanguages?.dart).toMatchObject({ + name: 'Dart', + aliases: ['dart'], + }) + expect(options.supportedLanguages?.hcl).toMatchObject({ + name: 'HCL', + aliases: ['hcl'], + }) + expect(options.supportedLanguages?.terraform).toMatchObject({ + name: 'Terraform', + aliases: ['terraform', 'tf', 'tfvars'], + }) + expect(options.supportedLanguages?.dockerfile).toMatchObject({ + name: 'Dockerfile', + aliases: ['dockerfile', 'docker'], + }) + expect(options.supportedLanguages?.php).toMatchObject({ + name: 'PHP', + aliases: ['php'], + }) + }) + it('loads the Go Shiki grammar for Go code blocks', async () => { vi.resetModules() @@ -92,6 +129,21 @@ describe('editor schema code block highlighting', () => { expect(highlighter?.getLoadedLanguages()).toContain('go') }) + it.each([ + ...EXTRA_CODE_BLOCK_LANGUAGES.map(language => [language.id, language.id]), + ['ps1', 'ps1'], + ['vb', 'vb'], + ['php', 'php'], + ])('loads the %s Shiki grammar for code blocks', async (language, loadedLanguage) => { + vi.resetModules() + + const { createTolariaCodeBlockOptions } = await import('./codeBlockOptions') + const highlighter = await createTolariaCodeBlockOptions().createHighlighter?.() + + await expect(highlighter?.loadLanguage(language)).resolves.toBeUndefined() + expect(highlighter?.getLoadedLanguages()).toContain(loadedLanguage) + }) + it('omits the Shiki highlighter when WebKit lacks precompiled regex flags', async () => { installLegacyWebKitRegExp() vi.resetModules() diff --git a/src/utils/codeBlockLanguage.test.ts b/src/utils/codeBlockLanguage.test.ts index 03d2357e..152ee2cc 100644 --- a/src/utils/codeBlockLanguage.test.ts +++ b/src/utils/codeBlockLanguage.test.ts @@ -29,6 +29,43 @@ describe('code block language inference', () => { }) }) + it('normalizes known fenced code block aliases from imported Markdown', () => { + const blocks = inferCodeBlockLanguages([ + { + type: 'codeBlock', + props: { language: 'ps1' }, + content: [{ type: 'text', text: 'Get-ChildItem' }], + children: [], + }, + { + type: 'codeBlock', + props: { language: 'vb' }, + content: [{ type: 'text', text: 'MsgBox "Hello"' }], + children: [], + }, + ]) + + expect(blocks[0]).toMatchObject({ + props: { language: 'powershell' }, + }) + expect(blocks[1]).toMatchObject({ + props: { language: 'vbscript' }, + }) + }) + + it('leaves unsupported explicit code block languages unchanged', () => { + const blocks = inferCodeBlockLanguages([{ + type: 'codeBlock', + props: { language: 'foolang' }, + content: [{ type: 'text', text: 'demo' }], + children: [], + }]) + + expect(blocks[0]).toMatchObject({ + props: { language: 'foolang' }, + }) + }) + it('updates unlabeled code blocks from plain text to the inferred language', () => { const blocks = inferCodeBlockLanguages([{ type: 'codeBlock', diff --git a/src/utils/codeBlockLanguage.ts b/src/utils/codeBlockLanguage.ts index 23216164..59763b12 100644 --- a/src/utils/codeBlockLanguage.ts +++ b/src/utils/codeBlockLanguage.ts @@ -1,3 +1,5 @@ +import { canonicalKnownCodeBlockLanguage } from './codeBlockLanguageCatalog' + type UnknownRecord = Record type LanguageDetector = (source: string) => boolean @@ -31,9 +33,9 @@ function textFromInlineContent(content: unknown): string { }).join('') } -function normalizedLanguage(props: unknown): string { +function readLanguage(props: unknown): string { if (!isRecord(props) || typeof props.language !== 'string') return '' - return props.language.trim().toLowerCase() + return props.language.trim() } function isPlainTextLanguage(language: string): boolean { @@ -87,10 +89,6 @@ function withInferredChildren(block: UnknownRecord, children: unknown): UnknownR return children === block.children ? block : { ...block, children } } -function shouldInferLanguage(block: UnknownRecord): boolean { - return block.type === 'codeBlock' && isPlainTextLanguage(normalizedLanguage(block.props)) -} - function withInferredLanguage(block: UnknownRecord, children: unknown, language: string): UnknownRecord { const props = isRecord(block.props) ? block.props : {} return { @@ -105,8 +103,15 @@ function withInferredLanguage(block: UnknownRecord, children: unknown, language: function inferBlockLanguage(block: UnknownRecord): UnknownRecord { const children = inferChildren(block.children) + const rawLanguage = readLanguage(block.props) + const language = rawLanguage.toLowerCase() + const canonicalLanguage = canonicalKnownCodeBlockLanguage(rawLanguage) - if (!shouldInferLanguage(block)) return withInferredChildren(block, children) + if (block.type === 'codeBlock' && canonicalLanguage && canonicalLanguage !== rawLanguage) { + return withInferredLanguage(block, children, canonicalLanguage) + } + + if (!isPlainTextLanguage(language) || block.type !== 'codeBlock') return withInferredChildren(block, children) const inferred = inferCodeBlockLanguage(textFromInlineContent(block.content)) if (!inferred) return withInferredChildren(block, children) diff --git a/src/utils/codeBlockLanguageCatalog.ts b/src/utils/codeBlockLanguageCatalog.ts new file mode 100644 index 00000000..b9f27950 --- /dev/null +++ b/src/utils/codeBlockLanguageCatalog.ts @@ -0,0 +1,70 @@ +export type CodeBlockLanguageOption = { + name: string + aliases: string[] +} + +export type CodeBlockLanguageCatalogEntry = CodeBlockLanguageOption & { + id: string +} + +export const GO_CODE_BLOCK_LANGUAGE = { + id: 'go', + name: 'Go', + aliases: ['go', 'golang'], +} as const satisfies CodeBlockLanguageCatalogEntry + +export const EXTRA_CODE_BLOCK_LANGUAGES = [ + { id: 'powershell', name: 'PowerShell', aliases: ['powershell', 'ps', 'ps1'] }, + { id: 'vbscript', name: 'VBScript', aliases: ['vbscript', 'vbs', 'vb', 'vba', 'visual-basic', 'visualbasic'] }, + { id: 'dart', name: 'Dart', aliases: ['dart'] }, + { id: 'groovy', name: 'Groovy', aliases: ['groovy'] }, + { id: 'matlab', name: 'MATLAB', aliases: ['matlab'] }, + { id: 'perl', name: 'Perl', aliases: ['perl', 'pl', 'pm'] }, + { id: 'elixir', name: 'Elixir', aliases: ['elixir', 'ex', 'exs'] }, + { id: 'erlang', name: 'Erlang', aliases: ['erlang', 'erl'] }, + { id: 'fsharp', name: 'F#', aliases: ['fsharp', 'f#', 'fs'] }, + { id: 'clojure', name: 'Clojure', aliases: ['clojure', 'clj'] }, + { id: 'asm', name: 'Assembly', aliases: ['asm', 'assembly'] }, + { id: 'zig', name: 'Zig', aliases: ['zig'] }, + { id: 'hcl', name: 'HCL', aliases: ['hcl'] }, + { id: 'terraform', name: 'Terraform', aliases: ['terraform', 'tf', 'tfvars'] }, + { id: 'dockerfile', name: 'Dockerfile', aliases: ['dockerfile', 'docker'] }, + { id: 'batch', name: 'Batch', aliases: ['batch', 'bat', 'cmd'] }, + { id: 'diff', name: 'Diff', aliases: ['diff', 'patch'] }, + { id: 'ini', name: 'INI', aliases: ['ini', 'properties'] }, + { id: 'toml', name: 'TOML', aliases: ['toml'] }, +] as const satisfies readonly CodeBlockLanguageCatalogEntry[] + +const KNOWN_CODE_BLOCK_LANGUAGES = [ + GO_CODE_BLOCK_LANGUAGE, + ...EXTRA_CODE_BLOCK_LANGUAGES, +] as const + +function knownLanguageAliases(): Array<[string, string]> { + const aliases: Array<[string, string]> = [] + for (const language of KNOWN_CODE_BLOCK_LANGUAGES) { + aliases.push([language.id, language.id]) + for (const alias of language.aliases) aliases.push([alias, language.id]) + } + return aliases +} + +const KNOWN_LANGUAGE_ID_BY_ALIAS = new Map(knownLanguageAliases()) + +export function codeBlockLanguageOptions( + languages: readonly CodeBlockLanguageCatalogEntry[], +): Record { + return Object.fromEntries( + languages.map((language) => [ + language.id, + { + name: language.name, + aliases: [...language.aliases], + }, + ]), + ) +} + +export function canonicalKnownCodeBlockLanguage(language: string): string | null { + return KNOWN_LANGUAGE_ID_BY_ALIAS.get(language.trim().toLowerCase()) ?? null +}