nuwiki-rs/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs

//! Vimwiki lexer.
//!
//! Hand-rolled, two-pass:
//!
//!   - **Block pass:** scan the source line by line, recognise structural
//!     constructs (headings, lists, tables, fences, comments, etc.), emit
//!     the matching block-level token, and within text-bearing lines invoke
//!     the inline pass on the line content.
//!   - **Inline pass:** scan a text run byte by byte, accumulate plain text,
//!     and emit inline marker tokens (bold, italic, code, links, …).
//!
//! Both passes share one `Vec<VimwikiToken>` so the parser sees a single,
//! ordered stream. The lexer is permissive: every delimiter is emitted; the
//! parser pairs them and falls back to literal text on mismatches. Spans are
//! stored as 0-indexed byte offsets.
//!
//! Multi-line constructs (`{{{ }}}`, `{{$ }}$`, `%%+ +%%`) flip a
//! `BlockMode` so subsequent lines are treated as raw content until the
//! matching closer is seen.

use std::collections::HashMap;

use crate::ast::{CheckboxState, Keyword, ListSymbol, Position, Span};
use crate::listsyms::ListSyms;
use crate::syntax::{Lexer, TokenStream};

/// A single vimwiki token. The `kind` carries the variant, `span` carries
/// 0-indexed byte offsets into the source.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VimwikiToken {
    pub kind: VimwikiTokenKind,
    pub span: Span,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VimwikiTokenKind {
    // ----- Line structure -----
    /// End of a non-empty line (`\n`).
    Newline,
    /// A line that contained only whitespace. Replaces both the (empty)
    /// content and the trailing newline.
    BlankLine,

    // ----- Headings -----
    /// Opening `=`s of a heading. `centered` is true when the line had
    /// leading whitespace before the `=` run.
    HeadingOpen {
        level: u8,
        centered: bool,
    },
    /// Closing `=`s on the same line as `HeadingOpen`.
    HeadingClose,

    // ----- Bare structure -----
    HorizontalRule,
    BlockquoteMarker,
    BlockquoteIndent,

    // ----- Lists -----
    ListMarker {
        symbol: ListSymbol,
        indent: u32,
    },
    Checkbox(CheckboxState),

    // ----- Definition lists -----
    DefinitionTermMarker,

    // ----- Tables -----
    /// `|` cell separator.
    TableSep,
    /// A header-separator row, e.g. `|---|---|`. Carries per-cell
    /// Markdown-style alignment markers (`:--`, `--:`, `:--:`).
    TableHeaderRow(Vec<crate::ast::TableAlign>),
    /// A cell whose only content is `>`, meaning "merge with cell to the left".
    TableColSpan,
    /// A cell whose only content is `\/`, meaning "merge with cell above".
    TableRowSpan,

    // ----- Multi-line fences -----
    PreformattedOpen {
        language: Option<String>,
        attrs: HashMap<String, String>,
    },
    PreformattedClose,
    PreformattedLine(String),

    MathBlockOpen {
        environment: Option<String>,
    },
    MathBlockClose,
    MathBlockLine(String),

    /// Single-line comment: `%% ...`
    CommentLine(String),
    /// Opening of a multi-line comment: `%%+`
    CommentMultiOpen,
    CommentMultiClose,
    CommentMultiLine(String),

    // ----- Tags -----
    /// `:tag1:tag2:` — colon-delimited tag line. The lexer treats this as
    /// block-level (must be the entire trimmed content of the line).
    /// Scope (file / heading / standalone) is the parser's job.
    Tag(Vec<String>),

    // ----- Page placeholders -----
    PlaceholderTitle(Option<String>),
    PlaceholderNohtml,
    PlaceholderTemplate(Option<String>),
    PlaceholderDate(Option<String>),

    // ----- Inline content -----
    Text(String),
    BoldDelim,
    ItalicDelim,
    StrikethroughDelim,
    SuperscriptDelim,
    SubscriptDelim,
    Code(String),
    MathInline(String),
    Keyword(Keyword),

    // ----- Links / transclusions -----
    WikiLinkOpen,
    WikiLinkClose,
    WikiLinkSep,
    TransclusionOpen,
    TransclusionClose,
    TransclusionSep,
    RawUrl(String),

    /// Lex error — never aborts the whole document.
    Error(String),
}

#[derive(Debug, Default, Clone)]
pub struct VimwikiLexer {
    listsyms: ListSyms,
}

impl VimwikiLexer {
    pub fn new() -> Self {
        Self::default()
    }

    /// Recognise checkbox glyphs from a custom palette
    /// (vimwiki's `g:vimwiki_listsyms`) instead of the default `" .oOX"`.
    pub fn with_listsyms(mut self, listsyms: ListSyms) -> Self {
        self.listsyms = listsyms;
        self
    }
}

impl Lexer for VimwikiLexer {
    type Token = VimwikiToken;

    fn lex(&self, text: &str) -> TokenStream<VimwikiToken> {
        let mut state = LexState::new(text, &self.listsyms);
        state.run();
        TokenStream::from_vec(state.tokens)
    }
}

// ===== Internal state machine =====

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BlockMode {
    Normal,
    Preformatted,
    MathBlock,
    MultilineComment,
}

struct LexState<'src> {
    src: &'src str,
    /// Byte offset of the start of the current line.
    line_start_offset: usize,
    /// Index of the current line (0-based).
    line: u32,
    tokens: Vec<VimwikiToken>,
    mode: BlockMode,
    listsyms: &'src ListSyms,
}

impl<'src> LexState<'src> {
    fn new(src: &'src str, listsyms: &'src ListSyms) -> Self {
        Self {
            src,
            line_start_offset: 0,
            line: 0,
            tokens: Vec::new(),
            mode: BlockMode::Normal,
            listsyms,
        }
    }

    fn run(&mut self) {
        while self.line_start_offset < self.src.len() {
            let line_text = self.current_line();
            let line_len = line_text.len();

            match self.mode {
                BlockMode::Normal => self.lex_normal_line(line_text),
                BlockMode::Preformatted => self.lex_preformatted_line(line_text),
                BlockMode::MathBlock => self.lex_math_block_line(line_text),
                BlockMode::MultilineComment => self.lex_multi_comment_line(line_text),
            }

            // Skip past the line's trailing newline (if any) and advance line counters.
            let after_content = self.line_start_offset + line_len;
            if after_content < self.src.len() && self.src.as_bytes()[after_content] == b'\n' {
                self.line_start_offset = after_content + 1;
                self.line += 1;
            } else {
                self.line_start_offset = after_content;
            }
        }
    }

    /// Slice of the current line, not including the trailing newline.
    fn current_line(&self) -> &'src str {
        let rest = &self.src[self.line_start_offset..];
        match rest.find('\n') {
            Some(idx) => &rest[..idx],
            None => rest,
        }
    }

    /// Position at byte offset `col` within the current line.
    fn pos_in_line(&self, col: u32) -> Position {
        Position {
            line: self.line,
            column: col,
            offset: self.line_start_offset + col as usize,
        }
    }

    fn line_start_pos(&self) -> Position {
        self.pos_in_line(0)
    }

    fn line_end_pos(&self, line_text: &str) -> Position {
        self.pos_in_line(line_text.len() as u32)
    }

    fn push(&mut self, kind: VimwikiTokenKind, span: Span) {
        self.tokens.push(VimwikiToken { kind, span });
    }

    /// Emit a token whose span runs from byte-col `start_col` to `end_col`
    /// within the current line.
    fn emit(&mut self, kind: VimwikiTokenKind, start_col: u32, end_col: u32) {
        let span = Span::new(self.pos_in_line(start_col), self.pos_in_line(end_col));
        self.push(kind, span);
    }

    fn emit_newline(&mut self, line_text: &str) {
        let nl_col = line_text.len() as u32;
        // Newline is just \n at column = line_text.len() spanning one byte.
        let span = Span::new(self.pos_in_line(nl_col), self.pos_in_line(nl_col + 1));
        self.push(VimwikiTokenKind::Newline, span);
    }

    // ===== Mode dispatch =====

    fn lex_preformatted_line(&mut self, line: &str) {
        // Closing fence: line whose first non-space content is `}}}`.
        if line.trim() == "}}}" {
            let start_col = line.find("}}}").unwrap() as u32;
            self.emit(
                VimwikiTokenKind::PreformattedClose,
                start_col,
                start_col + 3,
            );
            self.mode = BlockMode::Normal;
            self.emit_newline(line);
            return;
        }
        let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
        self.push(VimwikiTokenKind::PreformattedLine(line.to_owned()), span);
        self.emit_newline(line);
    }

    fn lex_math_block_line(&mut self, line: &str) {
        if line.trim() == "}}$" {
            let start_col = line.find("}}$").unwrap() as u32;
            self.emit(VimwikiTokenKind::MathBlockClose, start_col, start_col + 3);
            self.mode = BlockMode::Normal;
            self.emit_newline(line);
            return;
        }
        let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
        self.push(VimwikiTokenKind::MathBlockLine(line.to_owned()), span);
        self.emit_newline(line);
    }

    fn lex_multi_comment_line(&mut self, line: &str) {
        // Closer is `+%%`, possibly with content before it on the same line.
        if let Some(idx) = line.find("+%%") {
            let prefix = &line[..idx];
            if !prefix.is_empty() {
                let span = Span::new(self.pos_in_line(0), self.pos_in_line(prefix.len() as u32));
                self.push(VimwikiTokenKind::CommentMultiLine(prefix.to_owned()), span);
            }
            let close_col = idx as u32;
            self.emit(
                VimwikiTokenKind::CommentMultiClose,
                close_col,
                close_col + 3,
            );
            self.mode = BlockMode::Normal;
            self.emit_newline(line);
            return;
        }
        let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
        self.push(VimwikiTokenKind::CommentMultiLine(line.to_owned()), span);
        self.emit_newline(line);
    }

    fn lex_normal_line(&mut self, line: &str) {
        // Whitespace-only line → BlankLine (replaces both content and the
        // following newline).
        if line.bytes().all(|b| b == b' ' || b == b'\t') {
            let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
            self.push(VimwikiTokenKind::BlankLine, span);
            // Also account for the newline byte if present.
            // We don't emit a Newline because BlankLine subsumes both.
            return;
        }

        // Block patterns in priority order. The first match wins.
        if self.try_lex_multi_comment_open(line)
            || self.try_lex_single_comment(line)
            || self.try_lex_preformatted_open(line)
            || self.try_lex_math_block_open(line)
            || self.try_lex_placeholder(line)
            || self.try_lex_horizontal_rule(line)
            || self.try_lex_heading(line)
            || self.try_lex_tag(line)
            || self.try_lex_table_row(line)
            || self.try_lex_blockquote(line)
            || self.try_lex_list_item(line)
            || self.try_lex_definition_term(line)
        {
            return;
        }

        // Default: paragraph line — lex inline.
        self.lex_inline(line, 0, line.len() as u32);
        self.emit_newline(line);
    }

    // ===== Block patterns =====

    fn try_lex_multi_comment_open(&mut self, line: &str) -> bool {
        let trimmed = line.trim_start();
        if !trimmed.starts_with("%%+") {
            return false;
        }
        let indent = (line.len() - trimmed.len()) as u32;
        self.emit(VimwikiTokenKind::CommentMultiOpen, indent, indent + 3);
        // Same line might also contain `+%%` (one-line multi).
        let after_open = (indent + 3) as usize;
        let rest = &line[after_open..];
        if let Some(end_rel) = rest.find("+%%") {
            let prefix = &rest[..end_rel];
            if !prefix.is_empty() {
                let p_start = indent + 3;
                let p_end = p_start + prefix.len() as u32;
                self.push(
                    VimwikiTokenKind::CommentMultiLine(prefix.to_owned()),
                    Span::new(self.pos_in_line(p_start), self.pos_in_line(p_end)),
                );
            }
            let close_col = (after_open + end_rel) as u32;
            self.emit(
                VimwikiTokenKind::CommentMultiClose,
                close_col,
                close_col + 3,
            );
            self.emit_newline(line);
        } else {
            // Capture remainder of this line as content, then enter
            // multiline-comment mode for following lines.
            if !rest.is_empty() {
                let p_start = indent + 3;
                let p_end = p_start + rest.len() as u32;
                self.push(
                    VimwikiTokenKind::CommentMultiLine(rest.to_owned()),
                    Span::new(self.pos_in_line(p_start), self.pos_in_line(p_end)),
                );
            }
            self.emit_newline(line);
            self.mode = BlockMode::MultilineComment;
        }
        true
    }

    fn try_lex_single_comment(&mut self, line: &str) -> bool {
        let trimmed = line.trim_start();
        if !trimmed.starts_with("%%") || trimmed.starts_with("%%+") {
            return false;
        }
        let indent = (line.len() - trimmed.len()) as u32;
        let content = &trimmed[2..]; // skip "%%"
        let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line));
        self.push(VimwikiTokenKind::CommentLine(content.to_owned()), span);
        self.emit_newline(line);
        true
    }

    fn try_lex_preformatted_open(&mut self, line: &str) -> bool {
        let trimmed = line.trim_start();
        if !trimmed.starts_with("{{{") {
            return false;
        }
        let indent = (line.len() - trimmed.len()) as u32;
        // `{{{[lang][;key=val[;key=val]*]?` style. Vimwiki accepts a single
        // word after the fence (language hint) and/or class/attribute key=val
        // pairs separated by spaces. Keep parsing forgiving.
        let after = trimmed[3..].trim();
        let (language, attrs) = parse_fence_attrs(after);
        let kind = VimwikiTokenKind::PreformattedOpen { language, attrs };
        let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line));
        self.push(kind, span);
        self.mode = BlockMode::Preformatted;
        self.emit_newline(line);
        true
    }

    fn try_lex_math_block_open(&mut self, line: &str) -> bool {
        let trimmed = line.trim_start();
        if !trimmed.starts_with("{{$") {
            return false;
        }
        let indent = (line.len() - trimmed.len()) as u32;
        let after = trimmed[3..].trim();
        // `{{$%env%` — environment is between `%`s.
        let environment = after
            .strip_prefix('%')
            .and_then(|s| s.strip_suffix('%'))
            .map(str::to_owned);
        let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line));
        self.push(VimwikiTokenKind::MathBlockOpen { environment }, span);
        self.mode = BlockMode::MathBlock;
        self.emit_newline(line);
        true
    }

    fn try_lex_placeholder(&mut self, line: &str) -> bool {
        // Placeholders must start at column 0 (no leading whitespace).
        if !line.starts_with('%') {
            return false;
        }
        let kind = if let Some(rest) = line.strip_prefix("%title") {
            let value = rest.trim();
            let v = if value.is_empty() {
                None
            } else {
                Some(value.to_owned())
            };
            VimwikiTokenKind::PlaceholderTitle(v)
        } else if line.trim_end() == "%nohtml" {
            VimwikiTokenKind::PlaceholderNohtml
        } else if let Some(rest) = line.strip_prefix("%template") {
            let value = rest.trim();
            let v = if value.is_empty() {
                None
            } else {
                Some(value.to_owned())
            };
            VimwikiTokenKind::PlaceholderTemplate(v)
        } else if let Some(rest) = line.strip_prefix("%date") {
            let value = rest.trim();
            let v = if value.is_empty() {
                None
            } else {
                Some(value.to_owned())
            };
            VimwikiTokenKind::PlaceholderDate(v)
        } else {
            return false;
        };
        let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
        self.push(kind, span);
        self.emit_newline(line);
        true
    }

    fn try_lex_horizontal_rule(&mut self, line: &str) -> bool {
        let trimmed = line.trim();
        if trimmed.len() < 4 || !trimmed.bytes().all(|b| b == b'-') {
            return false;
        }
        // Four or more dashes. Allow surrounding whitespace.
        let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
        self.push(VimwikiTokenKind::HorizontalRule, span);
        self.emit_newline(line);
        true
    }

    fn try_lex_heading(&mut self, line: &str) -> bool {
        let bytes = line.as_bytes();
        let leading_ws = bytes
            .iter()
            .take_while(|&&b| b == b' ' || b == b'\t')
            .count();
        let centered = leading_ws > 0;

        // Count `=`s after leading whitespace.
        let mut level = 0usize;
        while level < 6 && leading_ws + level < bytes.len() && bytes[leading_ws + level] == b'=' {
            level += 1;
        }
        if level == 0 {
            return false;
        }
        // Must be followed by at least one space.
        if leading_ws + level >= bytes.len() || bytes[leading_ws + level] != b' ' {
            return false;
        }

        // Trailing `=`s of the same level.
        let trimmed_end = line.trim_end_matches([' ', '\t']);
        let trailing_eqs = trimmed_end
            .as_bytes()
            .iter()
            .rev()
            .take_while(|&&b| b == b'=')
            .count();
        if trailing_eqs != level {
            return false;
        }

        let title_start = leading_ws + level;
        let title_end = trimmed_end.len() - trailing_eqs;
        if title_end <= title_start {
            return false;
        }

        let open_col = leading_ws as u32;
        let open_end = (leading_ws + level) as u32;
        self.emit(
            VimwikiTokenKind::HeadingOpen {
                level: level as u8,
                centered,
            },
            open_col,
            open_end,
        );

        // Inline content between markers (trim one space on each side if present).
        let mut t_start = title_start;
        let mut t_end = title_end;
        if t_start < t_end && bytes[t_start] == b' ' {
            t_start += 1;
        }
        if t_end > t_start && bytes[t_end - 1] == b' ' {
            t_end -= 1;
        }
        if t_end > t_start {
            self.lex_inline(&line[t_start..t_end], t_start as u32, t_end as u32);
        }

        let close_col = title_end as u32;
        let close_end = (title_end + trailing_eqs) as u32;
        self.emit(VimwikiTokenKind::HeadingClose, close_col, close_end);
        self.emit_newline(line);
        true
    }

    fn try_lex_table_row(&mut self, line: &str) -> bool {
        let trimmed = line.trim();
        if !trimmed.starts_with('|') || !trimmed.ends_with('|') || trimmed.len() < 2 {
            return false;
        }

        // Header separator row: every cell is `-`s (with optional `:`
        // anchors at one or both ends for alignment) or whitespace.
        if trimmed
            .split('|')
            .filter(|s| !s.is_empty())
            .all(is_sep_cell)
        {
            let aligns: Vec<crate::ast::TableAlign> = trimmed
                .split('|')
                .filter(|s| !s.is_empty())
                .map(parse_sep_alignment)
                .collect();
            let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
            self.push(VimwikiTokenKind::TableHeaderRow(aligns), span);
            self.emit_newline(line);
            return true;
        }

        // Walk the line and emit TableSep tokens at each `|`, with cell
        // content lexed inline between them.
        let bytes = line.as_bytes();
        let mut i = 0usize;
        // Skip leading whitespace.
        while i < bytes.len() && (bytes[i] == b' ' || bytes[i] == b'\t') {
            i += 1;
        }
        // Trailing whitespace bound.
        let mut end = bytes.len();
        while end > i && (bytes[end - 1] == b' ' || bytes[end - 1] == b'\t') {
            end -= 1;
        }

        // Collect cell boundaries.
        let mut sep_positions = Vec::new();
        let mut k = i;
        while k < end {
            if bytes[k] == b'|' {
                sep_positions.push(k);
            }
            k += 1;
        }

        if sep_positions.len() < 2 {
            return false;
        }

        for window in sep_positions.windows(2) {
            let sep_col = window[0] as u32;
            self.emit(VimwikiTokenKind::TableSep, sep_col, sep_col + 1);
            let cell_start = window[0] + 1;
            let cell_end = window[1];
            let cell_text = &line[cell_start..cell_end];
            let cell_trimmed = cell_text.trim();
            match cell_trimmed {
                ">" => {
                    let s = (cell_start + cell_text.find('>').unwrap()) as u32;
                    self.emit(VimwikiTokenKind::TableColSpan, s, s + 1);
                }
                "\\/" => {
                    let s = (cell_start + cell_text.find("\\/").unwrap()) as u32;
                    self.emit(VimwikiTokenKind::TableRowSpan, s, s + 2);
                }
                _ if !cell_trimmed.is_empty() => {
                    self.lex_inline(cell_text, cell_start as u32, cell_end as u32);
                }
                _ => {}
            }
        }
        // Emit the last separator.
        let last_sep = *sep_positions.last().unwrap() as u32;
        self.emit(VimwikiTokenKind::TableSep, last_sep, last_sep + 1);
        self.emit_newline(line);
        true
    }

    fn try_lex_blockquote(&mut self, line: &str) -> bool {
        // `>` prefix style.
        let trimmed = line.trim_start();
        let indent = (line.len() - trimmed.len()) as u32;
        if let Some(rest) =
            trimmed
                .strip_prefix("> ")
                .or_else(|| if trimmed == ">" { Some("") } else { None })
        {
            self.emit(VimwikiTokenKind::BlockquoteMarker, indent, indent + 1);
            if !rest.is_empty() {
                let inline_start = indent + 2;
                let inline_end = inline_start + rest.len() as u32;
                self.lex_inline(rest, inline_start, inline_end);
            }
            self.emit_newline(line);
            return true;
        }

        // 4-space indent style. To avoid swallowing list items, only treat as
        // a blockquote when the indented content does NOT start with a list
        // marker.
        if let Some(rest) = line.strip_prefix("    ") {
            if rest.chars().next().is_none_or(|c| c == ' ' || c == '\t')
                || looks_like_list_marker(rest)
            {
                return false;
            }
            self.emit(VimwikiTokenKind::BlockquoteIndent, 0, 4);
            self.lex_inline(rest, 4, line.len() as u32);
            self.emit_newline(line);
            return true;
        }
        false
    }

    fn try_lex_list_item(&mut self, line: &str) -> bool {
        let trimmed = line.trim_start();
        let indent_bytes = line.len() - trimmed.len();
        let indent = indent_bytes as u32;

        let (symbol, marker_len) = match list_marker_at(trimmed) {
            Some(v) => v,
            None => return false,
        };

        // Marker must be followed by a space.
        if marker_len >= trimmed.len() || trimmed.as_bytes()[marker_len] != b' ' {
            return false;
        }

        let marker_start = indent;
        let marker_end = indent + marker_len as u32;
        self.emit(
            VimwikiTokenKind::ListMarker { symbol, indent },
            marker_start,
            marker_end,
        );

        // Past marker + the single required space.
        let mut cursor = marker_len + 1;
        let after = &trimmed[cursor..];

        // Optional checkbox `[ ]` / `[.]` / `[o]` / `[O]` / `[X]` / `[-]`,
        // followed by a space.
        if let Some((cb, cb_len)) = checkbox_at(after, self.listsyms) {
            let cb_start = indent + cursor as u32;
            let cb_end = cb_start + cb_len as u32;
            self.emit(VimwikiTokenKind::Checkbox(cb), cb_start, cb_end);
            cursor += cb_len;
            // Eat the trailing space if present.
            if cursor < trimmed.len() && trimmed.as_bytes()[cursor] == b' ' {
                cursor += 1;
            }
        }

        let inline_start = indent + cursor as u32;
        let inline_end = line.len() as u32;
        if (inline_end as usize) > (inline_start as usize) {
            let abs_start = self.line_start_offset + inline_start as usize;
            let abs_end = self.line_start_offset + inline_end as usize;
            let slice = &self.src[abs_start..abs_end];
            self.lex_inline(slice, inline_start, inline_end);
        }
        self.emit_newline(line);
        true
    }

    /// `:tag1:tag2:tag3:` — colon-delimited tag line (block-level).
    ///
    /// The whole trimmed line must match: starts and ends with `:`, with
    /// non-empty whitespace-free names between adjacent colons. Empty
    /// segments or whitespace inside tag names disqualify the line.
    /// Doesn't conflict with `::` (definition list marker) because that
    /// pattern requires text *before* the `::`.
    fn try_lex_tag(&mut self, line: &str) -> bool {
        let trimmed = line.trim();
        if trimmed.len() < 3 || !trimmed.starts_with(':') || !trimmed.ends_with(':') {
            return false;
        }
        // `:tag1:tag2:` splits as `["", "tag1", "tag2", ""]`; the empty
        // strings at the ends are expected, everything in between must be
        // non-empty and contain no whitespace.
        let parts: Vec<&str> = trimmed.split(':').collect();
        if parts.first() != Some(&"") || parts.last() != Some(&"") {
            return false;
        }
        let names = &parts[1..parts.len() - 1];
        if names.is_empty()
            || names
                .iter()
                .any(|t| t.is_empty() || t.chars().any(char::is_whitespace))
        {
            return false;
        }
        let indent = (line.len() - line.trim_start().len()) as u32;
        let end_col = indent + trimmed.len() as u32;
        self.emit(
            VimwikiTokenKind::Tag(names.iter().map(|s| (*s).to_owned()).collect()),
            indent,
            end_col,
        );
        self.emit_newline(line);
        true
    }

    fn try_lex_definition_term(&mut self, line: &str) -> bool {
        // `Term:: Definition` or `Term::` (continuation lines follow).
        // The `::` must be preceded by non-`:` text and followed by either
        // EOL or a single space + content.
        let bytes = line.as_bytes();
        let mut i = 0usize;
        let mut found = None;
        while i + 1 < bytes.len() {
            if bytes[i] == b':' && bytes[i + 1] == b':' {
                // Must not be preceded by another `:` (avoid `:::`).
                if i > 0 && bytes[i - 1] == b':' {
                    i += 1;
                    continue;
                }
                // Must not be followed by another `:`.
                if i + 2 < bytes.len() && bytes[i + 2] == b':' {
                    i += 1;
                    continue;
                }
                found = Some(i);
                break;
            }
            i += 1;
        }
        let split = match found {
            Some(s) if s > 0 => s,
            _ => return false,
        };

        // Term inline.
        self.lex_inline(&line[..split], 0, split as u32);
        let marker_col = split as u32;
        self.emit(
            VimwikiTokenKind::DefinitionTermMarker,
            marker_col,
            marker_col + 2,
        );

        let after = &line[split + 2..];
        if !after.is_empty() {
            let leading = after
                .bytes()
                .take_while(|b| *b == b' ' || *b == b'\t')
                .count();
            if leading > 0 {
                // Treat the post-`::` whitespace as a separator; lex the rest as inline.
                let inline_start = (split + 2 + leading) as u32;
                let inline_end = line.len() as u32;
                if inline_end > inline_start {
                    self.lex_inline(&after[leading..], inline_start, inline_end);
                }
            } else {
                // No space after `::`: still try to lex inline.
                let inline_start = (split + 2) as u32;
                self.lex_inline(after, inline_start, line.len() as u32);
            }
        }
        self.emit_newline(line);
        true
    }

    // ===== Inline pass =====
    //
    // Inline content lives at byte offsets [start_col, end_col) within the
    // current line. Span columns are relative to the line.

    fn lex_inline(&mut self, slice: &str, line_col_start: u32, _line_col_end: u32) {
        let bytes = slice.as_bytes();
        let mut buf = String::new();
        let mut buf_start: Option<u32> = None;
        let mut i = 0usize;

        let flush = |this: &mut Self, buf: &mut String, buf_start: &mut Option<u32>, end: u32| {
            if !buf.is_empty() {
                let s = buf_start.take().unwrap();
                this.emit(VimwikiTokenKind::Text(std::mem::take(buf)), s, end);
            }
        };

        while i < bytes.len() {
            let abs_col = line_col_start + i as u32;
            let rest = &slice[i..];

            // 1) Multi-char delimiters and constructs (longest match wins).
            if let Some(after_open) = rest.strip_prefix("[[") {
                flush(self, &mut buf, &mut buf_start, abs_col);
                self.emit(VimwikiTokenKind::WikiLinkOpen, abs_col, abs_col + 2);
                i += 2;
                // Lex until `]]` (absolute index into `slice`).
                let close_abs_in_slice = after_open.find("]]").map(|c| i + c);
                let inner_end = close_abs_in_slice.unwrap_or(slice.len());
                self.lex_link_body(slice, i, line_col_start, inner_end, true);
                i = inner_end;
                if close_abs_in_slice.is_some() {
                    let close_col = line_col_start + i as u32;
                    self.emit(VimwikiTokenKind::WikiLinkClose, close_col, close_col + 2);
                    i += 2;
                }
                continue;
            }
            if rest.starts_with("{{{") {
                // Inline {{{ is rare; fall through as text.
                buf_start.get_or_insert(abs_col);
                buf.push('{');
                i += 1;
                continue;
            }
            if rest.starts_with("{{") {
                flush(self, &mut buf, &mut buf_start, abs_col);
                self.emit(VimwikiTokenKind::TransclusionOpen, abs_col, abs_col + 2);
                i += 2;
                let close_rel = slice[i..].find("}}");
                let inner_end_rel = close_rel.map(|c| i + c).unwrap_or(slice.len());
                self.lex_link_body(slice, i, line_col_start, inner_end_rel, false);
                i = inner_end_rel;
                if close_rel.is_some() {
                    let close_col = line_col_start + i as u32;
                    self.emit(
                        VimwikiTokenKind::TransclusionClose,
                        close_col,
                        close_col + 2,
                    );
                    i += 2;
                }
                continue;
            }
            if rest.starts_with("~~") {
                flush(self, &mut buf, &mut buf_start, abs_col);
                self.emit(VimwikiTokenKind::StrikethroughDelim, abs_col, abs_col + 2);
                i += 2;
                continue;
            }
            if rest.starts_with(",,") {
                flush(self, &mut buf, &mut buf_start, abs_col);
                self.emit(VimwikiTokenKind::SubscriptDelim, abs_col, abs_col + 2);
                i += 2;
                continue;
            }

            // 2) Inline code: `...`
            if bytes[i] == b'`' {
                if let Some(end_rel) = slice[i + 1..].find('`') {
                    flush(self, &mut buf, &mut buf_start, abs_col);
                    let content = &slice[i + 1..i + 1 + end_rel];
                    let span_end = abs_col + (end_rel + 2) as u32;
                    self.emit(
                        VimwikiTokenKind::Code(content.to_owned()),
                        abs_col,
                        span_end,
                    );
                    i += end_rel + 2;
                    continue;
                }
            }

            // 3) Inline math: $...$  (single $ on each side; bail to text if unmatched)
            if bytes[i] == b'$' {
                if let Some(end_rel) = slice[i + 1..].find('$') {
                    flush(self, &mut buf, &mut buf_start, abs_col);
                    let content = &slice[i + 1..i + 1 + end_rel];
                    let span_end = abs_col + (end_rel + 2) as u32;
                    self.emit(
                        VimwikiTokenKind::MathInline(content.to_owned()),
                        abs_col,
                        span_end,
                    );
                    i += end_rel + 2;
                    continue;
                }
            }

            // 4) Single-char delimiters.
            let single = match bytes[i] {
                b'*' => Some(VimwikiTokenKind::BoldDelim),
                b'_' => Some(VimwikiTokenKind::ItalicDelim),
                b'^' => Some(VimwikiTokenKind::SuperscriptDelim),
                _ => None,
            };
            if let Some(kind) = single {
                flush(self, &mut buf, &mut buf_start, abs_col);
                self.emit(kind, abs_col, abs_col + 1);
                i += 1;
                continue;
            }

            // 5) Raw URL — at a word boundary.
            if is_word_boundary(bytes, i) {
                if let Some(url_len) = match_url(rest) {
                    flush(self, &mut buf, &mut buf_start, abs_col);
                    let url = &rest[..url_len];
                    self.emit(
                        VimwikiTokenKind::RawUrl(url.to_owned()),
                        abs_col,
                        abs_col + url_len as u32,
                    );
                    i += url_len;
                    continue;
                }
                // 6) Keyword.
                if let Some((kw_len, kw)) = match_keyword(rest) {
                    flush(self, &mut buf, &mut buf_start, abs_col);
                    self.emit(
                        VimwikiTokenKind::Keyword(kw),
                        abs_col,
                        abs_col + kw_len as u32,
                    );
                    i += kw_len;
                    continue;
                }
            }

            // 7) Default: accumulate into Text. Walk to next byte boundary.
            let ch = slice[i..].chars().next().unwrap();
            let ch_len = ch.len_utf8();
            buf_start.get_or_insert(abs_col);
            buf.push(ch);
            i += ch_len;
        }

        let end_col = line_col_start + slice.len() as u32;
        flush(self, &mut buf, &mut buf_start, end_col);
    }

    /// Lex inline content within `[[...]]` or `{{...}}`, where `|` is a
    /// separator rather than literal pipe. `is_wikilink` chooses the token
    /// variant for the separator.
    fn lex_link_body(
        &mut self,
        slice: &str,
        start: usize,
        line_col_start: u32,
        end: usize,
        is_wikilink: bool,
    ) {
        let bytes = slice.as_bytes();
        let mut buf = String::new();
        let mut buf_start: Option<u32> = None;
        let mut i = start;

        let flush =
            |this: &mut Self, buf: &mut String, buf_start: &mut Option<u32>, end_col: u32| {
                if !buf.is_empty() {
                    let s = buf_start.take().unwrap();
                    this.emit(VimwikiTokenKind::Text(std::mem::take(buf)), s, end_col);
                }
            };

        while i < end {
            let abs_col = line_col_start + i as u32;
            if bytes[i] == b'|' {
                flush(self, &mut buf, &mut buf_start, abs_col);
                let kind = if is_wikilink {
                    VimwikiTokenKind::WikiLinkSep
                } else {
                    VimwikiTokenKind::TransclusionSep
                };
                self.emit(kind, abs_col, abs_col + 1);
                i += 1;
                continue;
            }
            let ch = slice[i..].chars().next().unwrap();
            let ch_len = ch.len_utf8();
            buf_start.get_or_insert(abs_col);
            buf.push(ch);
            i += ch_len;
        }
        flush(self, &mut buf, &mut buf_start, line_col_start + end as u32);
    }
}

// ===== Helpers =====

/// Parse the bit after `{{{` on a fence line. Format is forgiving:
/// `lang` (single word) followed by optional `key=value` pairs separated
/// by spaces.
fn parse_fence_attrs(s: &str) -> (Option<String>, HashMap<String, String>) {
    let mut attrs = HashMap::new();
    let mut language: Option<String> = None;
    for tok in s.split_whitespace() {
        if let Some(eq) = tok.find('=') {
            let (k, v) = tok.split_at(eq);
            let v = &v[1..];
            // Strip optional surrounding quotes from value.
            let v = v.trim_matches('"').trim_matches('\'');
            attrs.insert(k.to_owned(), v.to_owned());
        } else if language.is_none() {
            language = Some(tok.to_owned());
        }
    }
    (language, attrs)
}

fn is_word_boundary(bytes: &[u8], i: usize) -> bool {
    if i == 0 {
        return true;
    }
    let prev = bytes[i - 1];
    !(prev.is_ascii_alphanumeric() || prev == b'_')
}

fn is_word_end(bytes: &[u8], i: usize) -> bool {
    if i >= bytes.len() {
        return true;
    }
    let next = bytes[i];
    !(next.is_ascii_alphanumeric() || next == b'_')
}

const URL_SCHEMES: &[&str] = &["https://", "http://", "ftp://", "mailto:", "file://"];

fn match_url(rest: &str) -> Option<usize> {
    let bytes = rest.as_bytes();
    let scheme_len = URL_SCHEMES.iter().find_map(|s| {
        if rest.starts_with(s) {
            Some(s.len())
        } else {
            None
        }
    })?;
    let mut end = scheme_len;
    while end < bytes.len() {
        let b = bytes[end];
        // Conservative URL char set; stops at whitespace and most punctuation
        // that's typically not part of a URL tail.
        let ok = b.is_ascii_alphanumeric()
            || matches!(
                b,
                b'-' | b'_'
                    | b'.'
                    | b'~'
                    | b'/'
                    | b':'
                    | b'?'
                    | b'#'
                    | b'['
                    | b']'
                    | b'@'
                    | b'!'
                    | b'$'
                    | b'&'
                    | b'\''
                    | b'('
                    | b')'
                    | b'*'
                    | b'+'
                    | b','
                    | b';'
                    | b'='
                    | b'%'
            );
        if !ok {
            break;
        }
        end += 1;
    }
    // Trim trailing punctuation that's almost always sentence punctuation.
    while end > scheme_len {
        match bytes[end - 1] {
            b'.' | b',' | b';' | b':' | b'!' | b'?' | b')' | b']' => end -= 1,
            _ => break,
        }
    }
    if end <= scheme_len {
        None
    } else {
        Some(end)
    }
}

const KEYWORDS: &[(&str, Keyword)] = &[
    ("TODO", Keyword::Todo),
    ("DONE", Keyword::Done),
    ("STARTED", Keyword::Started),
    ("FIXME", Keyword::Fixme),
    ("FIXED", Keyword::Fixed),
    ("XXX", Keyword::Xxx),
    ("STOPPED", Keyword::Stopped),
];

fn match_keyword(rest: &str) -> Option<(usize, Keyword)> {
    let bytes = rest.as_bytes();
    KEYWORDS.iter().find_map(|(text, kw)| {
        if rest.starts_with(text) && is_word_end(bytes, text.len()) {
            Some((text.len(), *kw))
        } else {
            None
        }
    })
}

fn list_marker_at(s: &str) -> Option<(ListSymbol, usize)> {
    let bytes = s.as_bytes();
    if bytes.is_empty() {
        return None;
    }
    // Single-char markers: -, *, #
    match bytes[0] {
        b'-' => return Some((ListSymbol::Dash, 1)),
        b'*' => return Some((ListSymbol::Star, 1)),
        b'#' => return Some((ListSymbol::Hash, 1)),
        _ => {}
    }
    // Numeric: `1.` or `1)`. Allow multi-digit.
    let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count();
    if digits > 0 && bytes.len() > digits {
        match bytes[digits] {
            b'.' => return Some((ListSymbol::Numeric, digits + 1)),
            b')' => return Some((ListSymbol::NumericParen, digits + 1)),
            _ => {}
        }
    }
    // Single-letter alpha or roman: `a)`, `A)`, `i)`, `I)`.
    if bytes.len() >= 2 && bytes[1] == b')' {
        let c = bytes[0];
        if c.is_ascii_lowercase() {
            // `i)` / `v)` / `x)` are roman by convention; without context we
            // treat any single lowercase letter as alpha. Roman vs alpha is
            // ambiguous from a single char; vimwiki itself relies on context.
            // Mark `i` as RomanParen as a small concession to the spec.
            let sym = if c == b'i' {
                ListSymbol::RomanParen
            } else {
                ListSymbol::AlphaParen
            };
            return Some((sym, 2));
        }
        if c.is_ascii_uppercase() {
            let sym = if c == b'I' {
                ListSymbol::RomanUpperParen
            } else {
                ListSymbol::AlphaUpperParen
            };
            return Some((sym, 2));
        }
    }
    None
}

fn looks_like_list_marker(s: &str) -> bool {
    list_marker_at(s).is_some_and(|(_, len)| s.as_bytes().get(len) == Some(&b' '))
}

/// Is this cell content a valid separator-cell body? Allows leading /
/// trailing `:` for alignment and inner runs of `-` plus whitespace.
fn is_sep_cell(cell: &str) -> bool {
    let t = cell.trim();
    if t.is_empty() {
        return false;
    }
    let body = t.trim_start_matches(':').trim_end_matches(':');
    !body.is_empty() && body.chars().all(|c| c == '-')
}

fn parse_sep_alignment(cell: &str) -> crate::ast::TableAlign {
    let t = cell.trim();
    let l = t.starts_with(':');
    let r = t.ends_with(':');
    match (l, r) {
        (true, true) => crate::ast::TableAlign::Center,
        (true, false) => crate::ast::TableAlign::Left,
        (false, true) => crate::ast::TableAlign::Right,
        _ => crate::ast::TableAlign::Default,
    }
}

/// Recognise a `[g]` checkbox at the start of `s`, where `g` is a single
/// glyph drawn from `listsyms` (or the rejected marker). Returns the bucketed
/// state and the byte length consumed (including the brackets).
fn checkbox_at(s: &str, listsyms: &ListSyms) -> Option<(CheckboxState, usize)> {
    let rest = s.strip_prefix('[')?;
    let mut chars = rest.char_indices();
    let (_, glyph) = chars.next()?;
    let (close_off, ']') = chars.next()? else {
        return None;
    };
    let state = listsyms.state_of(glyph)?;
    // `[` + glyph + `]`: 1 + glyph.len_utf8() + 1, i.e. the close bracket
    // offset within `rest` plus one for the opening bracket and one for `]`.
    Some((state, 1 + close_off + 1))
}