From 942dbe2aa8bad3e14457b16566bf31bb0665b0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Fr=C3=B3es=20Franco?= Date: Sun, 10 May 2026 17:20:46 +0000 Subject: [PATCH] phase 3: vimwiki lexer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-pass, hand-rolled lexer per SPEC.md §6.6. Block pass walks the source line by line; recognised constructs emit structural tokens, and text-bearing line content is fed to the inline pass. Multi-line fences (`{{{ }}}`, `{{$ }}$`, `%%+ +%%`) flip a `BlockMode` so subsequent lines are captured raw until the matching closer. VimwikiToken covers the full §9 feature checklist: - headings (1–6, centered), horizontal rule, blockquotes (`>` and 4-space), all list-marker variants, checkboxes, definition `::`, tables (separators, header row, col/row span) - preformatted fences (with language + key=val attrs), math blocks (with `%env%`), single- and multi-line comments, all four page placeholders (%title / %nohtml / %template / %date) - inline: bold/italic/strike/super/sub delimiters, inline code, inline math, all six keywords (word-bounded), wikilinks (with description separator), transclusions (with attr separator), raw URLs across http(s)/ftp/mailto/file schemes — trailing sentence punctuation stripped from URL spans Spans are byte-accurate per SPEC §6.5 (0-indexed line + byte column + absolute byte offset). The lexer is permissive: every delimiter is emitted, the parser will pair them and fall back to literal text on mismatches. Tests (41 cases) cover one example per construct in §9, plus word-boundary keyword detection, list-vs-bold disambiguation, indent-vs-list disambiguation, and a span-correctness check. README now carries a per-phase status table; SPEC top-line status advanced to Phase 3 complete. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 19 +- SPEC.md | 2 +- crates/nuwiki-core/src/syntax/mod.rs | 1 + .../nuwiki-core/src/syntax/vimwiki/lexer.rs | 1172 +++++++++++++++++ crates/nuwiki-core/src/syntax/vimwiki/mod.rs | 8 + crates/nuwiki-core/tests/vimwiki_lexer.rs | 592 +++++++++ 6 files changed, 1792 insertions(+), 2 deletions(-) create mode 100644 crates/nuwiki-core/src/syntax/vimwiki/lexer.rs create mode 100644 crates/nuwiki-core/src/syntax/vimwiki/mod.rs create mode 100644 crates/nuwiki-core/tests/vimwiki_lexer.rs diff --git a/README.md b/README.md index 92ad588..4b14088 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,27 @@ A Vim/Neovim plugin providing full vimwiki syntax support, implemented as a Rust-based language server (LSP). -> **Status:** Phase 0 — scaffolding. Not yet functional. +> **Status:** Phase 3 (vimwiki lexer) complete. Parser, renderer, and editor +> glue still pending — not yet functional end-to-end. See [`SPEC.md`](./SPEC.md) for the full project specification. +## Implementation status + +| Phase | Name | Status | +|---|---|---| +| 0 | Scaffolding | ✅ done | +| 1 | Core AST | ✅ done | +| 2 | Syntax Plugin Interface | ✅ done | +| 3 | Vimwiki Lexer | ✅ done | +| 4 | Vimwiki Parser | ⏳ next | +| 5 | Renderer | ⏳ | +| 6 | LSP Foundation | ⏳ | +| 7 | Semantic Tokens | ⏳ | +| 8 | Navigation | ⏳ | +| 9 | Editor Glue | ⏳ | +| 10 | CI/CD release pipeline | ⏳ | + ## Repository layout ``` diff --git a/SPEC.md b/SPEC.md index c712153..d421f04 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,7 +1,7 @@ # nuwiki — Project Specification > Last updated: 2026-05-10 -> Status: Phase 2 (Syntax Plugin Interface) complete — moving to Phase 3 (Vimwiki Lexer) +> Status: Phase 3 (Vimwiki Lexer) complete — moving to Phase 4 (Vimwiki Parser) --- diff --git a/crates/nuwiki-core/src/syntax/mod.rs b/crates/nuwiki-core/src/syntax/mod.rs index 92c6abb..4dffa33 100644 --- a/crates/nuwiki-core/src/syntax/mod.rs +++ b/crates/nuwiki-core/src/syntax/mod.rs @@ -8,6 +8,7 @@ //! See SPEC.md §6.3 (interface) and §6.6 (lexer strategy). pub mod registry; +pub mod vimwiki; pub use registry::SyntaxRegistry; diff --git a/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs b/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs new file mode 100644 index 0000000..d8bff89 --- /dev/null +++ b/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs @@ -0,0 +1,1172 @@ +//! Vimwiki lexer. +//! +//! Hand-rolled, two-pass per SPEC.md §6.6: +//! +//! - **Block pass:** scan the source line by line, recognise structural +//! constructs (headings, lists, tables, fences, comments, etc.), emit +//! the matching block-level token, and within text-bearing lines invoke +//! the inline pass on the line content. +//! - **Inline pass:** scan a text run byte by byte, accumulate plain text, +//! and emit inline marker tokens (bold, italic, code, links, …). +//! +//! Both passes share one `Vec` so the parser sees a single, +//! ordered stream. The lexer is permissive: every delimiter is emitted; the +//! parser pairs them and falls back to literal text on mismatches. Spans are +//! stored as 0-indexed byte offsets per SPEC.md §6.5. +//! +//! Multi-line constructs (`{{{ }}}`, `{{$ }}$`, `%%+ +%%`) flip a +//! `BlockMode` so subsequent lines are treated as raw content until the +//! matching closer is seen. + +use std::collections::HashMap; + +use crate::ast::{CheckboxState, Keyword, ListSymbol, Position, Span}; +use crate::syntax::{Lexer, TokenStream}; + +/// A single vimwiki token. The `kind` carries the variant, `span` carries +/// 0-indexed byte offsets into the source. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VimwikiToken { + pub kind: VimwikiTokenKind, + pub span: Span, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VimwikiTokenKind { + // ----- Line structure ----- + /// End of a non-empty line (`\n`). + Newline, + /// A line that contained only whitespace. Replaces both the (empty) + /// content and the trailing newline. + BlankLine, + + // ----- Headings ----- + /// Opening `=`s of a heading. `centered` is true when the line had + /// leading whitespace before the `=` run. + HeadingOpen { + level: u8, + centered: bool, + }, + /// Closing `=`s on the same line as `HeadingOpen`. + HeadingClose, + + // ----- Bare structure ----- + HorizontalRule, + BlockquoteMarker, + BlockquoteIndent, + + // ----- Lists ----- + ListMarker { + symbol: ListSymbol, + indent: u32, + }, + Checkbox(CheckboxState), + + // ----- Definition lists ----- + DefinitionTermMarker, + + // ----- Tables ----- + /// `|` cell separator. + TableSep, + /// A header-separator row, e.g. `|---|---|`. + TableHeaderRow, + /// A cell whose only content is `>`, meaning "merge with cell to the left". + TableColSpan, + /// A cell whose only content is `\/`, meaning "merge with cell above". + TableRowSpan, + + // ----- Multi-line fences ----- + PreformattedOpen { + language: Option, + attrs: HashMap, + }, + PreformattedClose, + PreformattedLine(String), + + MathBlockOpen { + environment: Option, + }, + MathBlockClose, + MathBlockLine(String), + + /// Single-line comment: `%% ...` + CommentLine(String), + /// Opening of a multi-line comment: `%%+` + CommentMultiOpen, + CommentMultiClose, + CommentMultiLine(String), + + // ----- Page placeholders (SPEC §9) ----- + PlaceholderTitle(Option), + PlaceholderNohtml, + PlaceholderTemplate(Option), + PlaceholderDate(Option), + + // ----- Inline content ----- + Text(String), + BoldDelim, + ItalicDelim, + StrikethroughDelim, + SuperscriptDelim, + SubscriptDelim, + Code(String), + MathInline(String), + Keyword(Keyword), + + // ----- Links / transclusions ----- + WikiLinkOpen, + WikiLinkClose, + WikiLinkSep, + TransclusionOpen, + TransclusionClose, + TransclusionSep, + RawUrl(String), + + /// Lex error — never aborts the whole document (SPEC §6.7). + Error(String), +} + +#[derive(Debug, Default, Clone)] +pub struct VimwikiLexer; + +impl VimwikiLexer { + pub fn new() -> Self { + Self + } +} + +impl Lexer for VimwikiLexer { + type Token = VimwikiToken; + + fn lex(&self, text: &str) -> TokenStream { + let mut state = LexState::new(text); + state.run(); + TokenStream::from_vec(state.tokens) + } +} + +// ===== Internal state machine ===== + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BlockMode { + Normal, + Preformatted, + MathBlock, + MultilineComment, +} + +struct LexState<'src> { + src: &'src str, + /// Byte offset of the start of the current line. + line_start_offset: usize, + /// Index of the current line (0-based). + line: u32, + tokens: Vec, + mode: BlockMode, +} + +impl<'src> LexState<'src> { + fn new(src: &'src str) -> Self { + Self { + src, + line_start_offset: 0, + line: 0, + tokens: Vec::new(), + mode: BlockMode::Normal, + } + } + + fn run(&mut self) { + while self.line_start_offset < self.src.len() { + let line_text = self.current_line(); + let line_len = line_text.len(); + + match self.mode { + BlockMode::Normal => self.lex_normal_line(line_text), + BlockMode::Preformatted => self.lex_preformatted_line(line_text), + BlockMode::MathBlock => self.lex_math_block_line(line_text), + BlockMode::MultilineComment => self.lex_multi_comment_line(line_text), + } + + // Skip past the line's trailing newline (if any) and advance line counters. + let after_content = self.line_start_offset + line_len; + if after_content < self.src.len() && self.src.as_bytes()[after_content] == b'\n' { + self.line_start_offset = after_content + 1; + self.line += 1; + } else { + self.line_start_offset = after_content; + } + } + } + + /// Slice of the current line, not including the trailing newline. + fn current_line(&self) -> &'src str { + let rest = &self.src[self.line_start_offset..]; + match rest.find('\n') { + Some(idx) => &rest[..idx], + None => rest, + } + } + + /// Position at byte offset `col` within the current line. + fn pos_in_line(&self, col: u32) -> Position { + Position { + line: self.line, + column: col, + offset: self.line_start_offset + col as usize, + } + } + + fn line_start_pos(&self) -> Position { + self.pos_in_line(0) + } + + fn line_end_pos(&self, line_text: &str) -> Position { + self.pos_in_line(line_text.len() as u32) + } + + fn push(&mut self, kind: VimwikiTokenKind, span: Span) { + self.tokens.push(VimwikiToken { kind, span }); + } + + /// Emit a token whose span runs from byte-col `start_col` to `end_col` + /// within the current line. + fn emit(&mut self, kind: VimwikiTokenKind, start_col: u32, end_col: u32) { + let span = Span::new(self.pos_in_line(start_col), self.pos_in_line(end_col)); + self.push(kind, span); + } + + fn emit_newline(&mut self, line_text: &str) { + let nl_col = line_text.len() as u32; + // Newline is just \n at column = line_text.len() spanning one byte. + let span = Span::new(self.pos_in_line(nl_col), self.pos_in_line(nl_col + 1)); + self.push(VimwikiTokenKind::Newline, span); + } + + // ===== Mode dispatch ===== + + fn lex_preformatted_line(&mut self, line: &str) { + // Closing fence: line whose first non-space content is `}}}`. + if line.trim() == "}}}" { + let start_col = line.find("}}}").unwrap() as u32; + self.emit( + VimwikiTokenKind::PreformattedClose, + start_col, + start_col + 3, + ); + self.mode = BlockMode::Normal; + self.emit_newline(line); + return; + } + let span = Span::new(self.line_start_pos(), self.line_end_pos(line)); + self.push(VimwikiTokenKind::PreformattedLine(line.to_owned()), span); + self.emit_newline(line); + } + + fn lex_math_block_line(&mut self, line: &str) { + if line.trim() == "}}$" { + let start_col = line.find("}}$").unwrap() as u32; + self.emit(VimwikiTokenKind::MathBlockClose, start_col, start_col + 3); + self.mode = BlockMode::Normal; + self.emit_newline(line); + return; + } + let span = Span::new(self.line_start_pos(), self.line_end_pos(line)); + self.push(VimwikiTokenKind::MathBlockLine(line.to_owned()), span); + self.emit_newline(line); + } + + fn lex_multi_comment_line(&mut self, line: &str) { + // Closer is `+%%`, possibly with content before it on the same line. + if let Some(idx) = line.find("+%%") { + let prefix = &line[..idx]; + if !prefix.is_empty() { + let span = Span::new(self.pos_in_line(0), self.pos_in_line(prefix.len() as u32)); + self.push(VimwikiTokenKind::CommentMultiLine(prefix.to_owned()), span); + } + let close_col = idx as u32; + self.emit( + VimwikiTokenKind::CommentMultiClose, + close_col, + close_col + 3, + ); + self.mode = BlockMode::Normal; + self.emit_newline(line); + return; + } + let span = Span::new(self.line_start_pos(), self.line_end_pos(line)); + self.push(VimwikiTokenKind::CommentMultiLine(line.to_owned()), span); + self.emit_newline(line); + } + + fn lex_normal_line(&mut self, line: &str) { + // Whitespace-only line → BlankLine (replaces both content and the + // following newline). + if line.bytes().all(|b| b == b' ' || b == b'\t') { + let span = Span::new(self.line_start_pos(), self.line_end_pos(line)); + self.push(VimwikiTokenKind::BlankLine, span); + // Also account for the newline byte if present. + // We don't emit a Newline because BlankLine subsumes both. + return; + } + + // Block patterns in priority order. The first match wins. + if self.try_lex_multi_comment_open(line) + || self.try_lex_single_comment(line) + || self.try_lex_preformatted_open(line) + || self.try_lex_math_block_open(line) + || self.try_lex_placeholder(line) + || self.try_lex_horizontal_rule(line) + || self.try_lex_heading(line) + || self.try_lex_table_row(line) + || self.try_lex_blockquote(line) + || self.try_lex_list_item(line) + || self.try_lex_definition_term(line) + { + return; + } + + // Default: paragraph line — lex inline. + self.lex_inline(line, 0, line.len() as u32); + self.emit_newline(line); + } + + // ===== Block patterns ===== + + fn try_lex_multi_comment_open(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if !trimmed.starts_with("%%+") { + return false; + } + let indent = (line.len() - trimmed.len()) as u32; + self.emit(VimwikiTokenKind::CommentMultiOpen, indent, indent + 3); + // Same line might also contain `+%%` (one-line multi). + let after_open = (indent + 3) as usize; + let rest = &line[after_open..]; + if let Some(end_rel) = rest.find("+%%") { + let prefix = &rest[..end_rel]; + if !prefix.is_empty() { + let p_start = indent + 3; + let p_end = p_start + prefix.len() as u32; + self.push( + VimwikiTokenKind::CommentMultiLine(prefix.to_owned()), + Span::new(self.pos_in_line(p_start), self.pos_in_line(p_end)), + ); + } + let close_col = (after_open + end_rel) as u32; + self.emit( + VimwikiTokenKind::CommentMultiClose, + close_col, + close_col + 3, + ); + self.emit_newline(line); + } else { + // Capture remainder of this line as content, then enter + // multiline-comment mode for following lines. + if !rest.is_empty() { + let p_start = indent + 3; + let p_end = p_start + rest.len() as u32; + self.push( + VimwikiTokenKind::CommentMultiLine(rest.to_owned()), + Span::new(self.pos_in_line(p_start), self.pos_in_line(p_end)), + ); + } + self.emit_newline(line); + self.mode = BlockMode::MultilineComment; + } + true + } + + fn try_lex_single_comment(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if !trimmed.starts_with("%%") || trimmed.starts_with("%%+") { + return false; + } + let indent = (line.len() - trimmed.len()) as u32; + let content = &trimmed[2..]; // skip "%%" + let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line)); + self.push(VimwikiTokenKind::CommentLine(content.to_owned()), span); + self.emit_newline(line); + true + } + + fn try_lex_preformatted_open(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if !trimmed.starts_with("{{{") { + return false; + } + let indent = (line.len() - trimmed.len()) as u32; + // `{{{[lang][;key=val[;key=val]*]?` style. Vimwiki accepts a single + // word after the fence (language hint) and/or class/attribute key=val + // pairs separated by spaces. Keep parsing forgiving. + let after = trimmed[3..].trim(); + let (language, attrs) = parse_fence_attrs(after); + let kind = VimwikiTokenKind::PreformattedOpen { language, attrs }; + let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line)); + self.push(kind, span); + self.mode = BlockMode::Preformatted; + self.emit_newline(line); + true + } + + fn try_lex_math_block_open(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + if !trimmed.starts_with("{{$") { + return false; + } + let indent = (line.len() - trimmed.len()) as u32; + let after = trimmed[3..].trim(); + // `{{$%env%` — environment is between `%`s. + let environment = after + .strip_prefix('%') + .and_then(|s| s.strip_suffix('%')) + .map(str::to_owned); + let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line)); + self.push(VimwikiTokenKind::MathBlockOpen { environment }, span); + self.mode = BlockMode::MathBlock; + self.emit_newline(line); + true + } + + fn try_lex_placeholder(&mut self, line: &str) -> bool { + // Placeholders must start at column 0 (no leading whitespace). + if !line.starts_with('%') { + return false; + } + let kind = if let Some(rest) = line.strip_prefix("%title") { + let value = rest.trim(); + let v = if value.is_empty() { + None + } else { + Some(value.to_owned()) + }; + VimwikiTokenKind::PlaceholderTitle(v) + } else if line.trim_end() == "%nohtml" { + VimwikiTokenKind::PlaceholderNohtml + } else if let Some(rest) = line.strip_prefix("%template") { + let value = rest.trim(); + let v = if value.is_empty() { + None + } else { + Some(value.to_owned()) + }; + VimwikiTokenKind::PlaceholderTemplate(v) + } else if let Some(rest) = line.strip_prefix("%date") { + let value = rest.trim(); + let v = if value.is_empty() { + None + } else { + Some(value.to_owned()) + }; + VimwikiTokenKind::PlaceholderDate(v) + } else { + return false; + }; + let span = Span::new(self.line_start_pos(), self.line_end_pos(line)); + self.push(kind, span); + self.emit_newline(line); + true + } + + fn try_lex_horizontal_rule(&mut self, line: &str) -> bool { + let trimmed = line.trim(); + if trimmed.len() < 4 || !trimmed.bytes().all(|b| b == b'-') { + return false; + } + // SPEC §9: four or more dashes. Allow surrounding whitespace. + let span = Span::new(self.line_start_pos(), self.line_end_pos(line)); + self.push(VimwikiTokenKind::HorizontalRule, span); + self.emit_newline(line); + true + } + + fn try_lex_heading(&mut self, line: &str) -> bool { + let bytes = line.as_bytes(); + let leading_ws = bytes + .iter() + .take_while(|&&b| b == b' ' || b == b'\t') + .count(); + let centered = leading_ws > 0; + + // Count `=`s after leading whitespace. + let mut level = 0usize; + while level < 6 && leading_ws + level < bytes.len() && bytes[leading_ws + level] == b'=' { + level += 1; + } + if level == 0 { + return false; + } + // Must be followed by at least one space. + if leading_ws + level >= bytes.len() || bytes[leading_ws + level] != b' ' { + return false; + } + + // Trailing `=`s of the same level. + let trimmed_end = line.trim_end_matches([' ', '\t']); + let trailing_eqs = trimmed_end + .as_bytes() + .iter() + .rev() + .take_while(|&&b| b == b'=') + .count(); + if trailing_eqs != level { + return false; + } + + let title_start = leading_ws + level; + let title_end = trimmed_end.len() - trailing_eqs; + if title_end <= title_start { + return false; + } + + let open_col = leading_ws as u32; + let open_end = (leading_ws + level) as u32; + self.emit( + VimwikiTokenKind::HeadingOpen { + level: level as u8, + centered, + }, + open_col, + open_end, + ); + + // Inline content between markers (trim one space on each side if present). + let mut t_start = title_start; + let mut t_end = title_end; + if t_start < t_end && bytes[t_start] == b' ' { + t_start += 1; + } + if t_end > t_start && bytes[t_end - 1] == b' ' { + t_end -= 1; + } + if t_end > t_start { + self.lex_inline(&line[t_start..t_end], t_start as u32, t_end as u32); + } + + let close_col = title_end as u32; + let close_end = (title_end + trailing_eqs) as u32; + self.emit(VimwikiTokenKind::HeadingClose, close_col, close_end); + self.emit_newline(line); + true + } + + fn try_lex_table_row(&mut self, line: &str) -> bool { + let trimmed = line.trim(); + if !trimmed.starts_with('|') || !trimmed.ends_with('|') || trimmed.len() < 2 { + return false; + } + + // Header separator row: every cell content is dashes (e.g. "|---|---|"). + if trimmed + .split('|') + .filter(|s| !s.is_empty()) + .all(|cell| cell.chars().all(|c| c == '-' || c == ' ' || c == '\t')) + { + let span = Span::new(self.line_start_pos(), self.line_end_pos(line)); + self.push(VimwikiTokenKind::TableHeaderRow, span); + self.emit_newline(line); + return true; + } + + // Walk the line and emit TableSep tokens at each `|`, with cell + // content lexed inline between them. + let bytes = line.as_bytes(); + let mut i = 0usize; + // Skip leading whitespace. + while i < bytes.len() && (bytes[i] == b' ' || bytes[i] == b'\t') { + i += 1; + } + // Trailing whitespace bound. + let mut end = bytes.len(); + while end > i && (bytes[end - 1] == b' ' || bytes[end - 1] == b'\t') { + end -= 1; + } + + // Collect cell boundaries. + let mut sep_positions = Vec::new(); + let mut k = i; + while k < end { + if bytes[k] == b'|' { + sep_positions.push(k); + } + k += 1; + } + + if sep_positions.len() < 2 { + return false; + } + + for window in sep_positions.windows(2) { + let sep_col = window[0] as u32; + self.emit(VimwikiTokenKind::TableSep, sep_col, sep_col + 1); + let cell_start = window[0] + 1; + let cell_end = window[1]; + let cell_text = &line[cell_start..cell_end]; + let cell_trimmed = cell_text.trim(); + match cell_trimmed { + ">" => { + let s = (cell_start + cell_text.find('>').unwrap()) as u32; + self.emit(VimwikiTokenKind::TableColSpan, s, s + 1); + } + "\\/" => { + let s = (cell_start + cell_text.find("\\/").unwrap()) as u32; + self.emit(VimwikiTokenKind::TableRowSpan, s, s + 2); + } + _ if !cell_trimmed.is_empty() => { + self.lex_inline(cell_text, cell_start as u32, cell_end as u32); + } + _ => {} + } + } + // Emit the last separator. + let last_sep = *sep_positions.last().unwrap() as u32; + self.emit(VimwikiTokenKind::TableSep, last_sep, last_sep + 1); + self.emit_newline(line); + true + } + + fn try_lex_blockquote(&mut self, line: &str) -> bool { + // `>` prefix style. + let trimmed = line.trim_start(); + let indent = (line.len() - trimmed.len()) as u32; + if let Some(rest) = + trimmed + .strip_prefix("> ") + .or_else(|| if trimmed == ">" { Some("") } else { None }) + { + self.emit(VimwikiTokenKind::BlockquoteMarker, indent, indent + 1); + if !rest.is_empty() { + let inline_start = indent + 2; + let inline_end = inline_start + rest.len() as u32; + self.lex_inline(rest, inline_start, inline_end); + } + self.emit_newline(line); + return true; + } + + // 4-space indent style. To avoid swallowing list items, only treat as + // a blockquote when the indented content does NOT start with a list + // marker. + if let Some(rest) = line.strip_prefix(" ") { + if rest.chars().next().is_none_or(|c| c == ' ' || c == '\t') + || looks_like_list_marker(rest) + { + return false; + } + self.emit(VimwikiTokenKind::BlockquoteIndent, 0, 4); + self.lex_inline(rest, 4, line.len() as u32); + self.emit_newline(line); + return true; + } + false + } + + fn try_lex_list_item(&mut self, line: &str) -> bool { + let trimmed = line.trim_start(); + let indent_bytes = line.len() - trimmed.len(); + let indent = indent_bytes as u32; + + let (symbol, marker_len) = match list_marker_at(trimmed) { + Some(v) => v, + None => return false, + }; + + // Marker must be followed by a space. + if marker_len >= trimmed.len() || trimmed.as_bytes()[marker_len] != b' ' { + return false; + } + + let marker_start = indent; + let marker_end = indent + marker_len as u32; + self.emit( + VimwikiTokenKind::ListMarker { symbol, indent }, + marker_start, + marker_end, + ); + + // Past marker + the single required space. + let mut cursor = marker_len + 1; + let after = &trimmed[cursor..]; + + // Optional checkbox `[ ]` / `[.]` / `[o]` / `[O]` / `[X]` / `[-]`, + // followed by a space. + if let Some((cb, cb_len)) = checkbox_at(after) { + let cb_start = indent + cursor as u32; + let cb_end = cb_start + cb_len as u32; + self.emit(VimwikiTokenKind::Checkbox(cb), cb_start, cb_end); + cursor += cb_len; + // Eat the trailing space if present. + if cursor < trimmed.len() && trimmed.as_bytes()[cursor] == b' ' { + cursor += 1; + } + } + + let inline_start = indent + cursor as u32; + let inline_end = line.len() as u32; + if (inline_end as usize) > (inline_start as usize) { + let abs_start = self.line_start_offset + inline_start as usize; + let abs_end = self.line_start_offset + inline_end as usize; + let slice = &self.src[abs_start..abs_end]; + self.lex_inline(slice, inline_start, inline_end); + } + self.emit_newline(line); + true + } + + fn try_lex_definition_term(&mut self, line: &str) -> bool { + // `Term:: Definition` or `Term::` (continuation lines follow). + // The `::` must be preceded by non-`:` text and followed by either + // EOL or a single space + content. + let bytes = line.as_bytes(); + let mut i = 0usize; + let mut found = None; + while i + 1 < bytes.len() { + if bytes[i] == b':' && bytes[i + 1] == b':' { + // Must not be preceded by another `:` (avoid `:::`). + if i > 0 && bytes[i - 1] == b':' { + i += 1; + continue; + } + // Must not be followed by another `:`. + if i + 2 < bytes.len() && bytes[i + 2] == b':' { + i += 1; + continue; + } + found = Some(i); + break; + } + i += 1; + } + let split = match found { + Some(s) if s > 0 => s, + _ => return false, + }; + + // Term inline. + self.lex_inline(&line[..split], 0, split as u32); + let marker_col = split as u32; + self.emit( + VimwikiTokenKind::DefinitionTermMarker, + marker_col, + marker_col + 2, + ); + + let after = &line[split + 2..]; + if !after.is_empty() { + let leading = after + .bytes() + .take_while(|b| *b == b' ' || *b == b'\t') + .count(); + if leading > 0 { + // Treat the post-`::` whitespace as a separator; lex the rest as inline. + let inline_start = (split + 2 + leading) as u32; + let inline_end = line.len() as u32; + if inline_end > inline_start { + self.lex_inline(&after[leading..], inline_start, inline_end); + } + } else { + // No space after `::`: still try to lex inline. + let inline_start = (split + 2) as u32; + self.lex_inline(after, inline_start, line.len() as u32); + } + } + self.emit_newline(line); + true + } + + // ===== Inline pass ===== + // + // Inline content lives at byte offsets [start_col, end_col) within the + // current line. Span columns are relative to the line. + + fn lex_inline(&mut self, slice: &str, line_col_start: u32, _line_col_end: u32) { + let bytes = slice.as_bytes(); + let mut buf = String::new(); + let mut buf_start: Option = None; + let mut i = 0usize; + + let flush = |this: &mut Self, buf: &mut String, buf_start: &mut Option, end: u32| { + if !buf.is_empty() { + let s = buf_start.take().unwrap(); + this.emit(VimwikiTokenKind::Text(std::mem::take(buf)), s, end); + } + }; + + while i < bytes.len() { + let abs_col = line_col_start + i as u32; + let rest = &slice[i..]; + + // 1) Multi-char delimiters and constructs (longest match wins). + if let Some(after_open) = rest.strip_prefix("[[") { + flush(self, &mut buf, &mut buf_start, abs_col); + self.emit(VimwikiTokenKind::WikiLinkOpen, abs_col, abs_col + 2); + i += 2; + // Lex until `]]` (absolute index into `slice`). + let close_abs_in_slice = after_open.find("]]").map(|c| i + c); + let inner_end = close_abs_in_slice.unwrap_or(slice.len()); + self.lex_link_body(slice, i, line_col_start, inner_end, true); + i = inner_end; + if close_abs_in_slice.is_some() { + let close_col = line_col_start + i as u32; + self.emit(VimwikiTokenKind::WikiLinkClose, close_col, close_col + 2); + i += 2; + } + continue; + } + if rest.starts_with("{{{") { + // Inline {{{ is rare; fall through as text. + buf_start.get_or_insert(abs_col); + buf.push('{'); + i += 1; + continue; + } + if rest.starts_with("{{") { + flush(self, &mut buf, &mut buf_start, abs_col); + self.emit(VimwikiTokenKind::TransclusionOpen, abs_col, abs_col + 2); + i += 2; + let close_rel = slice[i..].find("}}"); + let inner_end_rel = close_rel.map(|c| i + c).unwrap_or(slice.len()); + self.lex_link_body(slice, i, line_col_start, inner_end_rel, false); + i = inner_end_rel; + if close_rel.is_some() { + let close_col = line_col_start + i as u32; + self.emit( + VimwikiTokenKind::TransclusionClose, + close_col, + close_col + 2, + ); + i += 2; + } + continue; + } + if rest.starts_with("~~") { + flush(self, &mut buf, &mut buf_start, abs_col); + self.emit(VimwikiTokenKind::StrikethroughDelim, abs_col, abs_col + 2); + i += 2; + continue; + } + if rest.starts_with(",,") { + flush(self, &mut buf, &mut buf_start, abs_col); + self.emit(VimwikiTokenKind::SubscriptDelim, abs_col, abs_col + 2); + i += 2; + continue; + } + + // 2) Inline code: `...` + if bytes[i] == b'`' { + if let Some(end_rel) = slice[i + 1..].find('`') { + flush(self, &mut buf, &mut buf_start, abs_col); + let content = &slice[i + 1..i + 1 + end_rel]; + let span_end = abs_col + (end_rel + 2) as u32; + self.emit( + VimwikiTokenKind::Code(content.to_owned()), + abs_col, + span_end, + ); + i += end_rel + 2; + continue; + } + } + + // 3) Inline math: $...$ (single $ on each side; bail to text if unmatched) + if bytes[i] == b'$' { + if let Some(end_rel) = slice[i + 1..].find('$') { + flush(self, &mut buf, &mut buf_start, abs_col); + let content = &slice[i + 1..i + 1 + end_rel]; + let span_end = abs_col + (end_rel + 2) as u32; + self.emit( + VimwikiTokenKind::MathInline(content.to_owned()), + abs_col, + span_end, + ); + i += end_rel + 2; + continue; + } + } + + // 4) Single-char delimiters. + let single = match bytes[i] { + b'*' => Some(VimwikiTokenKind::BoldDelim), + b'_' => Some(VimwikiTokenKind::ItalicDelim), + b'^' => Some(VimwikiTokenKind::SuperscriptDelim), + _ => None, + }; + if let Some(kind) = single { + flush(self, &mut buf, &mut buf_start, abs_col); + self.emit(kind, abs_col, abs_col + 1); + i += 1; + continue; + } + + // 5) Raw URL — at a word boundary. + if is_word_boundary(bytes, i) { + if let Some(url_len) = match_url(rest) { + flush(self, &mut buf, &mut buf_start, abs_col); + let url = &rest[..url_len]; + self.emit( + VimwikiTokenKind::RawUrl(url.to_owned()), + abs_col, + abs_col + url_len as u32, + ); + i += url_len; + continue; + } + // 6) Keyword. + if let Some((kw_len, kw)) = match_keyword(rest) { + flush(self, &mut buf, &mut buf_start, abs_col); + self.emit( + VimwikiTokenKind::Keyword(kw), + abs_col, + abs_col + kw_len as u32, + ); + i += kw_len; + continue; + } + } + + // 7) Default: accumulate into Text. Walk to next byte boundary. + let ch = slice[i..].chars().next().unwrap(); + let ch_len = ch.len_utf8(); + buf_start.get_or_insert(abs_col); + buf.push(ch); + i += ch_len; + } + + let end_col = line_col_start + slice.len() as u32; + flush(self, &mut buf, &mut buf_start, end_col); + } + + /// Lex inline content within `[[...]]` or `{{...}}`, where `|` is a + /// separator rather than literal pipe. `is_wikilink` chooses the token + /// variant for the separator. + fn lex_link_body( + &mut self, + slice: &str, + start: usize, + line_col_start: u32, + end: usize, + is_wikilink: bool, + ) { + let bytes = slice.as_bytes(); + let mut buf = String::new(); + let mut buf_start: Option = None; + let mut i = start; + + let flush = + |this: &mut Self, buf: &mut String, buf_start: &mut Option, end_col: u32| { + if !buf.is_empty() { + let s = buf_start.take().unwrap(); + this.emit(VimwikiTokenKind::Text(std::mem::take(buf)), s, end_col); + } + }; + + while i < end { + let abs_col = line_col_start + i as u32; + if bytes[i] == b'|' { + flush(self, &mut buf, &mut buf_start, abs_col); + let kind = if is_wikilink { + VimwikiTokenKind::WikiLinkSep + } else { + VimwikiTokenKind::TransclusionSep + }; + self.emit(kind, abs_col, abs_col + 1); + i += 1; + continue; + } + let ch = slice[i..].chars().next().unwrap(); + let ch_len = ch.len_utf8(); + buf_start.get_or_insert(abs_col); + buf.push(ch); + i += ch_len; + } + flush(self, &mut buf, &mut buf_start, line_col_start + end as u32); + } +} + +// ===== Helpers ===== + +/// Parse the bit after `{{{` on a fence line. Format is forgiving: +/// `lang` (single word) followed by optional `key=value` pairs separated +/// by spaces. +fn parse_fence_attrs(s: &str) -> (Option, HashMap) { + let mut attrs = HashMap::new(); + let mut language: Option = None; + for tok in s.split_whitespace() { + if let Some(eq) = tok.find('=') { + let (k, v) = tok.split_at(eq); + let v = &v[1..]; + // Strip optional surrounding quotes from value. + let v = v.trim_matches('"').trim_matches('\''); + attrs.insert(k.to_owned(), v.to_owned()); + } else if language.is_none() { + language = Some(tok.to_owned()); + } + } + (language, attrs) +} + +fn is_word_boundary(bytes: &[u8], i: usize) -> bool { + if i == 0 { + return true; + } + let prev = bytes[i - 1]; + !(prev.is_ascii_alphanumeric() || prev == b'_') +} + +fn is_word_end(bytes: &[u8], i: usize) -> bool { + if i >= bytes.len() { + return true; + } + let next = bytes[i]; + !(next.is_ascii_alphanumeric() || next == b'_') +} + +const URL_SCHEMES: &[&str] = &["https://", "http://", "ftp://", "mailto:", "file://"]; + +fn match_url(rest: &str) -> Option { + let bytes = rest.as_bytes(); + let scheme_len = URL_SCHEMES.iter().find_map(|s| { + if rest.starts_with(s) { + Some(s.len()) + } else { + None + } + })?; + let mut end = scheme_len; + while end < bytes.len() { + let b = bytes[end]; + // Conservative URL char set; stops at whitespace and most punctuation + // that's typically not part of a URL tail. + let ok = b.is_ascii_alphanumeric() + || matches!( + b, + b'-' | b'_' + | b'.' + | b'~' + | b'/' + | b':' + | b'?' + | b'#' + | b'[' + | b']' + | b'@' + | b'!' + | b'$' + | b'&' + | b'\'' + | b'(' + | b')' + | b'*' + | b'+' + | b',' + | b';' + | b'=' + | b'%' + ); + if !ok { + break; + } + end += 1; + } + // Trim trailing punctuation that's almost always sentence punctuation. + while end > scheme_len { + match bytes[end - 1] { + b'.' | b',' | b';' | b':' | b'!' | b'?' | b')' | b']' => end -= 1, + _ => break, + } + } + if end <= scheme_len { + None + } else { + Some(end) + } +} + +const KEYWORDS: &[(&str, Keyword)] = &[ + ("TODO", Keyword::Todo), + ("DONE", Keyword::Done), + ("STARTED", Keyword::Started), + ("FIXME", Keyword::Fixme), + ("FIXED", Keyword::Fixed), + ("XXX", Keyword::Xxx), +]; + +fn match_keyword(rest: &str) -> Option<(usize, Keyword)> { + let bytes = rest.as_bytes(); + KEYWORDS.iter().find_map(|(text, kw)| { + if rest.starts_with(text) && is_word_end(bytes, text.len()) { + Some((text.len(), *kw)) + } else { + None + } + }) +} + +fn list_marker_at(s: &str) -> Option<(ListSymbol, usize)> { + let bytes = s.as_bytes(); + if bytes.is_empty() { + return None; + } + // Single-char markers: -, *, # + match bytes[0] { + b'-' => return Some((ListSymbol::Dash, 1)), + b'*' => return Some((ListSymbol::Star, 1)), + b'#' => return Some((ListSymbol::Hash, 1)), + _ => {} + } + // Numeric: `1.` or `1)`. Allow multi-digit. + let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count(); + if digits > 0 && bytes.len() > digits { + match bytes[digits] { + b'.' => return Some((ListSymbol::Numeric, digits + 1)), + b')' => return Some((ListSymbol::NumericParen, digits + 1)), + _ => {} + } + } + // Single-letter alpha or roman: `a)`, `A)`, `i)`, `I)`. + if bytes.len() >= 2 && bytes[1] == b')' { + let c = bytes[0]; + if c.is_ascii_lowercase() { + // `i)` / `v)` / `x)` are roman by convention; without context we + // treat any single lowercase letter as alpha. Roman vs alpha is + // ambiguous from a single char; vimwiki itself relies on context. + // Mark `i` as RomanParen as a small concession to the spec. + let sym = if c == b'i' { + ListSymbol::RomanParen + } else { + ListSymbol::AlphaParen + }; + return Some((sym, 2)); + } + if c.is_ascii_uppercase() { + let sym = if c == b'I' { + ListSymbol::RomanUpperParen + } else { + ListSymbol::AlphaUpperParen + }; + return Some((sym, 2)); + } + } + None +} + +fn looks_like_list_marker(s: &str) -> bool { + list_marker_at(s).is_some_and(|(_, len)| s.as_bytes().get(len) == Some(&b' ')) +} + +fn checkbox_at(s: &str) -> Option<(CheckboxState, usize)> { + let b = s.as_bytes(); + if b.len() < 3 || b[0] != b'[' || b[2] != b']' { + return None; + } + let cb = match b[1] { + b' ' => CheckboxState::Empty, + b'.' => CheckboxState::Quarter, + b'o' => CheckboxState::Half, + b'O' => CheckboxState::ThreeQuarters, + b'X' => CheckboxState::Done, + b'-' => CheckboxState::Rejected, + _ => return None, + }; + Some((cb, 3)) +} diff --git a/crates/nuwiki-core/src/syntax/vimwiki/mod.rs b/crates/nuwiki-core/src/syntax/vimwiki/mod.rs new file mode 100644 index 0000000..5dee08b --- /dev/null +++ b/crates/nuwiki-core/src/syntax/vimwiki/mod.rs @@ -0,0 +1,8 @@ +//! Vimwiki syntax plugin. +//! +//! Phase 3 lands the lexer; the parser arrives in Phase 4 and the full +//! `SyntaxPlugin` impl follows. + +pub mod lexer; + +pub use lexer::{VimwikiLexer, VimwikiToken, VimwikiTokenKind}; diff --git a/crates/nuwiki-core/tests/vimwiki_lexer.rs b/crates/nuwiki-core/tests/vimwiki_lexer.rs new file mode 100644 index 0000000..236506e --- /dev/null +++ b/crates/nuwiki-core/tests/vimwiki_lexer.rs @@ -0,0 +1,592 @@ +//! End-to-end tests for the vimwiki lexer. +//! +//! Each test feeds a small fragment to `VimwikiLexer::lex` and asserts the +//! exact token-kind sequence. A separate `spans_are_byte_accurate` test +//! pins down position semantics. + +use std::collections::HashMap; + +use nuwiki_core::ast::{CheckboxState, Keyword, ListSymbol, Position}; +use nuwiki_core::syntax::vimwiki::{VimwikiLexer, VimwikiTokenKind}; +use nuwiki_core::syntax::Lexer; + +use VimwikiTokenKind::*; + +fn lex(text: &str) -> Vec { + VimwikiLexer::new() + .lex(text) + .into_iter() + .map(|t| t.kind) + .collect() +} + +fn text(s: &str) -> VimwikiTokenKind { + Text(s.into()) +} + +// ===== Empty / whitespace ===== + +#[test] +fn empty_input_produces_no_tokens() { + assert!(lex("").is_empty()); +} + +#[test] +fn lone_newline_is_a_blank_line() { + assert_eq!(lex("\n"), vec![BlankLine]); +} + +#[test] +fn whitespace_only_line_is_a_blank_line() { + assert_eq!(lex(" \n"), vec![BlankLine]); +} + +#[test] +fn blank_lines_separate_paragraphs() { + assert_eq!( + lex("hello\n\nworld\n"), + vec![text("hello"), Newline, BlankLine, text("world"), Newline] + ); +} + +// ===== Headings ===== + +#[test] +fn headings_at_all_six_levels() { + let mut tokens = Vec::new(); + for level in 1..=6 { + let line = format!( + "{} L{} {}\n", + "=".repeat(level as usize), + level, + "=".repeat(level as usize) + ); + let lexed = lex(&line); + assert!(matches!( + lexed[0], + HeadingOpen { level: l, centered: false } if l == level + )); + tokens.extend(lexed); + } + assert!(tokens.iter().any(|t| matches!(t, HeadingClose))); +} + +#[test] +fn centered_heading_is_marked_centered() { + let lexed = lex(" = title =\n"); + assert!(matches!( + lexed[0], + HeadingOpen { + level: 1, + centered: true + } + )); + assert_eq!(lexed[1], text("title")); + assert!(matches!(lexed[2], HeadingClose)); +} + +#[test] +fn heading_with_inline_bold() { + let lexed = lex("== Hello *world* ==\n"); + assert_eq!( + lexed, + vec![ + HeadingOpen { + level: 2, + centered: false + }, + text("Hello "), + BoldDelim, + text("world"), + BoldDelim, + HeadingClose, + Newline, + ] + ); +} + +// ===== Horizontal rule ===== + +#[test] +fn horizontal_rule_requires_four_or_more_dashes() { + assert_eq!(lex("----\n"), vec![HorizontalRule, Newline]); + assert_eq!(lex("--------\n"), vec![HorizontalRule, Newline]); + // Three dashes: not a rule, parsed as text. + assert_eq!(lex("---\n"), vec![text("---"), Newline]); +} + +// ===== Comments ===== + +#[test] +fn single_line_comment() { + assert_eq!( + lex("%% hello\n"), + vec![CommentLine(" hello".into()), Newline] + ); +} + +#[test] +fn multiline_comment_spans_multiple_lines() { + assert_eq!( + lex("%%+ a\nb\n+%%\n"), + vec![ + CommentMultiOpen, + CommentMultiLine(" a".into()), + Newline, + CommentMultiLine("b".into()), + Newline, + CommentMultiClose, + Newline, + ] + ); +} + +#[test] +fn multiline_comment_one_liner() { + assert_eq!( + lex("%%+ inline +%%\n"), + vec![ + CommentMultiOpen, + CommentMultiLine(" inline ".into()), + CommentMultiClose, + Newline, + ] + ); +} + +// ===== Preformatted ===== + +#[test] +fn preformatted_block_captures_lines_verbatim() { + let lexed = lex("{{{\nfn main() {}\n}}}\n"); + assert!(matches!(lexed[0], PreformattedOpen { .. })); + assert_eq!(lexed[1], Newline); + assert_eq!(lexed[2], PreformattedLine("fn main() {}".into())); + assert_eq!(lexed[3], Newline); + assert_eq!(lexed[4], PreformattedClose); + assert_eq!(lexed[5], Newline); +} + +#[test] +fn preformatted_block_parses_language() { + let lexed = lex("{{{rust\nx\n}}}\n"); + let PreformattedOpen { language, .. } = &lexed[0] else { + panic!("expected PreformattedOpen"); + }; + assert_eq!(language.as_deref(), Some("rust")); +} + +#[test] +fn preformatted_block_parses_attrs() { + let lexed = lex("{{{rust class=\"hl\" key=val\nx\n}}}\n"); + let PreformattedOpen { language, attrs } = &lexed[0] else { + panic!("expected PreformattedOpen"); + }; + assert_eq!(language.as_deref(), Some("rust")); + let mut expected = HashMap::new(); + expected.insert("class".into(), "hl".into()); + expected.insert("key".into(), "val".into()); + assert_eq!(attrs, &expected); +} + +// ===== Math block ===== + +#[test] +fn math_block_captures_environment() { + let lexed = lex("{{$%align%\nx = 1\n}}$\n"); + assert_eq!( + lexed[0], + MathBlockOpen { + environment: Some("align".into()) + } + ); + assert_eq!(lexed[2], MathBlockLine("x = 1".into())); + assert_eq!(lexed[4], MathBlockClose); +} + +// ===== Placeholders ===== + +#[test] +fn all_four_placeholders() { + assert_eq!( + lex("%title My Page\n"), + vec![PlaceholderTitle(Some("My Page".into())), Newline] + ); + assert_eq!(lex("%nohtml\n"), vec![PlaceholderNohtml, Newline]); + assert_eq!( + lex("%template fancy\n"), + vec![PlaceholderTemplate(Some("fancy".into())), Newline] + ); + assert_eq!( + lex("%date 2026-05-10\n"), + vec![PlaceholderDate(Some("2026-05-10".into())), Newline] + ); +} + +// ===== Lists ===== + +#[test] +fn list_marker_dash() { + let lexed = lex("- item\n"); + assert_eq!( + lexed, + vec![ + ListMarker { + symbol: ListSymbol::Dash, + indent: 0 + }, + text("item"), + Newline, + ] + ); +} + +#[test] +fn list_marker_star_with_space_is_a_list_not_bold() { + let lexed = lex("* item\n"); + assert_eq!( + lexed[0], + ListMarker { + symbol: ListSymbol::Star, + indent: 0 + } + ); + assert_eq!(lexed[1], text("item")); +} + +#[test] +fn list_marker_kinds() { + let cases: &[(&str, ListSymbol)] = &[ + ("- a\n", ListSymbol::Dash), + ("* a\n", ListSymbol::Star), + ("# a\n", ListSymbol::Hash), + ("1. a\n", ListSymbol::Numeric), + ("1) a\n", ListSymbol::NumericParen), + ("a) a\n", ListSymbol::AlphaParen), + ("A) a\n", ListSymbol::AlphaUpperParen), + ("i) a\n", ListSymbol::RomanParen), + ("I) a\n", ListSymbol::RomanUpperParen), + ]; + for (line, expected) in cases { + let lexed = lex(line); + assert!( + matches!(lexed[0], ListMarker { symbol, .. } if symbol == *expected), + "input {line:?} expected {expected:?}, got {:?}", + lexed[0] + ); + } +} + +#[test] +fn list_marker_at_indent() { + let lexed = lex(" - nested\n"); + assert_eq!( + lexed[0], + ListMarker { + symbol: ListSymbol::Dash, + indent: 4 + } + ); +} + +#[test] +fn checkbox_states() { + let cases: &[(&str, CheckboxState)] = &[ + ("- [ ] a\n", CheckboxState::Empty), + ("- [.] a\n", CheckboxState::Quarter), + ("- [o] a\n", CheckboxState::Half), + ("- [O] a\n", CheckboxState::ThreeQuarters), + ("- [X] a\n", CheckboxState::Done), + ("- [-] a\n", CheckboxState::Rejected), + ]; + for (line, expected) in cases { + let lexed = lex(line); + assert!( + matches!(lexed[1], Checkbox(s) if s == *expected), + "input {line:?} expected {expected:?}, got {:?}", + lexed[1] + ); + } +} + +// ===== Blockquotes ===== + +#[test] +fn angle_blockquote() { + assert_eq!( + lex("> quoted\n"), + vec![BlockquoteMarker, text("quoted"), Newline] + ); +} + +#[test] +fn indent_blockquote() { + assert_eq!( + lex(" quoted\n"), + vec![BlockquoteIndent, text("quoted"), Newline] + ); +} + +#[test] +fn indent_with_list_marker_is_a_list_not_blockquote() { + let lexed = lex(" - item\n"); + assert!(matches!(lexed[0], ListMarker { indent: 4, .. })); +} + +// ===== Tables ===== + +#[test] +fn simple_table_row() { + assert_eq!( + lex("| a | b |\n"), + vec![ + TableSep, + text(" a "), + TableSep, + text(" b "), + TableSep, + Newline, + ] + ); +} + +#[test] +fn table_header_separator_row() { + assert_eq!(lex("|---|---|\n"), vec![TableHeaderRow, Newline]); +} + +#[test] +fn table_col_and_row_span_cells() { + let lexed = lex("| a | > | \\/ |\n"); + // sep, " a ", sep, ColSpan, sep, RowSpan, sep, newline + assert_eq!( + lexed, + vec![ + TableSep, + text(" a "), + TableSep, + TableColSpan, + TableSep, + TableRowSpan, + TableSep, + Newline, + ] + ); +} + +// ===== Definition list ===== + +#[test] +fn definition_term_with_inline_definition() { + let lexed = lex("Term:: Definition\n"); + assert_eq!( + lexed, + vec![ + text("Term"), + DefinitionTermMarker, + text("Definition"), + Newline, + ] + ); +} + +#[test] +fn definition_term_alone() { + let lexed = lex("Term::\n"); + assert_eq!(lexed, vec![text("Term"), DefinitionTermMarker, Newline]); +} + +// ===== Inline typefaces ===== + +#[test] +fn inline_bold_italic_strike_super_sub() { + assert_eq!( + lex("*b* _i_ ~~s~~ ^sup^ ,,sub,,\n"), + vec![ + BoldDelim, + text("b"), + BoldDelim, + text(" "), + ItalicDelim, + text("i"), + ItalicDelim, + text(" "), + StrikethroughDelim, + text("s"), + StrikethroughDelim, + text(" "), + SuperscriptDelim, + text("sup"), + SuperscriptDelim, + text(" "), + SubscriptDelim, + text("sub"), + SubscriptDelim, + Newline, + ] + ); +} + +#[test] +fn inline_code_captures_content() { + assert_eq!(lex("`x = 1`\n"), vec![Code("x = 1".into()), Newline]); +} + +#[test] +fn inline_math_captures_content() { + assert_eq!( + lex("$E = mc^2$\n"), + vec![MathInline("E = mc^2".into()), Newline] + ); +} + +// ===== Keywords ===== + +#[test] +fn all_six_keywords() { + let cases: &[(&str, Keyword)] = &[ + ("TODO", Keyword::Todo), + ("DONE", Keyword::Done), + ("STARTED", Keyword::Started), + ("FIXME", Keyword::Fixme), + ("FIXED", Keyword::Fixed), + ("XXX", Keyword::Xxx), + ]; + for (s, expected) in cases { + let line = format!("{s} stuff\n"); + let lexed = lex(&line); + assert!( + matches!(lexed[0], Keyword(k) if k == *expected), + "input {s:?} expected {expected:?}, got {:?}", + lexed[0] + ); + } +} + +#[test] +fn keyword_only_at_word_boundary() { + // Embedded TODO inside a longer word should NOT match as a keyword. + let lexed = lex("aTODOb\n"); + assert_eq!(lexed, vec![text("aTODOb"), Newline]); +} + +// ===== Wikilinks / transclusions / raw URLs ===== + +#[test] +fn bare_wikilink() { + assert_eq!( + lex("[[Page]]\n"), + vec![WikiLinkOpen, text("Page"), WikiLinkClose, Newline] + ); +} + +#[test] +fn described_wikilink() { + assert_eq!( + lex("[[Page|Description]]\n"), + vec![ + WikiLinkOpen, + text("Page"), + WikiLinkSep, + text("Description"), + WikiLinkClose, + Newline, + ] + ); +} + +#[test] +fn wikilink_with_anchor_is_lexed_as_text() { + // The lexer doesn't carve out anchors; the parser does. + let lexed = lex("[[Page#Anchor]]\n"); + assert_eq!( + lexed, + vec![WikiLinkOpen, text("Page#Anchor"), WikiLinkClose, Newline,] + ); +} + +#[test] +fn transclusion_with_alt_and_attrs() { + assert_eq!( + lex("{{img.png|alt|class=hi}}\n"), + vec![ + TransclusionOpen, + text("img.png"), + TransclusionSep, + text("alt"), + TransclusionSep, + text("class=hi"), + TransclusionClose, + Newline, + ] + ); +} + +#[test] +fn raw_urls_for_each_supported_scheme() { + for scheme in &[ + "https://example.com", + "http://example.com/path?q=1", + "ftp://archive.example.com/file.tar.gz", + "mailto:hi@example.com", + "file:///tmp/file.txt", + ] { + let line = format!("see {scheme} now\n"); + let lexed = lex(&line); + assert!( + lexed.iter().any(|t| matches!(t, RawUrl(u) if u == scheme)), + "scheme {scheme:?} not lexed; got {lexed:?}" + ); + } +} + +#[test] +fn raw_url_drops_trailing_sentence_punctuation() { + let lexed = lex("Visit https://example.com.\n"); + assert!( + lexed + .iter() + .any(|t| matches!(t, RawUrl(u) if u == "https://example.com")), + "got {lexed:?}" + ); +} + +// ===== Span correctness ===== + +#[test] +fn spans_are_byte_accurate() { + // Input layout: + // Line 0: "= H =" + // Line 1: "" (blank line) + // Line 2: "*bold*" + let src = "= H =\n\n*bold*\n"; + let tokens = VimwikiLexer::new().lex(src).into_vec(); + + // First token: HeadingOpen at line 0, col 0..1 + assert_eq!( + tokens[0].span.start, + Position { + line: 0, + column: 0, + offset: 0 + } + ); + assert_eq!( + tokens[0].span.end, + Position { + line: 0, + column: 1, + offset: 1 + } + ); + + // Find the BoldDelim opening on line 2. + let bold_open = tokens + .iter() + .find(|t| matches!(t.kind, BoldDelim)) + .expect("bold delim"); + assert_eq!(bold_open.span.start.line, 2); + assert_eq!(bold_open.span.start.column, 0); + // "= H =\n" = 6 bytes, "\n" = 1, so line 2 starts at offset 7. + assert_eq!(bold_open.span.start.offset, 7); +}