nuwiki/crates/nuwiki-core/tests/vimwiki_lexer.rs

//! End-to-end tests for the vimwiki lexer.
//!
//! Each test feeds a small fragment to `VimwikiLexer::lex` and asserts the
//! exact token-kind sequence. A separate `spans_are_byte_accurate` test
//! pins down position semantics.

use std::collections::HashMap;

use nuwiki_core::ast::{CheckboxState, Keyword, ListSymbol, Position};
use nuwiki_core::syntax::vimwiki::{VimwikiLexer, VimwikiTokenKind};
use nuwiki_core::syntax::Lexer;

use VimwikiTokenKind::*;

fn lex(text: &str) -> Vec<VimwikiTokenKind> {
    VimwikiLexer::new()
        .lex(text)
        .into_iter()
        .map(|t| t.kind)
        .collect()
}

fn text(s: &str) -> VimwikiTokenKind {
    Text(s.into())
}

// ===== Empty / whitespace =====

#[test]
fn empty_input_produces_no_tokens() {
    assert!(lex("").is_empty());
}

#[test]
fn lone_newline_is_a_blank_line() {
    assert_eq!(lex("\n"), vec![BlankLine]);
}

#[test]
fn whitespace_only_line_is_a_blank_line() {
    assert_eq!(lex("   \n"), vec![BlankLine]);
}

#[test]
fn blank_lines_separate_paragraphs() {
    assert_eq!(
        lex("hello\n\nworld\n"),
        vec![text("hello"), Newline, BlankLine, text("world"), Newline]
    );
}

// ===== Headings =====

#[test]
fn headings_at_all_six_levels() {
    let mut tokens = Vec::new();
    for level in 1..=6 {
        let line = format!(
            "{} L{} {}\n",
            "=".repeat(level as usize),
            level,
            "=".repeat(level as usize)
        );
        let lexed = lex(&line);
        assert!(matches!(
            lexed[0],
            HeadingOpen { level: l, centered: false } if l == level
        ));
        tokens.extend(lexed);
    }
    assert!(tokens.iter().any(|t| matches!(t, HeadingClose)));
}

#[test]
fn centered_heading_is_marked_centered() {
    let lexed = lex("  = title =\n");
    assert!(matches!(
        lexed[0],
        HeadingOpen {
            level: 1,
            centered: true
        }
    ));
    assert_eq!(lexed[1], text("title"));
    assert!(matches!(lexed[2], HeadingClose));
}

#[test]
fn heading_with_inline_bold() {
    let lexed = lex("== Hello *world* ==\n");
    assert_eq!(
        lexed,
        vec![
            HeadingOpen {
                level: 2,
                centered: false
            },
            text("Hello "),
            BoldDelim,
            text("world"),
            BoldDelim,
            HeadingClose,
            Newline,
        ]
    );
}

// ===== Horizontal rule =====

#[test]
fn horizontal_rule_requires_four_or_more_dashes() {
    assert_eq!(lex("----\n"), vec![HorizontalRule, Newline]);
    assert_eq!(lex("--------\n"), vec![HorizontalRule, Newline]);
    // Three dashes: not a rule, parsed as text.
    assert_eq!(lex("---\n"), vec![text("---"), Newline]);
}

// ===== Comments =====

#[test]
fn single_line_comment() {
    assert_eq!(
        lex("%% hello\n"),
        vec![CommentLine(" hello".into()), Newline]
    );
}

#[test]
fn multiline_comment_spans_multiple_lines() {
    assert_eq!(
        lex("%%+ a\nb\n+%%\n"),
        vec![
            CommentMultiOpen,
            CommentMultiLine(" a".into()),
            Newline,
            CommentMultiLine("b".into()),
            Newline,
            CommentMultiClose,
            Newline,
        ]
    );
}

#[test]
fn multiline_comment_one_liner() {
    assert_eq!(
        lex("%%+ inline +%%\n"),
        vec![
            CommentMultiOpen,
            CommentMultiLine(" inline ".into()),
            CommentMultiClose,
            Newline,
        ]
    );
}

// ===== Preformatted =====

#[test]
fn preformatted_block_captures_lines_verbatim() {
    let lexed = lex("{{{\nfn main() {}\n}}}\n");
    assert!(matches!(lexed[0], PreformattedOpen { .. }));
    assert_eq!(lexed[1], Newline);
    assert_eq!(lexed[2], PreformattedLine("fn main() {}".into()));
    assert_eq!(lexed[3], Newline);
    assert_eq!(lexed[4], PreformattedClose);
    assert_eq!(lexed[5], Newline);
}

#[test]
fn preformatted_block_parses_language() {
    let lexed = lex("{{{rust\nx\n}}}\n");
    let PreformattedOpen { language, .. } = &lexed[0] else {
        panic!("expected PreformattedOpen");
    };
    assert_eq!(language.as_deref(), Some("rust"));
}

#[test]
fn preformatted_block_parses_attrs() {
    let lexed = lex("{{{rust class=\"hl\" key=val\nx\n}}}\n");
    let PreformattedOpen { language, attrs } = &lexed[0] else {
        panic!("expected PreformattedOpen");
    };
    assert_eq!(language.as_deref(), Some("rust"));
    let mut expected = HashMap::new();
    expected.insert("class".into(), "hl".into());
    expected.insert("key".into(), "val".into());
    assert_eq!(attrs, &expected);
}

// ===== Math block =====

#[test]
fn math_block_captures_environment() {
    let lexed = lex("{{$%align%\nx = 1\n}}$\n");
    assert_eq!(
        lexed[0],
        MathBlockOpen {
            environment: Some("align".into())
        }
    );
    assert_eq!(lexed[2], MathBlockLine("x = 1".into()));
    assert_eq!(lexed[4], MathBlockClose);
}

// ===== Placeholders =====

#[test]
fn all_four_placeholders() {
    assert_eq!(
        lex("%title My Page\n"),
        vec![PlaceholderTitle(Some("My Page".into())), Newline]
    );
    assert_eq!(lex("%nohtml\n"), vec![PlaceholderNohtml, Newline]);
    assert_eq!(
        lex("%template fancy\n"),
        vec![PlaceholderTemplate(Some("fancy".into())), Newline]
    );
    assert_eq!(
        lex("%date 2026-05-10\n"),
        vec![PlaceholderDate(Some("2026-05-10".into())), Newline]
    );
}

// ===== Lists =====

#[test]
fn list_marker_dash() {
    let lexed = lex("- item\n");
    assert_eq!(
        lexed,
        vec![
            ListMarker {
                symbol: ListSymbol::Dash,
                indent: 0
            },
            text("item"),
            Newline,
        ]
    );
}

#[test]
fn list_marker_star_with_space_is_a_list_not_bold() {
    let lexed = lex("* item\n");
    assert_eq!(
        lexed[0],
        ListMarker {
            symbol: ListSymbol::Star,
            indent: 0
        }
    );
    assert_eq!(lexed[1], text("item"));
}

#[test]
fn list_marker_kinds() {
    let cases: &[(&str, ListSymbol)] = &[
        ("- a\n", ListSymbol::Dash),
        ("* a\n", ListSymbol::Star),
        ("# a\n", ListSymbol::Hash),
        ("1. a\n", ListSymbol::Numeric),
        ("1) a\n", ListSymbol::NumericParen),
        ("a) a\n", ListSymbol::AlphaParen),
        ("A) a\n", ListSymbol::AlphaUpperParen),
        ("i) a\n", ListSymbol::RomanParen),
        ("I) a\n", ListSymbol::RomanUpperParen),
    ];
    for (line, expected) in cases {
        let lexed = lex(line);
        assert!(
            matches!(lexed[0], ListMarker { symbol, .. } if symbol == *expected),
            "input {line:?} expected {expected:?}, got {:?}",
            lexed[0]
        );
    }
}

#[test]
fn list_marker_at_indent() {
    let lexed = lex("    - nested\n");
    assert_eq!(
        lexed[0],
        ListMarker {
            symbol: ListSymbol::Dash,
            indent: 4
        }
    );
}

#[test]
fn checkbox_states() {
    let cases: &[(&str, CheckboxState)] = &[
        ("- [ ] a\n", CheckboxState::Empty),
        ("- [.] a\n", CheckboxState::Quarter),
        ("- [o] a\n", CheckboxState::Half),
        ("- [O] a\n", CheckboxState::ThreeQuarters),
        ("- [X] a\n", CheckboxState::Done),
        ("- [-] a\n", CheckboxState::Rejected),
    ];
    for (line, expected) in cases {
        let lexed = lex(line);
        assert!(
            matches!(lexed[1], Checkbox(s) if s == *expected),
            "input {line:?} expected {expected:?}, got {:?}",
            lexed[1]
        );
    }
}

// ===== Blockquotes =====

#[test]
fn angle_blockquote() {
    assert_eq!(
        lex("> quoted\n"),
        vec![BlockquoteMarker, text("quoted"), Newline]
    );
}

#[test]
fn indent_blockquote() {
    assert_eq!(
        lex("    quoted\n"),
        vec![BlockquoteIndent, text("quoted"), Newline]
    );
}

#[test]
fn indent_with_list_marker_is_a_list_not_blockquote() {
    let lexed = lex("    - item\n");
    assert!(matches!(lexed[0], ListMarker { indent: 4, .. }));
}

// ===== Tables =====

#[test]
fn simple_table_row() {
    assert_eq!(
        lex("| a | b |\n"),
        vec![
            TableSep,
            text(" a "),
            TableSep,
            text(" b "),
            TableSep,
            Newline,
        ]
    );
}

#[test]
fn table_header_separator_row() {
    assert_eq!(lex("|---|---|\n"), vec![TableHeaderRow, Newline]);
}

#[test]
fn table_col_and_row_span_cells() {
    let lexed = lex("| a | > | \\/ |\n");
    // sep, " a ", sep, ColSpan, sep, RowSpan, sep, newline
    assert_eq!(
        lexed,
        vec![
            TableSep,
            text(" a "),
            TableSep,
            TableColSpan,
            TableSep,
            TableRowSpan,
            TableSep,
            Newline,
        ]
    );
}

// ===== Definition list =====

#[test]
fn definition_term_with_inline_definition() {
    let lexed = lex("Term:: Definition\n");
    assert_eq!(
        lexed,
        vec![
            text("Term"),
            DefinitionTermMarker,
            text("Definition"),
            Newline,
        ]
    );
}

#[test]
fn definition_term_alone() {
    let lexed = lex("Term::\n");
    assert_eq!(lexed, vec![text("Term"), DefinitionTermMarker, Newline]);
}

// ===== Inline typefaces =====

#[test]
fn inline_bold_italic_strike_super_sub() {
    assert_eq!(
        lex("*b* _i_ ~~s~~ ^sup^ ,,sub,,\n"),
        vec![
            BoldDelim,
            text("b"),
            BoldDelim,
            text(" "),
            ItalicDelim,
            text("i"),
            ItalicDelim,
            text(" "),
            StrikethroughDelim,
            text("s"),
            StrikethroughDelim,
            text(" "),
            SuperscriptDelim,
            text("sup"),
            SuperscriptDelim,
            text(" "),
            SubscriptDelim,
            text("sub"),
            SubscriptDelim,
            Newline,
        ]
    );
}

#[test]
fn inline_code_captures_content() {
    assert_eq!(lex("`x = 1`\n"), vec![Code("x = 1".into()), Newline]);
}

#[test]
fn inline_math_captures_content() {
    assert_eq!(
        lex("$E = mc^2$\n"),
        vec![MathInline("E = mc^2".into()), Newline]
    );
}

// ===== Keywords =====

#[test]
fn all_six_keywords() {
    let cases: &[(&str, Keyword)] = &[
        ("TODO", Keyword::Todo),
        ("DONE", Keyword::Done),
        ("STARTED", Keyword::Started),
        ("FIXME", Keyword::Fixme),
        ("FIXED", Keyword::Fixed),
        ("XXX", Keyword::Xxx),
    ];
    for (s, expected) in cases {
        let line = format!("{s} stuff\n");
        let lexed = lex(&line);
        assert!(
            matches!(lexed[0], Keyword(k) if k == *expected),
            "input {s:?} expected {expected:?}, got {:?}",
            lexed[0]
        );
    }
}

#[test]
fn keyword_only_at_word_boundary() {
    // Embedded TODO inside a longer word should NOT match as a keyword.
    let lexed = lex("aTODOb\n");
    assert_eq!(lexed, vec![text("aTODOb"), Newline]);
}

// ===== Wikilinks / transclusions / raw URLs =====

#[test]
fn bare_wikilink() {
    assert_eq!(
        lex("[[Page]]\n"),
        vec![WikiLinkOpen, text("Page"), WikiLinkClose, Newline]
    );
}

#[test]
fn described_wikilink() {
    assert_eq!(
        lex("[[Page|Description]]\n"),
        vec![
            WikiLinkOpen,
            text("Page"),
            WikiLinkSep,
            text("Description"),
            WikiLinkClose,
            Newline,
        ]
    );
}

#[test]
fn wikilink_with_anchor_is_lexed_as_text() {
    // The lexer doesn't carve out anchors; the parser does.
    let lexed = lex("[[Page#Anchor]]\n");
    assert_eq!(
        lexed,
        vec![WikiLinkOpen, text("Page#Anchor"), WikiLinkClose, Newline,]
    );
}

#[test]
fn transclusion_with_alt_and_attrs() {
    assert_eq!(
        lex("{{img.png|alt|class=hi}}\n"),
        vec![
            TransclusionOpen,
            text("img.png"),
            TransclusionSep,
            text("alt"),
            TransclusionSep,
            text("class=hi"),
            TransclusionClose,
            Newline,
        ]
    );
}

#[test]
fn raw_urls_for_each_supported_scheme() {
    for scheme in &[
        "https://example.com",
        "http://example.com/path?q=1",
        "ftp://archive.example.com/file.tar.gz",
        "mailto:hi@example.com",
        "file:///tmp/file.txt",
    ] {
        let line = format!("see {scheme} now\n");
        let lexed = lex(&line);
        assert!(
            lexed.iter().any(|t| matches!(t, RawUrl(u) if u == scheme)),
            "scheme {scheme:?} not lexed; got {lexed:?}"
        );
    }
}

#[test]
fn raw_url_drops_trailing_sentence_punctuation() {
    let lexed = lex("Visit https://example.com.\n");
    assert!(
        lexed
            .iter()
            .any(|t| matches!(t, RawUrl(u) if u == "https://example.com")),
        "got {lexed:?}"
    );
}

// ===== Span correctness =====

#[test]
fn spans_are_byte_accurate() {
    // Input layout:
    //   Line 0: "= H ="
    //   Line 1: ""        (blank line)
    //   Line 2: "*bold*"
    let src = "= H =\n\n*bold*\n";
    let tokens = VimwikiLexer::new().lex(src).into_vec();

    // First token: HeadingOpen at line 0, col 0..1
    assert_eq!(
        tokens[0].span.start,
        Position {
            line: 0,
            column: 0,
            offset: 0
        }
    );
    assert_eq!(
        tokens[0].span.end,
        Position {
            line: 0,
            column: 1,
            offset: 1
        }
    );

    // Find the BoldDelim opening on line 2.
    let bold_open = tokens
        .iter()
        .find(|t| matches!(t.kind, BoldDelim))
        .expect("bold delim");
    assert_eq!(bold_open.span.start.line, 2);
    assert_eq!(bold_open.span.start.column, 0);
    // "= H =\n" = 6 bytes, "\n" = 1, so line 2 starts at offset 7.
    assert_eq!(bold_open.span.start.offset, 7);
}