//! End-to-end tests for the vimwiki lexer. //! //! Each test feeds a small fragment to `VimwikiLexer::lex` and asserts the //! exact token-kind sequence. A separate `spans_are_byte_accurate` test //! pins down position semantics. use std::collections::HashMap; use nuwiki_core::ast::{CheckboxState, Keyword, ListSymbol, Position}; use nuwiki_core::syntax::vimwiki::{VimwikiLexer, VimwikiTokenKind}; use nuwiki_core::syntax::Lexer; use VimwikiTokenKind::*; fn lex(text: &str) -> Vec { VimwikiLexer::new() .lex(text) .into_iter() .map(|t| t.kind) .collect() } fn text(s: &str) -> VimwikiTokenKind { Text(s.into()) } // ===== Empty / whitespace ===== #[test] fn empty_input_produces_no_tokens() { assert!(lex("").is_empty()); } #[test] fn lone_newline_is_a_blank_line() { assert_eq!(lex("\n"), vec![BlankLine]); } #[test] fn whitespace_only_line_is_a_blank_line() { assert_eq!(lex(" \n"), vec![BlankLine]); } #[test] fn blank_lines_separate_paragraphs() { assert_eq!( lex("hello\n\nworld\n"), vec![text("hello"), Newline, BlankLine, text("world"), Newline] ); } // ===== Headings ===== #[test] fn headings_at_all_six_levels() { let mut tokens = Vec::new(); for level in 1..=6 { let line = format!( "{} L{} {}\n", "=".repeat(level as usize), level, "=".repeat(level as usize) ); let lexed = lex(&line); assert!(matches!( lexed[0], HeadingOpen { level: l, centered: false } if l == level )); tokens.extend(lexed); } assert!(tokens.iter().any(|t| matches!(t, HeadingClose))); } #[test] fn centered_heading_is_marked_centered() { let lexed = lex(" = title =\n"); assert!(matches!( lexed[0], HeadingOpen { level: 1, centered: true } )); assert_eq!(lexed[1], text("title")); assert!(matches!(lexed[2], HeadingClose)); } #[test] fn heading_with_inline_bold() { let lexed = lex("== Hello *world* ==\n"); assert_eq!( lexed, vec![ HeadingOpen { level: 2, centered: false }, text("Hello "), BoldDelim, text("world"), BoldDelim, HeadingClose, Newline, ] ); } // ===== Horizontal rule ===== #[test] fn horizontal_rule_requires_four_or_more_dashes() { assert_eq!(lex("----\n"), vec![HorizontalRule, Newline]); assert_eq!(lex("--------\n"), vec![HorizontalRule, Newline]); // Three dashes: not a rule, parsed as text. assert_eq!(lex("---\n"), vec![text("---"), Newline]); } // ===== Comments ===== #[test] fn single_line_comment() { assert_eq!( lex("%% hello\n"), vec![CommentLine(" hello".into()), Newline] ); } #[test] fn multiline_comment_spans_multiple_lines() { assert_eq!( lex("%%+ a\nb\n+%%\n"), vec![ CommentMultiOpen, CommentMultiLine(" a".into()), Newline, CommentMultiLine("b".into()), Newline, CommentMultiClose, Newline, ] ); } #[test] fn multiline_comment_one_liner() { assert_eq!( lex("%%+ inline +%%\n"), vec![ CommentMultiOpen, CommentMultiLine(" inline ".into()), CommentMultiClose, Newline, ] ); } // ===== Preformatted ===== #[test] fn preformatted_block_captures_lines_verbatim() { let lexed = lex("{{{\nfn main() {}\n}}}\n"); assert!(matches!(lexed[0], PreformattedOpen { .. })); assert_eq!(lexed[1], Newline); assert_eq!(lexed[2], PreformattedLine("fn main() {}".into())); assert_eq!(lexed[3], Newline); assert_eq!(lexed[4], PreformattedClose); assert_eq!(lexed[5], Newline); } #[test] fn preformatted_block_parses_language() { let lexed = lex("{{{rust\nx\n}}}\n"); let PreformattedOpen { language, .. } = &lexed[0] else { panic!("expected PreformattedOpen"); }; assert_eq!(language.as_deref(), Some("rust")); } #[test] fn preformatted_block_parses_attrs() { let lexed = lex("{{{rust class=\"hl\" key=val\nx\n}}}\n"); let PreformattedOpen { language, attrs } = &lexed[0] else { panic!("expected PreformattedOpen"); }; assert_eq!(language.as_deref(), Some("rust")); let mut expected = HashMap::new(); expected.insert("class".into(), "hl".into()); expected.insert("key".into(), "val".into()); assert_eq!(attrs, &expected); } // ===== Math block ===== #[test] fn math_block_captures_environment() { let lexed = lex("{{$%align%\nx = 1\n}}$\n"); assert_eq!( lexed[0], MathBlockOpen { environment: Some("align".into()) } ); assert_eq!(lexed[2], MathBlockLine("x = 1".into())); assert_eq!(lexed[4], MathBlockClose); } // ===== Placeholders ===== #[test] fn all_four_placeholders() { assert_eq!( lex("%title My Page\n"), vec![PlaceholderTitle(Some("My Page".into())), Newline] ); assert_eq!(lex("%nohtml\n"), vec![PlaceholderNohtml, Newline]); assert_eq!( lex("%template fancy\n"), vec![PlaceholderTemplate(Some("fancy".into())), Newline] ); assert_eq!( lex("%date 2026-05-10\n"), vec![PlaceholderDate(Some("2026-05-10".into())), Newline] ); } // ===== Lists ===== #[test] fn list_marker_dash() { let lexed = lex("- item\n"); assert_eq!( lexed, vec![ ListMarker { symbol: ListSymbol::Dash, indent: 0 }, text("item"), Newline, ] ); } #[test] fn list_marker_star_with_space_is_a_list_not_bold() { let lexed = lex("* item\n"); assert_eq!( lexed[0], ListMarker { symbol: ListSymbol::Star, indent: 0 } ); assert_eq!(lexed[1], text("item")); } #[test] fn list_marker_kinds() { let cases: &[(&str, ListSymbol)] = &[ ("- a\n", ListSymbol::Dash), ("* a\n", ListSymbol::Star), ("# a\n", ListSymbol::Hash), ("1. a\n", ListSymbol::Numeric), ("1) a\n", ListSymbol::NumericParen), ("a) a\n", ListSymbol::AlphaParen), ("A) a\n", ListSymbol::AlphaUpperParen), ("i) a\n", ListSymbol::RomanParen), ("I) a\n", ListSymbol::RomanUpperParen), ]; for (line, expected) in cases { let lexed = lex(line); assert!( matches!(lexed[0], ListMarker { symbol, .. } if symbol == *expected), "input {line:?} expected {expected:?}, got {:?}", lexed[0] ); } } #[test] fn list_marker_at_indent() { let lexed = lex(" - nested\n"); assert_eq!( lexed[0], ListMarker { symbol: ListSymbol::Dash, indent: 4 } ); } #[test] fn checkbox_states() { let cases: &[(&str, CheckboxState)] = &[ ("- [ ] a\n", CheckboxState::Empty), ("- [.] a\n", CheckboxState::Quarter), ("- [o] a\n", CheckboxState::Half), ("- [O] a\n", CheckboxState::ThreeQuarters), ("- [X] a\n", CheckboxState::Done), ("- [-] a\n", CheckboxState::Rejected), ]; for (line, expected) in cases { let lexed = lex(line); assert!( matches!(lexed[1], Checkbox(s) if s == *expected), "input {line:?} expected {expected:?}, got {:?}", lexed[1] ); } } // ===== Blockquotes ===== #[test] fn angle_blockquote() { assert_eq!( lex("> quoted\n"), vec![BlockquoteMarker, text("quoted"), Newline] ); } #[test] fn indent_blockquote() { assert_eq!( lex(" quoted\n"), vec![BlockquoteIndent, text("quoted"), Newline] ); } #[test] fn indent_with_list_marker_is_a_list_not_blockquote() { let lexed = lex(" - item\n"); assert!(matches!(lexed[0], ListMarker { indent: 4, .. })); } // ===== Tables ===== #[test] fn simple_table_row() { assert_eq!( lex("| a | b |\n"), vec![ TableSep, text(" a "), TableSep, text(" b "), TableSep, Newline, ] ); } #[test] fn table_header_separator_row() { use nuwiki_core::ast::TableAlign; assert_eq!( lex("|---|---|\n"), vec![ TableHeaderRow(vec![TableAlign::Default, TableAlign::Default]), Newline ] ); } #[test] fn table_col_and_row_span_cells() { let lexed = lex("| a | > | \\/ |\n"); // sep, " a ", sep, ColSpan, sep, RowSpan, sep, newline assert_eq!( lexed, vec![ TableSep, text(" a "), TableSep, TableColSpan, TableSep, TableRowSpan, TableSep, Newline, ] ); } // ===== Definition list ===== #[test] fn definition_term_with_inline_definition() { let lexed = lex("Term:: Definition\n"); assert_eq!( lexed, vec![ text("Term"), DefinitionTermMarker, text("Definition"), Newline, ] ); } #[test] fn definition_term_alone() { let lexed = lex("Term::\n"); assert_eq!(lexed, vec![text("Term"), DefinitionTermMarker, Newline]); } // ===== Inline typefaces ===== #[test] fn inline_bold_italic_strike_super_sub() { assert_eq!( lex("*b* _i_ ~~s~~ ^sup^ ,,sub,,\n"), vec![ BoldDelim, text("b"), BoldDelim, text(" "), ItalicDelim, text("i"), ItalicDelim, text(" "), StrikethroughDelim, text("s"), StrikethroughDelim, text(" "), SuperscriptDelim, text("sup"), SuperscriptDelim, text(" "), SubscriptDelim, text("sub"), SubscriptDelim, Newline, ] ); } #[test] fn inline_code_captures_content() { assert_eq!(lex("`x = 1`\n"), vec![Code("x = 1".into()), Newline]); } #[test] fn inline_math_captures_content() { assert_eq!( lex("$E = mc^2$\n"), vec![MathInline("E = mc^2".into()), Newline] ); } // ===== Keywords ===== #[test] fn all_six_keywords() { let cases: &[(&str, Keyword)] = &[ ("TODO", Keyword::Todo), ("DONE", Keyword::Done), ("STARTED", Keyword::Started), ("FIXME", Keyword::Fixme), ("FIXED", Keyword::Fixed), ("XXX", Keyword::Xxx), ]; for (s, expected) in cases { let line = format!("{s} stuff\n"); let lexed = lex(&line); assert!( matches!(lexed[0], Keyword(k) if k == *expected), "input {s:?} expected {expected:?}, got {:?}", lexed[0] ); } } #[test] fn keyword_only_at_word_boundary() { // Embedded TODO inside a longer word should NOT match as a keyword. let lexed = lex("aTODOb\n"); assert_eq!(lexed, vec![text("aTODOb"), Newline]); } // ===== Wikilinks / transclusions / raw URLs ===== #[test] fn bare_wikilink() { assert_eq!( lex("[[Page]]\n"), vec![WikiLinkOpen, text("Page"), WikiLinkClose, Newline] ); } #[test] fn described_wikilink() { assert_eq!( lex("[[Page|Description]]\n"), vec![ WikiLinkOpen, text("Page"), WikiLinkSep, text("Description"), WikiLinkClose, Newline, ] ); } #[test] fn wikilink_with_anchor_is_lexed_as_text() { // The lexer doesn't carve out anchors; the parser does. let lexed = lex("[[Page#Anchor]]\n"); assert_eq!( lexed, vec![WikiLinkOpen, text("Page#Anchor"), WikiLinkClose, Newline,] ); } #[test] fn transclusion_with_alt_and_attrs() { assert_eq!( lex("{{img.png|alt|class=hi}}\n"), vec![ TransclusionOpen, text("img.png"), TransclusionSep, text("alt"), TransclusionSep, text("class=hi"), TransclusionClose, Newline, ] ); } #[test] fn raw_urls_for_each_supported_scheme() { for scheme in &[ "https://example.com", "http://example.com/path?q=1", "ftp://archive.example.com/file.tar.gz", "mailto:hi@example.com", "file:///tmp/file.txt", ] { let line = format!("see {scheme} now\n"); let lexed = lex(&line); assert!( lexed.iter().any(|t| matches!(t, RawUrl(u) if u == scheme)), "scheme {scheme:?} not lexed; got {lexed:?}" ); } } #[test] fn raw_url_drops_trailing_sentence_punctuation() { let lexed = lex("Visit https://example.com.\n"); assert!( lexed .iter() .any(|t| matches!(t, RawUrl(u) if u == "https://example.com")), "got {lexed:?}" ); } // ===== Span correctness ===== #[test] fn spans_are_byte_accurate() { // Input layout: // Line 0: "= H =" // Line 1: "" (blank line) // Line 2: "*bold*" let src = "= H =\n\n*bold*\n"; let tokens = VimwikiLexer::new().lex(src).into_vec(); // First token: HeadingOpen at line 0, col 0..1 assert_eq!( tokens[0].span.start, Position { line: 0, column: 0, offset: 0 } ); assert_eq!( tokens[0].span.end, Position { line: 0, column: 1, offset: 1 } ); // Find the BoldDelim opening on line 2. let bold_open = tokens .iter() .find(|t| matches!(t.kind, BoldDelim)) .expect("bold delim"); assert_eq!(bold_open.span.start.line, 2); assert_eq!(bold_open.span.start.column, 0); // "= H =\n" = 6 bytes, "\n" = 1, so line 2 starts at offset 7. assert_eq!(bold_open.span.start.offset, 7); }