refactor(tests): group test files by feature, drop phase/cluster naming

Restructures the integration test layout so each file is named after what it covers, not the implementation phase it landed in. nuwiki-core (renames only): vimwiki_lexer → lexer vimwiki_parser → parser vimwiki_tags → tags vimwiki_table_alignment → table_alignment diary_period → diary list_continuation → lists transclusion_attrs → transclusion + table colspan/rowspan tests moved here from parity_cluster_1 nuwiki-lsp (renames + merges + one split): cluster_a_list_rewriters → commands_lists cluster_b_table_rewriters → commands_tables cluster_c_link_helpers + phase19_followlink_creates → commands_links phase13_rename_commands → commands_files phase17_colorize → commands_colorize phase17_html_export → html_export phase15_link_health → link_health phase18_multi_wiki → multi_wiki phase19_folding → folding nav → navigation lsp_helpers → helpers phase16_diary + diary_frequency → diary phase11_plumbing + parity_cluster_1 (config) → index_and_config tags_index_and_lsp + phase17_backfill → commands_tags phase14_edit_commands → split into commands_checkboxes, commands_headings, commands_tasks Net: 30 → 28 integration-test files. 456 → 455 tests (the one removed test was a dummy `paragraph_render_is_unchanged` whose only purpose was to keep a `ParagraphNode` import alive in parity_cluster_1; the import is exercised elsewhere now). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-13 01:30:55 +00:00
parent 5b6f789c8d
commit f4e086f981
30 changed files with 763 additions and 778 deletions
@@ -0,0 +1,599 @@
+//! End-to-end tests for the vimwiki lexer.
+//!
+//! Each test feeds a small fragment to `VimwikiLexer::lex` and asserts the
+//! exact token-kind sequence. A separate `spans_are_byte_accurate` test
+//! pins down position semantics.
+
+use std::collections::HashMap;
+
+use nuwiki_core::ast::{CheckboxState, Keyword, ListSymbol, Position};
+use nuwiki_core::syntax::vimwiki::{VimwikiLexer, VimwikiTokenKind};
+use nuwiki_core::syntax::Lexer;
+
+use VimwikiTokenKind::*;
+
+fn lex(text: &str) -> Vec<VimwikiTokenKind> {
+    VimwikiLexer::new()
+        .lex(text)
+        .into_iter()
+        .map(|t| t.kind)
+        .collect()
+}
+
+fn text(s: &str) -> VimwikiTokenKind {
+    Text(s.into())
+}
+
+// ===== Empty / whitespace =====
+
+#[test]
+fn empty_input_produces_no_tokens() {
+    assert!(lex("").is_empty());
+}
+
+#[test]
+fn lone_newline_is_a_blank_line() {
+    assert_eq!(lex("\n"), vec![BlankLine]);
+}
+
+#[test]
+fn whitespace_only_line_is_a_blank_line() {
+    assert_eq!(lex("   \n"), vec![BlankLine]);
+}
+
+#[test]
+fn blank_lines_separate_paragraphs() {
+    assert_eq!(
+        lex("hello\n\nworld\n"),
+        vec![text("hello"), Newline, BlankLine, text("world"), Newline]
+    );
+}
+
+// ===== Headings =====
+
+#[test]
+fn headings_at_all_six_levels() {
+    let mut tokens = Vec::new();
+    for level in 1..=6 {
+        let line = format!(
+            "{} L{} {}\n",
+            "=".repeat(level as usize),
+            level,
+            "=".repeat(level as usize)
+        );
+        let lexed = lex(&line);
+        assert!(matches!(
+            lexed[0],
+            HeadingOpen { level: l, centered: false } if l == level
+        ));
+        tokens.extend(lexed);
+    }
+    assert!(tokens.iter().any(|t| matches!(t, HeadingClose)));
+}
+
+#[test]
+fn centered_heading_is_marked_centered() {
+    let lexed = lex("  = title =\n");
+    assert!(matches!(
+        lexed[0],
+        HeadingOpen {
+            level: 1,
+            centered: true
+        }
+    ));
+    assert_eq!(lexed[1], text("title"));
+    assert!(matches!(lexed[2], HeadingClose));
+}
+
+#[test]
+fn heading_with_inline_bold() {
+    let lexed = lex("== Hello *world* ==\n");
+    assert_eq!(
+        lexed,
+        vec![
+            HeadingOpen {
+                level: 2,
+                centered: false
+            },
+            text("Hello "),
+            BoldDelim,
+            text("world"),
+            BoldDelim,
+            HeadingClose,
+            Newline,
+        ]
+    );
+}
+
+// ===== Horizontal rule =====
+
+#[test]
+fn horizontal_rule_requires_four_or_more_dashes() {
+    assert_eq!(lex("----\n"), vec![HorizontalRule, Newline]);
+    assert_eq!(lex("--------\n"), vec![HorizontalRule, Newline]);
+    // Three dashes: not a rule, parsed as text.
+    assert_eq!(lex("---\n"), vec![text("---"), Newline]);
+}
+
+// ===== Comments =====
+
+#[test]
+fn single_line_comment() {
+    assert_eq!(
+        lex("%% hello\n"),
+        vec![CommentLine(" hello".into()), Newline]
+    );
+}
+
+#[test]
+fn multiline_comment_spans_multiple_lines() {
+    assert_eq!(
+        lex("%%+ a\nb\n+%%\n"),
+        vec![
+            CommentMultiOpen,
+            CommentMultiLine(" a".into()),
+            Newline,
+            CommentMultiLine("b".into()),
+            Newline,
+            CommentMultiClose,
+            Newline,
+        ]
+    );
+}
+
+#[test]
+fn multiline_comment_one_liner() {
+    assert_eq!(
+        lex("%%+ inline +%%\n"),
+        vec![
+            CommentMultiOpen,
+            CommentMultiLine(" inline ".into()),
+            CommentMultiClose,
+            Newline,
+        ]
+    );
+}
+
+// ===== Preformatted =====
+
+#[test]
+fn preformatted_block_captures_lines_verbatim() {
+    let lexed = lex("{{{\nfn main() {}\n}}}\n");
+    assert!(matches!(lexed[0], PreformattedOpen { .. }));
+    assert_eq!(lexed[1], Newline);
+    assert_eq!(lexed[2], PreformattedLine("fn main() {}".into()));
+    assert_eq!(lexed[3], Newline);
+    assert_eq!(lexed[4], PreformattedClose);
+    assert_eq!(lexed[5], Newline);
+}
+
+#[test]
+fn preformatted_block_parses_language() {
+    let lexed = lex("{{{rust\nx\n}}}\n");
+    let PreformattedOpen { language, .. } = &lexed[0] else {
+        panic!("expected PreformattedOpen");
+    };
+    assert_eq!(language.as_deref(), Some("rust"));
+}
+
+#[test]
+fn preformatted_block_parses_attrs() {
+    let lexed = lex("{{{rust class=\"hl\" key=val\nx\n}}}\n");
+    let PreformattedOpen { language, attrs } = &lexed[0] else {
+        panic!("expected PreformattedOpen");
+    };
+    assert_eq!(language.as_deref(), Some("rust"));
+    let mut expected = HashMap::new();
+    expected.insert("class".into(), "hl".into());
+    expected.insert("key".into(), "val".into());
+    assert_eq!(attrs, &expected);
+}
+
+// ===== Math block =====
+
+#[test]
+fn math_block_captures_environment() {
+    let lexed = lex("{{$%align%\nx = 1\n}}$\n");
+    assert_eq!(
+        lexed[0],
+        MathBlockOpen {
+            environment: Some("align".into())
+        }
+    );
+    assert_eq!(lexed[2], MathBlockLine("x = 1".into()));
+    assert_eq!(lexed[4], MathBlockClose);
+}
+
+// ===== Placeholders =====
+
+#[test]
+fn all_four_placeholders() {
+    assert_eq!(
+        lex("%title My Page\n"),
+        vec![PlaceholderTitle(Some("My Page".into())), Newline]
+    );
+    assert_eq!(lex("%nohtml\n"), vec![PlaceholderNohtml, Newline]);
+    assert_eq!(
+        lex("%template fancy\n"),
+        vec![PlaceholderTemplate(Some("fancy".into())), Newline]
+    );
+    assert_eq!(
+        lex("%date 2026-05-10\n"),
+        vec![PlaceholderDate(Some("2026-05-10".into())), Newline]
+    );
+}
+
+// ===== Lists =====
+
+#[test]
+fn list_marker_dash() {
+    let lexed = lex("- item\n");
+    assert_eq!(
+        lexed,
+        vec![
+            ListMarker {
+                symbol: ListSymbol::Dash,
+                indent: 0
+            },
+            text("item"),
+            Newline,
+        ]
+    );
+}
+
+#[test]
+fn list_marker_star_with_space_is_a_list_not_bold() {
+    let lexed = lex("* item\n");
+    assert_eq!(
+        lexed[0],
+        ListMarker {
+            symbol: ListSymbol::Star,
+            indent: 0
+        }
+    );
+    assert_eq!(lexed[1], text("item"));
+}
+
+#[test]
+fn list_marker_kinds() {
+    let cases: &[(&str, ListSymbol)] = &[
+        ("- a\n", ListSymbol::Dash),
+        ("* a\n", ListSymbol::Star),
+        ("# a\n", ListSymbol::Hash),
+        ("1. a\n", ListSymbol::Numeric),
+        ("1) a\n", ListSymbol::NumericParen),
+        ("a) a\n", ListSymbol::AlphaParen),
+        ("A) a\n", ListSymbol::AlphaUpperParen),
+        ("i) a\n", ListSymbol::RomanParen),
+        ("I) a\n", ListSymbol::RomanUpperParen),
+    ];
+    for (line, expected) in cases {
+        let lexed = lex(line);
+        assert!(
+            matches!(lexed[0], ListMarker { symbol, .. } if symbol == *expected),
+            "input {line:?} expected {expected:?}, got {:?}",
+            lexed[0]
+        );
+    }
+}
+
+#[test]
+fn list_marker_at_indent() {
+    let lexed = lex("    - nested\n");
+    assert_eq!(
+        lexed[0],
+        ListMarker {
+            symbol: ListSymbol::Dash,
+            indent: 4
+        }
+    );
+}
+
+#[test]
+fn checkbox_states() {
+    let cases: &[(&str, CheckboxState)] = &[
+        ("- [ ] a\n", CheckboxState::Empty),
+        ("- [.] a\n", CheckboxState::Quarter),
+        ("- [o] a\n", CheckboxState::Half),
+        ("- [O] a\n", CheckboxState::ThreeQuarters),
+        ("- [X] a\n", CheckboxState::Done),
+        ("- [-] a\n", CheckboxState::Rejected),
+    ];
+    for (line, expected) in cases {
+        let lexed = lex(line);
+        assert!(
+            matches!(lexed[1], Checkbox(s) if s == *expected),
+            "input {line:?} expected {expected:?}, got {:?}",
+            lexed[1]
+        );
+    }
+}
+
+// ===== Blockquotes =====
+
+#[test]
+fn angle_blockquote() {
+    assert_eq!(
+        lex("> quoted\n"),
+        vec![BlockquoteMarker, text("quoted"), Newline]
+    );
+}
+
+#[test]
+fn indent_blockquote() {
+    assert_eq!(
+        lex("    quoted\n"),
+        vec![BlockquoteIndent, text("quoted"), Newline]
+    );
+}
+
+#[test]
+fn indent_with_list_marker_is_a_list_not_blockquote() {
+    let lexed = lex("    - item\n");
+    assert!(matches!(lexed[0], ListMarker { indent: 4, .. }));
+}
+
+// ===== Tables =====
+
+#[test]
+fn simple_table_row() {
+    assert_eq!(
+        lex("| a | b |\n"),
+        vec![
+            TableSep,
+            text(" a "),
+            TableSep,
+            text(" b "),
+            TableSep,
+            Newline,
+        ]
+    );
+}
+
+#[test]
+fn table_header_separator_row() {
+    use nuwiki_core::ast::TableAlign;
+    assert_eq!(
+        lex("|---|---|\n"),
+        vec![
+            TableHeaderRow(vec![TableAlign::Default, TableAlign::Default]),
+            Newline
+        ]
+    );
+}
+
+#[test]
+fn table_col_and_row_span_cells() {
+    let lexed = lex("| a | > | \\/ |\n");
+    // sep, " a ", sep, ColSpan, sep, RowSpan, sep, newline
+    assert_eq!(
+        lexed,
+        vec![
+            TableSep,
+            text(" a "),
+            TableSep,
+            TableColSpan,
+            TableSep,
+            TableRowSpan,
+            TableSep,
+            Newline,
+        ]
+    );
+}
+
+// ===== Definition list =====
+
+#[test]
+fn definition_term_with_inline_definition() {
+    let lexed = lex("Term:: Definition\n");
+    assert_eq!(
+        lexed,
+        vec![
+            text("Term"),
+            DefinitionTermMarker,
+            text("Definition"),
+            Newline,
+        ]
+    );
+}
+
+#[test]
+fn definition_term_alone() {
+    let lexed = lex("Term::\n");
+    assert_eq!(lexed, vec![text("Term"), DefinitionTermMarker, Newline]);
+}
+
+// ===== Inline typefaces =====
+
+#[test]
+fn inline_bold_italic_strike_super_sub() {
+    assert_eq!(
+        lex("*b* _i_ ~~s~~ ^sup^ ,,sub,,\n"),
+        vec![
+            BoldDelim,
+            text("b"),
+            BoldDelim,
+            text(" "),
+            ItalicDelim,
+            text("i"),
+            ItalicDelim,
+            text(" "),
+            StrikethroughDelim,
+            text("s"),
+            StrikethroughDelim,
+            text(" "),
+            SuperscriptDelim,
+            text("sup"),
+            SuperscriptDelim,
+            text(" "),
+            SubscriptDelim,
+            text("sub"),
+            SubscriptDelim,
+            Newline,
+        ]
+    );
+}
+
+#[test]
+fn inline_code_captures_content() {
+    assert_eq!(lex("`x = 1`\n"), vec![Code("x = 1".into()), Newline]);
+}
+
+#[test]
+fn inline_math_captures_content() {
+    assert_eq!(
+        lex("$E = mc^2$\n"),
+        vec![MathInline("E = mc^2".into()), Newline]
+    );
+}
+
+// ===== Keywords =====
+
+#[test]
+fn all_six_keywords() {
+    let cases: &[(&str, Keyword)] = &[
+        ("TODO", Keyword::Todo),
+        ("DONE", Keyword::Done),
+        ("STARTED", Keyword::Started),
+        ("FIXME", Keyword::Fixme),
+        ("FIXED", Keyword::Fixed),
+        ("XXX", Keyword::Xxx),
+    ];
+    for (s, expected) in cases {
+        let line = format!("{s} stuff\n");
+        let lexed = lex(&line);
+        assert!(
+            matches!(lexed[0], Keyword(k) if k == *expected),
+            "input {s:?} expected {expected:?}, got {:?}",
+            lexed[0]
+        );
+    }
+}
+
+#[test]
+fn keyword_only_at_word_boundary() {
+    // Embedded TODO inside a longer word should NOT match as a keyword.
+    let lexed = lex("aTODOb\n");
+    assert_eq!(lexed, vec![text("aTODOb"), Newline]);
+}
+
+// ===== Wikilinks / transclusions / raw URLs =====
+
+#[test]
+fn bare_wikilink() {
+    assert_eq!(
+        lex("[[Page]]\n"),
+        vec![WikiLinkOpen, text("Page"), WikiLinkClose, Newline]
+    );
+}
+
+#[test]
+fn described_wikilink() {
+    assert_eq!(
+        lex("[[Page|Description]]\n"),
+        vec![
+            WikiLinkOpen,
+            text("Page"),
+            WikiLinkSep,
+            text("Description"),
+            WikiLinkClose,
+            Newline,
+        ]
+    );
+}
+
+#[test]
+fn wikilink_with_anchor_is_lexed_as_text() {
+    // The lexer doesn't carve out anchors; the parser does.
+    let lexed = lex("[[Page#Anchor]]\n");
+    assert_eq!(
+        lexed,
+        vec![WikiLinkOpen, text("Page#Anchor"), WikiLinkClose, Newline,]
+    );
+}
+
+#[test]
+fn transclusion_with_alt_and_attrs() {
+    assert_eq!(
+        lex("{{img.png|alt|class=hi}}\n"),
+        vec![
+            TransclusionOpen,
+            text("img.png"),
+            TransclusionSep,
+            text("alt"),
+            TransclusionSep,
+            text("class=hi"),
+            TransclusionClose,
+            Newline,
+        ]
+    );
+}
+
+#[test]
+fn raw_urls_for_each_supported_scheme() {
+    for scheme in &[
+        "https://example.com",
+        "http://example.com/path?q=1",
+        "ftp://archive.example.com/file.tar.gz",
+        "mailto:hi@example.com",
+        "file:///tmp/file.txt",
+    ] {
+        let line = format!("see {scheme} now\n");
+        let lexed = lex(&line);
+        assert!(
+            lexed.iter().any(|t| matches!(t, RawUrl(u) if u == scheme)),
+            "scheme {scheme:?} not lexed; got {lexed:?}"
+        );
+    }
+}
+
+#[test]
+fn raw_url_drops_trailing_sentence_punctuation() {
+    let lexed = lex("Visit https://example.com.\n");
+    assert!(
+        lexed
+            .iter()
+            .any(|t| matches!(t, RawUrl(u) if u == "https://example.com")),
+        "got {lexed:?}"
+    );
+}
+
+// ===== Span correctness =====
+
+#[test]
+fn spans_are_byte_accurate() {
+    // Input layout:
+    //   Line 0: "= H ="
+    //   Line 1: ""        (blank line)
+    //   Line 2: "*bold*"
+    let src = "= H =\n\n*bold*\n";
+    let tokens = VimwikiLexer::new().lex(src).into_vec();
+
+    // First token: HeadingOpen at line 0, col 0..1
+    assert_eq!(
+        tokens[0].span.start,
+        Position {
+            line: 0,
+            column: 0,
+            offset: 0
+        }
+    );
+    assert_eq!(
+        tokens[0].span.end,
+        Position {
+            line: 0,
+            column: 1,
+            offset: 1
+        }
+    );
+
+    // Find the BoldDelim opening on line 2.
+    let bold_open = tokens
+        .iter()
+        .find(|t| matches!(t.kind, BoldDelim))
+        .expect("bold delim");
+    assert_eq!(bold_open.span.start.line, 2);
+    assert_eq!(bold_open.span.start.column, 0);
+    // "= H =\n" = 6 bytes, "\n" = 1, so line 2 starts at offset 7.
+    assert_eq!(bold_open.span.start.offset, 7);
+}