feat: initial nuwiki Rust workspace (core, lsp, ls)

crates/nuwiki-lsp/tests/semantic_tokens.rs

@@ -0,0 +1,312 @@
+//! Tests for the semantic-token builder, encoder, range filter, and
+//! UTF-8/UTF-16 conversion.
+
+use nuwiki_core::syntax::vimwiki::VimwikiSyntax;
+use nuwiki_core::syntax::SyntaxPlugin;
+use nuwiki_lsp::semantic_tokens::{
+    build_data, collect_tokens, encode, legend, LineIndex, RawToken, MOD_CENTERED, MOD_LEVEL1,
+    MOD_LEVEL2, TOKEN_BOLD, TOKEN_CODE, TOKEN_CODE_BLOCK, TOKEN_COMMENT, TOKEN_DEFINITION_TERM,
+    TOKEN_HEADING, TOKEN_ITALIC, TOKEN_KEYWORD, TOKEN_MATH_BLOCK, TOKEN_TRANSCLUSION,
+    TOKEN_TYPE_NAMES, TOKEN_URL, TOKEN_WIKILINK,
+};
+use tower_lsp::lsp_types::{Position, Range};
+
+fn parse(src: &str) -> nuwiki_core::ast::DocumentNode {
+    VimwikiSyntax::new().parse(src)
+}
+
+fn first_with_kind(tokens: &[RawToken], kind: u32) -> Option<&RawToken> {
+    tokens.iter().find(|t| t.kind == kind)
+}
+
+// ===== Legend =====
+
+#[test]
+fn legend_lists_all_custom_token_types() {
+    let l = legend();
+    assert_eq!(l.token_types.len(), TOKEN_TYPE_NAMES.len());
+    let names: Vec<String> = l
+        .token_types
+        .iter()
+        .map(|t| t.as_str().to_owned())
+        .collect();
+    assert!(names.contains(&"vimwikiHeading".to_string()));
+    assert!(names.contains(&"vimwikiWikilink".to_string()));
+    assert!(names.contains(&"vimwikiTransclusion".to_string()));
+}
+
+// ===== Per-construct emission =====
+
+#[test]
+fn heading_emits_one_token_with_level_modifier() {
+    let doc = parse("== Hi ==\n");
+    let toks = collect_tokens(&doc, "== Hi ==\n");
+    let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
+    assert_eq!(h.mods & MOD_LEVEL2, MOD_LEVEL2);
+    assert_eq!(h.mods & MOD_CENTERED, 0);
+}
+
+#[test]
+fn centered_heading_carries_centered_modifier() {
+    let src = "  = Title =\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
+    assert_eq!(h.mods & MOD_LEVEL1, MOD_LEVEL1);
+    assert_eq!(h.mods & MOD_CENTERED, MOD_CENTERED);
+}
+
+#[test]
+fn heading_does_not_emit_inner_inline_tokens() {
+    // Headings render as a single styled span, so bold inside is shadowed.
+    let src = "= Hi *bold* =\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_HEADING));
+    assert!(
+        !toks.iter().any(|t| t.kind == TOKEN_BOLD),
+        "should not have emitted Bold inside a Heading"
+    );
+}
+
+#[test]
+fn paragraph_emits_inline_tokens_only() {
+    let src = "see *bold* and `code`\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_BOLD));
+    assert!(toks.iter().any(|t| t.kind == TOKEN_CODE));
+    assert!(!toks.iter().any(|t| t.kind == TOKEN_HEADING));
+}
+
+#[test]
+fn italic_token_for_underscored_run() {
+    let src = "_emphasis_\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_ITALIC));
+}
+
+#[test]
+fn keyword_token_for_each_keyword() {
+    let src = "TODO write tests, FIXME later\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    let kw_count = toks.iter().filter(|t| t.kind == TOKEN_KEYWORD).count();
+    assert_eq!(kw_count, 2);
+}
+
+#[test]
+fn wikilink_and_external_link_get_different_types() {
+    let src = "[[Page]] and [[https://example.com|docs]]\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_WIKILINK));
+    assert!(
+        toks.iter()
+            .any(|t| t.kind == nuwiki_lsp::semantic_tokens::TOKEN_EXTERNAL_LINK),
+        "URL inside [[...]] should be vimwikiExternalLink, got {toks:?}"
+    );
+}
+
+#[test]
+fn raw_url_emits_url_token() {
+    let src = "see https://example.com here\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_URL));
+}
+
+#[test]
+fn transclusion_emits_transclusion_token() {
+    let src = "{{img.png|alt}}\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_TRANSCLUSION));
+}
+
+#[test]
+fn comment_token_for_single_line_comment() {
+    let src = "%% hidden\nstuff\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_COMMENT));
+}
+
+#[test]
+fn definition_term_token() {
+    let src = "Term:: Definition\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    assert!(toks.iter().any(|t| t.kind == TOKEN_DEFINITION_TERM));
+}
+
+// ===== Multi-line fences =====
+
+#[test]
+fn preformatted_block_splits_into_one_token_per_line() {
+    let src = "{{{rust\nlet x = 1;\nlet y = 2;\n}}}\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    let code_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_CODE_BLOCK).collect();
+    // Spans line 0 (open fence) through line 3 (close fence) = 4 lines, but
+    // empty trailing slices are dropped, so we expect 4 line tokens.
+    assert!(
+        code_tokens.len() >= 3,
+        "expected multi-line split, got {code_tokens:?}"
+    );
+    // No token should cross a line boundary.
+    for t in &code_tokens {
+        assert!(t.byte_len > 0);
+    }
+}
+
+#[test]
+fn math_block_splits_into_one_token_per_line() {
+    let src = "{{$\nx=1\ny=2\n}}$\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    let math_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_MATH_BLOCK).collect();
+    assert!(math_tokens.len() >= 3);
+}
+
+// ===== Sorting =====
+
+#[test]
+fn tokens_are_sorted_by_line_then_column() {
+    let src = "= H =\n*bold* and _italic_\n";
+    let doc = parse(src);
+    let toks = collect_tokens(&doc, src);
+    for w in toks.windows(2) {
+        let a = (w[0].line, w[0].byte_col);
+        let b = (w[1].line, w[1].byte_col);
+        assert!(a <= b, "tokens not sorted: {a:?} then {b:?}");
+    }
+}
+
+// ===== Encoding =====
+
+#[test]
+fn encode_produces_correct_delta_quintuples() {
+    let src = "= H =\n*bold*\n";
+    let raws = vec![
+        RawToken {
+            line: 0,
+            byte_col: 0,
+            byte_len: 5,
+            kind: TOKEN_HEADING,
+            mods: MOD_LEVEL1,
+        },
+        RawToken {
+            line: 1,
+            byte_col: 0,
+            byte_len: 6,
+            kind: TOKEN_BOLD,
+            mods: 0,
+        },
+    ];
+    let idx = LineIndex::new(src);
+    let data = encode(&raws, src, &idx, true);
+    assert_eq!(data.len(), 10);
+    // First token: delta_line=0, delta_col=0, len=5, type=HEADING, mods=LEVEL1
+    assert_eq!(&data[0..5], &[0, 0, 5, TOKEN_HEADING, MOD_LEVEL1]);
+    // Second token: delta_line=1 (new line), delta_col=0 (absolute), len=6, type=BOLD, mods=0
+    assert_eq!(&data[5..10], &[1, 0, 6, TOKEN_BOLD, 0]);
+}
+
+#[test]
+fn encode_uses_relative_column_within_same_line() {
+    let src = "abc def ghi\n";
+    let raws = vec![
+        RawToken {
+            line: 0,
+            byte_col: 0,
+            byte_len: 3,
+            kind: TOKEN_BOLD,
+            mods: 0,
+        },
+        RawToken {
+            line: 0,
+            byte_col: 4,
+            byte_len: 3,
+            kind: TOKEN_ITALIC,
+            mods: 0,
+        },
+        RawToken {
+            line: 0,
+            byte_col: 8,
+            byte_len: 3,
+            kind: TOKEN_CODE,
+            mods: 0,
+        },
+    ];
+    let idx = LineIndex::new(src);
+    let data = encode(&raws, src, &idx, true);
+    assert_eq!(&data[0..5], &[0, 0, 3, TOKEN_BOLD, 0]);
+    // Second: delta_col = 4 - 0 = 4
+    assert_eq!(&data[5..10], &[0, 4, 3, TOKEN_ITALIC, 0]);
+    // Third: delta_col = 8 - 4 = 4
+    assert_eq!(&data[10..15], &[0, 4, 3, TOKEN_CODE, 0]);
+}
+
+#[test]
+fn encode_converts_to_utf16_for_multibyte_chars() {
+    // "_é_" — italic delimiters around é. Span covers 4 bytes (1 + 2 + 1)
+    // but 3 UTF-16 code units.
+    let src = "_é_\n";
+    let raws = vec![RawToken {
+        line: 0,
+        byte_col: 0,
+        byte_len: 4,
+        kind: TOKEN_ITALIC,
+        mods: 0,
+    }];
+    let idx = LineIndex::new(src);
+    let data = encode(&raws, src, &idx, false);
+    // length should be in UTF-16 code units (3), not bytes (4).
+    assert_eq!(data[2], 3);
+}
+
+// ===== Range filter =====
+
+#[test]
+fn range_filter_keeps_only_overlapping_tokens() {
+    let src = "= L1 =\n= L2 =\n= L3 =\n";
+    let doc = parse(src);
+    // Restrict to line 1 only.
+    let range = Range {
+        start: Position {
+            line: 1,
+            character: 0,
+        },
+        end: Position {
+            line: 2,
+            character: 0,
+        },
+    };
+    let data = build_data(&doc, src, true, Some(range));
+    // Expect exactly 1 heading token (the L2 one).
+    assert_eq!(data.len(), 5, "got {data:?}");
+    // Verify it's on line 1 (delta_line = 1 from prev_line = 0).
+    assert_eq!(data[0], 1);
+}
+
+#[test]
+fn empty_document_produces_empty_token_stream() {
+    let doc = parse("");
+    let data = build_data(&doc, "", true, None);
+    assert!(data.is_empty());
+}
+
+// ===== End-to-end through builder =====
+
+#[test]
+fn build_data_matches_collect_then_encode() {
+    let src = "= H1 =\n_em_ TODO\n";
+    let doc = parse(src);
+    let direct = build_data(&doc, src, true, None);
+    let raws = collect_tokens(&doc, src);
+    let idx = LineIndex::new(src);
+    let manual = encode(&raws, src, &idx, true);
+    assert_eq!(direct, manual);
+}