nuwiki-rs/crates/nuwiki-lsp/tests/semantic_tokens.rs

//! Tests for the semantic-token builder, encoder, range filter, and
//! UTF-8/UTF-16 conversion.

use nuwiki_core::syntax::vimwiki::VimwikiSyntax;
use nuwiki_core::syntax::SyntaxPlugin;
use nuwiki_lsp::semantic_tokens::{
    build_data, collect_tokens, encode, legend, LineIndex, RawToken, MOD_CENTERED, MOD_LEVEL1,
    MOD_LEVEL2, TOKEN_BOLD, TOKEN_CODE, TOKEN_CODE_BLOCK, TOKEN_COMMENT, TOKEN_DEFINITION_TERM,
    TOKEN_HEADING, TOKEN_ITALIC, TOKEN_KEYWORD, TOKEN_MATH_BLOCK, TOKEN_TRANSCLUSION,
    TOKEN_TYPE_NAMES, TOKEN_URL, TOKEN_WIKILINK,
};
use tower_lsp::lsp_types::{Position, Range};

fn parse(src: &str) -> nuwiki_core::ast::DocumentNode {
    VimwikiSyntax::new().parse(src)
}

fn first_with_kind(tokens: &[RawToken], kind: u32) -> Option<&RawToken> {
    tokens.iter().find(|t| t.kind == kind)
}

// ===== Legend =====

#[test]
fn legend_lists_all_custom_token_types() {
    let l = legend();
    assert_eq!(l.token_types.len(), TOKEN_TYPE_NAMES.len());
    let names: Vec<String> = l
        .token_types
        .iter()
        .map(|t| t.as_str().to_owned())
        .collect();
    assert!(names.contains(&"vimwikiHeading".to_string()));
    assert!(names.contains(&"vimwikiWikilink".to_string()));
    assert!(names.contains(&"vimwikiTransclusion".to_string()));
}

// ===== Per-construct emission =====

#[test]
fn heading_emits_one_token_with_level_modifier() {
    let doc = parse("== Hi ==\n");
    let toks = collect_tokens(&doc, "== Hi ==\n");
    let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
    assert_eq!(h.mods & MOD_LEVEL2, MOD_LEVEL2);
    assert_eq!(h.mods & MOD_CENTERED, 0);
}

#[test]
fn centered_heading_carries_centered_modifier() {
    let src = "  = Title =\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
    assert_eq!(h.mods & MOD_LEVEL1, MOD_LEVEL1);
    assert_eq!(h.mods & MOD_CENTERED, MOD_CENTERED);
}

#[test]
fn heading_does_not_emit_inner_inline_tokens() {
    // Headings render as a single styled span, so bold inside is shadowed.
    let src = "= Hi *bold* =\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_HEADING));
    assert!(
        !toks.iter().any(|t| t.kind == TOKEN_BOLD),
        "should not have emitted Bold inside a Heading"
    );
}

#[test]
fn paragraph_emits_inline_tokens_only() {
    let src = "see *bold* and `code`\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_BOLD));
    assert!(toks.iter().any(|t| t.kind == TOKEN_CODE));
    assert!(!toks.iter().any(|t| t.kind == TOKEN_HEADING));
}

#[test]
fn italic_token_for_underscored_run() {
    let src = "_emphasis_\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_ITALIC));
}

#[test]
fn keyword_token_for_each_keyword() {
    let src = "TODO write tests, FIXME later\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    let kw_count = toks.iter().filter(|t| t.kind == TOKEN_KEYWORD).count();
    assert_eq!(kw_count, 2);
}

#[test]
fn wikilink_and_external_link_get_different_types() {
    let src = "[[Page]] and [[https://example.com|docs]]\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_WIKILINK));
    assert!(
        toks.iter()
            .any(|t| t.kind == nuwiki_lsp::semantic_tokens::TOKEN_EXTERNAL_LINK),
        "URL inside [[...]] should be vimwikiExternalLink, got {toks:?}"
    );
}

#[test]
fn raw_url_emits_url_token() {
    let src = "see https://example.com here\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_URL));
}

#[test]
fn transclusion_emits_transclusion_token() {
    let src = "{{img.png|alt}}\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_TRANSCLUSION));
}

#[test]
fn comment_token_for_single_line_comment() {
    let src = "%% hidden\nstuff\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_COMMENT));
}

#[test]
fn definition_term_token() {
    let src = "Term:: Definition\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    assert!(toks.iter().any(|t| t.kind == TOKEN_DEFINITION_TERM));
}

// ===== Multi-line fences =====

#[test]
fn preformatted_block_splits_into_one_token_per_line() {
    let src = "{{{rust\nlet x = 1;\nlet y = 2;\n}}}\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    let code_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_CODE_BLOCK).collect();
    // Spans line 0 (open fence) through line 3 (close fence) = 4 lines, but
    // empty trailing slices are dropped, so we expect 4 line tokens.
    assert!(
        code_tokens.len() >= 3,
        "expected multi-line split, got {code_tokens:?}"
    );
    // No token should cross a line boundary.
    for t in &code_tokens {
        assert!(t.byte_len > 0);
    }
}

#[test]
fn math_block_splits_into_one_token_per_line() {
    let src = "{{$\nx=1\ny=2\n}}$\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    let math_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_MATH_BLOCK).collect();
    assert!(math_tokens.len() >= 3);
}

// ===== Sorting =====

#[test]
fn tokens_are_sorted_by_line_then_column() {
    let src = "= H =\n*bold* and _italic_\n";
    let doc = parse(src);
    let toks = collect_tokens(&doc, src);
    for w in toks.windows(2) {
        let a = (w[0].line, w[0].byte_col);
        let b = (w[1].line, w[1].byte_col);
        assert!(a <= b, "tokens not sorted: {a:?} then {b:?}");
    }
}

// ===== Encoding =====

#[test]
fn encode_produces_correct_delta_quintuples() {
    let src = "= H =\n*bold*\n";
    let raws = vec![
        RawToken {
            line: 0,
            byte_col: 0,
            byte_len: 5,
            kind: TOKEN_HEADING,
            mods: MOD_LEVEL1,
        },
        RawToken {
            line: 1,
            byte_col: 0,
            byte_len: 6,
            kind: TOKEN_BOLD,
            mods: 0,
        },
    ];
    let idx = LineIndex::new(src);
    let data = encode(&raws, src, &idx, true);
    assert_eq!(data.len(), 10);
    // First token: delta_line=0, delta_col=0, len=5, type=HEADING, mods=LEVEL1
    assert_eq!(&data[0..5], &[0, 0, 5, TOKEN_HEADING, MOD_LEVEL1]);
    // Second token: delta_line=1 (new line), delta_col=0 (absolute), len=6, type=BOLD, mods=0
    assert_eq!(&data[5..10], &[1, 0, 6, TOKEN_BOLD, 0]);
}

#[test]
fn encode_uses_relative_column_within_same_line() {
    let src = "abc def ghi\n";
    let raws = vec![
        RawToken {
            line: 0,
            byte_col: 0,
            byte_len: 3,
            kind: TOKEN_BOLD,
            mods: 0,
        },
        RawToken {
            line: 0,
            byte_col: 4,
            byte_len: 3,
            kind: TOKEN_ITALIC,
            mods: 0,
        },
        RawToken {
            line: 0,
            byte_col: 8,
            byte_len: 3,
            kind: TOKEN_CODE,
            mods: 0,
        },
    ];
    let idx = LineIndex::new(src);
    let data = encode(&raws, src, &idx, true);
    assert_eq!(&data[0..5], &[0, 0, 3, TOKEN_BOLD, 0]);
    // Second: delta_col = 4 - 0 = 4
    assert_eq!(&data[5..10], &[0, 4, 3, TOKEN_ITALIC, 0]);
    // Third: delta_col = 8 - 4 = 4
    assert_eq!(&data[10..15], &[0, 4, 3, TOKEN_CODE, 0]);
}

#[test]
fn encode_converts_to_utf16_for_multibyte_chars() {
    // "_é_" — italic delimiters around é. Span covers 4 bytes (1 + 2 + 1)
    // but 3 UTF-16 code units.
    let src = "_é_\n";
    let raws = vec![RawToken {
        line: 0,
        byte_col: 0,
        byte_len: 4,
        kind: TOKEN_ITALIC,
        mods: 0,
    }];
    let idx = LineIndex::new(src);
    let data = encode(&raws, src, &idx, false);
    // length should be in UTF-16 code units (3), not bytes (4).
    assert_eq!(data[2], 3);
}

// ===== Range filter =====

#[test]
fn range_filter_keeps_only_overlapping_tokens() {
    let src = "= L1 =\n= L2 =\n= L3 =\n";
    let doc = parse(src);
    // Restrict to line 1 only.
    let range = Range {
        start: Position {
            line: 1,
            character: 0,
        },
        end: Position {
            line: 2,
            character: 0,
        },
    };
    let data = build_data(&doc, src, true, Some(range));
    // Expect exactly 1 heading token (the L2 one).
    assert_eq!(data.len(), 5, "got {data:?}");
    // Verify it's on line 1 (delta_line = 1 from prev_line = 0).
    assert_eq!(data[0], 1);
}

#[test]
fn empty_document_produces_empty_token_stream() {
    let doc = parse("");
    let data = build_data(&doc, "", true, None);
    assert!(data.is_empty());
}

// ===== End-to-end through builder =====

#[test]
fn build_data_matches_collect_then_encode() {
    let src = "= H1 =\n_em_ TODO\n";
    let doc = parse(src);
    let direct = build_data(&doc, src, true, None);
    let raws = collect_tokens(&doc, src);
    let idx = LineIndex::new(src);
    let manual = encode(&raws, src, &idx, true);
    assert_eq!(direct, manual);
}