313 lines
9.2 KiB
Rust
313 lines
9.2 KiB
Rust
|
//! Tests for the semantic-token builder, encoder, range filter, and
|
||
|
|
//! UTF-8/UTF-16 conversion.
|
||
|
|
|
||
|
|
use nuwiki_core::syntax::vimwiki::VimwikiSyntax;
|
||
|
|
use nuwiki_core::syntax::SyntaxPlugin;
|
||
|
|
use nuwiki_lsp::semantic_tokens::{
|
||
|
|
build_data, collect_tokens, encode, legend, LineIndex, RawToken, MOD_CENTERED, MOD_LEVEL1,
|
||
|
|
MOD_LEVEL2, TOKEN_BOLD, TOKEN_CODE, TOKEN_CODE_BLOCK, TOKEN_COMMENT, TOKEN_DEFINITION_TERM,
|
||
|
|
TOKEN_HEADING, TOKEN_ITALIC, TOKEN_KEYWORD, TOKEN_MATH_BLOCK, TOKEN_TRANSCLUSION,
|
||
|
|
TOKEN_TYPE_NAMES, TOKEN_URL, TOKEN_WIKILINK,
|
||
|
|
};
|
||
|
|
use tower_lsp::lsp_types::{Position, Range};
|
||
|
|
|
||
|
|
fn parse(src: &str) -> nuwiki_core::ast::DocumentNode {
|
||
|
|
VimwikiSyntax::new().parse(src)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn first_with_kind(tokens: &[RawToken], kind: u32) -> Option<&RawToken> {
|
||
|
|
tokens.iter().find(|t| t.kind == kind)
|
||
|
|
}
|
||
|
|
|
||
|
|
// ===== Legend =====
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn legend_lists_all_custom_token_types() {
|
||
|
|
let l = legend();
|
||
|
|
assert_eq!(l.token_types.len(), TOKEN_TYPE_NAMES.len());
|
||
|
|
let names: Vec<String> = l
|
||
|
|
.token_types
|
||
|
|
.iter()
|
||
|
|
.map(|t| t.as_str().to_owned())
|
||
|
|
.collect();
|
||
|
|
assert!(names.contains(&"vimwikiHeading".to_string()));
|
||
|
|
assert!(names.contains(&"vimwikiWikilink".to_string()));
|
||
|
|
assert!(names.contains(&"vimwikiTransclusion".to_string()));
|
||
|
|
}
|
||
|
|
|
||
|
|
// ===== Per-construct emission =====
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn heading_emits_one_token_with_level_modifier() {
|
||
|
|
let doc = parse("== Hi ==\n");
|
||
|
|
let toks = collect_tokens(&doc, "== Hi ==\n");
|
||
|
|
let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
|
||
|
|
assert_eq!(h.mods & MOD_LEVEL2, MOD_LEVEL2);
|
||
|
|
assert_eq!(h.mods & MOD_CENTERED, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn centered_heading_carries_centered_modifier() {
|
||
|
|
let src = " = Title =\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
|
||
|
|
assert_eq!(h.mods & MOD_LEVEL1, MOD_LEVEL1);
|
||
|
|
assert_eq!(h.mods & MOD_CENTERED, MOD_CENTERED);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn heading_does_not_emit_inner_inline_tokens() {
|
||
|
|
// Headings render as a single styled span, so bold inside is shadowed.
|
||
|
|
let src = "= Hi *bold* =\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_HEADING));
|
||
|
|
assert!(
|
||
|
|
!toks.iter().any(|t| t.kind == TOKEN_BOLD),
|
||
|
|
"should not have emitted Bold inside a Heading"
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn paragraph_emits_inline_tokens_only() {
|
||
|
|
let src = "see *bold* and `code`\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_BOLD));
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_CODE));
|
||
|
|
assert!(!toks.iter().any(|t| t.kind == TOKEN_HEADING));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn italic_token_for_underscored_run() {
|
||
|
|
let src = "_emphasis_\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_ITALIC));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn keyword_token_for_each_keyword() {
|
||
|
|
let src = "TODO write tests, FIXME later\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
let kw_count = toks.iter().filter(|t| t.kind == TOKEN_KEYWORD).count();
|
||
|
|
assert_eq!(kw_count, 2);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn wikilink_and_external_link_get_different_types() {
|
||
|
|
let src = "[[Page]] and [[https://example.com|docs]]\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_WIKILINK));
|
||
|
|
assert!(
|
||
|
|
toks.iter()
|
||
|
|
.any(|t| t.kind == nuwiki_lsp::semantic_tokens::TOKEN_EXTERNAL_LINK),
|
||
|
|
"URL inside [[...]] should be vimwikiExternalLink, got {toks:?}"
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn raw_url_emits_url_token() {
|
||
|
|
let src = "see https://example.com here\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_URL));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn transclusion_emits_transclusion_token() {
|
||
|
|
let src = "{{img.png|alt}}\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_TRANSCLUSION));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn comment_token_for_single_line_comment() {
|
||
|
|
let src = "%% hidden\nstuff\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_COMMENT));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn definition_term_token() {
|
||
|
|
let src = "Term:: Definition\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
assert!(toks.iter().any(|t| t.kind == TOKEN_DEFINITION_TERM));
|
||
|
|
}
|
||
|
|
|
||
|
|
// ===== Multi-line fences =====
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn preformatted_block_splits_into_one_token_per_line() {
|
||
|
|
let src = "{{{rust\nlet x = 1;\nlet y = 2;\n}}}\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
let code_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_CODE_BLOCK).collect();
|
||
|
|
// Spans line 0 (open fence) through line 3 (close fence) = 4 lines, but
|
||
|
|
// empty trailing slices are dropped, so we expect 4 line tokens.
|
||
|
|
assert!(
|
||
|
|
code_tokens.len() >= 3,
|
||
|
|
"expected multi-line split, got {code_tokens:?}"
|
||
|
|
);
|
||
|
|
// No token should cross a line boundary.
|
||
|
|
for t in &code_tokens {
|
||
|
|
assert!(t.byte_len > 0);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn math_block_splits_into_one_token_per_line() {
|
||
|
|
let src = "{{$\nx=1\ny=2\n}}$\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
let math_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_MATH_BLOCK).collect();
|
||
|
|
assert!(math_tokens.len() >= 3);
|
||
|
|
}
|
||
|
|
|
||
|
|
// ===== Sorting =====
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn tokens_are_sorted_by_line_then_column() {
|
||
|
|
let src = "= H =\n*bold* and _italic_\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let toks = collect_tokens(&doc, src);
|
||
|
|
for w in toks.windows(2) {
|
||
|
|
let a = (w[0].line, w[0].byte_col);
|
||
|
|
let b = (w[1].line, w[1].byte_col);
|
||
|
|
assert!(a <= b, "tokens not sorted: {a:?} then {b:?}");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ===== Encoding =====
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn encode_produces_correct_delta_quintuples() {
|
||
|
|
let src = "= H =\n*bold*\n";
|
||
|
|
let raws = vec![
|
||
|
|
RawToken {
|
||
|
|
line: 0,
|
||
|
|
byte_col: 0,
|
||
|
|
byte_len: 5,
|
||
|
|
kind: TOKEN_HEADING,
|
||
|
|
mods: MOD_LEVEL1,
|
||
|
|
},
|
||
|
|
RawToken {
|
||
|
|
line: 1,
|
||
|
|
byte_col: 0,
|
||
|
|
byte_len: 6,
|
||
|
|
kind: TOKEN_BOLD,
|
||
|
|
mods: 0,
|
||
|
|
},
|
||
|
|
];
|
||
|
|
let idx = LineIndex::new(src);
|
||
|
|
let data = encode(&raws, src, &idx, true);
|
||
|
|
assert_eq!(data.len(), 10);
|
||
|
|
// First token: delta_line=0, delta_col=0, len=5, type=HEADING, mods=LEVEL1
|
||
|
|
assert_eq!(&data[0..5], &[0, 0, 5, TOKEN_HEADING, MOD_LEVEL1]);
|
||
|
|
// Second token: delta_line=1 (new line), delta_col=0 (absolute), len=6, type=BOLD, mods=0
|
||
|
|
assert_eq!(&data[5..10], &[1, 0, 6, TOKEN_BOLD, 0]);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn encode_uses_relative_column_within_same_line() {
|
||
|
|
let src = "abc def ghi\n";
|
||
|
|
let raws = vec![
|
||
|
|
RawToken {
|
||
|
|
line: 0,
|
||
|
|
byte_col: 0,
|
||
|
|
byte_len: 3,
|
||
|
|
kind: TOKEN_BOLD,
|
||
|
|
mods: 0,
|
||
|
|
},
|
||
|
|
RawToken {
|
||
|
|
line: 0,
|
||
|
|
byte_col: 4,
|
||
|
|
byte_len: 3,
|
||
|
|
kind: TOKEN_ITALIC,
|
||
|
|
mods: 0,
|
||
|
|
},
|
||
|
|
RawToken {
|
||
|
|
line: 0,
|
||
|
|
byte_col: 8,
|
||
|
|
byte_len: 3,
|
||
|
|
kind: TOKEN_CODE,
|
||
|
|
mods: 0,
|
||
|
|
},
|
||
|
|
];
|
||
|
|
let idx = LineIndex::new(src);
|
||
|
|
let data = encode(&raws, src, &idx, true);
|
||
|
|
assert_eq!(&data[0..5], &[0, 0, 3, TOKEN_BOLD, 0]);
|
||
|
|
// Second: delta_col = 4 - 0 = 4
|
||
|
|
assert_eq!(&data[5..10], &[0, 4, 3, TOKEN_ITALIC, 0]);
|
||
|
|
// Third: delta_col = 8 - 4 = 4
|
||
|
|
assert_eq!(&data[10..15], &[0, 4, 3, TOKEN_CODE, 0]);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn encode_converts_to_utf16_for_multibyte_chars() {
|
||
|
|
// "_é_" — italic delimiters around é. Span covers 4 bytes (1 + 2 + 1)
|
||
|
|
// but 3 UTF-16 code units.
|
||
|
|
let src = "_é_\n";
|
||
|
|
let raws = vec![RawToken {
|
||
|
|
line: 0,
|
||
|
|
byte_col: 0,
|
||
|
|
byte_len: 4,
|
||
|
|
kind: TOKEN_ITALIC,
|
||
|
|
mods: 0,
|
||
|
|
}];
|
||
|
|
let idx = LineIndex::new(src);
|
||
|
|
let data = encode(&raws, src, &idx, false);
|
||
|
|
// length should be in UTF-16 code units (3), not bytes (4).
|
||
|
|
assert_eq!(data[2], 3);
|
||
|
|
}
|
||
|
|
|
||
|
|
// ===== Range filter =====
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn range_filter_keeps_only_overlapping_tokens() {
|
||
|
|
let src = "= L1 =\n= L2 =\n= L3 =\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
// Restrict to line 1 only.
|
||
|
|
let range = Range {
|
||
|
|
start: Position {
|
||
|
|
line: 1,
|
||
|
|
character: 0,
|
||
|
|
},
|
||
|
|
end: Position {
|
||
|
|
line: 2,
|
||
|
|
character: 0,
|
||
|
|
},
|
||
|
|
};
|
||
|
|
let data = build_data(&doc, src, true, Some(range));
|
||
|
|
// Expect exactly 1 heading token (the L2 one).
|
||
|
|
assert_eq!(data.len(), 5, "got {data:?}");
|
||
|
|
// Verify it's on line 1 (delta_line = 1 from prev_line = 0).
|
||
|
|
assert_eq!(data[0], 1);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn empty_document_produces_empty_token_stream() {
|
||
|
|
let doc = parse("");
|
||
|
|
let data = build_data(&doc, "", true, None);
|
||
|
|
assert!(data.is_empty());
|
||
|
|
}
|
||
|
|
|
||
|
|
// ===== End-to-end through builder =====
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn build_data_matches_collect_then_encode() {
|
||
|
|
let src = "= H1 =\n_em_ TODO\n";
|
||
|
|
let doc = parse(src);
|
||
|
|
let direct = build_data(&doc, src, true, None);
|
||
|
|
let raws = collect_tokens(&doc, src);
|
||
|
|
let idx = LineIndex::new(src);
|
||
|
|
let manual = encode(&raws, src, &idx, true);
|
||
|
|
assert_eq!(direct, manual);
|
||
|
|
}
|