phase 7: semantic tokens for syntax highlighting
`textDocument/semanticTokens/full` + `/range` per SPEC §6.9. Token
type scheme settled in P7: custom `vimwiki*` types
(`vimwikiHeading`, `vimwikiBold`, `vimwikiCodeBlock`, `vimwikiKeyword`,
`vimwikiWikilink`, `vimwikiTransclusion`, …) plus `level1`..`level6`
and `centered` modifiers. Default highlight groups for these arrive
in Phase 9.
Pipeline (`semantic_tokens.rs`):
- `legend()` builds the `SemanticTokensLegend` advertised in
`initialize` capabilities.
- `LineIndex` precomputes byte offsets per line so multi-line span
splitting and UTF-16 conversion stay linear.
- `collect_tokens` walks the AST. Blocks that are visually distinct
(heading, preformatted, math-block, comment) emit one token over
their span; multi-line ones get split per line because the LSP wire
format can't represent a token that crosses a newline. Container
blocks (paragraph, list item, table cell, blockquote) recurse into
inlines instead of wrapping them, so we end up with a non-overlapping
tile of inline tokens. Headings shadow nested inlines (one heading
token covers the line, no Bold-inside-Heading).
- `encode` produces the LSP delta-quintuple stream
`(deltaLine, deltaStart, length, type, mods)`. UTF-16 mode converts
byte offsets/lengths to code-unit counts on the fly (BMP + surrogate
pairs).
- `build_data` is the entry point used by both handlers; the `range`
variant filters tokens by overlap in the same encoding the client
requested.
Capability declared as `SemanticTokensOptions { full = true,
range = true, legend = … }`. Both handlers reuse the document store.
P7 resolved in SPEC §11 — custom vimwiki types with rationale and a
note that Phase 9 will ship default highlight-group definitions in
`syntax/nuwiki.vim` and the Lua glue.
Tests (21 new): legend integrity, per-construct emission for every
inline + block kind, heading shadowing of inner inlines, multi-line
code/math block splitting, sort order, delta encoding (same line and
crossing lines), UTF-16 length conversion for é, range filter,
empty-doc edge case, end-to-end `build_data` parity.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,8 @@
|
||||
//!
|
||||
//! Re-parses are full per change (incremental parsing deferred post-v1).
|
||||
|
||||
pub mod semantic_tokens;
|
||||
|
||||
use std::io;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
@@ -25,8 +27,10 @@ use tower_lsp::lsp_types::{
|
||||
Diagnostic, DiagnosticSeverity, DidChangeTextDocumentParams, DidCloseTextDocumentParams,
|
||||
DidOpenTextDocumentParams, DocumentSymbol, DocumentSymbolParams, DocumentSymbolResponse,
|
||||
InitializeParams, InitializeResult, InitializedParams, MessageType, OneOf,
|
||||
PositionEncodingKind, ServerCapabilities, ServerInfo, SymbolKind, TextDocumentSyncCapability,
|
||||
TextDocumentSyncKind, Url,
|
||||
PositionEncodingKind, SemanticTokens, SemanticTokensFullOptions, SemanticTokensOptions,
|
||||
SemanticTokensParams, SemanticTokensRangeParams, SemanticTokensRangeResult,
|
||||
SemanticTokensResult, SemanticTokensServerCapabilities, ServerCapabilities, ServerInfo,
|
||||
SymbolKind, TextDocumentSyncCapability, TextDocumentSyncKind, Url, WorkDoneProgressOptions,
|
||||
};
|
||||
use tower_lsp::{Client, LanguageServer, LspService, Server};
|
||||
|
||||
@@ -136,6 +140,16 @@ impl LanguageServer for Backend {
|
||||
TextDocumentSyncKind::FULL,
|
||||
)),
|
||||
document_symbol_provider: Some(OneOf::Left(true)),
|
||||
semantic_tokens_provider: Some(
|
||||
SemanticTokensServerCapabilities::SemanticTokensOptions(
|
||||
SemanticTokensOptions {
|
||||
work_done_progress_options: WorkDoneProgressOptions::default(),
|
||||
legend: semantic_tokens::legend(),
|
||||
range: Some(true),
|
||||
full: Some(SemanticTokensFullOptions::Bool(true)),
|
||||
},
|
||||
),
|
||||
),
|
||||
..ServerCapabilities::default()
|
||||
},
|
||||
server_info: Some(ServerInfo {
|
||||
@@ -199,6 +213,50 @@ impl LanguageServer for Backend {
|
||||
headings_to_symbols(&doc.ast, &doc.text, self.use_utf8.load(Ordering::Relaxed));
|
||||
Ok(Some(DocumentSymbolResponse::Nested(symbols)))
|
||||
}
|
||||
|
||||
async fn semantic_tokens_full(
|
||||
&self,
|
||||
params: SemanticTokensParams,
|
||||
) -> LspResult<Option<SemanticTokensResult>> {
|
||||
let uri = ¶ms.text_document.uri;
|
||||
let Some(doc) = self.documents.get(uri) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let utf8 = self.use_utf8.load(Ordering::Relaxed);
|
||||
let data = semantic_tokens::build_data(&doc.ast, &doc.text, utf8, None);
|
||||
Ok(Some(SemanticTokensResult::Tokens(SemanticTokens {
|
||||
result_id: None,
|
||||
data: pack_data(&data),
|
||||
})))
|
||||
}
|
||||
|
||||
async fn semantic_tokens_range(
|
||||
&self,
|
||||
params: SemanticTokensRangeParams,
|
||||
) -> LspResult<Option<SemanticTokensRangeResult>> {
|
||||
let uri = ¶ms.text_document.uri;
|
||||
let Some(doc) = self.documents.get(uri) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let utf8 = self.use_utf8.load(Ordering::Relaxed);
|
||||
let data = semantic_tokens::build_data(&doc.ast, &doc.text, utf8, Some(params.range));
|
||||
Ok(Some(SemanticTokensRangeResult::Tokens(SemanticTokens {
|
||||
result_id: None,
|
||||
data: pack_data(&data),
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
fn pack_data(flat: &[u32]) -> Vec<tower_lsp::lsp_types::SemanticToken> {
|
||||
flat.chunks_exact(5)
|
||||
.map(|c| tower_lsp::lsp_types::SemanticToken {
|
||||
delta_line: c[0],
|
||||
delta_start: c[1],
|
||||
length: c[2],
|
||||
token_type: c[3],
|
||||
token_modifiers_bitset: c[4],
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ===== Pure helpers =====
|
||||
|
||||
@@ -0,0 +1,387 @@
|
||||
//! Semantic-token emission for the vimwiki AST.
|
||||
//!
|
||||
//! Token-type scheme settled in P7: custom `vimwiki*` types plus
|
||||
//! `level1`..`level6` + `centered` modifiers (SPEC §11). Default highlight
|
||||
//! groups for these arrive in Phase 9.
|
||||
//!
|
||||
//! Pipeline:
|
||||
//!
|
||||
//! 1. Walk the AST, emitting one or more `RawToken`s per AST node. Block
|
||||
//! structure that "wraps" text (paragraph, list item, table cell)
|
||||
//! recurses into inlines instead of emitting a wrapping token, so we
|
||||
//! end up with a non-overlapping tile of inline tokens. Block nodes
|
||||
//! that are themselves visually distinct (heading, code block, math
|
||||
//! block, comment) emit one token across their whole span.
|
||||
//! 2. Multi-line spans are split into one token per line — the LSP wire
|
||||
//! format can't express a single token that crosses a newline.
|
||||
//! 3. Tokens are sorted by `(line, byte_col)` and encoded as the LSP
|
||||
//! delta-quintuple stream `(deltaLine, deltaStart, length, type, mods)`.
|
||||
//! When the client doesn't support UTF-8 encoding, byte offsets are
|
||||
//! converted to UTF-16 code-unit counts on the fly.
|
||||
|
||||
use nuwiki_core::ast::{
|
||||
BlockNode, BlockquoteNode, DocumentNode, HeadingNode, InlineNode, ListItemNode, ListNode, Span,
|
||||
TableNode,
|
||||
};
|
||||
use tower_lsp::lsp_types::{Range, SemanticTokenModifier, SemanticTokenType, SemanticTokensLegend};
|
||||
|
||||
// ===== Legend =====
|
||||
|
||||
pub const TOKEN_HEADING: u32 = 0;
|
||||
pub const TOKEN_BOLD: u32 = 1;
|
||||
pub const TOKEN_ITALIC: u32 = 2;
|
||||
pub const TOKEN_BOLD_ITALIC: u32 = 3;
|
||||
pub const TOKEN_STRIKETHROUGH: u32 = 4;
|
||||
pub const TOKEN_SUPERSCRIPT: u32 = 5;
|
||||
pub const TOKEN_SUBSCRIPT: u32 = 6;
|
||||
pub const TOKEN_CODE: u32 = 7;
|
||||
pub const TOKEN_CODE_BLOCK: u32 = 8;
|
||||
pub const TOKEN_MATH: u32 = 9;
|
||||
pub const TOKEN_MATH_BLOCK: u32 = 10;
|
||||
pub const TOKEN_KEYWORD: u32 = 11;
|
||||
pub const TOKEN_COLOR: u32 = 12;
|
||||
pub const TOKEN_WIKILINK: u32 = 13;
|
||||
pub const TOKEN_EXTERNAL_LINK: u32 = 14;
|
||||
pub const TOKEN_TRANSCLUSION: u32 = 15;
|
||||
pub const TOKEN_URL: u32 = 16;
|
||||
pub const TOKEN_COMMENT: u32 = 17;
|
||||
pub const TOKEN_DEFINITION_TERM: u32 = 18;
|
||||
|
||||
pub const TOKEN_TYPE_NAMES: &[&str] = &[
|
||||
"vimwikiHeading",
|
||||
"vimwikiBold",
|
||||
"vimwikiItalic",
|
||||
"vimwikiBoldItalic",
|
||||
"vimwikiStrikethrough",
|
||||
"vimwikiSuperscript",
|
||||
"vimwikiSubscript",
|
||||
"vimwikiCode",
|
||||
"vimwikiCodeBlock",
|
||||
"vimwikiMath",
|
||||
"vimwikiMathBlock",
|
||||
"vimwikiKeyword",
|
||||
"vimwikiColor",
|
||||
"vimwikiWikilink",
|
||||
"vimwikiExternalLink",
|
||||
"vimwikiTransclusion",
|
||||
"vimwikiUrl",
|
||||
"vimwikiComment",
|
||||
"vimwikiDefinitionTerm",
|
||||
];
|
||||
|
||||
pub const MOD_LEVEL1: u32 = 1 << 0;
|
||||
pub const MOD_LEVEL2: u32 = 1 << 1;
|
||||
pub const MOD_LEVEL3: u32 = 1 << 2;
|
||||
pub const MOD_LEVEL4: u32 = 1 << 3;
|
||||
pub const MOD_LEVEL5: u32 = 1 << 4;
|
||||
pub const MOD_LEVEL6: u32 = 1 << 5;
|
||||
pub const MOD_CENTERED: u32 = 1 << 6;
|
||||
|
||||
pub const TOKEN_MODIFIER_NAMES: &[&str] = &[
|
||||
"level1", "level2", "level3", "level4", "level5", "level6", "centered",
|
||||
];
|
||||
|
||||
pub fn legend() -> SemanticTokensLegend {
|
||||
SemanticTokensLegend {
|
||||
token_types: TOKEN_TYPE_NAMES
|
||||
.iter()
|
||||
.map(|s| SemanticTokenType::new(s))
|
||||
.collect(),
|
||||
token_modifiers: TOKEN_MODIFIER_NAMES
|
||||
.iter()
|
||||
.map(|s| SemanticTokenModifier::new(s))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Raw tokens =====
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct RawToken {
|
||||
pub line: u32,
|
||||
pub byte_col: u32,
|
||||
pub byte_len: u32,
|
||||
pub kind: u32,
|
||||
pub mods: u32,
|
||||
}
|
||||
|
||||
/// Index of byte offsets at the start of each line. Built once per request
|
||||
/// so multi-line span splitting and UTF-16 conversion stay linear in
|
||||
/// document size.
|
||||
pub struct LineIndex {
|
||||
pub starts: Vec<usize>,
|
||||
pub total: usize,
|
||||
}
|
||||
|
||||
impl LineIndex {
|
||||
pub fn new(text: &str) -> Self {
|
||||
let mut starts = vec![0];
|
||||
for (i, b) in text.bytes().enumerate() {
|
||||
if b == b'\n' {
|
||||
starts.push(i + 1);
|
||||
}
|
||||
}
|
||||
Self {
|
||||
starts,
|
||||
total: text.len(),
|
||||
}
|
||||
}
|
||||
|
||||
fn line_byte_range(&self, line: u32) -> (usize, usize) {
|
||||
let start = self
|
||||
.starts
|
||||
.get(line as usize)
|
||||
.copied()
|
||||
.unwrap_or(self.total);
|
||||
let end = self
|
||||
.starts
|
||||
.get(line as usize + 1)
|
||||
.map(|&s| s.saturating_sub(1))
|
||||
.unwrap_or(self.total);
|
||||
(start, end)
|
||||
}
|
||||
|
||||
pub fn line_str<'a>(&self, line: u32, text: &'a str) -> &'a str {
|
||||
let (s, e) = self.line_byte_range(line);
|
||||
&text[s..e]
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Collection =====
|
||||
|
||||
pub fn collect_tokens(doc: &DocumentNode, text: &str) -> Vec<RawToken> {
|
||||
let idx = LineIndex::new(text);
|
||||
let mut out = Vec::new();
|
||||
for block in &doc.children {
|
||||
emit_block(block, text, &idx, &mut out);
|
||||
}
|
||||
out.sort_by_key(|t| (t.line, t.byte_col));
|
||||
out
|
||||
}
|
||||
|
||||
fn emit_block(block: &BlockNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
match block {
|
||||
BlockNode::Heading(h) => emit_heading(h, text, idx, out),
|
||||
BlockNode::Paragraph(p) => emit_inlines(&p.children, text, idx, out),
|
||||
BlockNode::HorizontalRule(_) => {}
|
||||
BlockNode::Blockquote(BlockquoteNode { children, .. }) => {
|
||||
for child in children {
|
||||
emit_block(child, text, idx, out);
|
||||
}
|
||||
}
|
||||
BlockNode::Preformatted(p) => split_span(p.span, text, idx, TOKEN_CODE_BLOCK, 0, out),
|
||||
BlockNode::MathBlock(m) => split_span(m.span, text, idx, TOKEN_MATH_BLOCK, 0, out),
|
||||
BlockNode::List(ListNode { items, .. }) => {
|
||||
for item in items {
|
||||
emit_list_item(item, text, idx, out);
|
||||
}
|
||||
}
|
||||
BlockNode::DefinitionList(dl) => {
|
||||
for item in &dl.items {
|
||||
if let Some(term) = &item.term {
|
||||
if let (Some(first), Some(last)) = (term.first(), term.last()) {
|
||||
let s = Span::new(span_of_inline(first).start, span_of_inline(last).end);
|
||||
split_span(s, text, idx, TOKEN_DEFINITION_TERM, 0, out);
|
||||
}
|
||||
}
|
||||
for def in &item.definitions {
|
||||
emit_inlines(def, text, idx, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
BlockNode::Table(TableNode { rows, .. }) => {
|
||||
for row in rows {
|
||||
for cell in &row.cells {
|
||||
emit_inlines(&cell.children, text, idx, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
BlockNode::Comment(c) => split_span(c.span, text, idx, TOKEN_COMMENT, 0, out),
|
||||
BlockNode::Error(_) => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_heading(h: &HeadingNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
let mut mods = 0u32;
|
||||
mods |= match h.level {
|
||||
1 => MOD_LEVEL1,
|
||||
2 => MOD_LEVEL2,
|
||||
3 => MOD_LEVEL3,
|
||||
4 => MOD_LEVEL4,
|
||||
5 => MOD_LEVEL5,
|
||||
_ => MOD_LEVEL6,
|
||||
};
|
||||
if h.centered {
|
||||
mods |= MOD_CENTERED;
|
||||
}
|
||||
split_span(h.span, text, idx, TOKEN_HEADING, mods, out);
|
||||
}
|
||||
|
||||
fn emit_list_item(item: &ListItemNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
emit_inlines(&item.children, text, idx, out);
|
||||
if let Some(sublist) = &item.sublist {
|
||||
for child in &sublist.items {
|
||||
emit_list_item(child, text, idx, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_inlines(inlines: &[InlineNode], text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
for n in inlines {
|
||||
let (kind, span) = match n {
|
||||
InlineNode::Text(_) => continue,
|
||||
InlineNode::Bold(b) => (TOKEN_BOLD, b.span),
|
||||
InlineNode::Italic(i) => (TOKEN_ITALIC, i.span),
|
||||
InlineNode::BoldItalic(bi) => (TOKEN_BOLD_ITALIC, bi.span),
|
||||
InlineNode::Strikethrough(s) => (TOKEN_STRIKETHROUGH, s.span),
|
||||
InlineNode::Code(c) => (TOKEN_CODE, c.span),
|
||||
InlineNode::Superscript(s) => (TOKEN_SUPERSCRIPT, s.span),
|
||||
InlineNode::Subscript(s) => (TOKEN_SUBSCRIPT, s.span),
|
||||
InlineNode::MathInline(m) => (TOKEN_MATH, m.span),
|
||||
InlineNode::Keyword(k) => (TOKEN_KEYWORD, k.span),
|
||||
InlineNode::Color(c) => (TOKEN_COLOR, c.span),
|
||||
InlineNode::WikiLink(w) => (TOKEN_WIKILINK, w.span),
|
||||
InlineNode::ExternalLink(e) => (TOKEN_EXTERNAL_LINK, e.span),
|
||||
InlineNode::Transclusion(t) => (TOKEN_TRANSCLUSION, t.span),
|
||||
InlineNode::RawUrl(u) => (TOKEN_URL, u.span),
|
||||
};
|
||||
split_span(span, text, idx, kind, 0, out);
|
||||
}
|
||||
}
|
||||
|
||||
fn span_of_inline(node: &InlineNode) -> Span {
|
||||
match node {
|
||||
InlineNode::Text(n) => n.span,
|
||||
InlineNode::Bold(n) => n.span,
|
||||
InlineNode::Italic(n) => n.span,
|
||||
InlineNode::BoldItalic(n) => n.span,
|
||||
InlineNode::Strikethrough(n) => n.span,
|
||||
InlineNode::Code(n) => n.span,
|
||||
InlineNode::Superscript(n) => n.span,
|
||||
InlineNode::Subscript(n) => n.span,
|
||||
InlineNode::MathInline(n) => n.span,
|
||||
InlineNode::Keyword(n) => n.span,
|
||||
InlineNode::Color(n) => n.span,
|
||||
InlineNode::WikiLink(n) => n.span,
|
||||
InlineNode::ExternalLink(n) => n.span,
|
||||
InlineNode::Transclusion(n) => n.span,
|
||||
InlineNode::RawUrl(n) => n.span,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a `Span` (potentially multi-line) into one or more single-line
|
||||
/// `RawToken`s. The LSP wire format can't express tokens that span newlines,
|
||||
/// so we split on line boundaries.
|
||||
pub fn split_span(
|
||||
span: Span,
|
||||
text: &str,
|
||||
idx: &LineIndex,
|
||||
kind: u32,
|
||||
mods: u32,
|
||||
out: &mut Vec<RawToken>,
|
||||
) {
|
||||
if span.start.line == span.end.line {
|
||||
let len = span.end.column.saturating_sub(span.start.column);
|
||||
if len > 0 {
|
||||
out.push(RawToken {
|
||||
line: span.start.line,
|
||||
byte_col: span.start.column,
|
||||
byte_len: len,
|
||||
kind,
|
||||
mods,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
for line in span.start.line..=span.end.line {
|
||||
let line_str = idx.line_str(line, text);
|
||||
let line_len = line_str.len() as u32;
|
||||
let start_col = if line == span.start.line {
|
||||
span.start.column
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let end_col = if line == span.end.line {
|
||||
span.end.column.min(line_len)
|
||||
} else {
|
||||
line_len
|
||||
};
|
||||
if end_col > start_col {
|
||||
out.push(RawToken {
|
||||
line,
|
||||
byte_col: start_col,
|
||||
byte_len: end_col - start_col,
|
||||
kind,
|
||||
mods,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Encoding =====
|
||||
|
||||
/// LSP wire-format encoder. Produces a flat `Vec<u32>` of
|
||||
/// `(deltaLine, deltaStart, length, type, modifiers)` quintuples.
|
||||
///
|
||||
/// `utf8 = true` means the client opted into UTF-8 encoding, so byte offsets
|
||||
/// pass through directly. Otherwise positions and lengths are converted to
|
||||
/// UTF-16 code units per the LSP default.
|
||||
pub fn encode(raws: &[RawToken], text: &str, idx: &LineIndex, utf8: bool) -> Vec<u32> {
|
||||
let mut data = Vec::with_capacity(raws.len() * 5);
|
||||
let mut prev_line = 0u32;
|
||||
let mut prev_col = 0u32;
|
||||
for tok in raws {
|
||||
let (col, len) = if utf8 {
|
||||
(tok.byte_col, tok.byte_len)
|
||||
} else {
|
||||
let line_str = idx.line_str(tok.line, text);
|
||||
let start = utf16_count(line_str, tok.byte_col as usize);
|
||||
let end = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize);
|
||||
(start, end - start)
|
||||
};
|
||||
let delta_line = tok.line - prev_line;
|
||||
let delta_col = if delta_line == 0 { col - prev_col } else { col };
|
||||
data.extend_from_slice(&[delta_line, delta_col, len, tok.kind, tok.mods]);
|
||||
prev_line = tok.line;
|
||||
prev_col = col;
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
fn utf16_count(s: &str, byte_offset: usize) -> u32 {
|
||||
let upto = byte_offset.min(s.len());
|
||||
s[..upto].chars().map(char::len_utf16).sum::<usize>() as u32
|
||||
}
|
||||
|
||||
/// Build the encoded data stream for either the full document or a range
|
||||
/// request.
|
||||
pub fn build_data(doc: &DocumentNode, text: &str, utf8: bool, range: Option<Range>) -> Vec<u32> {
|
||||
let idx = LineIndex::new(text);
|
||||
let mut tokens = collect_tokens(doc, text);
|
||||
if let Some(r) = range {
|
||||
tokens.retain(|t| token_overlaps_range(t, &r, text, &idx, utf8));
|
||||
}
|
||||
encode(&tokens, text, &idx, utf8)
|
||||
}
|
||||
|
||||
fn token_overlaps_range(
|
||||
tok: &RawToken,
|
||||
range: &Range,
|
||||
text: &str,
|
||||
idx: &LineIndex,
|
||||
utf8: bool,
|
||||
) -> bool {
|
||||
let (start_char, end_char) = if utf8 {
|
||||
(tok.byte_col, tok.byte_col + tok.byte_len)
|
||||
} else {
|
||||
let line_str = idx.line_str(tok.line, text);
|
||||
let s = utf16_count(line_str, tok.byte_col as usize);
|
||||
let e = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize);
|
||||
(s, e)
|
||||
};
|
||||
let tok_start = (tok.line, start_char);
|
||||
let tok_end = (tok.line, end_char);
|
||||
let r_start = (range.start.line, range.start.character);
|
||||
let r_end = (range.end.line, range.end.character);
|
||||
tok_end > r_start && tok_start < r_end
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
//! Tests for the semantic-token builder, encoder, range filter, and
|
||||
//! UTF-8/UTF-16 conversion.
|
||||
|
||||
use nuwiki_core::syntax::vimwiki::VimwikiSyntax;
|
||||
use nuwiki_core::syntax::SyntaxPlugin;
|
||||
use nuwiki_lsp::semantic_tokens::{
|
||||
build_data, collect_tokens, encode, legend, LineIndex, RawToken, MOD_CENTERED, MOD_LEVEL1,
|
||||
MOD_LEVEL2, TOKEN_BOLD, TOKEN_CODE, TOKEN_CODE_BLOCK, TOKEN_COMMENT, TOKEN_DEFINITION_TERM,
|
||||
TOKEN_HEADING, TOKEN_ITALIC, TOKEN_KEYWORD, TOKEN_MATH_BLOCK, TOKEN_TRANSCLUSION,
|
||||
TOKEN_TYPE_NAMES, TOKEN_URL, TOKEN_WIKILINK,
|
||||
};
|
||||
use tower_lsp::lsp_types::{Position, Range};
|
||||
|
||||
fn parse(src: &str) -> nuwiki_core::ast::DocumentNode {
|
||||
VimwikiSyntax::new().parse(src)
|
||||
}
|
||||
|
||||
fn first_with_kind(tokens: &[RawToken], kind: u32) -> Option<&RawToken> {
|
||||
tokens.iter().find(|t| t.kind == kind)
|
||||
}
|
||||
|
||||
// ===== Legend =====
|
||||
|
||||
#[test]
|
||||
fn legend_lists_all_custom_token_types() {
|
||||
let l = legend();
|
||||
assert_eq!(l.token_types.len(), TOKEN_TYPE_NAMES.len());
|
||||
let names: Vec<String> = l
|
||||
.token_types
|
||||
.iter()
|
||||
.map(|t| t.as_str().to_owned())
|
||||
.collect();
|
||||
assert!(names.contains(&"vimwikiHeading".to_string()));
|
||||
assert!(names.contains(&"vimwikiWikilink".to_string()));
|
||||
assert!(names.contains(&"vimwikiTransclusion".to_string()));
|
||||
}
|
||||
|
||||
// ===== Per-construct emission =====
|
||||
|
||||
#[test]
|
||||
fn heading_emits_one_token_with_level_modifier() {
|
||||
let doc = parse("== Hi ==\n");
|
||||
let toks = collect_tokens(&doc, "== Hi ==\n");
|
||||
let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
|
||||
assert_eq!(h.mods & MOD_LEVEL2, MOD_LEVEL2);
|
||||
assert_eq!(h.mods & MOD_CENTERED, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn centered_heading_carries_centered_modifier() {
|
||||
let src = " = Title =\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token");
|
||||
assert_eq!(h.mods & MOD_LEVEL1, MOD_LEVEL1);
|
||||
assert_eq!(h.mods & MOD_CENTERED, MOD_CENTERED);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heading_does_not_emit_inner_inline_tokens() {
|
||||
// Headings render as a single styled span, so bold inside is shadowed.
|
||||
let src = "= Hi *bold* =\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_HEADING));
|
||||
assert!(
|
||||
!toks.iter().any(|t| t.kind == TOKEN_BOLD),
|
||||
"should not have emitted Bold inside a Heading"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paragraph_emits_inline_tokens_only() {
|
||||
let src = "see *bold* and `code`\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_BOLD));
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_CODE));
|
||||
assert!(!toks.iter().any(|t| t.kind == TOKEN_HEADING));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italic_token_for_underscored_run() {
|
||||
let src = "_emphasis_\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_ITALIC));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keyword_token_for_each_keyword() {
|
||||
let src = "TODO write tests, FIXME later\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
let kw_count = toks.iter().filter(|t| t.kind == TOKEN_KEYWORD).count();
|
||||
assert_eq!(kw_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wikilink_and_external_link_get_different_types() {
|
||||
let src = "[[Page]] and [[https://example.com|docs]]\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_WIKILINK));
|
||||
assert!(
|
||||
toks.iter()
|
||||
.any(|t| t.kind == nuwiki_lsp::semantic_tokens::TOKEN_EXTERNAL_LINK),
|
||||
"URL inside [[...]] should be vimwikiExternalLink, got {toks:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_url_emits_url_token() {
|
||||
let src = "see https://example.com here\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_URL));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transclusion_emits_transclusion_token() {
|
||||
let src = "{{img.png|alt}}\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_TRANSCLUSION));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_token_for_single_line_comment() {
|
||||
let src = "%% hidden\nstuff\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_COMMENT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn definition_term_token() {
|
||||
let src = "Term:: Definition\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
assert!(toks.iter().any(|t| t.kind == TOKEN_DEFINITION_TERM));
|
||||
}
|
||||
|
||||
// ===== Multi-line fences =====
|
||||
|
||||
#[test]
|
||||
fn preformatted_block_splits_into_one_token_per_line() {
|
||||
let src = "{{{rust\nlet x = 1;\nlet y = 2;\n}}}\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
let code_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_CODE_BLOCK).collect();
|
||||
// Spans line 0 (open fence) through line 3 (close fence) = 4 lines, but
|
||||
// empty trailing slices are dropped, so we expect 4 line tokens.
|
||||
assert!(
|
||||
code_tokens.len() >= 3,
|
||||
"expected multi-line split, got {code_tokens:?}"
|
||||
);
|
||||
// No token should cross a line boundary.
|
||||
for t in &code_tokens {
|
||||
assert!(t.byte_len > 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn math_block_splits_into_one_token_per_line() {
|
||||
let src = "{{$\nx=1\ny=2\n}}$\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
let math_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_MATH_BLOCK).collect();
|
||||
assert!(math_tokens.len() >= 3);
|
||||
}
|
||||
|
||||
// ===== Sorting =====
|
||||
|
||||
#[test]
|
||||
fn tokens_are_sorted_by_line_then_column() {
|
||||
let src = "= H =\n*bold* and _italic_\n";
|
||||
let doc = parse(src);
|
||||
let toks = collect_tokens(&doc, src);
|
||||
for w in toks.windows(2) {
|
||||
let a = (w[0].line, w[0].byte_col);
|
||||
let b = (w[1].line, w[1].byte_col);
|
||||
assert!(a <= b, "tokens not sorted: {a:?} then {b:?}");
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Encoding =====
|
||||
|
||||
#[test]
|
||||
fn encode_produces_correct_delta_quintuples() {
|
||||
let src = "= H =\n*bold*\n";
|
||||
let raws = vec![
|
||||
RawToken {
|
||||
line: 0,
|
||||
byte_col: 0,
|
||||
byte_len: 5,
|
||||
kind: TOKEN_HEADING,
|
||||
mods: MOD_LEVEL1,
|
||||
},
|
||||
RawToken {
|
||||
line: 1,
|
||||
byte_col: 0,
|
||||
byte_len: 6,
|
||||
kind: TOKEN_BOLD,
|
||||
mods: 0,
|
||||
},
|
||||
];
|
||||
let idx = LineIndex::new(src);
|
||||
let data = encode(&raws, src, &idx, true);
|
||||
assert_eq!(data.len(), 10);
|
||||
// First token: delta_line=0, delta_col=0, len=5, type=HEADING, mods=LEVEL1
|
||||
assert_eq!(&data[0..5], &[0, 0, 5, TOKEN_HEADING, MOD_LEVEL1]);
|
||||
// Second token: delta_line=1 (new line), delta_col=0 (absolute), len=6, type=BOLD, mods=0
|
||||
assert_eq!(&data[5..10], &[1, 0, 6, TOKEN_BOLD, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_uses_relative_column_within_same_line() {
|
||||
let src = "abc def ghi\n";
|
||||
let raws = vec![
|
||||
RawToken {
|
||||
line: 0,
|
||||
byte_col: 0,
|
||||
byte_len: 3,
|
||||
kind: TOKEN_BOLD,
|
||||
mods: 0,
|
||||
},
|
||||
RawToken {
|
||||
line: 0,
|
||||
byte_col: 4,
|
||||
byte_len: 3,
|
||||
kind: TOKEN_ITALIC,
|
||||
mods: 0,
|
||||
},
|
||||
RawToken {
|
||||
line: 0,
|
||||
byte_col: 8,
|
||||
byte_len: 3,
|
||||
kind: TOKEN_CODE,
|
||||
mods: 0,
|
||||
},
|
||||
];
|
||||
let idx = LineIndex::new(src);
|
||||
let data = encode(&raws, src, &idx, true);
|
||||
assert_eq!(&data[0..5], &[0, 0, 3, TOKEN_BOLD, 0]);
|
||||
// Second: delta_col = 4 - 0 = 4
|
||||
assert_eq!(&data[5..10], &[0, 4, 3, TOKEN_ITALIC, 0]);
|
||||
// Third: delta_col = 8 - 4 = 4
|
||||
assert_eq!(&data[10..15], &[0, 4, 3, TOKEN_CODE, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_converts_to_utf16_for_multibyte_chars() {
|
||||
// "_é_" — italic delimiters around é. Span covers 4 bytes (1 + 2 + 1)
|
||||
// but 3 UTF-16 code units.
|
||||
let src = "_é_\n";
|
||||
let raws = vec![RawToken {
|
||||
line: 0,
|
||||
byte_col: 0,
|
||||
byte_len: 4,
|
||||
kind: TOKEN_ITALIC,
|
||||
mods: 0,
|
||||
}];
|
||||
let idx = LineIndex::new(src);
|
||||
let data = encode(&raws, src, &idx, false);
|
||||
// length should be in UTF-16 code units (3), not bytes (4).
|
||||
assert_eq!(data[2], 3);
|
||||
}
|
||||
|
||||
// ===== Range filter =====
|
||||
|
||||
#[test]
|
||||
fn range_filter_keeps_only_overlapping_tokens() {
|
||||
let src = "= L1 =\n= L2 =\n= L3 =\n";
|
||||
let doc = parse(src);
|
||||
// Restrict to line 1 only.
|
||||
let range = Range {
|
||||
start: Position {
|
||||
line: 1,
|
||||
character: 0,
|
||||
},
|
||||
end: Position {
|
||||
line: 2,
|
||||
character: 0,
|
||||
},
|
||||
};
|
||||
let data = build_data(&doc, src, true, Some(range));
|
||||
// Expect exactly 1 heading token (the L2 one).
|
||||
assert_eq!(data.len(), 5, "got {data:?}");
|
||||
// Verify it's on line 1 (delta_line = 1 from prev_line = 0).
|
||||
assert_eq!(data[0], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_document_produces_empty_token_stream() {
|
||||
let doc = parse("");
|
||||
let data = build_data(&doc, "", true, None);
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
// ===== End-to-end through builder =====
|
||||
|
||||
#[test]
|
||||
fn build_data_matches_collect_then_encode() {
|
||||
let src = "= H1 =\n_em_ TODO\n";
|
||||
let doc = parse(src);
|
||||
let direct = build_data(&doc, src, true, None);
|
||||
let raws = collect_tokens(&doc, src);
|
||||
let idx = LineIndex::new(src);
|
||||
let manual = encode(&raws, src, &idx, true);
|
||||
assert_eq!(direct, manual);
|
||||
}
|
||||
Reference in New Issue
Block a user