diff --git a/README.md b/README.md index 6c9adb9..3d20988 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,10 @@ A Vim/Neovim plugin providing full vimwiki syntax support, implemented as a Rust-based language server (LSP). -> **Status:** Phase 6 (LSP foundation) complete. `nuwiki-ls` speaks LSP over -> stdio: didOpen/didChange/didClose, parse diagnostics, nested document -> outline. Editor glue, semantic tokens, and navigation still pending. +> **Status:** Phase 7 (semantic tokens) complete. The server now serves +> `semanticTokens/full` + `/range` with custom `vimwiki*` token types and +> `level1`..`level6` + `centered` modifiers. Navigation features and +> editor glue still pending. See [`SPEC.md`](./SPEC.md) for the full project specification. @@ -20,8 +21,8 @@ See [`SPEC.md`](./SPEC.md) for the full project specification. | 4 | Vimwiki Parser | ✅ done | | 5 | Renderer | ✅ done | | 6 | LSP Foundation | ✅ done | -| 7 | Semantic Tokens | ⏳ next | -| 8 | Navigation | ⏳ | +| 7 | Semantic Tokens | ✅ done | +| 8 | Navigation | ⏳ next | | 9 | Editor Glue | ⏳ | | 10 | CI/CD release pipeline | ⏳ | diff --git a/SPEC.md b/SPEC.md index 2bd6c22..16708a8 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,7 +1,7 @@ # nuwiki — Project Specification > Last updated: 2026-05-10 -> Status: Phase 6 (LSP Foundation) complete — moving to Phase 7 (Semantic Tokens) +> Status: Phase 7 (Semantic Tokens) complete — moving to Phase 8 (Navigation) --- @@ -590,7 +590,7 @@ These decisions are required before or during the phase indicated. | ~~P4~~ | ~~**Visitor pattern**~~ | ~~Phase 1~~ | ✅ **Defined in Phase 1** | Open-recursion `Visitor` trait + `walk_*` helpers | | P5 | **Minimum Neovim version** | Phase 9 | 0.8 (`vim.lsp.start`) · 0.11 (`vim.lsp.config`) | 0.8 = broader compat; 0.11 = cleaner Lua API | | P6 | **Minimum Vim version** | Phase 9 | Vim 8.2 · Vim 9.0 · Vim 9.1 | LSP client plugins work best on 9.0+; recommend documenting 9.1 as minimum | -| P7 | **Semantic token type mapping** | Phase 7 | Standard LSP types only · Custom token types | Custom types need client-side highlight group definitions; more flexible but more setup for users | +| ~~P7~~ | ~~**Semantic token type mapping**~~ | ~~Phase 7~~ | ✅ **Custom vimwiki-specific token types** | ~20 types (`vimwikiHeading`, `vimwikiBold`, …) + `level1`..`level6` + `centered` modifiers. Phase 9 ships default highlight groups in `syntax/nuwiki.vim` and the Lua glue. | | P8 | **Workspace indexing strategy** | Phase 8 | Eager on startup · Lazy + background | Eager blocks startup on large wikis; recommend lazy with progress notification | | P9 | **macOS runner** | Phase 10 | Stay manual · Register macOS runner | Revisit if a Mac is available; no action needed now | diff --git a/crates/nuwiki-lsp/src/lib.rs b/crates/nuwiki-lsp/src/lib.rs index 52fc5b2..d00aa88 100644 --- a/crates/nuwiki-lsp/src/lib.rs +++ b/crates/nuwiki-lsp/src/lib.rs @@ -14,6 +14,8 @@ //! //! Re-parses are full per change (incremental parsing deferred post-v1). +pub mod semantic_tokens; + use std::io; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -25,8 +27,10 @@ use tower_lsp::lsp_types::{ Diagnostic, DiagnosticSeverity, DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, DocumentSymbol, DocumentSymbolParams, DocumentSymbolResponse, InitializeParams, InitializeResult, InitializedParams, MessageType, OneOf, - PositionEncodingKind, ServerCapabilities, ServerInfo, SymbolKind, TextDocumentSyncCapability, - TextDocumentSyncKind, Url, + PositionEncodingKind, SemanticTokens, SemanticTokensFullOptions, SemanticTokensOptions, + SemanticTokensParams, SemanticTokensRangeParams, SemanticTokensRangeResult, + SemanticTokensResult, SemanticTokensServerCapabilities, ServerCapabilities, ServerInfo, + SymbolKind, TextDocumentSyncCapability, TextDocumentSyncKind, Url, WorkDoneProgressOptions, }; use tower_lsp::{Client, LanguageServer, LspService, Server}; @@ -136,6 +140,16 @@ impl LanguageServer for Backend { TextDocumentSyncKind::FULL, )), document_symbol_provider: Some(OneOf::Left(true)), + semantic_tokens_provider: Some( + SemanticTokensServerCapabilities::SemanticTokensOptions( + SemanticTokensOptions { + work_done_progress_options: WorkDoneProgressOptions::default(), + legend: semantic_tokens::legend(), + range: Some(true), + full: Some(SemanticTokensFullOptions::Bool(true)), + }, + ), + ), ..ServerCapabilities::default() }, server_info: Some(ServerInfo { @@ -199,6 +213,50 @@ impl LanguageServer for Backend { headings_to_symbols(&doc.ast, &doc.text, self.use_utf8.load(Ordering::Relaxed)); Ok(Some(DocumentSymbolResponse::Nested(symbols))) } + + async fn semantic_tokens_full( + &self, + params: SemanticTokensParams, + ) -> LspResult> { + let uri = ¶ms.text_document.uri; + let Some(doc) = self.documents.get(uri) else { + return Ok(None); + }; + let utf8 = self.use_utf8.load(Ordering::Relaxed); + let data = semantic_tokens::build_data(&doc.ast, &doc.text, utf8, None); + Ok(Some(SemanticTokensResult::Tokens(SemanticTokens { + result_id: None, + data: pack_data(&data), + }))) + } + + async fn semantic_tokens_range( + &self, + params: SemanticTokensRangeParams, + ) -> LspResult> { + let uri = ¶ms.text_document.uri; + let Some(doc) = self.documents.get(uri) else { + return Ok(None); + }; + let utf8 = self.use_utf8.load(Ordering::Relaxed); + let data = semantic_tokens::build_data(&doc.ast, &doc.text, utf8, Some(params.range)); + Ok(Some(SemanticTokensRangeResult::Tokens(SemanticTokens { + result_id: None, + data: pack_data(&data), + }))) + } +} + +fn pack_data(flat: &[u32]) -> Vec { + flat.chunks_exact(5) + .map(|c| tower_lsp::lsp_types::SemanticToken { + delta_line: c[0], + delta_start: c[1], + length: c[2], + token_type: c[3], + token_modifiers_bitset: c[4], + }) + .collect() } // ===== Pure helpers ===== diff --git a/crates/nuwiki-lsp/src/semantic_tokens.rs b/crates/nuwiki-lsp/src/semantic_tokens.rs new file mode 100644 index 0000000..f4680f3 --- /dev/null +++ b/crates/nuwiki-lsp/src/semantic_tokens.rs @@ -0,0 +1,387 @@ +//! Semantic-token emission for the vimwiki AST. +//! +//! Token-type scheme settled in P7: custom `vimwiki*` types plus +//! `level1`..`level6` + `centered` modifiers (SPEC §11). Default highlight +//! groups for these arrive in Phase 9. +//! +//! Pipeline: +//! +//! 1. Walk the AST, emitting one or more `RawToken`s per AST node. Block +//! structure that "wraps" text (paragraph, list item, table cell) +//! recurses into inlines instead of emitting a wrapping token, so we +//! end up with a non-overlapping tile of inline tokens. Block nodes +//! that are themselves visually distinct (heading, code block, math +//! block, comment) emit one token across their whole span. +//! 2. Multi-line spans are split into one token per line — the LSP wire +//! format can't express a single token that crosses a newline. +//! 3. Tokens are sorted by `(line, byte_col)` and encoded as the LSP +//! delta-quintuple stream `(deltaLine, deltaStart, length, type, mods)`. +//! When the client doesn't support UTF-8 encoding, byte offsets are +//! converted to UTF-16 code-unit counts on the fly. + +use nuwiki_core::ast::{ + BlockNode, BlockquoteNode, DocumentNode, HeadingNode, InlineNode, ListItemNode, ListNode, Span, + TableNode, +}; +use tower_lsp::lsp_types::{Range, SemanticTokenModifier, SemanticTokenType, SemanticTokensLegend}; + +// ===== Legend ===== + +pub const TOKEN_HEADING: u32 = 0; +pub const TOKEN_BOLD: u32 = 1; +pub const TOKEN_ITALIC: u32 = 2; +pub const TOKEN_BOLD_ITALIC: u32 = 3; +pub const TOKEN_STRIKETHROUGH: u32 = 4; +pub const TOKEN_SUPERSCRIPT: u32 = 5; +pub const TOKEN_SUBSCRIPT: u32 = 6; +pub const TOKEN_CODE: u32 = 7; +pub const TOKEN_CODE_BLOCK: u32 = 8; +pub const TOKEN_MATH: u32 = 9; +pub const TOKEN_MATH_BLOCK: u32 = 10; +pub const TOKEN_KEYWORD: u32 = 11; +pub const TOKEN_COLOR: u32 = 12; +pub const TOKEN_WIKILINK: u32 = 13; +pub const TOKEN_EXTERNAL_LINK: u32 = 14; +pub const TOKEN_TRANSCLUSION: u32 = 15; +pub const TOKEN_URL: u32 = 16; +pub const TOKEN_COMMENT: u32 = 17; +pub const TOKEN_DEFINITION_TERM: u32 = 18; + +pub const TOKEN_TYPE_NAMES: &[&str] = &[ + "vimwikiHeading", + "vimwikiBold", + "vimwikiItalic", + "vimwikiBoldItalic", + "vimwikiStrikethrough", + "vimwikiSuperscript", + "vimwikiSubscript", + "vimwikiCode", + "vimwikiCodeBlock", + "vimwikiMath", + "vimwikiMathBlock", + "vimwikiKeyword", + "vimwikiColor", + "vimwikiWikilink", + "vimwikiExternalLink", + "vimwikiTransclusion", + "vimwikiUrl", + "vimwikiComment", + "vimwikiDefinitionTerm", +]; + +pub const MOD_LEVEL1: u32 = 1 << 0; +pub const MOD_LEVEL2: u32 = 1 << 1; +pub const MOD_LEVEL3: u32 = 1 << 2; +pub const MOD_LEVEL4: u32 = 1 << 3; +pub const MOD_LEVEL5: u32 = 1 << 4; +pub const MOD_LEVEL6: u32 = 1 << 5; +pub const MOD_CENTERED: u32 = 1 << 6; + +pub const TOKEN_MODIFIER_NAMES: &[&str] = &[ + "level1", "level2", "level3", "level4", "level5", "level6", "centered", +]; + +pub fn legend() -> SemanticTokensLegend { + SemanticTokensLegend { + token_types: TOKEN_TYPE_NAMES + .iter() + .map(|s| SemanticTokenType::new(s)) + .collect(), + token_modifiers: TOKEN_MODIFIER_NAMES + .iter() + .map(|s| SemanticTokenModifier::new(s)) + .collect(), + } +} + +// ===== Raw tokens ===== + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RawToken { + pub line: u32, + pub byte_col: u32, + pub byte_len: u32, + pub kind: u32, + pub mods: u32, +} + +/// Index of byte offsets at the start of each line. Built once per request +/// so multi-line span splitting and UTF-16 conversion stay linear in +/// document size. +pub struct LineIndex { + pub starts: Vec, + pub total: usize, +} + +impl LineIndex { + pub fn new(text: &str) -> Self { + let mut starts = vec![0]; + for (i, b) in text.bytes().enumerate() { + if b == b'\n' { + starts.push(i + 1); + } + } + Self { + starts, + total: text.len(), + } + } + + fn line_byte_range(&self, line: u32) -> (usize, usize) { + let start = self + .starts + .get(line as usize) + .copied() + .unwrap_or(self.total); + let end = self + .starts + .get(line as usize + 1) + .map(|&s| s.saturating_sub(1)) + .unwrap_or(self.total); + (start, end) + } + + pub fn line_str<'a>(&self, line: u32, text: &'a str) -> &'a str { + let (s, e) = self.line_byte_range(line); + &text[s..e] + } +} + +// ===== Collection ===== + +pub fn collect_tokens(doc: &DocumentNode, text: &str) -> Vec { + let idx = LineIndex::new(text); + let mut out = Vec::new(); + for block in &doc.children { + emit_block(block, text, &idx, &mut out); + } + out.sort_by_key(|t| (t.line, t.byte_col)); + out +} + +fn emit_block(block: &BlockNode, text: &str, idx: &LineIndex, out: &mut Vec) { + match block { + BlockNode::Heading(h) => emit_heading(h, text, idx, out), + BlockNode::Paragraph(p) => emit_inlines(&p.children, text, idx, out), + BlockNode::HorizontalRule(_) => {} + BlockNode::Blockquote(BlockquoteNode { children, .. }) => { + for child in children { + emit_block(child, text, idx, out); + } + } + BlockNode::Preformatted(p) => split_span(p.span, text, idx, TOKEN_CODE_BLOCK, 0, out), + BlockNode::MathBlock(m) => split_span(m.span, text, idx, TOKEN_MATH_BLOCK, 0, out), + BlockNode::List(ListNode { items, .. }) => { + for item in items { + emit_list_item(item, text, idx, out); + } + } + BlockNode::DefinitionList(dl) => { + for item in &dl.items { + if let Some(term) = &item.term { + if let (Some(first), Some(last)) = (term.first(), term.last()) { + let s = Span::new(span_of_inline(first).start, span_of_inline(last).end); + split_span(s, text, idx, TOKEN_DEFINITION_TERM, 0, out); + } + } + for def in &item.definitions { + emit_inlines(def, text, idx, out); + } + } + } + BlockNode::Table(TableNode { rows, .. }) => { + for row in rows { + for cell in &row.cells { + emit_inlines(&cell.children, text, idx, out); + } + } + } + BlockNode::Comment(c) => split_span(c.span, text, idx, TOKEN_COMMENT, 0, out), + BlockNode::Error(_) => {} + } +} + +fn emit_heading(h: &HeadingNode, text: &str, idx: &LineIndex, out: &mut Vec) { + let mut mods = 0u32; + mods |= match h.level { + 1 => MOD_LEVEL1, + 2 => MOD_LEVEL2, + 3 => MOD_LEVEL3, + 4 => MOD_LEVEL4, + 5 => MOD_LEVEL5, + _ => MOD_LEVEL6, + }; + if h.centered { + mods |= MOD_CENTERED; + } + split_span(h.span, text, idx, TOKEN_HEADING, mods, out); +} + +fn emit_list_item(item: &ListItemNode, text: &str, idx: &LineIndex, out: &mut Vec) { + emit_inlines(&item.children, text, idx, out); + if let Some(sublist) = &item.sublist { + for child in &sublist.items { + emit_list_item(child, text, idx, out); + } + } +} + +fn emit_inlines(inlines: &[InlineNode], text: &str, idx: &LineIndex, out: &mut Vec) { + for n in inlines { + let (kind, span) = match n { + InlineNode::Text(_) => continue, + InlineNode::Bold(b) => (TOKEN_BOLD, b.span), + InlineNode::Italic(i) => (TOKEN_ITALIC, i.span), + InlineNode::BoldItalic(bi) => (TOKEN_BOLD_ITALIC, bi.span), + InlineNode::Strikethrough(s) => (TOKEN_STRIKETHROUGH, s.span), + InlineNode::Code(c) => (TOKEN_CODE, c.span), + InlineNode::Superscript(s) => (TOKEN_SUPERSCRIPT, s.span), + InlineNode::Subscript(s) => (TOKEN_SUBSCRIPT, s.span), + InlineNode::MathInline(m) => (TOKEN_MATH, m.span), + InlineNode::Keyword(k) => (TOKEN_KEYWORD, k.span), + InlineNode::Color(c) => (TOKEN_COLOR, c.span), + InlineNode::WikiLink(w) => (TOKEN_WIKILINK, w.span), + InlineNode::ExternalLink(e) => (TOKEN_EXTERNAL_LINK, e.span), + InlineNode::Transclusion(t) => (TOKEN_TRANSCLUSION, t.span), + InlineNode::RawUrl(u) => (TOKEN_URL, u.span), + }; + split_span(span, text, idx, kind, 0, out); + } +} + +fn span_of_inline(node: &InlineNode) -> Span { + match node { + InlineNode::Text(n) => n.span, + InlineNode::Bold(n) => n.span, + InlineNode::Italic(n) => n.span, + InlineNode::BoldItalic(n) => n.span, + InlineNode::Strikethrough(n) => n.span, + InlineNode::Code(n) => n.span, + InlineNode::Superscript(n) => n.span, + InlineNode::Subscript(n) => n.span, + InlineNode::MathInline(n) => n.span, + InlineNode::Keyword(n) => n.span, + InlineNode::Color(n) => n.span, + InlineNode::WikiLink(n) => n.span, + InlineNode::ExternalLink(n) => n.span, + InlineNode::Transclusion(n) => n.span, + InlineNode::RawUrl(n) => n.span, + } +} + +/// Convert a `Span` (potentially multi-line) into one or more single-line +/// `RawToken`s. The LSP wire format can't express tokens that span newlines, +/// so we split on line boundaries. +pub fn split_span( + span: Span, + text: &str, + idx: &LineIndex, + kind: u32, + mods: u32, + out: &mut Vec, +) { + if span.start.line == span.end.line { + let len = span.end.column.saturating_sub(span.start.column); + if len > 0 { + out.push(RawToken { + line: span.start.line, + byte_col: span.start.column, + byte_len: len, + kind, + mods, + }); + } + return; + } + for line in span.start.line..=span.end.line { + let line_str = idx.line_str(line, text); + let line_len = line_str.len() as u32; + let start_col = if line == span.start.line { + span.start.column + } else { + 0 + }; + let end_col = if line == span.end.line { + span.end.column.min(line_len) + } else { + line_len + }; + if end_col > start_col { + out.push(RawToken { + line, + byte_col: start_col, + byte_len: end_col - start_col, + kind, + mods, + }); + } + } +} + +// ===== Encoding ===== + +/// LSP wire-format encoder. Produces a flat `Vec` of +/// `(deltaLine, deltaStart, length, type, modifiers)` quintuples. +/// +/// `utf8 = true` means the client opted into UTF-8 encoding, so byte offsets +/// pass through directly. Otherwise positions and lengths are converted to +/// UTF-16 code units per the LSP default. +pub fn encode(raws: &[RawToken], text: &str, idx: &LineIndex, utf8: bool) -> Vec { + let mut data = Vec::with_capacity(raws.len() * 5); + let mut prev_line = 0u32; + let mut prev_col = 0u32; + for tok in raws { + let (col, len) = if utf8 { + (tok.byte_col, tok.byte_len) + } else { + let line_str = idx.line_str(tok.line, text); + let start = utf16_count(line_str, tok.byte_col as usize); + let end = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize); + (start, end - start) + }; + let delta_line = tok.line - prev_line; + let delta_col = if delta_line == 0 { col - prev_col } else { col }; + data.extend_from_slice(&[delta_line, delta_col, len, tok.kind, tok.mods]); + prev_line = tok.line; + prev_col = col; + } + data +} + +fn utf16_count(s: &str, byte_offset: usize) -> u32 { + let upto = byte_offset.min(s.len()); + s[..upto].chars().map(char::len_utf16).sum::() as u32 +} + +/// Build the encoded data stream for either the full document or a range +/// request. +pub fn build_data(doc: &DocumentNode, text: &str, utf8: bool, range: Option) -> Vec { + let idx = LineIndex::new(text); + let mut tokens = collect_tokens(doc, text); + if let Some(r) = range { + tokens.retain(|t| token_overlaps_range(t, &r, text, &idx, utf8)); + } + encode(&tokens, text, &idx, utf8) +} + +fn token_overlaps_range( + tok: &RawToken, + range: &Range, + text: &str, + idx: &LineIndex, + utf8: bool, +) -> bool { + let (start_char, end_char) = if utf8 { + (tok.byte_col, tok.byte_col + tok.byte_len) + } else { + let line_str = idx.line_str(tok.line, text); + let s = utf16_count(line_str, tok.byte_col as usize); + let e = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize); + (s, e) + }; + let tok_start = (tok.line, start_char); + let tok_end = (tok.line, end_char); + let r_start = (range.start.line, range.start.character); + let r_end = (range.end.line, range.end.character); + tok_end > r_start && tok_start < r_end +} diff --git a/crates/nuwiki-lsp/tests/semantic_tokens.rs b/crates/nuwiki-lsp/tests/semantic_tokens.rs new file mode 100644 index 0000000..faedb5a --- /dev/null +++ b/crates/nuwiki-lsp/tests/semantic_tokens.rs @@ -0,0 +1,312 @@ +//! Tests for the semantic-token builder, encoder, range filter, and +//! UTF-8/UTF-16 conversion. + +use nuwiki_core::syntax::vimwiki::VimwikiSyntax; +use nuwiki_core::syntax::SyntaxPlugin; +use nuwiki_lsp::semantic_tokens::{ + build_data, collect_tokens, encode, legend, LineIndex, RawToken, MOD_CENTERED, MOD_LEVEL1, + MOD_LEVEL2, TOKEN_BOLD, TOKEN_CODE, TOKEN_CODE_BLOCK, TOKEN_COMMENT, TOKEN_DEFINITION_TERM, + TOKEN_HEADING, TOKEN_ITALIC, TOKEN_KEYWORD, TOKEN_MATH_BLOCK, TOKEN_TRANSCLUSION, + TOKEN_TYPE_NAMES, TOKEN_URL, TOKEN_WIKILINK, +}; +use tower_lsp::lsp_types::{Position, Range}; + +fn parse(src: &str) -> nuwiki_core::ast::DocumentNode { + VimwikiSyntax::new().parse(src) +} + +fn first_with_kind(tokens: &[RawToken], kind: u32) -> Option<&RawToken> { + tokens.iter().find(|t| t.kind == kind) +} + +// ===== Legend ===== + +#[test] +fn legend_lists_all_custom_token_types() { + let l = legend(); + assert_eq!(l.token_types.len(), TOKEN_TYPE_NAMES.len()); + let names: Vec = l + .token_types + .iter() + .map(|t| t.as_str().to_owned()) + .collect(); + assert!(names.contains(&"vimwikiHeading".to_string())); + assert!(names.contains(&"vimwikiWikilink".to_string())); + assert!(names.contains(&"vimwikiTransclusion".to_string())); +} + +// ===== Per-construct emission ===== + +#[test] +fn heading_emits_one_token_with_level_modifier() { + let doc = parse("== Hi ==\n"); + let toks = collect_tokens(&doc, "== Hi ==\n"); + let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token"); + assert_eq!(h.mods & MOD_LEVEL2, MOD_LEVEL2); + assert_eq!(h.mods & MOD_CENTERED, 0); +} + +#[test] +fn centered_heading_carries_centered_modifier() { + let src = " = Title =\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + let h = first_with_kind(&toks, TOKEN_HEADING).expect("heading token"); + assert_eq!(h.mods & MOD_LEVEL1, MOD_LEVEL1); + assert_eq!(h.mods & MOD_CENTERED, MOD_CENTERED); +} + +#[test] +fn heading_does_not_emit_inner_inline_tokens() { + // Headings render as a single styled span, so bold inside is shadowed. + let src = "= Hi *bold* =\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_HEADING)); + assert!( + !toks.iter().any(|t| t.kind == TOKEN_BOLD), + "should not have emitted Bold inside a Heading" + ); +} + +#[test] +fn paragraph_emits_inline_tokens_only() { + let src = "see *bold* and `code`\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_BOLD)); + assert!(toks.iter().any(|t| t.kind == TOKEN_CODE)); + assert!(!toks.iter().any(|t| t.kind == TOKEN_HEADING)); +} + +#[test] +fn italic_token_for_underscored_run() { + let src = "_emphasis_\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_ITALIC)); +} + +#[test] +fn keyword_token_for_each_keyword() { + let src = "TODO write tests, FIXME later\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + let kw_count = toks.iter().filter(|t| t.kind == TOKEN_KEYWORD).count(); + assert_eq!(kw_count, 2); +} + +#[test] +fn wikilink_and_external_link_get_different_types() { + let src = "[[Page]] and [[https://example.com|docs]]\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_WIKILINK)); + assert!( + toks.iter() + .any(|t| t.kind == nuwiki_lsp::semantic_tokens::TOKEN_EXTERNAL_LINK), + "URL inside [[...]] should be vimwikiExternalLink, got {toks:?}" + ); +} + +#[test] +fn raw_url_emits_url_token() { + let src = "see https://example.com here\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_URL)); +} + +#[test] +fn transclusion_emits_transclusion_token() { + let src = "{{img.png|alt}}\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_TRANSCLUSION)); +} + +#[test] +fn comment_token_for_single_line_comment() { + let src = "%% hidden\nstuff\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_COMMENT)); +} + +#[test] +fn definition_term_token() { + let src = "Term:: Definition\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + assert!(toks.iter().any(|t| t.kind == TOKEN_DEFINITION_TERM)); +} + +// ===== Multi-line fences ===== + +#[test] +fn preformatted_block_splits_into_one_token_per_line() { + let src = "{{{rust\nlet x = 1;\nlet y = 2;\n}}}\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + let code_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_CODE_BLOCK).collect(); + // Spans line 0 (open fence) through line 3 (close fence) = 4 lines, but + // empty trailing slices are dropped, so we expect 4 line tokens. + assert!( + code_tokens.len() >= 3, + "expected multi-line split, got {code_tokens:?}" + ); + // No token should cross a line boundary. + for t in &code_tokens { + assert!(t.byte_len > 0); + } +} + +#[test] +fn math_block_splits_into_one_token_per_line() { + let src = "{{$\nx=1\ny=2\n}}$\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + let math_tokens: Vec<&RawToken> = toks.iter().filter(|t| t.kind == TOKEN_MATH_BLOCK).collect(); + assert!(math_tokens.len() >= 3); +} + +// ===== Sorting ===== + +#[test] +fn tokens_are_sorted_by_line_then_column() { + let src = "= H =\n*bold* and _italic_\n"; + let doc = parse(src); + let toks = collect_tokens(&doc, src); + for w in toks.windows(2) { + let a = (w[0].line, w[0].byte_col); + let b = (w[1].line, w[1].byte_col); + assert!(a <= b, "tokens not sorted: {a:?} then {b:?}"); + } +} + +// ===== Encoding ===== + +#[test] +fn encode_produces_correct_delta_quintuples() { + let src = "= H =\n*bold*\n"; + let raws = vec![ + RawToken { + line: 0, + byte_col: 0, + byte_len: 5, + kind: TOKEN_HEADING, + mods: MOD_LEVEL1, + }, + RawToken { + line: 1, + byte_col: 0, + byte_len: 6, + kind: TOKEN_BOLD, + mods: 0, + }, + ]; + let idx = LineIndex::new(src); + let data = encode(&raws, src, &idx, true); + assert_eq!(data.len(), 10); + // First token: delta_line=0, delta_col=0, len=5, type=HEADING, mods=LEVEL1 + assert_eq!(&data[0..5], &[0, 0, 5, TOKEN_HEADING, MOD_LEVEL1]); + // Second token: delta_line=1 (new line), delta_col=0 (absolute), len=6, type=BOLD, mods=0 + assert_eq!(&data[5..10], &[1, 0, 6, TOKEN_BOLD, 0]); +} + +#[test] +fn encode_uses_relative_column_within_same_line() { + let src = "abc def ghi\n"; + let raws = vec![ + RawToken { + line: 0, + byte_col: 0, + byte_len: 3, + kind: TOKEN_BOLD, + mods: 0, + }, + RawToken { + line: 0, + byte_col: 4, + byte_len: 3, + kind: TOKEN_ITALIC, + mods: 0, + }, + RawToken { + line: 0, + byte_col: 8, + byte_len: 3, + kind: TOKEN_CODE, + mods: 0, + }, + ]; + let idx = LineIndex::new(src); + let data = encode(&raws, src, &idx, true); + assert_eq!(&data[0..5], &[0, 0, 3, TOKEN_BOLD, 0]); + // Second: delta_col = 4 - 0 = 4 + assert_eq!(&data[5..10], &[0, 4, 3, TOKEN_ITALIC, 0]); + // Third: delta_col = 8 - 4 = 4 + assert_eq!(&data[10..15], &[0, 4, 3, TOKEN_CODE, 0]); +} + +#[test] +fn encode_converts_to_utf16_for_multibyte_chars() { + // "_é_" — italic delimiters around é. Span covers 4 bytes (1 + 2 + 1) + // but 3 UTF-16 code units. + let src = "_é_\n"; + let raws = vec![RawToken { + line: 0, + byte_col: 0, + byte_len: 4, + kind: TOKEN_ITALIC, + mods: 0, + }]; + let idx = LineIndex::new(src); + let data = encode(&raws, src, &idx, false); + // length should be in UTF-16 code units (3), not bytes (4). + assert_eq!(data[2], 3); +} + +// ===== Range filter ===== + +#[test] +fn range_filter_keeps_only_overlapping_tokens() { + let src = "= L1 =\n= L2 =\n= L3 =\n"; + let doc = parse(src); + // Restrict to line 1 only. + let range = Range { + start: Position { + line: 1, + character: 0, + }, + end: Position { + line: 2, + character: 0, + }, + }; + let data = build_data(&doc, src, true, Some(range)); + // Expect exactly 1 heading token (the L2 one). + assert_eq!(data.len(), 5, "got {data:?}"); + // Verify it's on line 1 (delta_line = 1 from prev_line = 0). + assert_eq!(data[0], 1); +} + +#[test] +fn empty_document_produces_empty_token_stream() { + let doc = parse(""); + let data = build_data(&doc, "", true, None); + assert!(data.is_empty()); +} + +// ===== End-to-end through builder ===== + +#[test] +fn build_data_matches_collect_then_encode() { + let src = "= H1 =\n_em_ TODO\n"; + let doc = parse(src); + let direct = build_data(&doc, src, true, None); + let raws = collect_tokens(&doc, src); + let idx = LineIndex::new(src); + let manual = encode(&raws, src, &idx, true); + assert_eq!(direct, manual); +}