//! Semantic-token emission for the vimwiki AST. //! //! Token-type scheme settled in P7: custom `vimwiki*` types plus //! `level1`..`level6` + `centered` modifiers (SPEC §11). Default highlight //! groups for these arrive in Phase 9. //! //! Pipeline: //! //! 1. Walk the AST, emitting one or more `RawToken`s per AST node. Block //! structure that "wraps" text (paragraph, list item, table cell) //! recurses into inlines instead of emitting a wrapping token, so we //! end up with a non-overlapping tile of inline tokens. Block nodes //! that are themselves visually distinct (heading, code block, math //! block, comment) emit one token across their whole span. //! 2. Multi-line spans are split into one token per line — the LSP wire //! format can't express a single token that crosses a newline. //! 3. Tokens are sorted by `(line, byte_col)` and encoded as the LSP //! delta-quintuple stream `(deltaLine, deltaStart, length, type, mods)`. //! When the client doesn't support UTF-8 encoding, byte offsets are //! converted to UTF-16 code-unit counts on the fly. use nuwiki_core::ast::{ BlockNode, BlockquoteNode, DocumentNode, HeadingNode, InlineNode, ListItemNode, ListNode, Span, TableNode, }; use tower_lsp::lsp_types::{Range, SemanticTokenModifier, SemanticTokenType, SemanticTokensLegend}; // ===== Legend ===== pub const TOKEN_HEADING: u32 = 0; pub const TOKEN_BOLD: u32 = 1; pub const TOKEN_ITALIC: u32 = 2; pub const TOKEN_BOLD_ITALIC: u32 = 3; pub const TOKEN_STRIKETHROUGH: u32 = 4; pub const TOKEN_SUPERSCRIPT: u32 = 5; pub const TOKEN_SUBSCRIPT: u32 = 6; pub const TOKEN_CODE: u32 = 7; pub const TOKEN_CODE_BLOCK: u32 = 8; pub const TOKEN_MATH: u32 = 9; pub const TOKEN_MATH_BLOCK: u32 = 10; pub const TOKEN_KEYWORD: u32 = 11; pub const TOKEN_COLOR: u32 = 12; pub const TOKEN_WIKILINK: u32 = 13; pub const TOKEN_EXTERNAL_LINK: u32 = 14; pub const TOKEN_TRANSCLUSION: u32 = 15; pub const TOKEN_URL: u32 = 16; pub const TOKEN_COMMENT: u32 = 17; pub const TOKEN_DEFINITION_TERM: u32 = 18; pub const TOKEN_TAG: u32 = 19; pub const TOKEN_TYPE_NAMES: &[&str] = &[ "vimwikiHeading", "vimwikiBold", "vimwikiItalic", "vimwikiBoldItalic", "vimwikiStrikethrough", "vimwikiSuperscript", "vimwikiSubscript", "vimwikiCode", "vimwikiCodeBlock", "vimwikiMath", "vimwikiMathBlock", "vimwikiKeyword", "vimwikiColor", "vimwikiWikilink", "vimwikiExternalLink", "vimwikiTransclusion", "vimwikiUrl", "vimwikiComment", "vimwikiDefinitionTerm", "vimwikiTag", ]; pub const MOD_LEVEL1: u32 = 1 << 0; pub const MOD_LEVEL2: u32 = 1 << 1; pub const MOD_LEVEL3: u32 = 1 << 2; pub const MOD_LEVEL4: u32 = 1 << 3; pub const MOD_LEVEL5: u32 = 1 << 4; pub const MOD_LEVEL6: u32 = 1 << 5; pub const MOD_CENTERED: u32 = 1 << 6; pub const TOKEN_MODIFIER_NAMES: &[&str] = &[ "level1", "level2", "level3", "level4", "level5", "level6", "centered", ]; pub fn legend() -> SemanticTokensLegend { SemanticTokensLegend { token_types: TOKEN_TYPE_NAMES .iter() .map(|s| SemanticTokenType::new(s)) .collect(), token_modifiers: TOKEN_MODIFIER_NAMES .iter() .map(|s| SemanticTokenModifier::new(s)) .collect(), } } // ===== Raw tokens ===== #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RawToken { pub line: u32, pub byte_col: u32, pub byte_len: u32, pub kind: u32, pub mods: u32, } /// Index of byte offsets at the start of each line. Built once per request /// so multi-line span splitting and UTF-16 conversion stay linear in /// document size. pub struct LineIndex { pub starts: Vec, pub total: usize, } impl LineIndex { pub fn new(text: &str) -> Self { let mut starts = vec![0]; for (i, b) in text.bytes().enumerate() { if b == b'\n' { starts.push(i + 1); } } Self { starts, total: text.len(), } } fn line_byte_range(&self, line: u32) -> (usize, usize) { let start = self .starts .get(line as usize) .copied() .unwrap_or(self.total); let end = self .starts .get(line as usize + 1) .map(|&s| s.saturating_sub(1)) .unwrap_or(self.total); (start, end) } pub fn line_str<'a>(&self, line: u32, text: &'a str) -> &'a str { let (s, e) = self.line_byte_range(line); &text[s..e] } } // ===== Collection ===== pub fn collect_tokens(doc: &DocumentNode, text: &str) -> Vec { let idx = LineIndex::new(text); let mut out = Vec::new(); for block in &doc.children { emit_block(block, text, &idx, &mut out); } out.sort_by_key(|t| (t.line, t.byte_col)); out } fn emit_block(block: &BlockNode, text: &str, idx: &LineIndex, out: &mut Vec) { match block { BlockNode::Heading(h) => emit_heading(h, text, idx, out), BlockNode::Paragraph(p) => emit_inlines(&p.children, text, idx, out), BlockNode::HorizontalRule(_) => {} BlockNode::Blockquote(BlockquoteNode { children, .. }) => { for child in children { emit_block(child, text, idx, out); } } BlockNode::Preformatted(p) => split_span(p.span, text, idx, TOKEN_CODE_BLOCK, 0, out), BlockNode::MathBlock(m) => split_span(m.span, text, idx, TOKEN_MATH_BLOCK, 0, out), BlockNode::List(ListNode { items, .. }) => { for item in items { emit_list_item(item, text, idx, out); } } BlockNode::DefinitionList(dl) => { for item in &dl.items { if let Some(term) = &item.term { if let (Some(first), Some(last)) = (term.first(), term.last()) { let s = Span::new(span_of_inline(first).start, span_of_inline(last).end); split_span(s, text, idx, TOKEN_DEFINITION_TERM, 0, out); } } for def in &item.definitions { emit_inlines(def, text, idx, out); } } } BlockNode::Table(TableNode { rows, .. }) => { for row in rows { for cell in &row.cells { emit_inlines(&cell.children, text, idx, out); } } } BlockNode::Comment(c) => split_span(c.span, text, idx, TOKEN_COMMENT, 0, out), BlockNode::Tag(t) => split_span(t.span, text, idx, TOKEN_TAG, 0, out), BlockNode::Error(_) => {} } } fn emit_heading(h: &HeadingNode, text: &str, idx: &LineIndex, out: &mut Vec) { let mut mods = 0u32; mods |= match h.level { 1 => MOD_LEVEL1, 2 => MOD_LEVEL2, 3 => MOD_LEVEL3, 4 => MOD_LEVEL4, 5 => MOD_LEVEL5, _ => MOD_LEVEL6, }; if h.centered { mods |= MOD_CENTERED; } split_span(h.span, text, idx, TOKEN_HEADING, mods, out); } fn emit_list_item(item: &ListItemNode, text: &str, idx: &LineIndex, out: &mut Vec) { emit_inlines(&item.children, text, idx, out); if let Some(sublist) = &item.sublist { for child in &sublist.items { emit_list_item(child, text, idx, out); } } } fn emit_inlines(inlines: &[InlineNode], text: &str, idx: &LineIndex, out: &mut Vec) { for n in inlines { let (kind, span) = match n { InlineNode::Text(_) => continue, InlineNode::Bold(b) => (TOKEN_BOLD, b.span), InlineNode::Italic(i) => (TOKEN_ITALIC, i.span), InlineNode::BoldItalic(bi) => (TOKEN_BOLD_ITALIC, bi.span), InlineNode::Strikethrough(s) => (TOKEN_STRIKETHROUGH, s.span), InlineNode::Code(c) => (TOKEN_CODE, c.span), InlineNode::Superscript(s) => (TOKEN_SUPERSCRIPT, s.span), InlineNode::Subscript(s) => (TOKEN_SUBSCRIPT, s.span), InlineNode::MathInline(m) => (TOKEN_MATH, m.span), InlineNode::Keyword(k) => (TOKEN_KEYWORD, k.span), InlineNode::Color(c) => (TOKEN_COLOR, c.span), InlineNode::WikiLink(w) => (TOKEN_WIKILINK, w.span), InlineNode::ExternalLink(e) => (TOKEN_EXTERNAL_LINK, e.span), InlineNode::Transclusion(t) => (TOKEN_TRANSCLUSION, t.span), InlineNode::RawUrl(u) => (TOKEN_URL, u.span), }; split_span(span, text, idx, kind, 0, out); } } fn span_of_inline(node: &InlineNode) -> Span { match node { InlineNode::Text(n) => n.span, InlineNode::Bold(n) => n.span, InlineNode::Italic(n) => n.span, InlineNode::BoldItalic(n) => n.span, InlineNode::Strikethrough(n) => n.span, InlineNode::Code(n) => n.span, InlineNode::Superscript(n) => n.span, InlineNode::Subscript(n) => n.span, InlineNode::MathInline(n) => n.span, InlineNode::Keyword(n) => n.span, InlineNode::Color(n) => n.span, InlineNode::WikiLink(n) => n.span, InlineNode::ExternalLink(n) => n.span, InlineNode::Transclusion(n) => n.span, InlineNode::RawUrl(n) => n.span, } } /// Convert a `Span` (potentially multi-line) into one or more single-line /// `RawToken`s. The LSP wire format can't express tokens that span newlines, /// so we split on line boundaries. pub fn split_span( span: Span, text: &str, idx: &LineIndex, kind: u32, mods: u32, out: &mut Vec, ) { if span.start.line == span.end.line { let len = span.end.column.saturating_sub(span.start.column); if len > 0 { out.push(RawToken { line: span.start.line, byte_col: span.start.column, byte_len: len, kind, mods, }); } return; } for line in span.start.line..=span.end.line { let line_str = idx.line_str(line, text); let line_len = line_str.len() as u32; let start_col = if line == span.start.line { span.start.column } else { 0 }; let end_col = if line == span.end.line { span.end.column.min(line_len) } else { line_len }; if end_col > start_col { out.push(RawToken { line, byte_col: start_col, byte_len: end_col - start_col, kind, mods, }); } } } // ===== Encoding ===== /// LSP wire-format encoder. Produces a flat `Vec` of /// `(deltaLine, deltaStart, length, type, modifiers)` quintuples. /// /// `utf8 = true` means the client opted into UTF-8 encoding, so byte offsets /// pass through directly. Otherwise positions and lengths are converted to /// UTF-16 code units per the LSP default. pub fn encode(raws: &[RawToken], text: &str, idx: &LineIndex, utf8: bool) -> Vec { let mut data = Vec::with_capacity(raws.len() * 5); let mut prev_line = 0u32; let mut prev_col = 0u32; for tok in raws { let (col, len) = if utf8 { (tok.byte_col, tok.byte_len) } else { let line_str = idx.line_str(tok.line, text); let start = utf16_count(line_str, tok.byte_col as usize); let end = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize); (start, end - start) }; let delta_line = tok.line - prev_line; let delta_col = if delta_line == 0 { col - prev_col } else { col }; data.extend_from_slice(&[delta_line, delta_col, len, tok.kind, tok.mods]); prev_line = tok.line; prev_col = col; } data } fn utf16_count(s: &str, byte_offset: usize) -> u32 { let upto = byte_offset.min(s.len()); s[..upto].chars().map(char::len_utf16).sum::() as u32 } /// Build the encoded data stream for either the full document or a range /// request. pub fn build_data(doc: &DocumentNode, text: &str, utf8: bool, range: Option) -> Vec { let idx = LineIndex::new(text); let mut tokens = collect_tokens(doc, text); if let Some(r) = range { tokens.retain(|t| token_overlaps_range(t, &r, text, &idx, utf8)); } encode(&tokens, text, &idx, utf8) } fn token_overlaps_range( tok: &RawToken, range: &Range, text: &str, idx: &LineIndex, utf8: bool, ) -> bool { let (start_char, end_char) = if utf8 { (tok.byte_col, tok.byte_col + tok.byte_len) } else { let line_str = idx.line_str(tok.line, text); let s = utf16_count(line_str, tok.byte_col as usize); let e = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize); (s, e) }; let tok_start = (tok.line, start_char); let tok_end = (tok.line, end_char); let r_start = (range.start.line, range.start.character); let r_end = (range.end.line, range.end.character); tok_end > r_start && tok_start < r_end }