gffranco
370 lines
12 KiB
Rust
370 lines
12 KiB
Rust
//! Semantic-token emission for the vimwiki AST.
|
|
//!
|
|
//! Token-type scheme: custom `vimwiki*` types plus
|
|
//! `level1`..`level6` + `centered` modifiers.
|
|
//!
|
|
//! Pipeline:
|
|
//!
|
|
//! 1. Walk the AST, emitting one or more `RawToken`s per AST node. Block
|
|
//! structure that "wraps" text (paragraph, list item, table cell)
|
|
//! recurses into inlines instead of emitting a wrapping token, so we
|
|
//! end up with a non-overlapping tile of inline tokens. Block nodes
|
|
//! that are themselves visually distinct (heading, code block, math
|
|
//! block, comment) emit one token across their whole span.
|
|
//! 2. Multi-line spans are split into one token per line — the LSP wire
|
|
//! format can't express a single token that crosses a newline.
|
|
//! 3. Tokens are sorted by `(line, byte_col)` and encoded as the LSP
|
|
//! delta-quintuple stream `(deltaLine, deltaStart, length, type, mods)`.
|
|
//! When the client doesn't support UTF-8 encoding, byte offsets are
|
|
//! converted to UTF-16 code-unit counts on the fly.
|
|
|
|
use nuwiki_core::ast::{
|
|
BlockNode, BlockquoteNode, DocumentNode, HeadingNode, InlineNode, ListItemNode, ListNode, Span,
|
|
TableNode,
|
|
};
|
|
use tower_lsp::lsp_types::{Range, SemanticTokenModifier, SemanticTokenType, SemanticTokensLegend};
|
|
|
|
// ===== Legend =====
|
|
|
|
pub const TOKEN_HEADING: u32 = 0;
|
|
pub const TOKEN_BOLD: u32 = 1;
|
|
pub const TOKEN_ITALIC: u32 = 2;
|
|
pub const TOKEN_BOLD_ITALIC: u32 = 3;
|
|
pub const TOKEN_STRIKETHROUGH: u32 = 4;
|
|
pub const TOKEN_SUPERSCRIPT: u32 = 5;
|
|
pub const TOKEN_SUBSCRIPT: u32 = 6;
|
|
pub const TOKEN_CODE: u32 = 7;
|
|
pub const TOKEN_CODE_BLOCK: u32 = 8;
|
|
pub const TOKEN_MATH: u32 = 9;
|
|
pub const TOKEN_MATH_BLOCK: u32 = 10;
|
|
pub const TOKEN_KEYWORD: u32 = 11;
|
|
pub const TOKEN_COLOR: u32 = 12;
|
|
pub const TOKEN_WIKILINK: u32 = 13;
|
|
pub const TOKEN_EXTERNAL_LINK: u32 = 14;
|
|
pub const TOKEN_TRANSCLUSION: u32 = 15;
|
|
pub const TOKEN_URL: u32 = 16;
|
|
pub const TOKEN_COMMENT: u32 = 17;
|
|
pub const TOKEN_DEFINITION_TERM: u32 = 18;
|
|
pub const TOKEN_TAG: u32 = 19;
|
|
|
|
pub const TOKEN_TYPE_NAMES: &[&str] = &[
|
|
"vimwikiHeading",
|
|
"vimwikiBold",
|
|
"vimwikiItalic",
|
|
"vimwikiBoldItalic",
|
|
"vimwikiStrikethrough",
|
|
"vimwikiSuperscript",
|
|
"vimwikiSubscript",
|
|
"vimwikiCode",
|
|
"vimwikiCodeBlock",
|
|
"vimwikiMath",
|
|
"vimwikiMathBlock",
|
|
"vimwikiKeyword",
|
|
"vimwikiColor",
|
|
"vimwikiWikilink",
|
|
"vimwikiExternalLink",
|
|
"vimwikiTransclusion",
|
|
"vimwikiUrl",
|
|
"vimwikiComment",
|
|
"vimwikiDefinitionTerm",
|
|
"vimwikiTag",
|
|
];
|
|
|
|
pub const MOD_LEVEL1: u32 = 1 << 0;
|
|
pub const MOD_LEVEL2: u32 = 1 << 1;
|
|
pub const MOD_LEVEL3: u32 = 1 << 2;
|
|
pub const MOD_LEVEL4: u32 = 1 << 3;
|
|
pub const MOD_LEVEL5: u32 = 1 << 4;
|
|
pub const MOD_LEVEL6: u32 = 1 << 5;
|
|
pub const MOD_CENTERED: u32 = 1 << 6;
|
|
|
|
pub const TOKEN_MODIFIER_NAMES: &[&str] = &[
|
|
"level1", "level2", "level3", "level4", "level5", "level6", "centered",
|
|
];
|
|
|
|
pub fn legend() -> SemanticTokensLegend {
|
|
SemanticTokensLegend {
|
|
token_types: TOKEN_TYPE_NAMES
|
|
.iter()
|
|
.map(|s| SemanticTokenType::new(s))
|
|
.collect(),
|
|
token_modifiers: TOKEN_MODIFIER_NAMES
|
|
.iter()
|
|
.map(|s| SemanticTokenModifier::new(s))
|
|
.collect(),
|
|
}
|
|
}
|
|
|
|
// ===== Raw tokens =====
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct RawToken {
|
|
pub line: u32,
|
|
pub byte_col: u32,
|
|
pub byte_len: u32,
|
|
pub kind: u32,
|
|
pub mods: u32,
|
|
}
|
|
|
|
/// Index of byte offsets at the start of each line. Built once per request
|
|
/// so multi-line span splitting and UTF-16 conversion stay linear in
|
|
/// document size.
|
|
pub struct LineIndex {
|
|
pub starts: Vec<usize>,
|
|
pub total: usize,
|
|
}
|
|
|
|
impl LineIndex {
|
|
pub fn new(text: &str) -> Self {
|
|
let mut starts = vec![0];
|
|
for (i, b) in text.bytes().enumerate() {
|
|
if b == b'\n' {
|
|
starts.push(i + 1);
|
|
}
|
|
}
|
|
Self {
|
|
starts,
|
|
total: text.len(),
|
|
}
|
|
}
|
|
|
|
fn line_byte_range(&self, line: u32) -> (usize, usize) {
|
|
let start = self
|
|
.starts
|
|
.get(line as usize)
|
|
.copied()
|
|
.unwrap_or(self.total);
|
|
let end = self
|
|
.starts
|
|
.get(line as usize + 1)
|
|
.map(|&s| s.saturating_sub(1))
|
|
.unwrap_or(self.total);
|
|
(start, end)
|
|
}
|
|
|
|
pub fn line_str<'a>(&self, line: u32, text: &'a str) -> &'a str {
|
|
let (s, e) = self.line_byte_range(line);
|
|
&text[s..e]
|
|
}
|
|
}
|
|
|
|
// ===== Collection =====
|
|
|
|
pub fn collect_tokens(doc: &DocumentNode, text: &str) -> Vec<RawToken> {
|
|
let idx = LineIndex::new(text);
|
|
let mut out = Vec::new();
|
|
for block in &doc.children {
|
|
emit_block(block, text, &idx, &mut out);
|
|
}
|
|
out.sort_by_key(|t| (t.line, t.byte_col));
|
|
out
|
|
}
|
|
|
|
fn emit_block(block: &BlockNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
|
match block {
|
|
BlockNode::Heading(h) => emit_heading(h, text, idx, out),
|
|
BlockNode::Paragraph(p) => emit_inlines(&p.children, text, idx, out),
|
|
BlockNode::HorizontalRule(_) => {}
|
|
BlockNode::Blockquote(BlockquoteNode { children, .. }) => {
|
|
for child in children {
|
|
emit_block(child, text, idx, out);
|
|
}
|
|
}
|
|
BlockNode::Preformatted(p) => split_span(p.span, text, idx, TOKEN_CODE_BLOCK, 0, out),
|
|
BlockNode::MathBlock(m) => split_span(m.span, text, idx, TOKEN_MATH_BLOCK, 0, out),
|
|
BlockNode::List(ListNode { items, .. }) => {
|
|
for item in items {
|
|
emit_list_item(item, text, idx, out);
|
|
}
|
|
}
|
|
BlockNode::DefinitionList(dl) => {
|
|
for item in &dl.items {
|
|
if let Some(term) = &item.term {
|
|
if let (Some(first), Some(last)) = (term.first(), term.last()) {
|
|
let s = Span::new(first.span().start, last.span().end);
|
|
split_span(s, text, idx, TOKEN_DEFINITION_TERM, 0, out);
|
|
}
|
|
}
|
|
for def in &item.definitions {
|
|
emit_inlines(def, text, idx, out);
|
|
}
|
|
}
|
|
}
|
|
BlockNode::Table(TableNode { rows, .. }) => {
|
|
for row in rows {
|
|
for cell in &row.cells {
|
|
emit_inlines(&cell.children, text, idx, out);
|
|
}
|
|
}
|
|
}
|
|
BlockNode::Comment(c) => split_span(c.span, text, idx, TOKEN_COMMENT, 0, out),
|
|
BlockNode::Tag(t) => split_span(t.span, text, idx, TOKEN_TAG, 0, out),
|
|
BlockNode::Error(_) => {}
|
|
}
|
|
}
|
|
|
|
fn emit_heading(h: &HeadingNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
|
let mut mods = 0u32;
|
|
mods |= match h.level {
|
|
1 => MOD_LEVEL1,
|
|
2 => MOD_LEVEL2,
|
|
3 => MOD_LEVEL3,
|
|
4 => MOD_LEVEL4,
|
|
5 => MOD_LEVEL5,
|
|
_ => MOD_LEVEL6,
|
|
};
|
|
if h.centered {
|
|
mods |= MOD_CENTERED;
|
|
}
|
|
split_span(h.span, text, idx, TOKEN_HEADING, mods, out);
|
|
}
|
|
|
|
fn emit_list_item(item: &ListItemNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
|
emit_inlines(&item.children, text, idx, out);
|
|
if let Some(sublist) = &item.sublist {
|
|
for child in &sublist.items {
|
|
emit_list_item(child, text, idx, out);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn emit_inlines(inlines: &[InlineNode], text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
|
for n in inlines {
|
|
let (kind, span) = match n {
|
|
InlineNode::Text(_) | InlineNode::SoftBreak(_) => continue,
|
|
InlineNode::Bold(b) => (TOKEN_BOLD, b.span),
|
|
InlineNode::Italic(i) => (TOKEN_ITALIC, i.span),
|
|
InlineNode::BoldItalic(bi) => (TOKEN_BOLD_ITALIC, bi.span),
|
|
InlineNode::Strikethrough(s) => (TOKEN_STRIKETHROUGH, s.span),
|
|
InlineNode::Code(c) => (TOKEN_CODE, c.span),
|
|
InlineNode::Superscript(s) => (TOKEN_SUPERSCRIPT, s.span),
|
|
InlineNode::Subscript(s) => (TOKEN_SUBSCRIPT, s.span),
|
|
InlineNode::MathInline(m) => (TOKEN_MATH, m.span),
|
|
InlineNode::Keyword(k) => (TOKEN_KEYWORD, k.span),
|
|
InlineNode::Color(c) => (TOKEN_COLOR, c.span),
|
|
InlineNode::WikiLink(w) => (TOKEN_WIKILINK, w.span),
|
|
InlineNode::ExternalLink(e) => (TOKEN_EXTERNAL_LINK, e.span),
|
|
InlineNode::Transclusion(t) => (TOKEN_TRANSCLUSION, t.span),
|
|
InlineNode::RawUrl(u) => (TOKEN_URL, u.span),
|
|
};
|
|
split_span(span, text, idx, kind, 0, out);
|
|
}
|
|
}
|
|
|
|
/// Convert a `Span` (potentially multi-line) into one or more single-line
|
|
/// `RawToken`s. The LSP wire format can't express tokens that span newlines,
|
|
/// so we split on line boundaries.
|
|
pub fn split_span(
|
|
span: Span,
|
|
text: &str,
|
|
idx: &LineIndex,
|
|
kind: u32,
|
|
mods: u32,
|
|
out: &mut Vec<RawToken>,
|
|
) {
|
|
if span.start.line == span.end.line {
|
|
let len = span.end.column.saturating_sub(span.start.column);
|
|
if len > 0 {
|
|
out.push(RawToken {
|
|
line: span.start.line,
|
|
byte_col: span.start.column,
|
|
byte_len: len,
|
|
kind,
|
|
mods,
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
for line in span.start.line..=span.end.line {
|
|
let line_str = idx.line_str(line, text);
|
|
let line_len = line_str.len() as u32;
|
|
let start_col = if line == span.start.line {
|
|
span.start.column
|
|
} else {
|
|
0
|
|
};
|
|
let end_col = if line == span.end.line {
|
|
span.end.column.min(line_len)
|
|
} else {
|
|
line_len
|
|
};
|
|
if end_col > start_col {
|
|
out.push(RawToken {
|
|
line,
|
|
byte_col: start_col,
|
|
byte_len: end_col - start_col,
|
|
kind,
|
|
mods,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// ===== Encoding =====
|
|
|
|
/// LSP wire-format encoder. Produces a flat `Vec<u32>` of
|
|
/// `(deltaLine, deltaStart, length, type, modifiers)` quintuples.
|
|
///
|
|
/// `utf8 = true` means the client opted into UTF-8 encoding, so byte offsets
|
|
/// pass through directly. Otherwise positions and lengths are converted to
|
|
/// UTF-16 code units per the LSP default.
|
|
pub fn encode(raws: &[RawToken], text: &str, idx: &LineIndex, utf8: bool) -> Vec<u32> {
|
|
let mut data = Vec::with_capacity(raws.len() * 5);
|
|
let mut prev_line = 0u32;
|
|
let mut prev_col = 0u32;
|
|
for tok in raws {
|
|
let (col, len) = if utf8 {
|
|
(tok.byte_col, tok.byte_len)
|
|
} else {
|
|
let line_str = idx.line_str(tok.line, text);
|
|
let start = utf16_count(line_str, tok.byte_col as usize);
|
|
let end = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize);
|
|
(start, end - start)
|
|
};
|
|
let delta_line = tok.line - prev_line;
|
|
let delta_col = if delta_line == 0 { col - prev_col } else { col };
|
|
data.extend_from_slice(&[delta_line, delta_col, len, tok.kind, tok.mods]);
|
|
prev_line = tok.line;
|
|
prev_col = col;
|
|
}
|
|
data
|
|
}
|
|
|
|
fn utf16_count(s: &str, byte_offset: usize) -> u32 {
|
|
let upto = byte_offset.min(s.len());
|
|
s[..upto].chars().map(char::len_utf16).sum::<usize>() as u32
|
|
}
|
|
|
|
/// Build the encoded data stream for either the full document or a range
|
|
/// request.
|
|
pub fn build_data(doc: &DocumentNode, text: &str, utf8: bool, range: Option<Range>) -> Vec<u32> {
|
|
let idx = LineIndex::new(text);
|
|
let mut tokens = collect_tokens(doc, text);
|
|
if let Some(r) = range {
|
|
tokens.retain(|t| token_overlaps_range(t, &r, text, &idx, utf8));
|
|
}
|
|
encode(&tokens, text, &idx, utf8)
|
|
}
|
|
|
|
fn token_overlaps_range(
|
|
tok: &RawToken,
|
|
range: &Range,
|
|
text: &str,
|
|
idx: &LineIndex,
|
|
utf8: bool,
|
|
) -> bool {
|
|
let (start_char, end_char) = if utf8 {
|
|
(tok.byte_col, tok.byte_col + tok.byte_len)
|
|
} else {
|
|
let line_str = idx.line_str(tok.line, text);
|
|
let s = utf16_count(line_str, tok.byte_col as usize);
|
|
let e = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize);
|
|
(s, e)
|
|
};
|
|
let tok_start = (tok.line, start_char);
|
|
let tok_end = (tok.line, end_char);
|
|
let r_start = (range.start.line, range.start.character);
|
|
let r_end = (range.end.line, range.end.character);
|
|
tok_end > r_start && tok_start < r_end
|
|
}
|