feat: initial nuwiki Rust workspace (core, lsp, ls)
This commit is contained in:
gffranco
@@ -0,0 +1,369 @@
|
||||
//! Semantic-token emission for the vimwiki AST.
|
||||
//!
|
||||
//! Token-type scheme: custom `vimwiki*` types plus
|
||||
//! `level1`..`level6` + `centered` modifiers.
|
||||
//!
|
||||
//! Pipeline:
|
||||
//!
|
||||
//! 1. Walk the AST, emitting one or more `RawToken`s per AST node. Block
|
||||
//! structure that "wraps" text (paragraph, list item, table cell)
|
||||
//! recurses into inlines instead of emitting a wrapping token, so we
|
||||
//! end up with a non-overlapping tile of inline tokens. Block nodes
|
||||
//! that are themselves visually distinct (heading, code block, math
|
||||
//! block, comment) emit one token across their whole span.
|
||||
//! 2. Multi-line spans are split into one token per line — the LSP wire
|
||||
//! format can't express a single token that crosses a newline.
|
||||
//! 3. Tokens are sorted by `(line, byte_col)` and encoded as the LSP
|
||||
//! delta-quintuple stream `(deltaLine, deltaStart, length, type, mods)`.
|
||||
//! When the client doesn't support UTF-8 encoding, byte offsets are
|
||||
//! converted to UTF-16 code-unit counts on the fly.
|
||||
|
||||
use nuwiki_core::ast::{
|
||||
BlockNode, BlockquoteNode, DocumentNode, HeadingNode, InlineNode, ListItemNode, ListNode, Span,
|
||||
TableNode,
|
||||
};
|
||||
use tower_lsp::lsp_types::{Range, SemanticTokenModifier, SemanticTokenType, SemanticTokensLegend};
|
||||
|
||||
// ===== Legend =====
|
||||
|
||||
pub const TOKEN_HEADING: u32 = 0;
|
||||
pub const TOKEN_BOLD: u32 = 1;
|
||||
pub const TOKEN_ITALIC: u32 = 2;
|
||||
pub const TOKEN_BOLD_ITALIC: u32 = 3;
|
||||
pub const TOKEN_STRIKETHROUGH: u32 = 4;
|
||||
pub const TOKEN_SUPERSCRIPT: u32 = 5;
|
||||
pub const TOKEN_SUBSCRIPT: u32 = 6;
|
||||
pub const TOKEN_CODE: u32 = 7;
|
||||
pub const TOKEN_CODE_BLOCK: u32 = 8;
|
||||
pub const TOKEN_MATH: u32 = 9;
|
||||
pub const TOKEN_MATH_BLOCK: u32 = 10;
|
||||
pub const TOKEN_KEYWORD: u32 = 11;
|
||||
pub const TOKEN_COLOR: u32 = 12;
|
||||
pub const TOKEN_WIKILINK: u32 = 13;
|
||||
pub const TOKEN_EXTERNAL_LINK: u32 = 14;
|
||||
pub const TOKEN_TRANSCLUSION: u32 = 15;
|
||||
pub const TOKEN_URL: u32 = 16;
|
||||
pub const TOKEN_COMMENT: u32 = 17;
|
||||
pub const TOKEN_DEFINITION_TERM: u32 = 18;
|
||||
pub const TOKEN_TAG: u32 = 19;
|
||||
|
||||
pub const TOKEN_TYPE_NAMES: &[&str] = &[
|
||||
"vimwikiHeading",
|
||||
"vimwikiBold",
|
||||
"vimwikiItalic",
|
||||
"vimwikiBoldItalic",
|
||||
"vimwikiStrikethrough",
|
||||
"vimwikiSuperscript",
|
||||
"vimwikiSubscript",
|
||||
"vimwikiCode",
|
||||
"vimwikiCodeBlock",
|
||||
"vimwikiMath",
|
||||
"vimwikiMathBlock",
|
||||
"vimwikiKeyword",
|
||||
"vimwikiColor",
|
||||
"vimwikiWikilink",
|
||||
"vimwikiExternalLink",
|
||||
"vimwikiTransclusion",
|
||||
"vimwikiUrl",
|
||||
"vimwikiComment",
|
||||
"vimwikiDefinitionTerm",
|
||||
"vimwikiTag",
|
||||
];
|
||||
|
||||
pub const MOD_LEVEL1: u32 = 1 << 0;
|
||||
pub const MOD_LEVEL2: u32 = 1 << 1;
|
||||
pub const MOD_LEVEL3: u32 = 1 << 2;
|
||||
pub const MOD_LEVEL4: u32 = 1 << 3;
|
||||
pub const MOD_LEVEL5: u32 = 1 << 4;
|
||||
pub const MOD_LEVEL6: u32 = 1 << 5;
|
||||
pub const MOD_CENTERED: u32 = 1 << 6;
|
||||
|
||||
pub const TOKEN_MODIFIER_NAMES: &[&str] = &[
|
||||
"level1", "level2", "level3", "level4", "level5", "level6", "centered",
|
||||
];
|
||||
|
||||
pub fn legend() -> SemanticTokensLegend {
|
||||
SemanticTokensLegend {
|
||||
token_types: TOKEN_TYPE_NAMES
|
||||
.iter()
|
||||
.map(|s| SemanticTokenType::new(s))
|
||||
.collect(),
|
||||
token_modifiers: TOKEN_MODIFIER_NAMES
|
||||
.iter()
|
||||
.map(|s| SemanticTokenModifier::new(s))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Raw tokens =====
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct RawToken {
|
||||
pub line: u32,
|
||||
pub byte_col: u32,
|
||||
pub byte_len: u32,
|
||||
pub kind: u32,
|
||||
pub mods: u32,
|
||||
}
|
||||
|
||||
/// Index of byte offsets at the start of each line. Built once per request
|
||||
/// so multi-line span splitting and UTF-16 conversion stay linear in
|
||||
/// document size.
|
||||
pub struct LineIndex {
|
||||
pub starts: Vec<usize>,
|
||||
pub total: usize,
|
||||
}
|
||||
|
||||
impl LineIndex {
|
||||
pub fn new(text: &str) -> Self {
|
||||
let mut starts = vec![0];
|
||||
for (i, b) in text.bytes().enumerate() {
|
||||
if b == b'\n' {
|
||||
starts.push(i + 1);
|
||||
}
|
||||
}
|
||||
Self {
|
||||
starts,
|
||||
total: text.len(),
|
||||
}
|
||||
}
|
||||
|
||||
fn line_byte_range(&self, line: u32) -> (usize, usize) {
|
||||
let start = self
|
||||
.starts
|
||||
.get(line as usize)
|
||||
.copied()
|
||||
.unwrap_or(self.total);
|
||||
let end = self
|
||||
.starts
|
||||
.get(line as usize + 1)
|
||||
.map(|&s| s.saturating_sub(1))
|
||||
.unwrap_or(self.total);
|
||||
(start, end)
|
||||
}
|
||||
|
||||
pub fn line_str<'a>(&self, line: u32, text: &'a str) -> &'a str {
|
||||
let (s, e) = self.line_byte_range(line);
|
||||
&text[s..e]
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Collection =====
|
||||
|
||||
pub fn collect_tokens(doc: &DocumentNode, text: &str) -> Vec<RawToken> {
|
||||
let idx = LineIndex::new(text);
|
||||
let mut out = Vec::new();
|
||||
for block in &doc.children {
|
||||
emit_block(block, text, &idx, &mut out);
|
||||
}
|
||||
out.sort_by_key(|t| (t.line, t.byte_col));
|
||||
out
|
||||
}
|
||||
|
||||
fn emit_block(block: &BlockNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
match block {
|
||||
BlockNode::Heading(h) => emit_heading(h, text, idx, out),
|
||||
BlockNode::Paragraph(p) => emit_inlines(&p.children, text, idx, out),
|
||||
BlockNode::HorizontalRule(_) => {}
|
||||
BlockNode::Blockquote(BlockquoteNode { children, .. }) => {
|
||||
for child in children {
|
||||
emit_block(child, text, idx, out);
|
||||
}
|
||||
}
|
||||
BlockNode::Preformatted(p) => split_span(p.span, text, idx, TOKEN_CODE_BLOCK, 0, out),
|
||||
BlockNode::MathBlock(m) => split_span(m.span, text, idx, TOKEN_MATH_BLOCK, 0, out),
|
||||
BlockNode::List(ListNode { items, .. }) => {
|
||||
for item in items {
|
||||
emit_list_item(item, text, idx, out);
|
||||
}
|
||||
}
|
||||
BlockNode::DefinitionList(dl) => {
|
||||
for item in &dl.items {
|
||||
if let Some(term) = &item.term {
|
||||
if let (Some(first), Some(last)) = (term.first(), term.last()) {
|
||||
let s = Span::new(first.span().start, last.span().end);
|
||||
split_span(s, text, idx, TOKEN_DEFINITION_TERM, 0, out);
|
||||
}
|
||||
}
|
||||
for def in &item.definitions {
|
||||
emit_inlines(def, text, idx, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
BlockNode::Table(TableNode { rows, .. }) => {
|
||||
for row in rows {
|
||||
for cell in &row.cells {
|
||||
emit_inlines(&cell.children, text, idx, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
BlockNode::Comment(c) => split_span(c.span, text, idx, TOKEN_COMMENT, 0, out),
|
||||
BlockNode::Tag(t) => split_span(t.span, text, idx, TOKEN_TAG, 0, out),
|
||||
BlockNode::Error(_) => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_heading(h: &HeadingNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
let mut mods = 0u32;
|
||||
mods |= match h.level {
|
||||
1 => MOD_LEVEL1,
|
||||
2 => MOD_LEVEL2,
|
||||
3 => MOD_LEVEL3,
|
||||
4 => MOD_LEVEL4,
|
||||
5 => MOD_LEVEL5,
|
||||
_ => MOD_LEVEL6,
|
||||
};
|
||||
if h.centered {
|
||||
mods |= MOD_CENTERED;
|
||||
}
|
||||
split_span(h.span, text, idx, TOKEN_HEADING, mods, out);
|
||||
}
|
||||
|
||||
fn emit_list_item(item: &ListItemNode, text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
emit_inlines(&item.children, text, idx, out);
|
||||
if let Some(sublist) = &item.sublist {
|
||||
for child in &sublist.items {
|
||||
emit_list_item(child, text, idx, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_inlines(inlines: &[InlineNode], text: &str, idx: &LineIndex, out: &mut Vec<RawToken>) {
|
||||
for n in inlines {
|
||||
let (kind, span) = match n {
|
||||
InlineNode::Text(_) | InlineNode::SoftBreak(_) => continue,
|
||||
InlineNode::Bold(b) => (TOKEN_BOLD, b.span),
|
||||
InlineNode::Italic(i) => (TOKEN_ITALIC, i.span),
|
||||
InlineNode::BoldItalic(bi) => (TOKEN_BOLD_ITALIC, bi.span),
|
||||
InlineNode::Strikethrough(s) => (TOKEN_STRIKETHROUGH, s.span),
|
||||
InlineNode::Code(c) => (TOKEN_CODE, c.span),
|
||||
InlineNode::Superscript(s) => (TOKEN_SUPERSCRIPT, s.span),
|
||||
InlineNode::Subscript(s) => (TOKEN_SUBSCRIPT, s.span),
|
||||
InlineNode::MathInline(m) => (TOKEN_MATH, m.span),
|
||||
InlineNode::Keyword(k) => (TOKEN_KEYWORD, k.span),
|
||||
InlineNode::Color(c) => (TOKEN_COLOR, c.span),
|
||||
InlineNode::WikiLink(w) => (TOKEN_WIKILINK, w.span),
|
||||
InlineNode::ExternalLink(e) => (TOKEN_EXTERNAL_LINK, e.span),
|
||||
InlineNode::Transclusion(t) => (TOKEN_TRANSCLUSION, t.span),
|
||||
InlineNode::RawUrl(u) => (TOKEN_URL, u.span),
|
||||
};
|
||||
split_span(span, text, idx, kind, 0, out);
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a `Span` (potentially multi-line) into one or more single-line
|
||||
/// `RawToken`s. The LSP wire format can't express tokens that span newlines,
|
||||
/// so we split on line boundaries.
|
||||
pub fn split_span(
|
||||
span: Span,
|
||||
text: &str,
|
||||
idx: &LineIndex,
|
||||
kind: u32,
|
||||
mods: u32,
|
||||
out: &mut Vec<RawToken>,
|
||||
) {
|
||||
if span.start.line == span.end.line {
|
||||
let len = span.end.column.saturating_sub(span.start.column);
|
||||
if len > 0 {
|
||||
out.push(RawToken {
|
||||
line: span.start.line,
|
||||
byte_col: span.start.column,
|
||||
byte_len: len,
|
||||
kind,
|
||||
mods,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
for line in span.start.line..=span.end.line {
|
||||
let line_str = idx.line_str(line, text);
|
||||
let line_len = line_str.len() as u32;
|
||||
let start_col = if line == span.start.line {
|
||||
span.start.column
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let end_col = if line == span.end.line {
|
||||
span.end.column.min(line_len)
|
||||
} else {
|
||||
line_len
|
||||
};
|
||||
if end_col > start_col {
|
||||
out.push(RawToken {
|
||||
line,
|
||||
byte_col: start_col,
|
||||
byte_len: end_col - start_col,
|
||||
kind,
|
||||
mods,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Encoding =====
|
||||
|
||||
/// LSP wire-format encoder. Produces a flat `Vec<u32>` of
|
||||
/// `(deltaLine, deltaStart, length, type, modifiers)` quintuples.
|
||||
///
|
||||
/// `utf8 = true` means the client opted into UTF-8 encoding, so byte offsets
|
||||
/// pass through directly. Otherwise positions and lengths are converted to
|
||||
/// UTF-16 code units per the LSP default.
|
||||
pub fn encode(raws: &[RawToken], text: &str, idx: &LineIndex, utf8: bool) -> Vec<u32> {
|
||||
let mut data = Vec::with_capacity(raws.len() * 5);
|
||||
let mut prev_line = 0u32;
|
||||
let mut prev_col = 0u32;
|
||||
for tok in raws {
|
||||
let (col, len) = if utf8 {
|
||||
(tok.byte_col, tok.byte_len)
|
||||
} else {
|
||||
let line_str = idx.line_str(tok.line, text);
|
||||
let start = utf16_count(line_str, tok.byte_col as usize);
|
||||
let end = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize);
|
||||
(start, end - start)
|
||||
};
|
||||
let delta_line = tok.line - prev_line;
|
||||
let delta_col = if delta_line == 0 { col - prev_col } else { col };
|
||||
data.extend_from_slice(&[delta_line, delta_col, len, tok.kind, tok.mods]);
|
||||
prev_line = tok.line;
|
||||
prev_col = col;
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
fn utf16_count(s: &str, byte_offset: usize) -> u32 {
|
||||
let upto = byte_offset.min(s.len());
|
||||
s[..upto].chars().map(char::len_utf16).sum::<usize>() as u32
|
||||
}
|
||||
|
||||
/// Build the encoded data stream for either the full document or a range
|
||||
/// request.
|
||||
pub fn build_data(doc: &DocumentNode, text: &str, utf8: bool, range: Option<Range>) -> Vec<u32> {
|
||||
let idx = LineIndex::new(text);
|
||||
let mut tokens = collect_tokens(doc, text);
|
||||
if let Some(r) = range {
|
||||
tokens.retain(|t| token_overlaps_range(t, &r, text, &idx, utf8));
|
||||
}
|
||||
encode(&tokens, text, &idx, utf8)
|
||||
}
|
||||
|
||||
fn token_overlaps_range(
|
||||
tok: &RawToken,
|
||||
range: &Range,
|
||||
text: &str,
|
||||
idx: &LineIndex,
|
||||
utf8: bool,
|
||||
) -> bool {
|
||||
let (start_char, end_char) = if utf8 {
|
||||
(tok.byte_col, tok.byte_col + tok.byte_len)
|
||||
} else {
|
||||
let line_str = idx.line_str(tok.line, text);
|
||||
let s = utf16_count(line_str, tok.byte_col as usize);
|
||||
let e = utf16_count(line_str, (tok.byte_col + tok.byte_len) as usize);
|
||||
(s, e)
|
||||
};
|
||||
let tok_start = (tok.line, start_char);
|
||||
let tok_end = (tok.line, end_char);
|
||||
let r_start = (range.start.line, range.start.character);
|
||||
let r_end = (range.end.line, range.end.character);
|
||||
tok_end > r_start && tok_start < r_end
|
||||
}
|
||||
Reference in New Issue
Block a user