gffranco
1293 lines
45 KiB
Rust
1293 lines
45 KiB
Rust
//! Vimwiki lexer.
|
|
//!
|
|
//! Hand-rolled, two-pass:
|
|
//!
|
|
//! - **Block pass:** scan the source line by line, recognise structural
|
|
//! constructs (headings, lists, tables, fences, comments, etc.), emit
|
|
//! the matching block-level token, and within text-bearing lines invoke
|
|
//! the inline pass on the line content.
|
|
//! - **Inline pass:** scan a text run byte by byte, accumulate plain text,
|
|
//! and emit inline marker tokens (bold, italic, code, links, …).
|
|
//!
|
|
//! Both passes share one `Vec<VimwikiToken>` so the parser sees a single,
|
|
//! ordered stream. The lexer is permissive: every delimiter is emitted; the
|
|
//! parser pairs them and falls back to literal text on mismatches. Spans are
|
|
//! stored as 0-indexed byte offsets.
|
|
//!
|
|
//! Multi-line constructs (`{{{ }}}`, `{{$ }}$`, `%%+ +%%`) flip a
|
|
//! `BlockMode` so subsequent lines are treated as raw content until the
|
|
//! matching closer is seen.
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use crate::ast::{CheckboxState, Keyword, ListSymbol, Position, Span};
|
|
use crate::listsyms::ListSyms;
|
|
use crate::syntax::{Lexer, TokenStream};
|
|
|
|
/// A single vimwiki token. The `kind` carries the variant, `span` carries
|
|
/// 0-indexed byte offsets into the source.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct VimwikiToken {
|
|
pub kind: VimwikiTokenKind,
|
|
pub span: Span,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum VimwikiTokenKind {
|
|
// ----- Line structure -----
|
|
/// End of a non-empty line (`\n`).
|
|
Newline,
|
|
/// A line that contained only whitespace. Replaces both the (empty)
|
|
/// content and the trailing newline.
|
|
BlankLine,
|
|
|
|
// ----- Headings -----
|
|
/// Opening `=`s of a heading. `centered` is true when the line had
|
|
/// leading whitespace before the `=` run.
|
|
HeadingOpen {
|
|
level: u8,
|
|
centered: bool,
|
|
},
|
|
/// Closing `=`s on the same line as `HeadingOpen`.
|
|
HeadingClose,
|
|
|
|
// ----- Bare structure -----
|
|
HorizontalRule,
|
|
BlockquoteMarker,
|
|
BlockquoteIndent,
|
|
|
|
// ----- Lists -----
|
|
ListMarker {
|
|
symbol: ListSymbol,
|
|
indent: u32,
|
|
},
|
|
Checkbox(CheckboxState),
|
|
|
|
// ----- Definition lists -----
|
|
DefinitionTermMarker,
|
|
|
|
// ----- Tables -----
|
|
/// `|` cell separator.
|
|
TableSep,
|
|
/// A header-separator row, e.g. `|---|---|`. Carries per-cell
|
|
/// Markdown-style alignment markers (`:--`, `--:`, `:--:`).
|
|
TableHeaderRow(Vec<crate::ast::TableAlign>),
|
|
/// A cell whose only content is `>`, meaning "merge with cell to the left".
|
|
TableColSpan,
|
|
/// A cell whose only content is `\/`, meaning "merge with cell above".
|
|
TableRowSpan,
|
|
|
|
// ----- Multi-line fences -----
|
|
PreformattedOpen {
|
|
language: Option<String>,
|
|
attrs: HashMap<String, String>,
|
|
/// Verbatim text after `{{{` (trailing whitespace trimmed), preserved
|
|
/// so the HTML renderer can reproduce vimwiki's raw `<pre …>` attrs.
|
|
raw: String,
|
|
},
|
|
PreformattedClose,
|
|
PreformattedLine(String),
|
|
|
|
MathBlockOpen {
|
|
environment: Option<String>,
|
|
},
|
|
MathBlockClose,
|
|
MathBlockLine(String),
|
|
|
|
/// Single-line comment: `%% ...`
|
|
CommentLine(String),
|
|
/// Opening of a multi-line comment: `%%+`
|
|
CommentMultiOpen,
|
|
CommentMultiClose,
|
|
CommentMultiLine(String),
|
|
|
|
// ----- Tags -----
|
|
/// `:tag1:tag2:` — colon-delimited tag line. The lexer treats this as
|
|
/// block-level (must be the entire trimmed content of the line).
|
|
/// Scope (file / heading / standalone) is the parser's job.
|
|
Tag(Vec<String>),
|
|
|
|
// ----- Page placeholders -----
|
|
PlaceholderTitle(Option<String>),
|
|
PlaceholderNohtml,
|
|
PlaceholderTemplate(Option<String>),
|
|
PlaceholderDate(Option<String>),
|
|
|
|
// ----- Inline content -----
|
|
Text(String),
|
|
BoldDelim,
|
|
ItalicDelim,
|
|
StrikethroughDelim,
|
|
SuperscriptDelim,
|
|
SubscriptDelim,
|
|
Code(String),
|
|
MathInline(String),
|
|
Keyword(Keyword),
|
|
|
|
// ----- Links / transclusions -----
|
|
WikiLinkOpen,
|
|
WikiLinkClose,
|
|
WikiLinkSep,
|
|
TransclusionOpen,
|
|
TransclusionClose,
|
|
TransclusionSep,
|
|
RawUrl(String),
|
|
|
|
/// Lex error — never aborts the whole document.
|
|
Error(String),
|
|
}
|
|
|
|
#[derive(Debug, Default, Clone)]
|
|
pub struct VimwikiLexer {
|
|
listsyms: ListSyms,
|
|
}
|
|
|
|
impl VimwikiLexer {
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
/// Recognise checkbox glyphs from a custom palette
|
|
/// (vimwiki's `g:vimwiki_listsyms`) instead of the default `" .oOX"`.
|
|
pub fn with_listsyms(mut self, listsyms: ListSyms) -> Self {
|
|
self.listsyms = listsyms;
|
|
self
|
|
}
|
|
}
|
|
|
|
impl Lexer for VimwikiLexer {
|
|
type Token = VimwikiToken;
|
|
|
|
fn lex(&self, text: &str) -> TokenStream<VimwikiToken> {
|
|
let mut state = LexState::new(text, &self.listsyms);
|
|
state.run();
|
|
TokenStream::from_vec(state.tokens)
|
|
}
|
|
}
|
|
|
|
// ===== Internal state machine =====
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum BlockMode {
|
|
Normal,
|
|
Preformatted,
|
|
MathBlock,
|
|
MultilineComment,
|
|
}
|
|
|
|
struct LexState<'src> {
|
|
src: &'src str,
|
|
/// Byte offset of the start of the current line.
|
|
line_start_offset: usize,
|
|
/// Index of the current line (0-based).
|
|
line: u32,
|
|
tokens: Vec<VimwikiToken>,
|
|
mode: BlockMode,
|
|
listsyms: &'src ListSyms,
|
|
}
|
|
|
|
impl<'src> LexState<'src> {
|
|
fn new(src: &'src str, listsyms: &'src ListSyms) -> Self {
|
|
Self {
|
|
src,
|
|
line_start_offset: 0,
|
|
line: 0,
|
|
tokens: Vec::new(),
|
|
mode: BlockMode::Normal,
|
|
listsyms,
|
|
}
|
|
}
|
|
|
|
fn run(&mut self) {
|
|
while self.line_start_offset < self.src.len() {
|
|
let line_text = self.current_line();
|
|
let line_len = line_text.len();
|
|
|
|
match self.mode {
|
|
BlockMode::Normal => self.lex_normal_line(line_text),
|
|
BlockMode::Preformatted => self.lex_preformatted_line(line_text),
|
|
BlockMode::MathBlock => self.lex_math_block_line(line_text),
|
|
BlockMode::MultilineComment => self.lex_multi_comment_line(line_text),
|
|
}
|
|
|
|
// Skip past the line's trailing newline (if any) and advance line counters.
|
|
let after_content = self.line_start_offset + line_len;
|
|
if after_content < self.src.len() && self.src.as_bytes()[after_content] == b'\n' {
|
|
self.line_start_offset = after_content + 1;
|
|
self.line += 1;
|
|
} else {
|
|
self.line_start_offset = after_content;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Slice of the current line, not including the trailing newline.
|
|
fn current_line(&self) -> &'src str {
|
|
let rest = &self.src[self.line_start_offset..];
|
|
match rest.find('\n') {
|
|
Some(idx) => &rest[..idx],
|
|
None => rest,
|
|
}
|
|
}
|
|
|
|
/// Position at byte offset `col` within the current line.
|
|
fn pos_in_line(&self, col: u32) -> Position {
|
|
Position {
|
|
line: self.line,
|
|
column: col,
|
|
offset: self.line_start_offset + col as usize,
|
|
}
|
|
}
|
|
|
|
fn line_start_pos(&self) -> Position {
|
|
self.pos_in_line(0)
|
|
}
|
|
|
|
fn line_end_pos(&self, line_text: &str) -> Position {
|
|
self.pos_in_line(line_text.len() as u32)
|
|
}
|
|
|
|
fn push(&mut self, kind: VimwikiTokenKind, span: Span) {
|
|
self.tokens.push(VimwikiToken { kind, span });
|
|
}
|
|
|
|
/// Emit a token whose span runs from byte-col `start_col` to `end_col`
|
|
/// within the current line.
|
|
fn emit(&mut self, kind: VimwikiTokenKind, start_col: u32, end_col: u32) {
|
|
let span = Span::new(self.pos_in_line(start_col), self.pos_in_line(end_col));
|
|
self.push(kind, span);
|
|
}
|
|
|
|
fn emit_newline(&mut self, line_text: &str) {
|
|
let nl_col = line_text.len() as u32;
|
|
// Newline is just \n at column = line_text.len() spanning one byte.
|
|
let span = Span::new(self.pos_in_line(nl_col), self.pos_in_line(nl_col + 1));
|
|
self.push(VimwikiTokenKind::Newline, span);
|
|
}
|
|
|
|
// ===== Mode dispatch =====
|
|
|
|
fn lex_preformatted_line(&mut self, line: &str) {
|
|
// Closing fence: line whose first non-space content is `}}}`.
|
|
if line.trim() == "}}}" {
|
|
let start_col = line.find("}}}").unwrap() as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::PreformattedClose,
|
|
start_col,
|
|
start_col + 3,
|
|
);
|
|
self.mode = BlockMode::Normal;
|
|
self.emit_newline(line);
|
|
return;
|
|
}
|
|
let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::PreformattedLine(line.to_owned()), span);
|
|
self.emit_newline(line);
|
|
}
|
|
|
|
fn lex_math_block_line(&mut self, line: &str) {
|
|
if line.trim() == "}}$" {
|
|
let start_col = line.find("}}$").unwrap() as u32;
|
|
self.emit(VimwikiTokenKind::MathBlockClose, start_col, start_col + 3);
|
|
self.mode = BlockMode::Normal;
|
|
self.emit_newline(line);
|
|
return;
|
|
}
|
|
let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::MathBlockLine(line.to_owned()), span);
|
|
self.emit_newline(line);
|
|
}
|
|
|
|
fn lex_multi_comment_line(&mut self, line: &str) {
|
|
// Closer is `+%%`, possibly with content before it on the same line.
|
|
if let Some(idx) = line.find("+%%") {
|
|
let prefix = &line[..idx];
|
|
if !prefix.is_empty() {
|
|
let span = Span::new(self.pos_in_line(0), self.pos_in_line(prefix.len() as u32));
|
|
self.push(VimwikiTokenKind::CommentMultiLine(prefix.to_owned()), span);
|
|
}
|
|
let close_col = idx as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::CommentMultiClose,
|
|
close_col,
|
|
close_col + 3,
|
|
);
|
|
self.mode = BlockMode::Normal;
|
|
self.emit_newline(line);
|
|
return;
|
|
}
|
|
let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::CommentMultiLine(line.to_owned()), span);
|
|
self.emit_newline(line);
|
|
}
|
|
|
|
fn lex_normal_line(&mut self, line: &str) {
|
|
// Whitespace-only line → BlankLine (replaces both content and the
|
|
// following newline).
|
|
if line.bytes().all(|b| b == b' ' || b == b'\t') {
|
|
let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::BlankLine, span);
|
|
// Also account for the newline byte if present.
|
|
// We don't emit a Newline because BlankLine subsumes both.
|
|
return;
|
|
}
|
|
|
|
// Block patterns in priority order. The first match wins.
|
|
if self.try_lex_multi_comment_open(line)
|
|
|| self.try_lex_single_comment(line)
|
|
|| self.try_lex_preformatted_open(line)
|
|
|| self.try_lex_math_block_open(line)
|
|
|| self.try_lex_placeholder(line)
|
|
|| self.try_lex_horizontal_rule(line)
|
|
|| self.try_lex_heading(line)
|
|
|| self.try_lex_tag(line)
|
|
|| self.try_lex_table_row(line)
|
|
|| self.try_lex_blockquote(line)
|
|
|| self.try_lex_list_item(line)
|
|
|| self.try_lex_definition_term(line)
|
|
{
|
|
return;
|
|
}
|
|
|
|
// Default: paragraph line — lex inline.
|
|
self.lex_inline(line, 0, line.len() as u32);
|
|
self.emit_newline(line);
|
|
}
|
|
|
|
// ===== Block patterns =====
|
|
|
|
fn try_lex_multi_comment_open(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim_start();
|
|
if !trimmed.starts_with("%%+") {
|
|
return false;
|
|
}
|
|
let indent = (line.len() - trimmed.len()) as u32;
|
|
self.emit(VimwikiTokenKind::CommentMultiOpen, indent, indent + 3);
|
|
// Same line might also contain `+%%` (one-line multi).
|
|
let after_open = (indent + 3) as usize;
|
|
let rest = &line[after_open..];
|
|
if let Some(end_rel) = rest.find("+%%") {
|
|
let prefix = &rest[..end_rel];
|
|
if !prefix.is_empty() {
|
|
let p_start = indent + 3;
|
|
let p_end = p_start + prefix.len() as u32;
|
|
self.push(
|
|
VimwikiTokenKind::CommentMultiLine(prefix.to_owned()),
|
|
Span::new(self.pos_in_line(p_start), self.pos_in_line(p_end)),
|
|
);
|
|
}
|
|
let close_col = (after_open + end_rel) as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::CommentMultiClose,
|
|
close_col,
|
|
close_col + 3,
|
|
);
|
|
self.emit_newline(line);
|
|
} else {
|
|
// Capture remainder of this line as content, then enter
|
|
// multiline-comment mode for following lines.
|
|
if !rest.is_empty() {
|
|
let p_start = indent + 3;
|
|
let p_end = p_start + rest.len() as u32;
|
|
self.push(
|
|
VimwikiTokenKind::CommentMultiLine(rest.to_owned()),
|
|
Span::new(self.pos_in_line(p_start), self.pos_in_line(p_end)),
|
|
);
|
|
}
|
|
self.emit_newline(line);
|
|
self.mode = BlockMode::MultilineComment;
|
|
}
|
|
true
|
|
}
|
|
|
|
fn try_lex_single_comment(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim_start();
|
|
if !trimmed.starts_with("%%") || trimmed.starts_with("%%+") {
|
|
return false;
|
|
}
|
|
let indent = (line.len() - trimmed.len()) as u32;
|
|
let content = &trimmed[2..]; // skip "%%"
|
|
let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::CommentLine(content.to_owned()), span);
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_preformatted_open(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim_start();
|
|
if !trimmed.starts_with("{{{") {
|
|
return false;
|
|
}
|
|
let indent = (line.len() - trimmed.len()) as u32;
|
|
// `{{{[lang][;key=val[;key=val]*]?` style. Vimwiki accepts a single
|
|
// word after the fence (language hint) and/or class/attribute key=val
|
|
// pairs separated by spaces. Keep parsing forgiving.
|
|
let after = trimmed[3..].trim();
|
|
let (language, attrs) = parse_fence_attrs(after);
|
|
// vimwiki keeps everything after `{{{` (minus trailing whitespace) and
|
|
// drops it verbatim into the `<pre>` tag, so capture it unparsed.
|
|
let raw = trimmed[3..].trim_end().to_string();
|
|
let kind = VimwikiTokenKind::PreformattedOpen {
|
|
language,
|
|
attrs,
|
|
raw,
|
|
};
|
|
let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line));
|
|
self.push(kind, span);
|
|
self.mode = BlockMode::Preformatted;
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_math_block_open(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim_start();
|
|
if !trimmed.starts_with("{{$") {
|
|
return false;
|
|
}
|
|
let indent = (line.len() - trimmed.len()) as u32;
|
|
let after = trimmed[3..].trim();
|
|
// `{{$%env%` — environment is between `%`s.
|
|
let environment = after
|
|
.strip_prefix('%')
|
|
.and_then(|s| s.strip_suffix('%'))
|
|
.map(str::to_owned);
|
|
let span = Span::new(self.pos_in_line(indent), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::MathBlockOpen { environment }, span);
|
|
self.mode = BlockMode::MathBlock;
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_placeholder(&mut self, line: &str) -> bool {
|
|
// Placeholders must start at column 0 (no leading whitespace).
|
|
if !line.starts_with('%') {
|
|
return false;
|
|
}
|
|
let kind = if let Some(rest) = line.strip_prefix("%title") {
|
|
let value = rest.trim();
|
|
let v = if value.is_empty() {
|
|
None
|
|
} else {
|
|
Some(value.to_owned())
|
|
};
|
|
VimwikiTokenKind::PlaceholderTitle(v)
|
|
} else if line.trim_end() == "%nohtml" {
|
|
VimwikiTokenKind::PlaceholderNohtml
|
|
} else if let Some(rest) = line.strip_prefix("%template") {
|
|
let value = rest.trim();
|
|
let v = if value.is_empty() {
|
|
None
|
|
} else {
|
|
Some(value.to_owned())
|
|
};
|
|
VimwikiTokenKind::PlaceholderTemplate(v)
|
|
} else if let Some(rest) = line.strip_prefix("%date") {
|
|
let value = rest.trim();
|
|
let v = if value.is_empty() {
|
|
None
|
|
} else {
|
|
Some(value.to_owned())
|
|
};
|
|
VimwikiTokenKind::PlaceholderDate(v)
|
|
} else {
|
|
return false;
|
|
};
|
|
let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
|
|
self.push(kind, span);
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_horizontal_rule(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim();
|
|
if trimmed.len() < 4 || !trimmed.bytes().all(|b| b == b'-') {
|
|
return false;
|
|
}
|
|
// Four or more dashes. Allow surrounding whitespace.
|
|
let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::HorizontalRule, span);
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_heading(&mut self, line: &str) -> bool {
|
|
let bytes = line.as_bytes();
|
|
let leading_ws = bytes
|
|
.iter()
|
|
.take_while(|&&b| b == b' ' || b == b'\t')
|
|
.count();
|
|
let centered = leading_ws > 0;
|
|
|
|
// Count `=`s after leading whitespace.
|
|
let mut level = 0usize;
|
|
while level < 6 && leading_ws + level < bytes.len() && bytes[leading_ws + level] == b'=' {
|
|
level += 1;
|
|
}
|
|
if level == 0 {
|
|
return false;
|
|
}
|
|
// Must be followed by at least one space.
|
|
if leading_ws + level >= bytes.len() || bytes[leading_ws + level] != b' ' {
|
|
return false;
|
|
}
|
|
|
|
// Trailing `=`s of the same level.
|
|
let trimmed_end = line.trim_end_matches([' ', '\t']);
|
|
let trailing_eqs = trimmed_end
|
|
.as_bytes()
|
|
.iter()
|
|
.rev()
|
|
.take_while(|&&b| b == b'=')
|
|
.count();
|
|
if trailing_eqs != level {
|
|
return false;
|
|
}
|
|
|
|
let title_start = leading_ws + level;
|
|
let title_end = trimmed_end.len() - trailing_eqs;
|
|
if title_end <= title_start {
|
|
return false;
|
|
}
|
|
|
|
let open_col = leading_ws as u32;
|
|
let open_end = (leading_ws + level) as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::HeadingOpen {
|
|
level: level as u8,
|
|
centered,
|
|
},
|
|
open_col,
|
|
open_end,
|
|
);
|
|
|
|
// Inline content between markers (trim one space on each side if present).
|
|
let mut t_start = title_start;
|
|
let mut t_end = title_end;
|
|
if t_start < t_end && bytes[t_start] == b' ' {
|
|
t_start += 1;
|
|
}
|
|
if t_end > t_start && bytes[t_end - 1] == b' ' {
|
|
t_end -= 1;
|
|
}
|
|
if t_end > t_start {
|
|
self.lex_inline(&line[t_start..t_end], t_start as u32, t_end as u32);
|
|
}
|
|
|
|
let close_col = title_end as u32;
|
|
let close_end = (title_end + trailing_eqs) as u32;
|
|
self.emit(VimwikiTokenKind::HeadingClose, close_col, close_end);
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_table_row(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim();
|
|
if !trimmed.starts_with('|') || !trimmed.ends_with('|') || trimmed.len() < 2 {
|
|
return false;
|
|
}
|
|
|
|
// Header separator row: every cell is `-`s (with optional `:`
|
|
// anchors at one or both ends for alignment) or whitespace.
|
|
if trimmed
|
|
.split('|')
|
|
.filter(|s| !s.is_empty())
|
|
.all(is_sep_cell)
|
|
{
|
|
let aligns: Vec<crate::ast::TableAlign> = trimmed
|
|
.split('|')
|
|
.filter(|s| !s.is_empty())
|
|
.map(parse_sep_alignment)
|
|
.collect();
|
|
let span = Span::new(self.line_start_pos(), self.line_end_pos(line));
|
|
self.push(VimwikiTokenKind::TableHeaderRow(aligns), span);
|
|
self.emit_newline(line);
|
|
return true;
|
|
}
|
|
|
|
// Walk the line and emit TableSep tokens at each `|`, with cell
|
|
// content lexed inline between them.
|
|
let bytes = line.as_bytes();
|
|
let mut i = 0usize;
|
|
// Skip leading whitespace.
|
|
while i < bytes.len() && (bytes[i] == b' ' || bytes[i] == b'\t') {
|
|
i += 1;
|
|
}
|
|
// Trailing whitespace bound.
|
|
let mut end = bytes.len();
|
|
while end > i && (bytes[end - 1] == b' ' || bytes[end - 1] == b'\t') {
|
|
end -= 1;
|
|
}
|
|
|
|
// Collect cell boundaries.
|
|
let mut sep_positions = Vec::new();
|
|
let mut k = i;
|
|
while k < end {
|
|
if bytes[k] == b'|' {
|
|
sep_positions.push(k);
|
|
}
|
|
k += 1;
|
|
}
|
|
|
|
if sep_positions.len() < 2 {
|
|
return false;
|
|
}
|
|
|
|
for window in sep_positions.windows(2) {
|
|
let sep_col = window[0] as u32;
|
|
self.emit(VimwikiTokenKind::TableSep, sep_col, sep_col + 1);
|
|
let cell_start = window[0] + 1;
|
|
let cell_end = window[1];
|
|
let cell_text = &line[cell_start..cell_end];
|
|
let cell_trimmed = cell_text.trim();
|
|
match cell_trimmed {
|
|
">" => {
|
|
let s = (cell_start + cell_text.find('>').unwrap()) as u32;
|
|
self.emit(VimwikiTokenKind::TableColSpan, s, s + 1);
|
|
}
|
|
"\\/" => {
|
|
let s = (cell_start + cell_text.find("\\/").unwrap()) as u32;
|
|
self.emit(VimwikiTokenKind::TableRowSpan, s, s + 2);
|
|
}
|
|
_ if !cell_trimmed.is_empty() => {
|
|
self.lex_inline(cell_text, cell_start as u32, cell_end as u32);
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
// Emit the last separator.
|
|
let last_sep = *sep_positions.last().unwrap() as u32;
|
|
self.emit(VimwikiTokenKind::TableSep, last_sep, last_sep + 1);
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_blockquote(&mut self, line: &str) -> bool {
|
|
// `>` prefix style.
|
|
let trimmed = line.trim_start();
|
|
let indent = (line.len() - trimmed.len()) as u32;
|
|
if let Some(rest) =
|
|
trimmed
|
|
.strip_prefix("> ")
|
|
.or_else(|| if trimmed == ">" { Some("") } else { None })
|
|
{
|
|
self.emit(VimwikiTokenKind::BlockquoteMarker, indent, indent + 1);
|
|
if !rest.is_empty() {
|
|
let inline_start = indent + 2;
|
|
let inline_end = inline_start + rest.len() as u32;
|
|
self.lex_inline(rest, inline_start, inline_end);
|
|
}
|
|
self.emit_newline(line);
|
|
return true;
|
|
}
|
|
|
|
// 4-space indent style. To avoid swallowing list items, only treat as
|
|
// a blockquote when the indented content does NOT start with a list
|
|
// marker.
|
|
if let Some(rest) = line.strip_prefix(" ") {
|
|
if rest.chars().next().is_none_or(|c| c == ' ' || c == '\t')
|
|
|| looks_like_list_marker(rest)
|
|
{
|
|
return false;
|
|
}
|
|
self.emit(VimwikiTokenKind::BlockquoteIndent, 0, 4);
|
|
self.lex_inline(rest, 4, line.len() as u32);
|
|
self.emit_newline(line);
|
|
return true;
|
|
}
|
|
false
|
|
}
|
|
|
|
fn try_lex_list_item(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim_start();
|
|
let indent_bytes = line.len() - trimmed.len();
|
|
let indent = indent_bytes as u32;
|
|
|
|
let (symbol, marker_len) = match list_marker_at(trimmed) {
|
|
Some(v) => v,
|
|
None => return false,
|
|
};
|
|
|
|
// Marker must be followed by a space.
|
|
if marker_len >= trimmed.len() || trimmed.as_bytes()[marker_len] != b' ' {
|
|
return false;
|
|
}
|
|
|
|
let marker_start = indent;
|
|
let marker_end = indent + marker_len as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::ListMarker { symbol, indent },
|
|
marker_start,
|
|
marker_end,
|
|
);
|
|
|
|
// Past marker + the single required space.
|
|
let mut cursor = marker_len + 1;
|
|
let after = &trimmed[cursor..];
|
|
|
|
// Optional checkbox `[ ]` / `[.]` / `[o]` / `[O]` / `[X]` / `[-]`,
|
|
// followed by a space.
|
|
if let Some((cb, cb_len)) = checkbox_at(after, self.listsyms) {
|
|
let cb_start = indent + cursor as u32;
|
|
let cb_end = cb_start + cb_len as u32;
|
|
self.emit(VimwikiTokenKind::Checkbox(cb), cb_start, cb_end);
|
|
cursor += cb_len;
|
|
// Eat the trailing space if present.
|
|
if cursor < trimmed.len() && trimmed.as_bytes()[cursor] == b' ' {
|
|
cursor += 1;
|
|
}
|
|
}
|
|
|
|
let inline_start = indent + cursor as u32;
|
|
let inline_end = line.len() as u32;
|
|
if (inline_end as usize) > (inline_start as usize) {
|
|
let abs_start = self.line_start_offset + inline_start as usize;
|
|
let abs_end = self.line_start_offset + inline_end as usize;
|
|
let slice = &self.src[abs_start..abs_end];
|
|
self.lex_inline(slice, inline_start, inline_end);
|
|
}
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
/// `:tag1:tag2:tag3:` — colon-delimited tag line (block-level).
|
|
///
|
|
/// The whole trimmed line must match: starts and ends with `:`, with
|
|
/// non-empty whitespace-free names between adjacent colons. Empty
|
|
/// segments or whitespace inside tag names disqualify the line.
|
|
/// Doesn't conflict with `::` (definition list marker) because that
|
|
/// pattern requires text *before* the `::`.
|
|
fn try_lex_tag(&mut self, line: &str) -> bool {
|
|
let trimmed = line.trim();
|
|
if trimmed.len() < 3 || !trimmed.starts_with(':') || !trimmed.ends_with(':') {
|
|
return false;
|
|
}
|
|
// `:tag1:tag2:` splits as `["", "tag1", "tag2", ""]`; the empty
|
|
// strings at the ends are expected, everything in between must be
|
|
// non-empty and contain no whitespace.
|
|
let parts: Vec<&str> = trimmed.split(':').collect();
|
|
if parts.first() != Some(&"") || parts.last() != Some(&"") {
|
|
return false;
|
|
}
|
|
let names = &parts[1..parts.len() - 1];
|
|
if names.is_empty()
|
|
|| names
|
|
.iter()
|
|
.any(|t| t.is_empty() || t.chars().any(char::is_whitespace))
|
|
{
|
|
return false;
|
|
}
|
|
let indent = (line.len() - line.trim_start().len()) as u32;
|
|
let end_col = indent + trimmed.len() as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::Tag(names.iter().map(|s| (*s).to_owned()).collect()),
|
|
indent,
|
|
end_col,
|
|
);
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
fn try_lex_definition_term(&mut self, line: &str) -> bool {
|
|
// `Term:: Definition` or `Term::` (continuation lines follow).
|
|
// The `::` must be preceded by non-`:` text and followed by either
|
|
// EOL or a single space + content.
|
|
let bytes = line.as_bytes();
|
|
let mut i = 0usize;
|
|
let mut found = None;
|
|
while i + 1 < bytes.len() {
|
|
if bytes[i] == b':' && bytes[i + 1] == b':' {
|
|
// Must not be preceded by another `:` (avoid `:::`).
|
|
if i > 0 && bytes[i - 1] == b':' {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
// Must not be followed by another `:`.
|
|
if i + 2 < bytes.len() && bytes[i + 2] == b':' {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
found = Some(i);
|
|
break;
|
|
}
|
|
i += 1;
|
|
}
|
|
let split = match found {
|
|
Some(s) if s > 0 => s,
|
|
_ => return false,
|
|
};
|
|
|
|
// Term inline.
|
|
self.lex_inline(&line[..split], 0, split as u32);
|
|
let marker_col = split as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::DefinitionTermMarker,
|
|
marker_col,
|
|
marker_col + 2,
|
|
);
|
|
|
|
let after = &line[split + 2..];
|
|
if !after.is_empty() {
|
|
let leading = after
|
|
.bytes()
|
|
.take_while(|b| *b == b' ' || *b == b'\t')
|
|
.count();
|
|
if leading > 0 {
|
|
// Treat the post-`::` whitespace as a separator; lex the rest as inline.
|
|
let inline_start = (split + 2 + leading) as u32;
|
|
let inline_end = line.len() as u32;
|
|
if inline_end > inline_start {
|
|
self.lex_inline(&after[leading..], inline_start, inline_end);
|
|
}
|
|
} else {
|
|
// No space after `::`: still try to lex inline.
|
|
let inline_start = (split + 2) as u32;
|
|
self.lex_inline(after, inline_start, line.len() as u32);
|
|
}
|
|
}
|
|
self.emit_newline(line);
|
|
true
|
|
}
|
|
|
|
// ===== Inline pass =====
|
|
//
|
|
// Inline content lives at byte offsets [start_col, end_col) within the
|
|
// current line. Span columns are relative to the line.
|
|
|
|
fn lex_inline(&mut self, slice: &str, line_col_start: u32, _line_col_end: u32) {
|
|
let bytes = slice.as_bytes();
|
|
let mut buf = String::new();
|
|
let mut buf_start: Option<u32> = None;
|
|
let mut i = 0usize;
|
|
|
|
let flush = |this: &mut Self, buf: &mut String, buf_start: &mut Option<u32>, end: u32| {
|
|
if !buf.is_empty() {
|
|
let s = buf_start.take().unwrap();
|
|
this.emit(VimwikiTokenKind::Text(std::mem::take(buf)), s, end);
|
|
}
|
|
};
|
|
|
|
while i < bytes.len() {
|
|
let abs_col = line_col_start + i as u32;
|
|
let rest = &slice[i..];
|
|
|
|
// 1) Multi-char delimiters and constructs (longest match wins).
|
|
if let Some(after_open) = rest.strip_prefix("[[") {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
self.emit(VimwikiTokenKind::WikiLinkOpen, abs_col, abs_col + 2);
|
|
i += 2;
|
|
// Lex until the closing `]]`. A single `]` inside the body —
|
|
// e.g. a `[TICKET-1]` in the description — is literal text, so
|
|
// we balance inner `[ ]` rather than stopping at the first `]]`.
|
|
let close_abs_in_slice = find_wikilink_close(after_open).map(|c| i + c);
|
|
let inner_end = close_abs_in_slice.unwrap_or(slice.len());
|
|
self.lex_link_body(slice, i, line_col_start, inner_end, true);
|
|
i = inner_end;
|
|
if close_abs_in_slice.is_some() {
|
|
let close_col = line_col_start + i as u32;
|
|
self.emit(VimwikiTokenKind::WikiLinkClose, close_col, close_col + 2);
|
|
i += 2;
|
|
}
|
|
continue;
|
|
}
|
|
if rest.starts_with("{{{") {
|
|
// Inline {{{ is rare; fall through as text.
|
|
buf_start.get_or_insert(abs_col);
|
|
buf.push('{');
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if rest.starts_with("{{") {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
self.emit(VimwikiTokenKind::TransclusionOpen, abs_col, abs_col + 2);
|
|
i += 2;
|
|
let close_rel = slice[i..].find("}}");
|
|
let inner_end_rel = close_rel.map(|c| i + c).unwrap_or(slice.len());
|
|
self.lex_link_body(slice, i, line_col_start, inner_end_rel, false);
|
|
i = inner_end_rel;
|
|
if close_rel.is_some() {
|
|
let close_col = line_col_start + i as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::TransclusionClose,
|
|
close_col,
|
|
close_col + 2,
|
|
);
|
|
i += 2;
|
|
}
|
|
continue;
|
|
}
|
|
if rest.starts_with("~~") {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
self.emit(VimwikiTokenKind::StrikethroughDelim, abs_col, abs_col + 2);
|
|
i += 2;
|
|
continue;
|
|
}
|
|
if rest.starts_with(",,") {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
self.emit(VimwikiTokenKind::SubscriptDelim, abs_col, abs_col + 2);
|
|
i += 2;
|
|
continue;
|
|
}
|
|
|
|
// 2) Inline code: `...`
|
|
if bytes[i] == b'`' {
|
|
if let Some(end_rel) = slice[i + 1..].find('`') {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
let content = &slice[i + 1..i + 1 + end_rel];
|
|
let span_end = abs_col + (end_rel + 2) as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::Code(content.to_owned()),
|
|
abs_col,
|
|
span_end,
|
|
);
|
|
i += end_rel + 2;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// 3) Inline math: $...$ (single $ on each side; bail to text if unmatched)
|
|
if bytes[i] == b'$' {
|
|
if let Some(end_rel) = slice[i + 1..].find('$') {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
let content = &slice[i + 1..i + 1 + end_rel];
|
|
let span_end = abs_col + (end_rel + 2) as u32;
|
|
self.emit(
|
|
VimwikiTokenKind::MathInline(content.to_owned()),
|
|
abs_col,
|
|
span_end,
|
|
);
|
|
i += end_rel + 2;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// 4) Single-char delimiters.
|
|
let single = match bytes[i] {
|
|
b'*' => Some(VimwikiTokenKind::BoldDelim),
|
|
b'_' => Some(VimwikiTokenKind::ItalicDelim),
|
|
b'^' => Some(VimwikiTokenKind::SuperscriptDelim),
|
|
_ => None,
|
|
};
|
|
if let Some(kind) = single {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
self.emit(kind, abs_col, abs_col + 1);
|
|
i += 1;
|
|
continue;
|
|
}
|
|
|
|
// 5) Raw URL — at a word boundary.
|
|
if is_word_boundary(bytes, i) {
|
|
if let Some(url_len) = match_url(rest) {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
let url = &rest[..url_len];
|
|
self.emit(
|
|
VimwikiTokenKind::RawUrl(url.to_owned()),
|
|
abs_col,
|
|
abs_col + url_len as u32,
|
|
);
|
|
i += url_len;
|
|
continue;
|
|
}
|
|
// 6) Keyword.
|
|
if let Some((kw_len, kw)) = match_keyword(rest) {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
self.emit(
|
|
VimwikiTokenKind::Keyword(kw),
|
|
abs_col,
|
|
abs_col + kw_len as u32,
|
|
);
|
|
i += kw_len;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// 7) Default: accumulate into Text. Walk to next byte boundary.
|
|
let ch = slice[i..].chars().next().unwrap();
|
|
let ch_len = ch.len_utf8();
|
|
buf_start.get_or_insert(abs_col);
|
|
buf.push(ch);
|
|
i += ch_len;
|
|
}
|
|
|
|
let end_col = line_col_start + slice.len() as u32;
|
|
flush(self, &mut buf, &mut buf_start, end_col);
|
|
}
|
|
|
|
/// Lex inline content within `[[...]]` or `{{...}}`, where `|` is a
|
|
/// separator rather than literal pipe. `is_wikilink` chooses the token
|
|
/// variant for the separator.
|
|
fn lex_link_body(
|
|
&mut self,
|
|
slice: &str,
|
|
start: usize,
|
|
line_col_start: u32,
|
|
end: usize,
|
|
is_wikilink: bool,
|
|
) {
|
|
let bytes = slice.as_bytes();
|
|
let mut buf = String::new();
|
|
let mut buf_start: Option<u32> = None;
|
|
let mut i = start;
|
|
|
|
let flush =
|
|
|this: &mut Self, buf: &mut String, buf_start: &mut Option<u32>, end_col: u32| {
|
|
if !buf.is_empty() {
|
|
let s = buf_start.take().unwrap();
|
|
this.emit(VimwikiTokenKind::Text(std::mem::take(buf)), s, end_col);
|
|
}
|
|
};
|
|
|
|
while i < end {
|
|
let abs_col = line_col_start + i as u32;
|
|
if bytes[i] == b'|' {
|
|
flush(self, &mut buf, &mut buf_start, abs_col);
|
|
let kind = if is_wikilink {
|
|
VimwikiTokenKind::WikiLinkSep
|
|
} else {
|
|
VimwikiTokenKind::TransclusionSep
|
|
};
|
|
self.emit(kind, abs_col, abs_col + 1);
|
|
i += 1;
|
|
continue;
|
|
}
|
|
let ch = slice[i..].chars().next().unwrap();
|
|
let ch_len = ch.len_utf8();
|
|
buf_start.get_or_insert(abs_col);
|
|
buf.push(ch);
|
|
i += ch_len;
|
|
}
|
|
flush(self, &mut buf, &mut buf_start, line_col_start + end as u32);
|
|
}
|
|
}
|
|
|
|
// ===== Helpers =====
|
|
|
|
/// Parse the bit after `{{{` on a fence line. Format is forgiving:
|
|
/// `lang` (single word) followed by optional `key=value` pairs separated
|
|
/// by spaces.
|
|
/// Byte offset of the closing `]]` for a wikilink body that begins at the start
|
|
/// of `s` (just after the opening `[[`). Inner `[ ]` pairs are balanced so a
|
|
/// bracketed description like `[[page|Task [B-1]]]` keeps the `[B-1]` and closes
|
|
/// at the final `]]`. A stray single `]` (no matching `[`) is treated as literal
|
|
/// body text. Returns `None` if no closing `]]` is found.
|
|
fn find_wikilink_close(s: &str) -> Option<usize> {
|
|
let b = s.as_bytes();
|
|
let mut depth: u32 = 0;
|
|
let mut i = 0;
|
|
while i < b.len() {
|
|
match b[i] {
|
|
b'[' => depth += 1,
|
|
b']' if depth > 0 => depth -= 1, // closes an inner '['
|
|
b']' if i + 1 < b.len() && b[i + 1] == b']' => return Some(i),
|
|
_ => {}
|
|
}
|
|
i += 1;
|
|
}
|
|
None
|
|
}
|
|
|
|
fn parse_fence_attrs(s: &str) -> (Option<String>, HashMap<String, String>) {
|
|
let mut attrs = HashMap::new();
|
|
let mut language: Option<String> = None;
|
|
for tok in s.split_whitespace() {
|
|
if let Some(eq) = tok.find('=') {
|
|
let (k, v) = tok.split_at(eq);
|
|
let v = &v[1..];
|
|
// Strip optional surrounding quotes from value.
|
|
let v = v.trim_matches('"').trim_matches('\'');
|
|
attrs.insert(k.to_owned(), v.to_owned());
|
|
} else if language.is_none() {
|
|
language = Some(tok.to_owned());
|
|
}
|
|
}
|
|
(language, attrs)
|
|
}
|
|
|
|
fn is_word_boundary(bytes: &[u8], i: usize) -> bool {
|
|
if i == 0 {
|
|
return true;
|
|
}
|
|
let prev = bytes[i - 1];
|
|
!(prev.is_ascii_alphanumeric() || prev == b'_')
|
|
}
|
|
|
|
fn is_word_end(bytes: &[u8], i: usize) -> bool {
|
|
if i >= bytes.len() {
|
|
return true;
|
|
}
|
|
let next = bytes[i];
|
|
!(next.is_ascii_alphanumeric() || next == b'_')
|
|
}
|
|
|
|
const URL_SCHEMES: &[&str] = &["https://", "http://", "ftp://", "mailto:", "file://"];
|
|
|
|
fn match_url(rest: &str) -> Option<usize> {
|
|
let bytes = rest.as_bytes();
|
|
let scheme_len = URL_SCHEMES.iter().find_map(|s| {
|
|
if rest.starts_with(s) {
|
|
Some(s.len())
|
|
} else {
|
|
None
|
|
}
|
|
})?;
|
|
let mut end = scheme_len;
|
|
while end < bytes.len() {
|
|
let b = bytes[end];
|
|
// Conservative URL char set; stops at whitespace and most punctuation
|
|
// that's typically not part of a URL tail.
|
|
let ok = b.is_ascii_alphanumeric()
|
|
|| matches!(
|
|
b,
|
|
b'-' | b'_'
|
|
| b'.'
|
|
| b'~'
|
|
| b'/'
|
|
| b':'
|
|
| b'?'
|
|
| b'#'
|
|
| b'['
|
|
| b']'
|
|
| b'@'
|
|
| b'!'
|
|
| b'$'
|
|
| b'&'
|
|
| b'\''
|
|
| b'('
|
|
| b')'
|
|
| b'*'
|
|
| b'+'
|
|
| b','
|
|
| b';'
|
|
| b'='
|
|
| b'%'
|
|
);
|
|
if !ok {
|
|
break;
|
|
}
|
|
end += 1;
|
|
}
|
|
// Trim trailing punctuation that's almost always sentence punctuation.
|
|
while end > scheme_len {
|
|
match bytes[end - 1] {
|
|
b'.' | b',' | b';' | b':' | b'!' | b'?' | b')' | b']' => end -= 1,
|
|
_ => break,
|
|
}
|
|
}
|
|
if end <= scheme_len {
|
|
None
|
|
} else {
|
|
Some(end)
|
|
}
|
|
}
|
|
|
|
const KEYWORDS: &[(&str, Keyword)] = &[
|
|
("TODO", Keyword::Todo),
|
|
("DONE", Keyword::Done),
|
|
("STARTED", Keyword::Started),
|
|
("FIXME", Keyword::Fixme),
|
|
("FIXED", Keyword::Fixed),
|
|
("XXX", Keyword::Xxx),
|
|
("STOPPED", Keyword::Stopped),
|
|
];
|
|
|
|
fn match_keyword(rest: &str) -> Option<(usize, Keyword)> {
|
|
let bytes = rest.as_bytes();
|
|
KEYWORDS.iter().find_map(|(text, kw)| {
|
|
if rest.starts_with(text) && is_word_end(bytes, text.len()) {
|
|
Some((text.len(), *kw))
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
}
|
|
|
|
fn list_marker_at(s: &str) -> Option<(ListSymbol, usize)> {
|
|
let bytes = s.as_bytes();
|
|
if bytes.is_empty() {
|
|
return None;
|
|
}
|
|
// Single-char markers: -, *, #
|
|
match bytes[0] {
|
|
b'-' => return Some((ListSymbol::Dash, 1)),
|
|
b'*' => return Some((ListSymbol::Star, 1)),
|
|
b'#' => return Some((ListSymbol::Hash, 1)),
|
|
_ => {}
|
|
}
|
|
// Numeric: `1.` or `1)`. Allow multi-digit.
|
|
let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count();
|
|
if digits > 0 && bytes.len() > digits {
|
|
match bytes[digits] {
|
|
b'.' => return Some((ListSymbol::Numeric, digits + 1)),
|
|
b')' => return Some((ListSymbol::NumericParen, digits + 1)),
|
|
_ => {}
|
|
}
|
|
}
|
|
// Single-letter alpha or roman: `a)`, `A)`, `i)`, `I)`.
|
|
if bytes.len() >= 2 && bytes[1] == b')' {
|
|
let c = bytes[0];
|
|
if c.is_ascii_lowercase() {
|
|
// `i)` / `v)` / `x)` are roman by convention; without context we
|
|
// treat any single lowercase letter as alpha. Roman vs alpha is
|
|
// ambiguous from a single char; vimwiki itself relies on context.
|
|
// Mark `i` as RomanParen as a small concession to the spec.
|
|
let sym = if c == b'i' {
|
|
ListSymbol::RomanParen
|
|
} else {
|
|
ListSymbol::AlphaParen
|
|
};
|
|
return Some((sym, 2));
|
|
}
|
|
if c.is_ascii_uppercase() {
|
|
let sym = if c == b'I' {
|
|
ListSymbol::RomanUpperParen
|
|
} else {
|
|
ListSymbol::AlphaUpperParen
|
|
};
|
|
return Some((sym, 2));
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn looks_like_list_marker(s: &str) -> bool {
|
|
list_marker_at(s).is_some_and(|(_, len)| s.as_bytes().get(len) == Some(&b' '))
|
|
}
|
|
|
|
/// Is this cell content a valid separator-cell body? Allows leading /
|
|
/// trailing `:` for alignment and inner runs of `-` plus whitespace.
|
|
fn is_sep_cell(cell: &str) -> bool {
|
|
let t = cell.trim();
|
|
if t.is_empty() {
|
|
return false;
|
|
}
|
|
let body = t.trim_start_matches(':').trim_end_matches(':');
|
|
!body.is_empty() && body.chars().all(|c| c == '-')
|
|
}
|
|
|
|
fn parse_sep_alignment(cell: &str) -> crate::ast::TableAlign {
|
|
let t = cell.trim();
|
|
let l = t.starts_with(':');
|
|
let r = t.ends_with(':');
|
|
match (l, r) {
|
|
(true, true) => crate::ast::TableAlign::Center,
|
|
(true, false) => crate::ast::TableAlign::Left,
|
|
(false, true) => crate::ast::TableAlign::Right,
|
|
_ => crate::ast::TableAlign::Default,
|
|
}
|
|
}
|
|
|
|
/// Recognise a `[g]` checkbox at the start of `s`, where `g` is a single
|
|
/// glyph drawn from `listsyms` (or the rejected marker). Returns the bucketed
|
|
/// state and the byte length consumed (including the brackets).
|
|
fn checkbox_at(s: &str, listsyms: &ListSyms) -> Option<(CheckboxState, usize)> {
|
|
let rest = s.strip_prefix('[')?;
|
|
let mut chars = rest.char_indices();
|
|
let (_, glyph) = chars.next()?;
|
|
let (close_off, ']') = chars.next()? else {
|
|
return None;
|
|
};
|
|
let state = listsyms.state_of(glyph)?;
|
|
// `[` + glyph + `]`: 1 + glyph.len_utf8() + 1, i.e. the close bracket
|
|
// offset within `rest` plus one for the opening bracket and one for `]`.
|
|
Some((state, 1 + close_off + 1))
|
|
}
|