fix(lexer): don't split table cells on | inside links/code

The table-row lexer collected every `|` as a cell separator, so a pipe
inside a `[[url|title]]` wikilink (or `{{…}}` transclusion / `` `…` ``
inline code) was mistaken for a column boundary — mangling the link and
adding a phantom column in the AST and HTML export.

Replace the naive scan in `try_lex_table_row` with `cell_bar_positions`,
which skips pipes inside those constructs (matching the regions
`lex_inline` tokenises as a unit). Unclosed constructs fall back to
literal openers so a malformed cell can't swallow the rest of the row,
and the scan only slices at ASCII openers to stay UTF-8 safe.

Backslash-escaped `\|` is intentionally left unhandled: the inline lexer
has no escape handling, so honouring it here would leave a stray
backslash in the output. Documented as a matched follow-up.

Adds lexer + HTML-export regression tests.

Closes #1

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lzhq43v1qpkZczN65dqXa3
This commit is contained in:
2026-07-02 12:30:15 +00:00
parent 7da13ffc5c
commit 18f1ddfe7f
3 changed files with 104 additions and 9 deletions
+49 -9
View File
@@ -617,15 +617,10 @@ impl<'src> LexState<'src> {
end -= 1;
}
// Collect cell boundaries.
let mut sep_positions = Vec::new();
let mut k = i;
while k < end {
if bytes[k] == b'|' {
sep_positions.push(k);
}
k += 1;
}
// Collect cell boundaries. A `|` inside a `[[…]]` wiklink, a `{{…}}`
// transclusion, or `` `…` `` inline code is literal cell content, not
// a separator, so those regions are skipped (see `cell_bar_positions`).
let sep_positions = cell_bar_positions(line, i, end);
if sep_positions.len() < 2 {
return false;
@@ -1068,6 +1063,51 @@ impl<'src> LexState<'src> {
/// bracketed description like `[[page|Task [B-1]]]` keeps the `[B-1]` and closes
/// at the final `]]`. A stray single `]` (no matching `[`) is treated as literal
/// body text. Returns `None` if no closing `]]` is found.
/// Byte offsets of the `|` characters in `line[i..end]` that act as
/// table-cell separators. Pipes inside a `[[…]]` wikilink, a `{{…}}`
/// transclusion, or `` `…` `` inline code are literal cell content — they
/// mirror the regions [`VimwikiLexer::lex_inline`] tokenises as a single unit
/// — so they are not reported as separators. An *unclosed* construct is
/// treated as a literal opener (it does not start a skip region), so a
/// malformed cell can never swallow the rest of the row.
///
/// Note: a backslash-escaped `\|` is intentionally *not* honoured here,
/// because the inline lexer has no escape handling and would render the
/// backslash literally; supporting it would need a matching change there.
fn cell_bar_positions(line: &str, i: usize, end: usize) -> Vec<usize> {
let bytes = line.as_bytes();
let mut positions = Vec::new();
let mut k = i;
while k < end {
// These openers are ASCII, so `bytes[k]` never matches a UTF-8
// continuation byte and every slice below starts on a char boundary.
match bytes[k] {
b'[' if bytes.get(k + 1) == Some(&b'[') => {
if let Some(close) = find_wikilink_close(&line[k + 2..end]) {
k += close + 4; // `[[` + inner + `]]`
continue;
}
}
b'{' if bytes.get(k + 1) == Some(&b'{') => {
if let Some(close) = line[k + 2..end].find("}}") {
k += close + 4; // `{{` + inner + `}}`
continue;
}
}
b'`' => {
if let Some(close) = line[k + 1..end].find('`') {
k += close + 2; // opening + inner + closing backtick
continue;
}
}
b'|' => positions.push(k),
_ => {}
}
k += 1;
}
positions
}
fn find_wikilink_close(s: &str) -> Option<usize> {
let b = s.as_bytes();
let mut depth: u32 = 0;