fix(lexer): don't split table cells on | inside links/code
The table-row lexer collected every `|` as a cell separator, so a pipe
inside a `[[url|title]]` wikilink (or `{{…}}` transclusion / `` `…` ``
inline code) was mistaken for a column boundary — mangling the link and
adding a phantom column in the AST and HTML export.
Replace the naive scan in `try_lex_table_row` with `cell_bar_positions`,
which skips pipes inside those constructs (matching the regions
`lex_inline` tokenises as a unit). Unclosed constructs fall back to
literal openers so a malformed cell can't swallow the rest of the row,
and the scan only slices at ASCII openers to stay UTF-8 safe.
Backslash-escaped `\|` is intentionally left unhandled: the inline lexer
has no escape handling, so honouring it here would leave a stray
backslash in the output. Documented as a matched follow-up.
Adds lexer + HTML-export regression tests.
Closes #1
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lzhq43v1qpkZczN65dqXa3
This commit is contained in:
@@ -617,15 +617,10 @@ impl<'src> LexState<'src> {
|
|||||||
end -= 1;
|
end -= 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Collect cell boundaries.
|
// Collect cell boundaries. A `|` inside a `[[…]]` wiklink, a `{{…}}`
|
||||||
let mut sep_positions = Vec::new();
|
// transclusion, or `` `…` `` inline code is literal cell content, not
|
||||||
let mut k = i;
|
// a separator, so those regions are skipped (see `cell_bar_positions`).
|
||||||
while k < end {
|
let sep_positions = cell_bar_positions(line, i, end);
|
||||||
if bytes[k] == b'|' {
|
|
||||||
sep_positions.push(k);
|
|
||||||
}
|
|
||||||
k += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if sep_positions.len() < 2 {
|
if sep_positions.len() < 2 {
|
||||||
return false;
|
return false;
|
||||||
@@ -1068,6 +1063,51 @@ impl<'src> LexState<'src> {
|
|||||||
/// bracketed description like `[[page|Task [B-1]]]` keeps the `[B-1]` and closes
|
/// bracketed description like `[[page|Task [B-1]]]` keeps the `[B-1]` and closes
|
||||||
/// at the final `]]`. A stray single `]` (no matching `[`) is treated as literal
|
/// at the final `]]`. A stray single `]` (no matching `[`) is treated as literal
|
||||||
/// body text. Returns `None` if no closing `]]` is found.
|
/// body text. Returns `None` if no closing `]]` is found.
|
||||||
|
/// Byte offsets of the `|` characters in `line[i..end]` that act as
|
||||||
|
/// table-cell separators. Pipes inside a `[[…]]` wikilink, a `{{…}}`
|
||||||
|
/// transclusion, or `` `…` `` inline code are literal cell content — they
|
||||||
|
/// mirror the regions [`VimwikiLexer::lex_inline`] tokenises as a single unit
|
||||||
|
/// — so they are not reported as separators. An *unclosed* construct is
|
||||||
|
/// treated as a literal opener (it does not start a skip region), so a
|
||||||
|
/// malformed cell can never swallow the rest of the row.
|
||||||
|
///
|
||||||
|
/// Note: a backslash-escaped `\|` is intentionally *not* honoured here,
|
||||||
|
/// because the inline lexer has no escape handling and would render the
|
||||||
|
/// backslash literally; supporting it would need a matching change there.
|
||||||
|
fn cell_bar_positions(line: &str, i: usize, end: usize) -> Vec<usize> {
|
||||||
|
let bytes = line.as_bytes();
|
||||||
|
let mut positions = Vec::new();
|
||||||
|
let mut k = i;
|
||||||
|
while k < end {
|
||||||
|
// These openers are ASCII, so `bytes[k]` never matches a UTF-8
|
||||||
|
// continuation byte and every slice below starts on a char boundary.
|
||||||
|
match bytes[k] {
|
||||||
|
b'[' if bytes.get(k + 1) == Some(&b'[') => {
|
||||||
|
if let Some(close) = find_wikilink_close(&line[k + 2..end]) {
|
||||||
|
k += close + 4; // `[[` + inner + `]]`
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b'{' if bytes.get(k + 1) == Some(&b'{') => {
|
||||||
|
if let Some(close) = line[k + 2..end].find("}}") {
|
||||||
|
k += close + 4; // `{{` + inner + `}}`
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b'`' => {
|
||||||
|
if let Some(close) = line[k + 1..end].find('`') {
|
||||||
|
k += close + 2; // opening + inner + closing backtick
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b'|' => positions.push(k),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
k += 1;
|
||||||
|
}
|
||||||
|
positions
|
||||||
|
}
|
||||||
|
|
||||||
fn find_wikilink_close(s: &str) -> Option<usize> {
|
fn find_wikilink_close(s: &str) -> Option<usize> {
|
||||||
let b = s.as_bytes();
|
let b = s.as_bytes();
|
||||||
let mut depth: u32 = 0;
|
let mut depth: u32 = 0;
|
||||||
|
|||||||
@@ -245,6 +245,19 @@ fn table_with_header_separator() {
|
|||||||
assert!(out.contains("<td> 1 </td>"));
|
assert!(out.contains("<td> 1 </td>"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn table_cell_with_wikilink_pipe_exports_one_cell() {
|
||||||
|
// Regression: the `|` in `[[url|title]]` must stay inside the link, so the
|
||||||
|
// row keeps two columns and the link renders as a single anchor. Before the
|
||||||
|
// fix the cell split into `[[https://example.com` + `My Site]]`, adding a
|
||||||
|
// phantom <td> and breaking the link.
|
||||||
|
let out = render("| Name | Link |\n|---|---|\n| foo | [[https://example.com|My Site]] |\n");
|
||||||
|
assert!(
|
||||||
|
out.contains("<td> foo </td><td> <a href=\"https://example.com\">My Site</a> </td>"),
|
||||||
|
"got: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn single_comment_becomes_html_comment() {
|
fn single_comment_becomes_html_comment() {
|
||||||
let out = render("%% hidden\n");
|
let out = render("%% hidden\n");
|
||||||
|
|||||||
@@ -454,6 +454,48 @@ fn table_col_and_row_span_cells() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn table_cell_with_wikilink_pipe_is_not_split() {
|
||||||
|
// The `|` inside `[[url|title]]` is part of the link, not a cell
|
||||||
|
// boundary: the row must lex to exactly two cells (three TableSeps).
|
||||||
|
assert_eq!(
|
||||||
|
lex("| foo | [[https://example.com|My Site]] |\n"),
|
||||||
|
vec![
|
||||||
|
TableSep,
|
||||||
|
text(" foo "),
|
||||||
|
TableSep,
|
||||||
|
text(" "),
|
||||||
|
WikiLinkOpen,
|
||||||
|
text("https://example.com"),
|
||||||
|
WikiLinkSep,
|
||||||
|
text("My Site"),
|
||||||
|
WikiLinkClose,
|
||||||
|
text(" "),
|
||||||
|
TableSep,
|
||||||
|
Newline,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn table_cell_with_transclusion_and_code_pipes_are_not_split() {
|
||||||
|
// `|` inside `{{…}}` and inside `` `…` `` are both literal.
|
||||||
|
let lexed = lex("| {{a|b}} | `x|y` |\n");
|
||||||
|
let seps = lexed.iter().filter(|t| matches!(t, TableSep)).count();
|
||||||
|
assert_eq!(seps, 3, "expected 2 cells: {lexed:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn table_cell_with_unclosed_wikilink_falls_back_to_pipe_split() {
|
||||||
|
// A malformed, unclosed `[[` must not swallow the rest of the row —
|
||||||
|
// the pipes still delimit cells.
|
||||||
|
let seps = lex("| [[oops | tail |\n")
|
||||||
|
.iter()
|
||||||
|
.filter(|t| matches!(t, TableSep))
|
||||||
|
.count();
|
||||||
|
assert_eq!(seps, 3);
|
||||||
|
}
|
||||||
|
|
||||||
// ===== Definition list =====
|
// ===== Definition list =====
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
Reference in New Issue
Block a user