From 18f1ddfe7f87eb205d49b84708e6f1894823d3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Fr=C3=B3es=20Franco?= Date: Thu, 2 Jul 2026 12:30:15 +0000 Subject: [PATCH] fix(lexer): don't split table cells on `|` inside links/code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The table-row lexer collected every `|` as a cell separator, so a pipe inside a `[[url|title]]` wikilink (or `{{…}}` transclusion / `` `…` `` inline code) was mistaken for a column boundary — mangling the link and adding a phantom column in the AST and HTML export. Replace the naive scan in `try_lex_table_row` with `cell_bar_positions`, which skips pipes inside those constructs (matching the regions `lex_inline` tokenises as a unit). Unclosed constructs fall back to literal openers so a malformed cell can't swallow the rest of the row, and the scan only slices at ASCII openers to stay UTF-8 safe. Backslash-escaped `\|` is intentionally left unhandled: the inline lexer has no escape handling, so honouring it here would leave a stray backslash in the output. Documented as a matched follow-up. Adds lexer + HTML-export regression tests. Closes #1 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Lzhq43v1qpkZczN65dqXa3 --- .../nuwiki-core/src/syntax/vimwiki/lexer.rs | 58 ++++++++++++++++--- crates/nuwiki-core/tests/html_renderer.rs | 13 +++++ crates/nuwiki-core/tests/lexer.rs | 42 ++++++++++++++ 3 files changed, 104 insertions(+), 9 deletions(-) diff --git a/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs b/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs index 2d4b1af..82b7400 100644 --- a/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs +++ b/crates/nuwiki-core/src/syntax/vimwiki/lexer.rs @@ -617,15 +617,10 @@ impl<'src> LexState<'src> { end -= 1; } - // Collect cell boundaries. - let mut sep_positions = Vec::new(); - let mut k = i; - while k < end { - if bytes[k] == b'|' { - sep_positions.push(k); - } - k += 1; - } + // Collect cell boundaries. A `|` inside a `[[…]]` wiklink, a `{{…}}` + // transclusion, or `` `…` `` inline code is literal cell content, not + // a separator, so those regions are skipped (see `cell_bar_positions`). + let sep_positions = cell_bar_positions(line, i, end); if sep_positions.len() < 2 { return false; @@ -1068,6 +1063,51 @@ impl<'src> LexState<'src> { /// bracketed description like `[[page|Task [B-1]]]` keeps the `[B-1]` and closes /// at the final `]]`. A stray single `]` (no matching `[`) is treated as literal /// body text. Returns `None` if no closing `]]` is found. +/// Byte offsets of the `|` characters in `line[i..end]` that act as +/// table-cell separators. Pipes inside a `[[…]]` wikilink, a `{{…}}` +/// transclusion, or `` `…` `` inline code are literal cell content — they +/// mirror the regions [`VimwikiLexer::lex_inline`] tokenises as a single unit +/// — so they are not reported as separators. An *unclosed* construct is +/// treated as a literal opener (it does not start a skip region), so a +/// malformed cell can never swallow the rest of the row. +/// +/// Note: a backslash-escaped `\|` is intentionally *not* honoured here, +/// because the inline lexer has no escape handling and would render the +/// backslash literally; supporting it would need a matching change there. +fn cell_bar_positions(line: &str, i: usize, end: usize) -> Vec { + let bytes = line.as_bytes(); + let mut positions = Vec::new(); + let mut k = i; + while k < end { + // These openers are ASCII, so `bytes[k]` never matches a UTF-8 + // continuation byte and every slice below starts on a char boundary. + match bytes[k] { + b'[' if bytes.get(k + 1) == Some(&b'[') => { + if let Some(close) = find_wikilink_close(&line[k + 2..end]) { + k += close + 4; // `[[` + inner + `]]` + continue; + } + } + b'{' if bytes.get(k + 1) == Some(&b'{') => { + if let Some(close) = line[k + 2..end].find("}}") { + k += close + 4; // `{{` + inner + `}}` + continue; + } + } + b'`' => { + if let Some(close) = line[k + 1..end].find('`') { + k += close + 2; // opening + inner + closing backtick + continue; + } + } + b'|' => positions.push(k), + _ => {} + } + k += 1; + } + positions +} + fn find_wikilink_close(s: &str) -> Option { let b = s.as_bytes(); let mut depth: u32 = 0; diff --git a/crates/nuwiki-core/tests/html_renderer.rs b/crates/nuwiki-core/tests/html_renderer.rs index 713728e..ec4ae4c 100644 --- a/crates/nuwiki-core/tests/html_renderer.rs +++ b/crates/nuwiki-core/tests/html_renderer.rs @@ -245,6 +245,19 @@ fn table_with_header_separator() { assert!(out.contains(" 1 ")); } +#[test] +fn table_cell_with_wikilink_pipe_exports_one_cell() { + // Regression: the `|` in `[[url|title]]` must stay inside the link, so the + // row keeps two columns and the link renders as a single anchor. Before the + // fix the cell split into `[[https://example.com` + `My Site]]`, adding a + // phantom and breaking the link. + let out = render("| Name | Link |\n|---|---|\n| foo | [[https://example.com|My Site]] |\n"); + assert!( + out.contains(" foo My Site "), + "got: {out}" + ); +} + #[test] fn single_comment_becomes_html_comment() { let out = render("%% hidden\n"); diff --git a/crates/nuwiki-core/tests/lexer.rs b/crates/nuwiki-core/tests/lexer.rs index b3aab42..6da2482 100644 --- a/crates/nuwiki-core/tests/lexer.rs +++ b/crates/nuwiki-core/tests/lexer.rs @@ -454,6 +454,48 @@ fn table_col_and_row_span_cells() { ); } +#[test] +fn table_cell_with_wikilink_pipe_is_not_split() { + // The `|` inside `[[url|title]]` is part of the link, not a cell + // boundary: the row must lex to exactly two cells (three TableSeps). + assert_eq!( + lex("| foo | [[https://example.com|My Site]] |\n"), + vec![ + TableSep, + text(" foo "), + TableSep, + text(" "), + WikiLinkOpen, + text("https://example.com"), + WikiLinkSep, + text("My Site"), + WikiLinkClose, + text(" "), + TableSep, + Newline, + ] + ); +} + +#[test] +fn table_cell_with_transclusion_and_code_pipes_are_not_split() { + // `|` inside `{{…}}` and inside `` `…` `` are both literal. + let lexed = lex("| {{a|b}} | `x|y` |\n"); + let seps = lexed.iter().filter(|t| matches!(t, TableSep)).count(); + assert_eq!(seps, 3, "expected 2 cells: {lexed:?}"); +} + +#[test] +fn table_cell_with_unclosed_wikilink_falls_back_to_pipe_split() { + // A malformed, unclosed `[[` must not swallow the rest of the row — + // the pipes still delimit cells. + let seps = lex("| [[oops | tail |\n") + .iter() + .filter(|t| matches!(t, TableSep)) + .count(); + assert_eq!(seps, 3); +} + // ===== Definition list ===== #[test]