phase 2: syntax plugin interface and registry

Add the Lexer/Parser/SyntaxPlugin traits and the SyntaxRegistry per
SPEC.md §6.3.

- `Lexer<Token>` and `Parser<Token>` are typed building blocks: each
  syntax owns its own token alphabet (vimwiki today, markdown later).
- `TokenStream<T>` is a `Vec<T>` newtype so the eager-vs-lazy choice
  (§6.6) can flip later without breaking callers.
- `SyntaxPlugin` is type-erased: it exposes id / display_name /
  file_extensions / parse(text) so the registry can hold heterogeneous
  plugins behind a single trait object. Implementations chain their
  Lexer + Parser inside parse().
- `SyntaxPlugin: Send + Sync` so the LSP server can stash plugins in
  an Arc and share them across handler tasks.
- `SyntaxRegistry` stores `Arc<dyn SyntaxPlugin>`, supports lookup by
  id and by file extension, iteration in registration order.

Tests cover TokenStream round-trip, full lex→parse chain through a
mock plugin, registry lookup hits and misses, and a compile-time
assert_send_sync::<SyntaxRegistry>().

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

SPEC.md

@@ -1,7 +1,7 @@
 # nuwiki — Project Specification
 
 > Last updated: 2026-05-10
-> Status: Phase 1 (Core AST) complete — moving to Phase 2 (Syntax Plugin Interface)
+> Status: Phase 2 (Syntax Plugin Interface) complete — moving to Phase 3 (Vimwiki Lexer)
 
 ---
 

crates/nuwiki-core/src/lib.rs

@@ -4,3 +4,4 @@
 //! `nuwiki-ls`. See SPEC.md §6.2 for the dependency rules.
 
 pub mod ast;
+pub mod syntax;

crates/nuwiki-core/src/syntax/mod.rs

@@ -0,0 +1,116 @@
+//! Syntax plugin interface.
+//!
+//! Each syntax (vimwiki today, markdown later) implements the same shape:
+//! a `Lexer` produces a `TokenStream`, a `Parser` consumes it and produces a
+//! `DocumentNode`. A `SyntaxPlugin` is the type-erased facade that the LSP
+//! layer holds in a `SyntaxRegistry`.
+//!
+//! See SPEC.md §6.3 (interface) and §6.6 (lexer strategy).
+
+pub mod registry;
+
+pub use registry::SyntaxRegistry;
+
+use crate::ast::DocumentNode;
+
+/// Eager token buffer produced by a `Lexer`. The newtype keeps the door
+/// open to a lazy/streaming variant later (SPEC.md §6.6) without breaking
+/// callers.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct TokenStream<T> {
+    tokens: Vec<T>,
+}
+
+impl<T> TokenStream<T> {
+    pub fn new() -> Self {
+        Self { tokens: Vec::new() }
+    }
+
+    pub fn from_vec(tokens: Vec<T>) -> Self {
+        Self { tokens }
+    }
+
+    pub fn into_vec(self) -> Vec<T> {
+        self.tokens
+    }
+
+    pub fn as_slice(&self) -> &[T] {
+        &self.tokens
+    }
+
+    pub fn len(&self) -> usize {
+        self.tokens.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.tokens.is_empty()
+    }
+
+    pub fn iter(&self) -> std::slice::Iter<'_, T> {
+        self.tokens.iter()
+    }
+
+    pub fn push(&mut self, token: T) {
+        self.tokens.push(token);
+    }
+}
+
+impl<T> From<Vec<T>> for TokenStream<T> {
+    fn from(tokens: Vec<T>) -> Self {
+        Self::from_vec(tokens)
+    }
+}
+
+impl<T> IntoIterator for TokenStream<T> {
+    type Item = T;
+    type IntoIter = std::vec::IntoIter<T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.tokens.into_iter()
+    }
+}
+
+impl<'a, T> IntoIterator for &'a TokenStream<T> {
+    type Item = &'a T;
+    type IntoIter = std::slice::Iter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.tokens.iter()
+    }
+}
+
+/// Lexer half of a syntax plugin. Operates on raw source text and emits a
+/// `TokenStream` whose token type is syntax-specific (e.g. `VimwikiToken`).
+pub trait Lexer {
+    type Token;
+
+    fn lex(&self, text: &str) -> TokenStream<Self::Token>;
+}
+
+/// Parser half of a syntax plugin. Consumes a `TokenStream` and produces a
+/// `DocumentNode`. Per SPEC.md §6.7 parsing is resilient — malformed input
+/// becomes `BlockNode::Error(ErrorNode)`, never a hard failure.
+pub trait Parser {
+    type Token;
+
+    fn parse(&self, tokens: TokenStream<Self::Token>) -> DocumentNode;
+}
+
+/// Type-erased syntax plugin held by the `SyntaxRegistry`.
+///
+/// Implementations typically own a `Lexer` and a `Parser` and chain them
+/// inside `parse`. The token type is deliberately hidden so the registry can
+/// hold heterogeneous plugins behind a single trait object.
+///
+/// `Send + Sync` is required — the LSP server stores plugins behind an `Arc`
+/// and shares them across request handler tasks (SPEC.md §6.3).
+pub trait SyntaxPlugin: Send + Sync {
+    fn id(&self) -> &str;
+    fn display_name(&self) -> &str;
+
+    /// File extensions associated with this syntax. Each entry includes the
+    /// leading dot, e.g. `[".wiki"]`. See SPEC.md §6.3.
+    fn file_extensions(&self) -> &[&str];
+
+    fn parse(&self, text: &str) -> DocumentNode;
+}

crates/nuwiki-core/src/syntax/registry.rs

@@ -0,0 +1,64 @@
+//! Registry of syntax plugins. See SPEC.md §6.3.
+//!
+//! Plugins are stored behind `Arc<dyn SyntaxPlugin>` so the LSP layer can
+//! hand a cheap clone to per-document tasks without re-locking the registry.
+
+use std::sync::Arc;
+
+use super::SyntaxPlugin;
+
+#[derive(Default, Clone)]
+pub struct SyntaxRegistry {
+    plugins: Vec<Arc<dyn SyntaxPlugin>>,
+}
+
+impl SyntaxRegistry {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Register a plugin owned directly. The most common shape — implementer
+    /// hands over a value, registry takes ownership.
+    pub fn register<P>(&mut self, plugin: P)
+    where
+        P: SyntaxPlugin + 'static,
+    {
+        self.plugins.push(Arc::new(plugin));
+    }
+
+    /// Register a pre-arc'd plugin. Useful when the same plugin instance
+    /// also needs to be held elsewhere.
+    pub fn register_arc(&mut self, plugin: Arc<dyn SyntaxPlugin>) {
+        self.plugins.push(plugin);
+    }
+
+    /// Look up a plugin by its `id`.
+    pub fn get(&self, id: &str) -> Option<&dyn SyntaxPlugin> {
+        self.plugins
+            .iter()
+            .find(|p| p.id() == id)
+            .map(|p| p.as_ref())
+    }
+
+    /// Look up a plugin by file extension. `ext` should include the leading
+    /// dot (e.g. `".wiki"`) — matches the plugin's `file_extensions()`.
+    pub fn detect_from_extension(&self, ext: &str) -> Option<&dyn SyntaxPlugin> {
+        self.plugins
+            .iter()
+            .find(|p| p.file_extensions().iter().any(|e| *e == ext))
+            .map(|p| p.as_ref())
+    }
+
+    /// Iterate over registered plugins in registration order.
+    pub fn iter(&self) -> impl Iterator<Item = &dyn SyntaxPlugin> + '_ {
+        self.plugins.iter().map(|p| p.as_ref())
+    }
+
+    pub fn len(&self) -> usize {
+        self.plugins.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.plugins.is_empty()
+    }
+}

crates/nuwiki-core/tests/syntax.rs

@@ -0,0 +1,144 @@
+//! Integration tests for the syntax plugin interface.
+
+use nuwiki_core::ast::{
+    inline::InlineNode, BlockNode, DocumentNode, ParagraphNode, Span, TextNode,
+};
+use nuwiki_core::syntax::{Lexer, Parser, SyntaxPlugin, SyntaxRegistry, TokenStream};
+
+// --- A minimal mock plugin: tokenises the input on whitespace, then wraps
+// the joined text in a single Paragraph. Just enough surface to exercise
+// every trait + the registry.
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct MockToken(String);
+
+struct MockLexer;
+impl Lexer for MockLexer {
+    type Token = MockToken;
+
+    fn lex(&self, text: &str) -> TokenStream<MockToken> {
+        TokenStream::from_vec(
+            text.split_whitespace()
+                .map(|w| MockToken(w.to_owned()))
+                .collect(),
+        )
+    }
+}
+
+struct MockParser;
+impl Parser for MockParser {
+    type Token = MockToken;
+
+    fn parse(&self, tokens: TokenStream<MockToken>) -> DocumentNode {
+        let joined = tokens
+            .iter()
+            .map(|t| t.0.as_str())
+            .collect::<Vec<_>>()
+            .join(" ");
+        let paragraph = BlockNode::Paragraph(ParagraphNode {
+            span: Span::default(),
+            children: vec![InlineNode::Text(TextNode {
+                span: Span::default(),
+                content: joined,
+            })],
+        });
+        DocumentNode {
+            span: Span::default(),
+            children: vec![paragraph],
+            metadata: Default::default(),
+        }
+    }
+}
+
+struct MockPlugin;
+impl SyntaxPlugin for MockPlugin {
+    fn id(&self) -> &str {
+        "mock"
+    }
+    fn display_name(&self) -> &str {
+        "Mock Syntax"
+    }
+    fn file_extensions(&self) -> &[&str] {
+        &[".mock", ".mk"]
+    }
+    fn parse(&self, text: &str) -> DocumentNode {
+        MockParser.parse(MockLexer.lex(text))
+    }
+}
+
+#[test]
+fn token_stream_roundtrip() {
+    let mut s = TokenStream::<u32>::new();
+    assert!(s.is_empty());
+    s.push(1);
+    s.push(2);
+    s.push(3);
+    assert_eq!(s.len(), 3);
+    assert_eq!(s.as_slice(), &[1, 2, 3]);
+    assert_eq!(s.iter().sum::<u32>(), 6);
+    assert_eq!((&s).into_iter().sum::<u32>(), 6);
+    assert_eq!(s.into_vec(), vec![1, 2, 3]);
+
+    let from_vec: TokenStream<&str> = vec!["a", "b"].into();
+    assert_eq!(from_vec.len(), 2);
+    let collected: Vec<&str> = from_vec.into_iter().collect();
+    assert_eq!(collected, vec!["a", "b"]);
+}
+
+#[test]
+fn lexer_and_parser_chain_through_plugin() {
+    let plugin = MockPlugin;
+    let doc = plugin.parse("  hello   world  ");
+
+    let BlockNode::Paragraph(p) = &doc.children[0] else {
+        panic!("expected a paragraph, got {:?}", doc.children[0]);
+    };
+    let InlineNode::Text(t) = &p.children[0] else {
+        panic!("expected a text node");
+    };
+    assert_eq!(t.content, "hello world");
+}
+
+#[test]
+fn registry_lookup_by_id_and_extension() {
+    let mut registry = SyntaxRegistry::new();
+    assert!(registry.is_empty());
+
+    registry.register(MockPlugin);
+    assert_eq!(registry.len(), 1);
+
+    let by_id = registry
+        .get("mock")
+        .expect("plugin should be findable by id");
+    assert_eq!(by_id.display_name(), "Mock Syntax");
+
+    let by_ext = registry
+        .detect_from_extension(".mk")
+        .expect("plugin should be findable by secondary extension");
+    assert_eq!(by_ext.id(), "mock");
+
+    assert!(registry.get("nonexistent").is_none());
+    assert!(registry.detect_from_extension(".unknown").is_none());
+
+    // Iteration order matches registration order.
+    let ids: Vec<&str> = registry.iter().map(|p| p.id()).collect();
+    assert_eq!(ids, vec!["mock"]);
+}
+
+#[test]
+fn registry_can_parse_through_trait_object() {
+    let mut registry = SyntaxRegistry::new();
+    registry.register(MockPlugin);
+
+    let plugin = registry.detect_from_extension(".mock").unwrap();
+    let doc = plugin.parse("a b c");
+    assert_eq!(doc.children.len(), 1);
+}
+
+/// Compile-time check: SyntaxRegistry must be Send + Sync so the LSP server
+/// can share it across handler tasks (SPEC.md §6.3).
+#[test]
+fn registry_is_send_and_sync() {
+    fn assert_send_sync<T: Send + Sync>() {}
+    assert_send_sync::<SyntaxRegistry>();
+}