phase 2: syntax plugin interface and registry

Add the Lexer/Parser/SyntaxPlugin traits and the SyntaxRegistry per
SPEC.md §6.3.

- `Lexer<Token>` and `Parser<Token>` are typed building blocks: each
  syntax owns its own token alphabet (vimwiki today, markdown later).
- `TokenStream<T>` is a `Vec<T>` newtype so the eager-vs-lazy choice
  (§6.6) can flip later without breaking callers.
- `SyntaxPlugin` is type-erased: it exposes id / display_name /
  file_extensions / parse(text) so the registry can hold heterogeneous
  plugins behind a single trait object. Implementations chain their
  Lexer + Parser inside parse().
- `SyntaxPlugin: Send + Sync` so the LSP server can stash plugins in
  an Arc and share them across handler tasks.
- `SyntaxRegistry` stores `Arc<dyn SyntaxPlugin>`, supports lookup by
  id and by file extension, iteration in registration order.

Tests cover TokenStream round-trip, full lex→parse chain through a
mock plugin, registry lookup hits and misses, and a compile-time
assert_send_sync::<SyntaxRegistry>().

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

crates/nuwiki-core/src/syntax/mod.rs

@@ -0,0 +1,116 @@
+//! Syntax plugin interface.
+//!
+//! Each syntax (vimwiki today, markdown later) implements the same shape:
+//! a `Lexer` produces a `TokenStream`, a `Parser` consumes it and produces a
+//! `DocumentNode`. A `SyntaxPlugin` is the type-erased facade that the LSP
+//! layer holds in a `SyntaxRegistry`.
+//!
+//! See SPEC.md §6.3 (interface) and §6.6 (lexer strategy).
+
+pub mod registry;
+
+pub use registry::SyntaxRegistry;
+
+use crate::ast::DocumentNode;
+
+/// Eager token buffer produced by a `Lexer`. The newtype keeps the door
+/// open to a lazy/streaming variant later (SPEC.md §6.6) without breaking
+/// callers.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct TokenStream<T> {
+    tokens: Vec<T>,
+}
+
+impl<T> TokenStream<T> {
+    pub fn new() -> Self {
+        Self { tokens: Vec::new() }
+    }
+
+    pub fn from_vec(tokens: Vec<T>) -> Self {
+        Self { tokens }
+    }
+
+    pub fn into_vec(self) -> Vec<T> {
+        self.tokens
+    }
+
+    pub fn as_slice(&self) -> &[T] {
+        &self.tokens
+    }
+
+    pub fn len(&self) -> usize {
+        self.tokens.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.tokens.is_empty()
+    }
+
+    pub fn iter(&self) -> std::slice::Iter<'_, T> {
+        self.tokens.iter()
+    }
+
+    pub fn push(&mut self, token: T) {
+        self.tokens.push(token);
+    }
+}
+
+impl<T> From<Vec<T>> for TokenStream<T> {
+    fn from(tokens: Vec<T>) -> Self {
+        Self::from_vec(tokens)
+    }
+}
+
+impl<T> IntoIterator for TokenStream<T> {
+    type Item = T;
+    type IntoIter = std::vec::IntoIter<T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.tokens.into_iter()
+    }
+}
+
+impl<'a, T> IntoIterator for &'a TokenStream<T> {
+    type Item = &'a T;
+    type IntoIter = std::slice::Iter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.tokens.iter()
+    }
+}
+
+/// Lexer half of a syntax plugin. Operates on raw source text and emits a
+/// `TokenStream` whose token type is syntax-specific (e.g. `VimwikiToken`).
+pub trait Lexer {
+    type Token;
+
+    fn lex(&self, text: &str) -> TokenStream<Self::Token>;
+}
+
+/// Parser half of a syntax plugin. Consumes a `TokenStream` and produces a
+/// `DocumentNode`. Per SPEC.md §6.7 parsing is resilient — malformed input
+/// becomes `BlockNode::Error(ErrorNode)`, never a hard failure.
+pub trait Parser {
+    type Token;
+
+    fn parse(&self, tokens: TokenStream<Self::Token>) -> DocumentNode;
+}
+
+/// Type-erased syntax plugin held by the `SyntaxRegistry`.
+///
+/// Implementations typically own a `Lexer` and a `Parser` and chain them
+/// inside `parse`. The token type is deliberately hidden so the registry can
+/// hold heterogeneous plugins behind a single trait object.
+///
+/// `Send + Sync` is required — the LSP server stores plugins behind an `Arc`
+/// and shares them across request handler tasks (SPEC.md §6.3).
+pub trait SyntaxPlugin: Send + Sync {
+    fn id(&self) -> &str;
+    fn display_name(&self) -> &str;
+
+    /// File extensions associated with this syntax. Each entry includes the
+    /// leading dot, e.g. `[".wiki"]`. See SPEC.md §6.3.
+    fn file_extensions(&self) -> &[&str];
+
+    fn parse(&self, text: &str) -> DocumentNode;
+}

crates/nuwiki-core/src/syntax/registry.rs

@@ -0,0 +1,64 @@
+//! Registry of syntax plugins. See SPEC.md §6.3.
+//!
+//! Plugins are stored behind `Arc<dyn SyntaxPlugin>` so the LSP layer can
+//! hand a cheap clone to per-document tasks without re-locking the registry.
+
+use std::sync::Arc;
+
+use super::SyntaxPlugin;
+
+#[derive(Default, Clone)]
+pub struct SyntaxRegistry {
+    plugins: Vec<Arc<dyn SyntaxPlugin>>,
+}
+
+impl SyntaxRegistry {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Register a plugin owned directly. The most common shape — implementer
+    /// hands over a value, registry takes ownership.
+    pub fn register<P>(&mut self, plugin: P)
+    where
+        P: SyntaxPlugin + 'static,
+    {
+        self.plugins.push(Arc::new(plugin));
+    }
+
+    /// Register a pre-arc'd plugin. Useful when the same plugin instance
+    /// also needs to be held elsewhere.
+    pub fn register_arc(&mut self, plugin: Arc<dyn SyntaxPlugin>) {
+        self.plugins.push(plugin);
+    }
+
+    /// Look up a plugin by its `id`.
+    pub fn get(&self, id: &str) -> Option<&dyn SyntaxPlugin> {
+        self.plugins
+            .iter()
+            .find(|p| p.id() == id)
+            .map(|p| p.as_ref())
+    }
+
+    /// Look up a plugin by file extension. `ext` should include the leading
+    /// dot (e.g. `".wiki"`) — matches the plugin's `file_extensions()`.
+    pub fn detect_from_extension(&self, ext: &str) -> Option<&dyn SyntaxPlugin> {
+        self.plugins
+            .iter()
+            .find(|p| p.file_extensions().iter().any(|e| *e == ext))
+            .map(|p| p.as_ref())
+    }
+
+    /// Iterate over registered plugins in registration order.
+    pub fn iter(&self) -> impl Iterator<Item = &dyn SyntaxPlugin> + '_ {
+        self.plugins.iter().map(|p| p.as_ref())
+    }
+
+    pub fn len(&self) -> usize {
+        self.plugins.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.plugins.is_empty()
+    }
+}