diff --git a/SPEC.md b/SPEC.md index 1a8fc44..c712153 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,7 +1,7 @@ # nuwiki — Project Specification > Last updated: 2026-05-10 -> Status: Phase 1 (Core AST) complete — moving to Phase 2 (Syntax Plugin Interface) +> Status: Phase 2 (Syntax Plugin Interface) complete — moving to Phase 3 (Vimwiki Lexer) --- diff --git a/crates/nuwiki-core/src/lib.rs b/crates/nuwiki-core/src/lib.rs index 54d0e55..a2e9f5d 100644 --- a/crates/nuwiki-core/src/lib.rs +++ b/crates/nuwiki-core/src/lib.rs @@ -4,3 +4,4 @@ //! `nuwiki-ls`. See SPEC.md §6.2 for the dependency rules. pub mod ast; +pub mod syntax; diff --git a/crates/nuwiki-core/src/syntax/mod.rs b/crates/nuwiki-core/src/syntax/mod.rs new file mode 100644 index 0000000..92c6abb --- /dev/null +++ b/crates/nuwiki-core/src/syntax/mod.rs @@ -0,0 +1,116 @@ +//! Syntax plugin interface. +//! +//! Each syntax (vimwiki today, markdown later) implements the same shape: +//! a `Lexer` produces a `TokenStream`, a `Parser` consumes it and produces a +//! `DocumentNode`. A `SyntaxPlugin` is the type-erased facade that the LSP +//! layer holds in a `SyntaxRegistry`. +//! +//! See SPEC.md §6.3 (interface) and §6.6 (lexer strategy). + +pub mod registry; + +pub use registry::SyntaxRegistry; + +use crate::ast::DocumentNode; + +/// Eager token buffer produced by a `Lexer`. The newtype keeps the door +/// open to a lazy/streaming variant later (SPEC.md §6.6) without breaking +/// callers. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TokenStream { + tokens: Vec, +} + +impl TokenStream { + pub fn new() -> Self { + Self { tokens: Vec::new() } + } + + pub fn from_vec(tokens: Vec) -> Self { + Self { tokens } + } + + pub fn into_vec(self) -> Vec { + self.tokens + } + + pub fn as_slice(&self) -> &[T] { + &self.tokens + } + + pub fn len(&self) -> usize { + self.tokens.len() + } + + pub fn is_empty(&self) -> bool { + self.tokens.is_empty() + } + + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.tokens.iter() + } + + pub fn push(&mut self, token: T) { + self.tokens.push(token); + } +} + +impl From> for TokenStream { + fn from(tokens: Vec) -> Self { + Self::from_vec(tokens) + } +} + +impl IntoIterator for TokenStream { + type Item = T; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.tokens.into_iter() + } +} + +impl<'a, T> IntoIterator for &'a TokenStream { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.tokens.iter() + } +} + +/// Lexer half of a syntax plugin. Operates on raw source text and emits a +/// `TokenStream` whose token type is syntax-specific (e.g. `VimwikiToken`). +pub trait Lexer { + type Token; + + fn lex(&self, text: &str) -> TokenStream; +} + +/// Parser half of a syntax plugin. Consumes a `TokenStream` and produces a +/// `DocumentNode`. Per SPEC.md §6.7 parsing is resilient — malformed input +/// becomes `BlockNode::Error(ErrorNode)`, never a hard failure. +pub trait Parser { + type Token; + + fn parse(&self, tokens: TokenStream) -> DocumentNode; +} + +/// Type-erased syntax plugin held by the `SyntaxRegistry`. +/// +/// Implementations typically own a `Lexer` and a `Parser` and chain them +/// inside `parse`. The token type is deliberately hidden so the registry can +/// hold heterogeneous plugins behind a single trait object. +/// +/// `Send + Sync` is required — the LSP server stores plugins behind an `Arc` +/// and shares them across request handler tasks (SPEC.md §6.3). +pub trait SyntaxPlugin: Send + Sync { + fn id(&self) -> &str; + fn display_name(&self) -> &str; + + /// File extensions associated with this syntax. Each entry includes the + /// leading dot, e.g. `[".wiki"]`. See SPEC.md §6.3. + fn file_extensions(&self) -> &[&str]; + + fn parse(&self, text: &str) -> DocumentNode; +} diff --git a/crates/nuwiki-core/src/syntax/registry.rs b/crates/nuwiki-core/src/syntax/registry.rs new file mode 100644 index 0000000..84de3d7 --- /dev/null +++ b/crates/nuwiki-core/src/syntax/registry.rs @@ -0,0 +1,64 @@ +//! Registry of syntax plugins. See SPEC.md §6.3. +//! +//! Plugins are stored behind `Arc` so the LSP layer can +//! hand a cheap clone to per-document tasks without re-locking the registry. + +use std::sync::Arc; + +use super::SyntaxPlugin; + +#[derive(Default, Clone)] +pub struct SyntaxRegistry { + plugins: Vec>, +} + +impl SyntaxRegistry { + pub fn new() -> Self { + Self::default() + } + + /// Register a plugin owned directly. The most common shape — implementer + /// hands over a value, registry takes ownership. + pub fn register

(&mut self, plugin: P) + where + P: SyntaxPlugin + 'static, + { + self.plugins.push(Arc::new(plugin)); + } + + /// Register a pre-arc'd plugin. Useful when the same plugin instance + /// also needs to be held elsewhere. + pub fn register_arc(&mut self, plugin: Arc) { + self.plugins.push(plugin); + } + + /// Look up a plugin by its `id`. + pub fn get(&self, id: &str) -> Option<&dyn SyntaxPlugin> { + self.plugins + .iter() + .find(|p| p.id() == id) + .map(|p| p.as_ref()) + } + + /// Look up a plugin by file extension. `ext` should include the leading + /// dot (e.g. `".wiki"`) — matches the plugin's `file_extensions()`. + pub fn detect_from_extension(&self, ext: &str) -> Option<&dyn SyntaxPlugin> { + self.plugins + .iter() + .find(|p| p.file_extensions().iter().any(|e| *e == ext)) + .map(|p| p.as_ref()) + } + + /// Iterate over registered plugins in registration order. + pub fn iter(&self) -> impl Iterator + '_ { + self.plugins.iter().map(|p| p.as_ref()) + } + + pub fn len(&self) -> usize { + self.plugins.len() + } + + pub fn is_empty(&self) -> bool { + self.plugins.is_empty() + } +} diff --git a/crates/nuwiki-core/tests/syntax.rs b/crates/nuwiki-core/tests/syntax.rs new file mode 100644 index 0000000..ebdc93d --- /dev/null +++ b/crates/nuwiki-core/tests/syntax.rs @@ -0,0 +1,144 @@ +//! Integration tests for the syntax plugin interface. + +use nuwiki_core::ast::{ + inline::InlineNode, BlockNode, DocumentNode, ParagraphNode, Span, TextNode, +}; +use nuwiki_core::syntax::{Lexer, Parser, SyntaxPlugin, SyntaxRegistry, TokenStream}; + +// --- A minimal mock plugin: tokenises the input on whitespace, then wraps +// the joined text in a single Paragraph. Just enough surface to exercise +// every trait + the registry. + +#[derive(Debug, Clone, PartialEq, Eq)] +struct MockToken(String); + +struct MockLexer; +impl Lexer for MockLexer { + type Token = MockToken; + + fn lex(&self, text: &str) -> TokenStream { + TokenStream::from_vec( + text.split_whitespace() + .map(|w| MockToken(w.to_owned())) + .collect(), + ) + } +} + +struct MockParser; +impl Parser for MockParser { + type Token = MockToken; + + fn parse(&self, tokens: TokenStream) -> DocumentNode { + let joined = tokens + .iter() + .map(|t| t.0.as_str()) + .collect::>() + .join(" "); + let paragraph = BlockNode::Paragraph(ParagraphNode { + span: Span::default(), + children: vec![InlineNode::Text(TextNode { + span: Span::default(), + content: joined, + })], + }); + DocumentNode { + span: Span::default(), + children: vec![paragraph], + metadata: Default::default(), + } + } +} + +struct MockPlugin; +impl SyntaxPlugin for MockPlugin { + fn id(&self) -> &str { + "mock" + } + fn display_name(&self) -> &str { + "Mock Syntax" + } + fn file_extensions(&self) -> &[&str] { + &[".mock", ".mk"] + } + fn parse(&self, text: &str) -> DocumentNode { + MockParser.parse(MockLexer.lex(text)) + } +} + +#[test] +fn token_stream_roundtrip() { + let mut s = TokenStream::::new(); + assert!(s.is_empty()); + s.push(1); + s.push(2); + s.push(3); + assert_eq!(s.len(), 3); + assert_eq!(s.as_slice(), &[1, 2, 3]); + assert_eq!(s.iter().sum::(), 6); + assert_eq!((&s).into_iter().sum::(), 6); + assert_eq!(s.into_vec(), vec![1, 2, 3]); + + let from_vec: TokenStream<&str> = vec!["a", "b"].into(); + assert_eq!(from_vec.len(), 2); + let collected: Vec<&str> = from_vec.into_iter().collect(); + assert_eq!(collected, vec!["a", "b"]); +} + +#[test] +fn lexer_and_parser_chain_through_plugin() { + let plugin = MockPlugin; + let doc = plugin.parse(" hello world "); + + let BlockNode::Paragraph(p) = &doc.children[0] else { + panic!("expected a paragraph, got {:?}", doc.children[0]); + }; + let InlineNode::Text(t) = &p.children[0] else { + panic!("expected a text node"); + }; + assert_eq!(t.content, "hello world"); +} + +#[test] +fn registry_lookup_by_id_and_extension() { + let mut registry = SyntaxRegistry::new(); + assert!(registry.is_empty()); + + registry.register(MockPlugin); + assert_eq!(registry.len(), 1); + + let by_id = registry + .get("mock") + .expect("plugin should be findable by id"); + assert_eq!(by_id.display_name(), "Mock Syntax"); + + let by_ext = registry + .detect_from_extension(".mk") + .expect("plugin should be findable by secondary extension"); + assert_eq!(by_ext.id(), "mock"); + + assert!(registry.get("nonexistent").is_none()); + assert!(registry.detect_from_extension(".unknown").is_none()); + + // Iteration order matches registration order. + let ids: Vec<&str> = registry.iter().map(|p| p.id()).collect(); + assert_eq!(ids, vec!["mock"]); +} + +#[test] +fn registry_can_parse_through_trait_object() { + let mut registry = SyntaxRegistry::new(); + registry.register(MockPlugin); + + let plugin = registry.detect_from_extension(".mock").unwrap(); + let doc = plugin.parse("a b c"); + assert_eq!(doc.children.len(), 1); +} + +/// Compile-time check: SyntaxRegistry must be Send + Sync so the LSP server +/// can share it across handler tasks (SPEC.md §6.3). +#[test] +fn registry_is_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); +}