nuwiki-rs/crates/nuwiki-core/src/syntax/mod.rs

//! Syntax plugin interface.
//!
//! Each syntax (vimwiki today, markdown later) implements the same shape:
//! a `Lexer` produces a `TokenStream`, a `Parser` consumes it and produces a
//! `DocumentNode`. A `SyntaxPlugin` is the type-erased facade that the LSP
//! layer holds in a `SyntaxRegistry`.

pub mod registry;
pub mod vimwiki;

pub use registry::SyntaxRegistry;

use crate::ast::DocumentNode;

/// Eager token buffer produced by a `Lexer`. The newtype keeps the door
/// open to a lazy/streaming variant later without breaking
/// callers.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TokenStream<T> {
    tokens: Vec<T>,
}

impl<T> TokenStream<T> {
    pub fn new() -> Self {
        Self { tokens: Vec::new() }
    }

    pub fn from_vec(tokens: Vec<T>) -> Self {
        Self { tokens }
    }

    pub fn into_vec(self) -> Vec<T> {
        self.tokens
    }

    pub fn as_slice(&self) -> &[T] {
        &self.tokens
    }

    pub fn len(&self) -> usize {
        self.tokens.len()
    }

    pub fn is_empty(&self) -> bool {
        self.tokens.is_empty()
    }

    pub fn iter(&self) -> std::slice::Iter<'_, T> {
        self.tokens.iter()
    }

    pub fn push(&mut self, token: T) {
        self.tokens.push(token);
    }
}

impl<T> From<Vec<T>> for TokenStream<T> {
    fn from(tokens: Vec<T>) -> Self {
        Self::from_vec(tokens)
    }
}

impl<T> IntoIterator for TokenStream<T> {
    type Item = T;
    type IntoIter = std::vec::IntoIter<T>;

    fn into_iter(self) -> Self::IntoIter {
        self.tokens.into_iter()
    }
}

impl<'a, T> IntoIterator for &'a TokenStream<T> {
    type Item = &'a T;
    type IntoIter = std::slice::Iter<'a, T>;

    fn into_iter(self) -> Self::IntoIter {
        self.tokens.iter()
    }
}

/// Lexer half of a syntax plugin. Operates on raw source text and emits a
/// `TokenStream` whose token type is syntax-specific (e.g. `VimwikiToken`).
pub trait Lexer {
    type Token;

    fn lex(&self, text: &str) -> TokenStream<Self::Token>;
}

/// Parser half of a syntax plugin. Consumes a `TokenStream` and produces a
/// `DocumentNode`. Parsing is resilient — malformed input
/// becomes `BlockNode::Error(ErrorNode)`, never a hard failure.
pub trait Parser {
    type Token;

    fn parse(&self, tokens: TokenStream<Self::Token>) -> DocumentNode;
}

/// Type-erased syntax plugin held by the `SyntaxRegistry`.
///
/// Implementations typically own a `Lexer` and a `Parser` and chain them
/// inside `parse`. The token type is deliberately hidden so the registry can
/// hold heterogeneous plugins behind a single trait object.
///
/// `Send + Sync` is required — the LSP server stores plugins behind an `Arc`
/// and shares them across request handler tasks.
pub trait SyntaxPlugin: Send + Sync {
    fn id(&self) -> &str;
    fn display_name(&self) -> &str;

    /// File extensions associated with this syntax. Each entry includes the
    /// leading dot, e.g. `[".wiki"]`.
    fn file_extensions(&self) -> &[&str];

    fn parse(&self, text: &str) -> DocumentNode;
}