rune/parse/
parser.rs

Help
1use core::fmt;
2use core::ops;
3
4use crate::alloc::VecDeque;
5use crate::ast::Spanned;
6use crate::ast::{Kind, OptionSpanned, Span, Token};
7use crate::compile::WithSpan;
8use crate::compile::{self, ErrorKind};
9use crate::macros::{TokenStream, TokenStreamIter};
10use crate::parse::{Advance, Lexer, Parse, Peek};
11use crate::shared::FixedVec;
12use crate::SourceId;
13
14/// Parser for the rune language.
15///
16/// # Examples
17///
18/// ```
19/// use rune::ast;
20/// use rune::SourceId;
21/// use rune::parse::Parser;
22///
23/// let mut parser = Parser::new("fn foo() {}", SourceId::empty(), false);
24/// let ast = parser.parse::<ast::ItemFn>()?;
25/// # Ok::<_, rune::support::Error>(())
26/// ```
27#[derive(Debug)]
28pub struct Parser<'a> {
29    peeker: Peeker<'a>,
30}
31
32impl<'a> Parser<'a> {
33    /// Construct a new parser around the given source.
34    ///
35    /// `shebang` indicates if the parser should try and parse a shebang or not.
36    pub fn new(source: &'a str, source_id: SourceId, shebang: bool) -> Self {
37        Self::with_source(
38            Source {
39                inner: SourceInner::Lexer(Lexer::new(source, source_id, shebang)),
40            },
41            Span::new(0u32, source.len()),
42        )
43    }
44
45    /// Construct a parser from a token stream. The second argument `span` is
46    /// the span to use if the stream is empty.
47    pub fn from_token_stream(token_stream: &'a TokenStream, span: Span) -> Self {
48        Self::with_source(
49            Source {
50                inner: SourceInner::TokenStream(token_stream.iter()),
51            },
52            span,
53        )
54    }
55
56    /// Parse a specific item from the parser.
57    pub fn parse<T>(&mut self) -> compile::Result<T>
58    where
59        T: Parse,
60    {
61        T::parse(self)
62    }
63
64    /// Parse a specific item from the parser and then expect end of input.
65    pub fn parse_all<T>(&mut self) -> compile::Result<T>
66    where
67        T: Parse,
68    {
69        let item = self.parse::<T>()?;
70        self.eof()?;
71        Ok(item)
72    }
73
74    /// Peek for the given token.
75    pub fn peek<T>(&mut self) -> compile::Result<bool>
76    where
77        T: Peek,
78    {
79        if let Some(error) = self.peeker.error.take() {
80            return Err(error);
81        }
82
83        let result = T::peek(&mut self.peeker);
84
85        if let Some(error) = self.peeker.error.take() {
86            return Err(error);
87        }
88
89        Ok(result)
90    }
91
92    /// Assert that the parser has reached its end-of-file.
93    pub fn eof(&mut self) -> compile::Result<()> {
94        if let Some(token) = self.peeker.at(0)? {
95            return Err(compile::Error::new(
96                token,
97                ErrorKind::ExpectedEof { actual: token.kind },
98            ));
99        }
100
101        Ok(())
102    }
103
104    /// Test if the parser is at end-of-file, after which there is no more input
105    /// to parse.
106    pub fn is_eof(&mut self) -> compile::Result<bool> {
107        Ok(self.peeker.at(0)?.is_none())
108    }
109
110    /// Construct a new parser with a source.
111    fn with_source(source: Source<'a>, span: Span) -> Self {
112        let default_span = source.span().unwrap_or(span);
113
114        Self {
115            peeker: Peeker {
116                source,
117                buf: VecDeque::new(),
118                error: None,
119                last: None,
120                default_span,
121            },
122        }
123    }
124
125    /// Try to consume a single thing matching `T`, returns `true` if any tokens
126    /// were consumed.
127    pub fn try_consume<T>(&mut self) -> compile::Result<bool>
128    where
129        T: Parse + Peek,
130    {
131        Ok(if self.peek::<T>()? {
132            self.parse::<T>()?;
133            true
134        } else {
135            false
136        })
137    }
138
139    /// Try to consume all things matching `T`, returns `true` if any tokens
140    /// were consumed.
141    pub fn try_consume_all<T>(&mut self) -> compile::Result<bool>
142    where
143        T: Parse + Peek,
144    {
145        let mut consumed = false;
146
147        while self.peek::<T>()? {
148            self.parse::<T>()?;
149            consumed = true;
150        }
151
152        Ok(consumed)
153    }
154
155    /// Get the span for the given range offset of tokens.
156    pub(crate) fn span(&mut self, range: ops::Range<usize>) -> Span {
157        self.span_at(range.start).join(self.span_at(range.end))
158    }
159
160    /// Access the interior peeker of the parser.
161    pub(crate) fn peeker(&mut self) -> &mut Peeker<'a> {
162        &mut self.peeker
163    }
164
165    /// Consume the next token from the parser.
166    #[allow(clippy::should_implement_trait)]
167    pub(crate) fn next(&mut self) -> compile::Result<Token> {
168        if let Some(error) = self.peeker.error.take() {
169            return Err(error);
170        }
171
172        if let Some(t) = self.peeker.buf.pop_front() {
173            return Ok(t);
174        }
175
176        match self.peeker.next()? {
177            Some(t) => Ok(t),
178            None => Err(compile::Error::new(
179                self.last_span().tail(),
180                ErrorKind::UnexpectedEof,
181            )),
182        }
183    }
184
185    /// Peek the token kind at the given position.
186    pub(crate) fn nth(&mut self, n: usize) -> compile::Result<Kind> {
187        if let Some(t) = self.peeker.at(n)? {
188            Ok(t.kind)
189        } else {
190            Ok(Kind::Eof)
191        }
192    }
193
194    /// Get the span for the given offset.
195    pub(crate) fn span_at(&mut self, n: usize) -> Span {
196        if let Ok(Some(t)) = self.peeker.at(n) {
197            t.span
198        } else {
199            self.last_span().tail()
200        }
201    }
202
203    /// Get the token at the given offset.
204    pub(crate) fn tok_at(&mut self, n: usize) -> compile::Result<Token> {
205        Ok(if let Some(t) = self.peeker.at(n)? {
206            t
207        } else {
208            Token {
209                kind: Kind::Eof,
210                span: self.last_span().tail(),
211            }
212        })
213    }
214
215    /// The last known span in this parser.
216    pub(crate) fn last_span(&self) -> Span {
217        self.peeker.last_span()
218    }
219}
220
221/// Construct used to peek a parser.
222#[derive(Debug)]
223pub struct Peeker<'a> {
224    /// The source being processed.
225    source: Source<'a>,
226    /// The buffer of tokens seen.
227    buf: VecDeque<Token>,
228    // NB: parse errors encountered during peeking.
229    error: Option<compile::Error>,
230    /// The last span we encountered. Used to provide better EOF diagnostics.
231    last: Option<Span>,
232    /// The default span to use in case no better one is available.
233    default_span: Span,
234}
235
236impl Peeker<'_> {
237    /// Peek the token kind at the given position.
238    pub(crate) fn nth(&mut self, n: usize) -> Kind {
239        // Error tripped already, this peeker returns nothing but errors from
240        // here on out.
241        if self.error.is_some() {
242            return Kind::Error;
243        }
244
245        match self.at(n) {
246            Ok(t) => match t {
247                Some(t) => t.kind,
248                None => Kind::Eof,
249            },
250            Err(error) => {
251                self.error = Some(error);
252                Kind::Error
253            }
254        }
255    }
256
257    /// Peek an array.
258    pub(crate) fn array<const N: usize>(&mut self) -> FixedVec<Token, N> {
259        let mut vec = FixedVec::new();
260
261        if N == 0 {
262            return vec;
263        }
264
265        if let Err(error) = self.fill(N) {
266            self.error = Some(error);
267        }
268
269        let mut it = 0..N;
270
271        for (&tok, _) in self.buf.iter().zip(it.by_ref()) {
272            _ = vec.try_push(tok);
273        }
274
275        if let Some(error) = &self.error {
276            for _ in it {
277                _ = vec.try_push(Token {
278                    kind: Kind::Error,
279                    span: error.span(),
280                });
281            }
282        } else {
283            for _ in it {
284                _ = vec.try_push(Token {
285                    kind: Kind::Eof,
286                    span: self.last_span(),
287                });
288            }
289        }
290
291        vec
292    }
293
294    /// Test if we are at end of file.
295    pub(crate) fn is_eof(&mut self) -> bool {
296        match self.at(0) {
297            Ok(t) => t.is_none(),
298            Err(error) => {
299                self.error = Some(error);
300                false
301            }
302        }
303    }
304
305    /// Advance the internals of the peeker and return the next token (without
306    /// buffering).
307    fn next(&mut self) -> compile::Result<Option<Token>> {
308        loop {
309            let Some(token) = self.source.next()? else {
310                return Ok(None);
311            };
312
313            match token.kind {
314                Kind::Comment | Kind::Whitespace => {
315                    continue;
316                }
317                Kind::MultilineComment(term) => {
318                    if !term {
319                        return Err(compile::Error::new(
320                            token.span,
321                            ErrorKind::ExpectedMultilineCommentTerm,
322                        ));
323                    }
324
325                    continue;
326                }
327                _ => (),
328            }
329
330            return Ok(Some(token));
331        }
332    }
333
334    /// Make sure there are at least `n` items in the buffer, and return the
335    /// item at that point.
336    fn at(&mut self, n: usize) -> compile::Result<Option<Token>> {
337        self.fill(n)?;
338        Ok(self.buf.get(n).copied())
339    }
340
341    fn fill(&mut self, n: usize) -> compile::Result<()> {
342        if let Some(error) = self.error.take() {
343            return Err(error);
344        }
345
346        while self.buf.len() <= n {
347            let Some(tok) = self.next()? else {
348                break;
349            };
350
351            self.last = Some(tok.span);
352            self.buf.try_push_back(tok).with_span(tok.span)?;
353        }
354
355        Ok(())
356    }
357
358    /// The last known span in this parser.
359    fn last_span(&self) -> Span {
360        self.last.unwrap_or(self.default_span)
361    }
362}
363
364/// A source adapter.
365pub(crate) struct Source<'a> {
366    inner: SourceInner<'a>,
367}
368
369impl Source<'_> {
370    /// Get the span of the source.
371    fn span(&self) -> Option<Span> {
372        match &self.inner {
373            SourceInner::Lexer(lexer) => Some(lexer.span()),
374            SourceInner::TokenStream(token_stream) => token_stream.option_span(),
375        }
376    }
377
378    /// Get the next token in the stream.
379    fn next(&mut self) -> compile::Result<Option<Token>> {
380        match &mut self.inner {
381            SourceInner::Lexer(lexer) => lexer.next(),
382            SourceInner::TokenStream(token_stream) => Ok(token_stream.next()),
383        }
384    }
385}
386
387impl fmt::Debug for Source<'_> {
388    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
389        fmt::Debug::fmt(&self.inner, f)
390    }
391}
392
393#[derive(Debug)]
394enum SourceInner<'a> {
395    Lexer(Lexer<'a>),
396    TokenStream(TokenStreamIter<'a>),
397}
398
399impl Advance for Parser<'_> {
400    type Error = compile::Error;
401
402    #[inline]
403    fn advance(&mut self, n: usize) -> Result<(), Self::Error> {
404        for _ in 0..n {
405            self.next()?;
406        }
407
408        Ok(())
409    }
410}
rune/parse/parser.rs

rune/parse/
parser.rs
