rune/parse/
lexer.rs

Help
1#[cfg(test)]
2mod tests;
3
4use core::fmt;
5use core::mem::take;
6
7use unicode_ident::{is_xid_continue, is_xid_start};
8
9use crate::alloc::{self, Vec, VecDeque};
10use crate::ast;
11use crate::ast::Span;
12use crate::compile::{self, ErrorKind};
13use crate::SourceId;
14
15/// Lexer for the rune language.
16#[derive(Debug)]
17pub struct Lexer<'a> {
18    /// The source identifier of the lexed data.
19    source_id: SourceId,
20    /// Source iterator.
21    iter: SourceIter<'a>,
22    /// Current lexer mode.
23    modes: LexerModes,
24    /// Buffered tokens.
25    buffer: VecDeque<ast::Token>,
26    /// If the lexer should try and lex a shebang.
27    shebang: bool,
28    /// If we should synthesise doc attributes.
29    process: bool,
30}
31
32impl<'a> Lexer<'a> {
33    /// Construct a new lexer over the given source.
34    pub(crate) fn new(source: &'a str, source_id: SourceId, shebang: bool) -> Self {
35        Self {
36            iter: SourceIter::new(source),
37            source_id,
38            modes: LexerModes::default(),
39            buffer: VecDeque::new(),
40            shebang,
41            process: true,
42        }
43    }
44
45    /// Disable docs synthesizing.
46    pub(crate) fn without_processing(self) -> Self {
47        Self {
48            process: false,
49            ..self
50        }
51    }
52
53    /// Access the span of the lexer.
54    pub(crate) fn span(&self) -> Span {
55        self.iter.span_to_len(0)
56    }
57
58    /// Denote whether the next sequence of characters begin a doc comment.
59    ///
60    /// The lexer should have just identified a regular comment (`//_`, `/*_`, where _ is the
61    /// cursor's current position).
62    ///
63    /// Returns a tuple (doc, inner), referring to if a doc comment was found and if the ! character
64    /// was used to denote an inner comment.
65    fn check_doc_comment(&mut self, ch: char) -> (bool, bool) {
66        match self.iter.peek() {
67            Some(c) if c == ch => {
68                // The character following this must not be another of the provided character.
69                // If it is, it's probably a separator, not a doc comment.
70                match self.iter.peek2() {
71                    Some(c) if c == ch => (false, false),
72                    _ => (true, false),
73                }
74            }
75            Some('!') => (true, true),
76            _ => (false, false),
77        }
78    }
79
80    fn emit_doc_attribute(
81        &mut self,
82        inner: bool,
83        span: Span,
84        docstring_span: Span,
85    ) -> alloc::Result<()> {
86        // outer: #[doc = ...]
87        // inner: #![doc = ...]
88
89        self.buffer
90            .try_push_back(ast::Token { kind: K![#], span })?;
91
92        if inner {
93            self.buffer
94                .try_push_back(ast::Token { kind: K![!], span })?;
95        }
96
97        self.buffer.try_push_back(ast::Token {
98            kind: K!['['],
99            span,
100        })?;
101
102        self.buffer.try_push_back(ast::Token {
103            kind: ast::Kind::Ident(ast::LitSource::BuiltIn(ast::BuiltIn::Doc)),
104            span,
105        })?;
106
107        self.buffer
108            .try_push_back(ast::Token { kind: K![=], span })?;
109
110        self.buffer.try_push_back(ast::Token {
111            kind: ast::Kind::Str(ast::StrSource::Text(ast::StrText {
112                source_id: self.source_id,
113                escaped: false,
114                wrapped: false,
115            })),
116            span: docstring_span,
117        })?;
118
119        self.buffer.try_push_back(ast::Token {
120            kind: K![']'],
121            span,
122        })?;
123
124        Ok(())
125    }
126
127    fn emit_builtin_attribute(&mut self, span: Span) -> alloc::Result<()> {
128        self.buffer
129            .try_push_back(ast::Token { kind: K![#], span })?;
130
131        self.buffer.try_push_back(ast::Token {
132            kind: K!['['],
133            span,
134        })?;
135
136        self.buffer.try_push_back(ast::Token {
137            kind: ast::Kind::Ident(ast::LitSource::BuiltIn(ast::BuiltIn::BuiltIn)),
138            span,
139        })?;
140
141        self.buffer.try_push_back(ast::Token {
142            kind: K!['('],
143            span,
144        })?;
145
146        self.buffer.try_push_back(ast::Token {
147            kind: ast::Kind::Ident(ast::LitSource::BuiltIn(ast::BuiltIn::Literal)),
148            span,
149        })?;
150
151        self.buffer.try_push_back(ast::Token {
152            kind: K![')'],
153            span,
154        })?;
155
156        self.buffer.try_push_back(ast::Token {
157            kind: K![']'],
158            span,
159        })?;
160
161        Ok(())
162    }
163
164    fn next_ident(&mut self, start: usize) -> compile::Result<Option<ast::Token>> {
165        while let Some(c) = self.iter.peek() {
166            if !is_xid_continue(c) {
167                break;
168            }
169
170            self.iter.next();
171        }
172
173        match self.iter.source_from(start) {
174            ("_", span) => Ok(Some(ast::Token {
175                span,
176                kind: ast::Kind::Underscore,
177            })),
178            (ident, span) => {
179                let kind = ast::Kind::from_keyword(ident)
180                    .unwrap_or(ast::Kind::Ident(ast::LitSource::Text(self.source_id)));
181                Ok(Some(ast::Token { kind, span }))
182            }
183        }
184    }
185
186    /// Consume a number literal.
187    fn next_number_literal(
188        &mut self,
189        c: char,
190        start: usize,
191    ) -> compile::Result<Option<ast::Token>> {
192        let (base, number_start) = 'ok: {
193            if let ('0', Some(m)) = (c, self.iter.peek()) {
194                let number = match m {
195                    'x' => ast::NumberBase::Hex,
196                    'b' => ast::NumberBase::Binary,
197                    'o' => ast::NumberBase::Octal,
198                    _ => break 'ok (ast::NumberBase::Decimal, start),
199                };
200
201                self.iter.next();
202                (number, self.iter.pos())
203            } else {
204                (ast::NumberBase::Decimal, start)
205            }
206        };
207
208        let mut is_fractional = false;
209        let mut has_exponent = false;
210
211        let mut split = None;
212
213        while let Some(c) = self.iter.peek() {
214            match c {
215                // NB: We need to avoid exponent check for hex number bases,
216                // since 'e' is a legal hex literal.
217                'e' if !has_exponent && !matches!(base, ast::NumberBase::Hex) => {
218                    self.iter.next();
219                    has_exponent = true;
220                    is_fractional = true;
221
222                    // Negative or explicitly positive exponent.
223                    if matches!(self.iter.peek(), Some('-') | Some('+')) {
224                        self.iter.next();
225                    }
226                }
227                '.' if !is_fractional => {
228                    if let Some(p2) = self.iter.peek2() {
229                        // NB: only skip if the next peek matches:
230                        // * the beginning of an ident.
231                        // * `..`, which is a range expression.
232                        //
233                        // Our goal is otherwise to consume as much alphanumeric
234                        // content as possible to provide better diagnostics.
235                        // But we must treat these cases differently since field
236                        // accesses might be instance fn calls, and range
237                        // expressions should work.
238                        if matches!(p2, 'a'..='z' | 'A'..='Z' | '_' | '.') {
239                            break;
240                        }
241                    }
242
243                    self.iter.next();
244                    is_fractional = true;
245                }
246                // NB: Allows for underscores to pass through number literals,
247                // so that they can be used to break up large numbers in a
248                // natural manner.
249                '_' => {
250                    self.iter.next();
251                }
252                c if c.is_alphanumeric() => {
253                    if split.is_none()
254                        && matches!((c, base), ('u' | 'i', _) | ('f', ast::NumberBase::Decimal))
255                    {
256                        split = Some(self.iter.pos());
257                    }
258
259                    self.iter.next();
260                }
261                _ => break,
262            }
263        }
264
265        let end = split.unwrap_or(self.iter.pos());
266
267        Ok(Some(ast::Token {
268            kind: ast::Kind::Number(ast::NumberSource::Text(ast::NumberText {
269                source_id: self.source_id,
270                is_fractional,
271                base,
272                number: Span::new(number_start, end),
273                suffix: Span::new(end, self.iter.pos()),
274            })),
275            span: self.iter.span_to_pos(start),
276        }))
277    }
278
279    /// Consume a string literal.
280    fn next_char_or_label(&mut self, start: usize) -> compile::Result<Option<ast::Token>> {
281        let mut is_label = true;
282        let mut count = 0;
283
284        while let Some((s, c)) = self.iter.peek_with_pos() {
285            match c {
286                '\\' => {
287                    self.iter.next();
288
289                    if self.iter.next().is_none() {
290                        return Err(compile::Error::new(
291                            self.iter.span_to_pos(s),
292                            ErrorKind::ExpectedEscape,
293                        ));
294                    }
295
296                    is_label = false;
297                    count += 1;
298                }
299                '\'' => {
300                    is_label = false;
301                    self.iter.next();
302                    break;
303                }
304                // components of labels.
305                '0'..='9' | 'a'..='z' => {
306                    self.iter.next();
307                    count += 1;
308                }
309                c if c.is_control() => {
310                    let span = self.iter.span_to_pos(start);
311                    return Err(compile::Error::new(span, ErrorKind::UnterminatedCharLit));
312                }
313                _ if is_label && count > 0 => {
314                    break;
315                }
316                _ => {
317                    is_label = false;
318                    self.iter.next();
319                    count += 1;
320                }
321            }
322        }
323
324        if count == 0 {
325            let span = self.iter.span_to_len(start);
326
327            if !is_label {
328                return Err(compile::Error::new(span, ErrorKind::ExpectedCharClose));
329            }
330
331            return Err(compile::Error::new(span, ErrorKind::ExpectedCharOrLabel));
332        }
333
334        if is_label {
335            Ok(Some(ast::Token {
336                kind: ast::Kind::Label(ast::LitSource::Text(self.source_id)),
337                span: self.iter.span_to_pos(start),
338            }))
339        } else {
340            Ok(Some(ast::Token {
341                kind: ast::Kind::Char(ast::CopySource::Text(self.source_id)),
342                span: self.iter.span_to_pos(start),
343            }))
344        }
345    }
346
347    /// Consume a string literal.
348    fn next_lit_byte(&mut self, start: usize) -> compile::Result<Option<ast::Token>> {
349        loop {
350            let (s, c) = match self.iter.next_with_pos() {
351                Some(c) => c,
352                None => {
353                    return Err(compile::Error::new(
354                        self.iter.span_to_pos(start),
355                        ErrorKind::ExpectedByteClose,
356                    ));
357                }
358            };
359
360            match c {
361                '\\' => {
362                    if self.iter.next().is_none() {
363                        return Err(compile::Error::new(
364                            self.iter.span_to_pos(s),
365                            ErrorKind::ExpectedEscape,
366                        ));
367                    }
368                }
369                '\'' => {
370                    break;
371                }
372                c if c.is_control() => {
373                    let span = self.iter.span_to_pos(start);
374                    return Err(compile::Error::new(span, ErrorKind::UnterminatedByteLit));
375                }
376                _ => (),
377            }
378        }
379
380        Ok(Some(ast::Token {
381            kind: ast::Kind::Byte(ast::CopySource::Text(self.source_id)),
382            span: self.iter.span_to_pos(start),
383        }))
384    }
385
386    /// Consume a string literal.
387    fn next_str(
388        &mut self,
389        start: usize,
390        error_kind: impl FnOnce() -> ErrorKind + Copy,
391        kind: impl FnOnce(ast::StrSource) -> ast::Kind,
392    ) -> compile::Result<Option<ast::Token>> {
393        let mut escaped = false;
394
395        loop {
396            let (s, c) = match self.iter.next_with_pos() {
397                Some(next) => next,
398                None => {
399                    return Err(compile::Error::new(
400                        self.iter.span_to_pos(start),
401                        error_kind(),
402                    ));
403                }
404            };
405
406            match c {
407                '"' => break,
408                '\\' => {
409                    if self.iter.next().is_none() {
410                        return Err(compile::Error::new(
411                            self.iter.span_to_pos(s),
412                            ErrorKind::ExpectedEscape,
413                        ));
414                    }
415
416                    escaped = true;
417                }
418                _ => (),
419            }
420        }
421
422        Ok(Some(ast::Token {
423            kind: kind(ast::StrSource::Text(ast::StrText {
424                source_id: self.source_id,
425                escaped,
426                wrapped: true,
427            })),
428            span: self.iter.span_to_pos(start),
429        }))
430    }
431
432    /// Consume the entire line.
433    fn consume_line(&mut self) {
434        while !matches!(self.iter.peek(), Some('\n') | None) {
435            self.iter.next();
436        }
437    }
438
439    /// Consume whitespace.
440    fn consume_whitespace(&mut self) {
441        while let Some(c) = self.iter.peek() {
442            if !c.is_whitespace() {
443                break;
444            }
445
446            self.iter.next();
447        }
448    }
449
450    /// Consume a multiline comment and indicate if it's terminated correctly.
451    fn consume_multiline_comment(&mut self) -> bool {
452        self.iter.next();
453        self.iter.next();
454
455        let mut cur = self.iter.next();
456
457        while let Some(a) = cur {
458            cur = self.iter.next();
459
460            if matches!((a, cur), ('*', Some('/'))) {
461                return true;
462            }
463        }
464
465        false
466    }
467
468    fn template_next(&mut self) -> compile::Result<()> {
469        let start = self.iter.pos();
470        let mut escaped = false;
471
472        while let Some((s, c)) = self.iter.peek_with_pos() {
473            match c {
474                '$' => {
475                    let expressions = self.modes.expression_count(&self.iter, start)?;
476
477                    let span = self.iter.span_to_pos(start);
478                    let had_string = start != self.iter.pos();
479                    let start = self.iter.pos();
480
481                    self.iter.next();
482
483                    match self.iter.next_with_pos() {
484                        Some((_, '{')) => (),
485                        Some((start, c)) => {
486                            let span = self.iter.span_to_pos(start);
487                            return Err(compile::Error::new(span, ErrorKind::UnexpectedChar { c }));
488                        }
489                        None => {
490                            let span = self.iter.span_to_len(start);
491                            return Err(compile::Error::new(span, ErrorKind::UnexpectedEof));
492                        }
493                    }
494
495                    if had_string {
496                        if *expressions > 0 {
497                            self.buffer.try_push_back(ast::Token {
498                                kind: ast::Kind::Comma,
499                                span,
500                            })?;
501                        }
502
503                        self.buffer.try_push_back(ast::Token {
504                            kind: ast::Kind::Str(ast::StrSource::Text(ast::StrText {
505                                source_id: self.source_id,
506                                escaped: take(&mut escaped),
507                                wrapped: false,
508                            })),
509                            span,
510                        })?;
511
512                        *expressions += 1;
513                    }
514
515                    if *expressions > 0 {
516                        self.buffer.try_push_back(ast::Token {
517                            kind: ast::Kind::Comma,
518                            span: self.iter.span_to_pos(start),
519                        })?;
520                    }
521
522                    self.modes.push(LexerMode::Default(1))?;
523                    return Ok(());
524                }
525                '\\' => {
526                    self.iter.next();
527
528                    if self.iter.next().is_none() {
529                        return Err(compile::Error::new(
530                            self.iter.span_to_pos(s),
531                            ErrorKind::ExpectedEscape,
532                        ));
533                    }
534
535                    escaped = true;
536                }
537                '`' => {
538                    let span = self.iter.span_to_pos(start);
539                    let had_string = start != self.iter.pos();
540                    let start = self.iter.pos();
541                    self.iter.next();
542
543                    let expressions = self.modes.expression_count(&self.iter, start)?;
544
545                    if had_string {
546                        if *expressions > 0 {
547                            self.buffer.try_push_back(ast::Token {
548                                kind: ast::Kind::Comma,
549                                span,
550                            })?;
551                        }
552
553                        self.buffer.try_push_back(ast::Token {
554                            kind: ast::Kind::Str(ast::StrSource::Text(ast::StrText {
555                                source_id: self.source_id,
556                                escaped: take(&mut escaped),
557                                wrapped: false,
558                            })),
559                            span,
560                        })?;
561
562                        *expressions += 1;
563                    }
564
565                    self.buffer.try_push_back(ast::Token {
566                        kind: K![')'],
567                        span: self.iter.span_to_pos(start),
568                    })?;
569
570                    self.buffer.try_push_back(ast::Token {
571                        kind: ast::Kind::Close(ast::Delimiter::Empty),
572                        span: self.iter.span_to_pos(start),
573                    })?;
574
575                    let expressions = *expressions;
576                    self.modes
577                        .pop(&self.iter, LexerMode::Template(expressions))?;
578
579                    return Ok(());
580                }
581                _ => {
582                    self.iter.next();
583                }
584            }
585        }
586
587        Err(compile::Error::new(
588            self.iter.point_span(),
589            ErrorKind::UnexpectedEof,
590        ))
591    }
592
593    /// Consume the next token from the lexer.
594    #[allow(clippy::should_implement_trait)]
595    pub(crate) fn next(&mut self) -> compile::Result<Option<ast::Token>> {
596        'outer: loop {
597            if let Some(token) = self.buffer.pop_front() {
598                return Ok(Some(token));
599            }
600
601            let mode = self.modes.last();
602
603            let level = match mode {
604                LexerMode::Template(..) => {
605                    self.template_next()?;
606                    continue;
607                }
608                LexerMode::Default(level) => level,
609            };
610
611            let (start, c) = match self.iter.next_with_pos() {
612                Some(next) => next,
613                None => {
614                    self.modes.pop(&self.iter, LexerMode::Default(0))?;
615                    return Ok(None);
616                }
617            };
618
619            // Added here specifically to avoid skipping over leading whitespace
620            // tokens just below. We only ever want to parse shebangs which are
621            // the first two leading characters in any input.
622            if self.shebang {
623                self.shebang = false;
624
625                if matches!((c, self.iter.peek()), ('#', Some('!'))) {
626                    self.consume_line();
627
628                    return Ok(Some(ast::Token {
629                        kind: ast::Kind::Shebang(ast::LitSource::Text(self.source_id)),
630                        span: self.iter.span_to_pos(start),
631                    }));
632                }
633            }
634
635            if char::is_whitespace(c) {
636                self.consume_whitespace();
637
638                return Ok(Some(ast::Token {
639                    kind: ast::Kind::Whitespace,
640                    span: self.iter.span_to_pos(start),
641                }));
642            }
643
644            // This loop is useful, at least until it's rewritten.
645            #[allow(clippy::never_loop)]
646            let kind = loop {
647                if let Some(c2) = self.iter.peek() {
648                    match (c, c2) {
649                        ('+', '=') => {
650                            self.iter.next();
651                            break ast::Kind::PlusEq;
652                        }
653                        ('-', '=') => {
654                            self.iter.next();
655                            break ast::Kind::DashEq;
656                        }
657                        ('*', '=') => {
658                            self.iter.next();
659                            break ast::Kind::StarEq;
660                        }
661                        ('/', '=') => {
662                            self.iter.next();
663                            break ast::Kind::SlashEq;
664                        }
665                        ('%', '=') => {
666                            self.iter.next();
667                            break ast::Kind::PercEq;
668                        }
669                        ('&', '=') => {
670                            self.iter.next();
671                            break ast::Kind::AmpEq;
672                        }
673                        ('^', '=') => {
674                            self.iter.next();
675                            break ast::Kind::CaretEq;
676                        }
677                        ('|', '=') => {
678                            self.iter.next();
679                            break ast::Kind::PipeEq;
680                        }
681                        ('/', '/') => {
682                            self.iter.next();
683                            let (doc, inner) = self.check_doc_comment('/');
684
685                            self.consume_line();
686
687                            if self.process && doc {
688                                // docstring span drops the first 3 characters (/// or //!)
689                                let span = self.iter.span_to_pos(start);
690                                self.emit_doc_attribute(inner, span, span.trim_start(3))?;
691                                continue 'outer;
692                            } else {
693                                break ast::Kind::Comment;
694                            }
695                        }
696                        ('/', '*') => {
697                            self.iter.next();
698                            let (doc, inner) = self.check_doc_comment('*');
699                            let term = self.consume_multiline_comment();
700
701                            if !term {
702                                break ast::Kind::MultilineComment(false);
703                            }
704
705                            if self.process && doc {
706                                // docstring span drops the first 3 characters (/** or /*!)
707                                // drop the last two characters to remove */
708                                let span = self.iter.span_to_pos(start);
709                                self.emit_doc_attribute(
710                                    inner,
711                                    span,
712                                    span.trim_start(3).trim_end(2),
713                                )?;
714                                continue 'outer;
715                            } else {
716                                break ast::Kind::MultilineComment(true);
717                            }
718                        }
719                        (':', ':') => {
720                            self.iter.next();
721                            break ast::Kind::ColonColon;
722                        }
723                        ('<', '=') => {
724                            self.iter.next();
725                            break ast::Kind::LtEq;
726                        }
727                        ('>', '=') => {
728                            self.iter.next();
729                            break ast::Kind::GtEq;
730                        }
731                        ('=', '=') => {
732                            self.iter.next();
733                            break ast::Kind::EqEq;
734                        }
735                        ('!', '=') => {
736                            self.iter.next();
737                            break ast::Kind::BangEq;
738                        }
739                        ('&', '&') => {
740                            self.iter.next();
741                            break ast::Kind::AmpAmp;
742                        }
743                        ('|', '|') => {
744                            self.iter.next();
745                            break ast::Kind::PipePipe;
746                        }
747                        ('<', '<') => {
748                            self.iter.next();
749
750                            break if matches!(self.iter.peek(), Some('=')) {
751                                self.iter.next();
752                                ast::Kind::LtLtEq
753                            } else {
754                                ast::Kind::LtLt
755                            };
756                        }
757                        ('>', '>') => {
758                            self.iter.next();
759
760                            break if matches!(self.iter.peek(), Some('=')) {
761                                self.iter.next();
762                                ast::Kind::GtGtEq
763                            } else {
764                                ast::Kind::GtGt
765                            };
766                        }
767                        ('.', '.') => {
768                            self.iter.next();
769
770                            break if matches!(self.iter.peek(), Some('=')) {
771                                self.iter.next();
772                                ast::Kind::DotDotEq
773                            } else {
774                                ast::Kind::DotDot
775                            };
776                        }
777                        ('=', '>') => {
778                            self.iter.next();
779                            break ast::Kind::Rocket;
780                        }
781                        ('-', '>') => {
782                            self.iter.next();
783                            break ast::Kind::Arrow;
784                        }
785                        ('b', '\'') => {
786                            self.iter.next();
787                            self.iter.next();
788                            return self.next_lit_byte(start);
789                        }
790                        ('b', '"') => {
791                            self.iter.next();
792                            return self.next_str(
793                                start,
794                                || ErrorKind::UnterminatedByteStrLit,
795                                ast::Kind::ByteStr,
796                            );
797                        }
798                        _ => (),
799                    }
800                }
801
802                break match c {
803                    '(' => ast::Kind::Open(ast::Delimiter::Parenthesis),
804                    ')' => ast::Kind::Close(ast::Delimiter::Parenthesis),
805                    '{' => {
806                        if level > 0 {
807                            self.modes.push(LexerMode::Default(level + 1))?;
808                        }
809
810                        ast::Kind::Open(ast::Delimiter::Brace)
811                    }
812                    '}' => {
813                        if level > 0 {
814                            self.modes.pop(&self.iter, LexerMode::Default(level))?;
815
816                            // NB: end of expression in template.
817                            if level == 1 {
818                                let expressions = self.modes.expression_count(&self.iter, start)?;
819                                *expressions += 1;
820                                continue 'outer;
821                            }
822                        }
823
824                        ast::Kind::Close(ast::Delimiter::Brace)
825                    }
826                    '[' => ast::Kind::Open(ast::Delimiter::Bracket),
827                    ']' => ast::Kind::Close(ast::Delimiter::Bracket),
828                    ',' => ast::Kind::Comma,
829                    ':' => ast::Kind::Colon,
830                    '#' => ast::Kind::Pound,
831                    '.' => ast::Kind::Dot,
832                    ';' => ast::Kind::SemiColon,
833                    '=' => ast::Kind::Eq,
834                    '+' => ast::Kind::Plus,
835                    '-' => ast::Kind::Dash,
836                    '/' => ast::Kind::Div,
837                    '*' => ast::Kind::Star,
838                    '&' => ast::Kind::Amp,
839                    '>' => ast::Kind::Gt,
840                    '<' => ast::Kind::Lt,
841                    '!' => ast::Kind::Bang,
842                    '?' => ast::Kind::QuestionMark,
843                    '|' => ast::Kind::Pipe,
844                    '%' => ast::Kind::Perc,
845                    '^' => ast::Kind::Caret,
846                    '@' => ast::Kind::At,
847                    '$' => ast::Kind::Dollar,
848                    '~' => ast::Kind::Tilde,
849                    c if c == '_' || is_xid_start(c) => {
850                        return self.next_ident(start);
851                    }
852                    '0'..='9' => {
853                        return self.next_number_literal(c, start);
854                    }
855                    '"' => {
856                        return self.next_str(
857                            start,
858                            || ErrorKind::UnterminatedStrLit,
859                            ast::Kind::Str,
860                        );
861                    }
862                    '`' => {
863                        if self.process {
864                            let span = self.iter.span_to_pos(start);
865
866                            self.buffer.try_push_back(ast::Token {
867                                kind: ast::Kind::Open(ast::Delimiter::Empty),
868                                span,
869                            })?;
870
871                            self.emit_builtin_attribute(span)?;
872
873                            self.buffer.try_push_back(ast::Token {
874                                kind: ast::Kind::Ident(ast::LitSource::BuiltIn(
875                                    ast::BuiltIn::Template,
876                                )),
877                                span,
878                            })?;
879
880                            self.buffer
881                                .try_push_back(ast::Token { kind: K![!], span })?;
882
883                            self.buffer.try_push_back(ast::Token {
884                                kind: K!['('],
885                                span,
886                            })?;
887
888                            self.modes.push(LexerMode::Template(0))?;
889                            continue 'outer;
890                        }
891
892                        let mut level = 0u32;
893
894                        while let Some(c) = self.iter.next() {
895                            let n = match c {
896                                '{' => 1i32,
897                                '}' => -1i32,
898                                '\\' => {
899                                    _ = self.next();
900                                    continue;
901                                }
902                                '`' if level == 0 => break,
903                                _ => 0,
904                            };
905
906                            level = level.wrapping_add_signed(n);
907                        }
908
909                        ast::Kind::TemplateString
910                    }
911                    '\'' => {
912                        return self.next_char_or_label(start);
913                    }
914                    _ => {
915                        let span = self.iter.span_to_pos(start);
916                        return Err(compile::Error::new(span, ErrorKind::UnexpectedChar { c }));
917                    }
918                };
919            };
920
921            return Ok(Some(ast::Token {
922                kind,
923                span: self.iter.span_to_pos(start),
924            }));
925        }
926    }
927}
928
929#[derive(Debug, Clone)]
930struct SourceIter<'a> {
931    source: &'a str,
932    cursor: usize,
933}
934
935impl<'a> SourceIter<'a> {
936    fn new(source: &'a str) -> Self {
937        Self { source, cursor: 0 }
938    }
939
940    /// Get the current character position of the iterator.
941    #[inline]
942    fn pos(&self) -> usize {
943        self.cursor
944    }
945
946    /// Get the source from the given start, to the current position.
947    fn source_from(&self, start: usize) -> (&'a str, Span) {
948        let end = self.pos();
949        let span = Span::new(start, end);
950        (&self.source[start..end], span)
951    }
952
953    /// Get the current point span.
954    fn point_span(&self) -> Span {
955        Span::point(self.pos())
956    }
957
958    /// Get the span from the given start, to the current position.
959    fn span_to_pos(&self, start: usize) -> Span {
960        Span::new(start, self.pos())
961    }
962
963    /// Get the end span from the given start to the end of the source.
964    fn span_to_len(&self, start: usize) -> Span {
965        Span::new(start, self.source.len())
966    }
967
968    /// Peek the next index.
969    fn peek(&self) -> Option<char> {
970        self.source.get(self.cursor..)?.chars().next()
971    }
972
973    /// Peek the next next char.
974    fn peek2(&self) -> Option<char> {
975        let mut it = self.source.get(self.cursor..)?.chars();
976        it.next()?;
977        it.next()
978    }
979
980    /// Peek the next character with position.
981    fn peek_with_pos(&self) -> Option<(usize, char)> {
982        self.clone().next_with_pos()
983    }
984
985    /// Next with position.
986    fn next_with_pos(&mut self) -> Option<(usize, char)> {
987        let p = self.pos();
988        let c = self.next()?;
989        Some((p, c))
990    }
991}
992
993impl Iterator for SourceIter<'_> {
994    type Item = char;
995
996    /// Consume the next character.
997    fn next(&mut self) -> Option<Self::Item> {
998        let c = self.source.get(self.cursor..)?.chars().next()?;
999        self.cursor += c.len_utf8();
1000        Some(c)
1001    }
1002}
1003
1004#[derive(Debug, Default)]
1005struct LexerModes {
1006    modes: Vec<LexerMode>,
1007}
1008
1009impl LexerModes {
1010    /// Get the last mode.
1011    fn last(&self) -> LexerMode {
1012        self.modes.last().copied().unwrap_or_default()
1013    }
1014
1015    /// Push the given lexer mode.
1016    fn push(&mut self, mode: LexerMode) -> alloc::Result<()> {
1017        self.modes.try_push(mode)
1018    }
1019
1020    /// Pop the expected lexer mode.
1021    fn pop(&mut self, iter: &SourceIter<'_>, expected: LexerMode) -> compile::Result<()> {
1022        let actual = self.modes.pop().unwrap_or_default();
1023
1024        if actual != expected {
1025            return Err(compile::Error::new(
1026                iter.point_span(),
1027                ErrorKind::BadLexerMode { actual, expected },
1028            ));
1029        }
1030
1031        Ok(())
1032    }
1033
1034    /// Get the expression count.
1035    fn expression_count<'a>(
1036        &'a mut self,
1037        iter: &SourceIter<'_>,
1038        start: usize,
1039    ) -> compile::Result<&'a mut usize> {
1040        match self.modes.last_mut() {
1041            Some(LexerMode::Template(expression)) => Ok(expression),
1042            _ => {
1043                let span = iter.span_to_pos(start);
1044                Err(compile::Error::new(
1045                    span,
1046                    ErrorKind::BadLexerMode {
1047                        actual: LexerMode::default(),
1048                        expected: LexerMode::Template(0),
1049                    },
1050                ))
1051            }
1052        }
1053    }
1054}
1055
1056/// The mode of the lexer.
1057#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1058pub enum LexerMode {
1059    /// Default mode, boolean indicating if we are inside a template or not.
1060    Default(usize),
1061    /// We are parsing a template string.
1062    Template(usize),
1063}
1064
1065impl Default for LexerMode {
1066    fn default() -> Self {
1067        Self::Default(0)
1068    }
1069}
1070
1071impl fmt::Display for LexerMode {
1072    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1073        match self {
1074            LexerMode::Default(..) => {
1075                write!(f, "default")?;
1076            }
1077            LexerMode::Template(..) => {
1078                write!(f, "template")?;
1079            }
1080        }
1081
1082        Ok(())
1083    }
1084}
rune/parse/lexer.rs

rune/parse/
lexer.rs
