rune/ast/
token.rs

1use core::ascii;
2use core::fmt;
3
4use crate::ast::prelude::*;
5use crate::ast::{Kind, Span, Spanned};
6use crate::compile;
7use crate::macros::{MacroContext, SyntheticId, ToTokens, TokenStream};
8use crate::parse::{Expectation, IntoExpectation, Parse, Parser, Peek};
9use crate::SourceId;
10
11/// A single token encountered during parsing.
12#[derive(TryClone, Clone, Copy, PartialEq, Eq, Hash)]
13#[try_clone(copy)]
14#[non_exhaustive]
15pub struct Token {
16    /// The span of the token.
17    pub span: Span,
18    /// The kind of the token.
19    pub kind: Kind,
20}
21
22impl Token {
23    /// Format the current token to a formatter.
24    pub(crate) fn token_fmt(&self, cx: &MacroContext, f: &mut fmt::Formatter<'_>) -> fmt::Result {
25        match &self.kind {
26            Kind::Eof | Kind::Error => {
27                // NB: marker tokens can't be formatted.
28                return Err(fmt::Error);
29            }
30            Kind::Ident(s) => {
31                let literal = cx.literal_source(*s, self.span).ok_or(fmt::Error)?;
32                write!(f, "{literal}")?;
33            }
34            Kind::Label(s) => {
35                let literal = cx.literal_source(*s, self.span).ok_or(fmt::Error)?;
36                write!(f, "'{literal}")?;
37            }
38            Kind::Byte(s) => match s {
39                CopySource::Text(source_id) => {
40                    let s = cx
41                        .idx
42                        .q
43                        .sources
44                        .source(*source_id, self.span)
45                        .ok_or(fmt::Error)?;
46                    write!(f, "{s}")?;
47                }
48                CopySource::Inline(b) => {
49                    write!(f, "{b:?}")?;
50                }
51            },
52            Kind::ByteStr(s) => match s {
53                StrSource::Text(text) => {
54                    let span = if text.wrapped {
55                        self.span.narrow(1u32)
56                    } else {
57                        self.span
58                    };
59
60                    let s = cx
61                        .idx
62                        .q
63                        .sources
64                        .source(text.source_id, span)
65                        .ok_or(fmt::Error)?;
66
67                    write!(f, "b\"{s}\"")?;
68                }
69                StrSource::Synthetic(id) => {
70                    let b = cx.idx.q.storage.get_byte_string(*id).ok_or(fmt::Error)?;
71                    write!(f, "{}", FormatBytes(b))?;
72                }
73            },
74            Kind::Str(s) => match s {
75                StrSource::Text(text) => {
76                    let span = if text.wrapped {
77                        self.span.narrow(1u32)
78                    } else {
79                        self.span
80                    };
81
82                    let s = cx
83                        .idx
84                        .q
85                        .sources
86                        .source(text.source_id, span)
87                        .ok_or(fmt::Error)?;
88                    write!(f, "\"{s}\"")?;
89                }
90                StrSource::Synthetic(id) => {
91                    let s = cx.idx.q.storage.get_string(*id).ok_or(fmt::Error)?;
92                    write!(f, "{s:?}")?;
93                }
94            },
95            Kind::Char(s) => match s {
96                CopySource::Text(source_id) => {
97                    let s = cx
98                        .idx
99                        .q
100                        .sources
101                        .source(*source_id, self.span)
102                        .ok_or(fmt::Error)?;
103                    write!(f, "{s}")?;
104                }
105                CopySource::Inline(c) => {
106                    write!(f, "{c:?}")?;
107                }
108            },
109            Kind::Number(s) => match s {
110                NumberSource::Text(text) => {
111                    let s = cx
112                        .idx
113                        .q
114                        .sources
115                        .source(text.source_id, self.span)
116                        .ok_or(fmt::Error)?;
117                    write!(f, "{s}")?;
118                }
119                NumberSource::Synthetic(id) => {
120                    let n = cx.idx.q.storage.get_number(*id).ok_or(fmt::Error)?;
121                    write!(f, "{n}")?;
122                }
123            },
124            other => {
125                let s = other.as_literal_str().ok_or(fmt::Error)?;
126                write!(f, "{s}")?;
127            }
128        }
129
130        return Ok(());
131
132        struct FormatBytes<'a>(&'a [u8]);
133
134        impl fmt::Display for FormatBytes<'_> {
135            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
136                write!(f, "b\"")?;
137
138                for b in bytes_escape_default(self.0) {
139                    write!(f, "{}", b as char)?;
140                }
141
142                write!(f, "\"")?;
143                Ok(())
144            }
145        }
146
147        fn bytes_escape_default(bytes: &[u8]) -> impl Iterator<Item = u8> + '_ {
148            bytes.iter().copied().flat_map(ascii::escape_default)
149        }
150    }
151}
152
153impl fmt::Debug for Token {
154    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
155        write!(f, "{:?}@{:?}", self.kind, self.span)
156    }
157}
158
159impl Parse for Token {
160    fn parse(p: &mut Parser<'_>) -> compile::Result<Self> {
161        p.next()
162    }
163}
164
165impl Peek for Token {
166    fn peek(p: &mut super::prelude::Peeker<'_>) -> bool {
167        !p.is_eof()
168    }
169}
170
171impl ToTokens for Token {
172    fn to_tokens(
173        &self,
174        _: &mut MacroContext<'_, '_, '_>,
175        stream: &mut TokenStream,
176    ) -> alloc::Result<()> {
177        stream.push(*self)
178    }
179}
180
181impl Spanned for Token {
182    fn span(&self) -> Span {
183        self.span
184    }
185}
186
187impl IntoExpectation for Token {
188    fn into_expectation(self) -> Expectation {
189        self.kind.into_expectation()
190    }
191}
192
193/// The value of a number literal.
194#[derive(Debug, Clone, Copy, TryClone)]
195#[non_exhaustive]
196#[try_clone(copy)]
197pub enum NumberValue {
198    /// A float literal number.
199    Float(f64),
200    /// An integer literal number.
201    Integer(i128),
202}
203
204/// The literal size of a number.
205///
206/// If this comes from a `u8` or `i8` suffix it would be `S8`.
207#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
208#[try_clone(copy)]
209#[non_exhaustive]
210pub enum NumberSize {
211    /// A 8-bit sized integer.
212    S8,
213    /// A 16-bit sized integer.
214    S16,
215    /// A 32-bit sized integer.
216    S32,
217    /// A 64-bit sized integer.
218    S64,
219}
220
221impl NumberSize {
222    pub(crate) fn signed_in(&self, value: i64) -> bool {
223        self.signed_min() <= value && value <= self.signed_max()
224    }
225
226    pub(crate) fn unsigned_in(&self, value: u64) -> bool {
227        self.unsigned_min() <= value && value <= self.unsigned_max()
228    }
229
230    pub(crate) fn signed_min(&self) -> i64 {
231        match self {
232            Self::S8 => i8::MIN as i64,
233            Self::S16 => i16::MIN as i64,
234            Self::S32 => i32::MIN as i64,
235            Self::S64 => i64::MIN,
236        }
237    }
238
239    pub(crate) fn signed_max(&self) -> i64 {
240        match self {
241            Self::S8 => i8::MAX as i64,
242            Self::S16 => i16::MAX as i64,
243            Self::S32 => i32::MAX as i64,
244            Self::S64 => i64::MAX,
245        }
246    }
247
248    pub(crate) fn unsigned_min(&self) -> u64 {
249        match self {
250            Self::S8 => u8::MIN as u64,
251            Self::S16 => u16::MIN as u64,
252            Self::S32 => u32::MIN as u64,
253            Self::S64 => u64::MIN,
254        }
255    }
256
257    pub(crate) fn unsigned_max(&self) -> u64 {
258        match self {
259            Self::S8 => u8::MAX as u64,
260            Self::S16 => u16::MAX as u64,
261            Self::S32 => u32::MAX as u64,
262            Self::S64 => u64::MAX,
263        }
264    }
265}
266
267/// The suffix of a number.
268#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
269#[try_clone(copy)]
270#[non_exhaustive]
271pub enum NumberSuffix {
272    /// The `u64` suffix.
273    Unsigned(Span, NumberSize),
274    /// The `i64` suffix.
275    Signed(Span, NumberSize),
276    /// The `f64` suffix.
277    Float(Span),
278}
279
280/// A resolved number literal.
281#[derive(Debug, Clone, Copy, TryClone)]
282#[non_exhaustive]
283#[try_clone(copy)]
284pub struct Number {
285    /// The parsed number value.
286    pub value: NumberValue,
287    /// The parsed number suffix.
288    pub suffix: Option<NumberSuffix>,
289}
290
291impl Number {
292    /// Convert into a 32-bit unsigned number.
293    pub(crate) fn as_u32(&self) -> Option<u32> {
294        match self.value {
295            NumberValue::Integer(n) => u32::try_from(n).ok(),
296            _ => None,
297        }
298    }
299
300    /// Convert into usize.
301    pub(crate) fn as_usize(&self) -> Option<usize> {
302        match self.value {
303            NumberValue::Integer(n) => usize::try_from(n).ok(),
304            _ => None,
305        }
306    }
307
308    /// Try to convert number into a tuple index.
309    pub(crate) fn as_tuple_index(&self) -> Option<usize> {
310        match self.value {
311            NumberValue::Integer(n) => usize::try_from(n).ok(),
312            _ => None,
313        }
314    }
315}
316
317macro_rules! from_unsigned {
318    ($($ty:ty),*) => {
319        $(
320            impl From<$ty> for Number {
321                #[inline]
322                fn from(value: $ty) -> Self {
323                    Self {
324                        value: NumberValue::Integer(value as i128),
325                        suffix: None,
326                    }
327                }
328            }
329        )*
330    };
331}
332
333macro_rules! from_signed {
334    ($($ty:ty),*) => {
335        $(
336            impl From<$ty> for Number {
337                #[inline]
338                fn from(value: $ty) -> Self {
339                    Self {
340                        value: NumberValue::Integer(value as i128),
341                        suffix: None,
342                    }
343                }
344            }
345        )*
346    };
347}
348
349from_unsigned!(usize, u8, u16, u32, u64);
350from_signed!(isize, i8, i16, i32, i64);
351
352impl From<f32> for Number {
353    #[inline]
354    fn from(value: f32) -> Self {
355        Self {
356            value: NumberValue::Float(value as f64),
357            suffix: None,
358        }
359    }
360}
361
362impl From<f64> for Number {
363    #[inline]
364    fn from(value: f64) -> Self {
365        Self {
366            value: NumberValue::Float(value),
367            suffix: None,
368        }
369    }
370}
371
372impl fmt::Display for Number {
373    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
374        match &self.value {
375            NumberValue::Float(n) => n.fmt(f),
376            NumberValue::Integer(n) => n.fmt(f),
377        }
378    }
379}
380
381/// The kind of a number literal.
382#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
383#[try_clone(copy)]
384#[non_exhaustive]
385pub enum NumberBase {
386    /// A decimal number literal, like `3.14`.
387    Decimal,
388    /// A hex literal, like `0xffff`.
389    Hex,
390    /// An octal literal, like `0o7711`.
391    Octal,
392    /// A binary literal, like `0b110011`.
393    Binary,
394}
395
396impl fmt::Display for NumberBase {
397    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
398        match *self {
399            Self::Decimal => write!(fmt, "decimal"),
400            Self::Hex => write!(fmt, "hex"),
401            Self::Octal => write!(fmt, "octal"),
402            Self::Binary => write!(fmt, "binary"),
403        }
404    }
405}
406
407/// A built-in identifiers that do not have a source.
408///
409/// This is necessary to synthesize identifiers in the lexer since there's not
410/// storage available, nor is the identifier reflected in the source.
411#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
412#[try_clone(copy)]
413#[non_exhaustive]
414pub enum BuiltIn {
415    /// `template`.
416    Template,
417    /// `formatspec`.
418    Format,
419    /// `builtin`.
420    BuiltIn,
421    /// `literal`.
422    Literal,
423    /// `doc`.
424    Doc,
425}
426
427impl BuiltIn {
428    /// Coerce into static string.
429    pub(crate) fn as_str(self) -> &'static str {
430        match self {
431            Self::Template => "template",
432            Self::Format => "formatspec",
433            Self::BuiltIn => "builtin",
434            Self::Literal => "literal",
435            Self::Doc => "doc",
436        }
437    }
438}
439
440impl fmt::Display for BuiltIn {
441    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
442        f.write_str(self.as_str())
443    }
444}
445
446/// The kind of the identifier.
447#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
448#[try_clone(copy)]
449#[non_exhaustive]
450pub enum LitSource {
451    /// The identifier is from the source text.
452    Text(SourceId),
453    /// The identifier is synthetic (generated in a macro).
454    Synthetic(SyntheticId),
455    /// Built-in strings.
456    BuiltIn(BuiltIn),
457}
458
459/// The source of the literal string. This need to be treated separately from
460/// [LitSource] because it might encompass special things like quoting and
461/// escaping.
462#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
463#[try_clone(copy)]
464#[non_exhaustive]
465pub enum StrSource {
466    /// The literal string source is from the source text.
467    Text(StrText),
468    /// The string source is synthetic (generated in a macro).
469    Synthetic(SyntheticId),
470}
471
472/// Configuration for a literal string.
473#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
474#[try_clone(copy)]
475#[non_exhaustive]
476pub struct StrText {
477    /// The source of the text.
478    pub source_id: SourceId,
479    /// Indicates if the string is escaped or not.
480    pub escaped: bool,
481    /// Indicated if the buffer is wrapped or not.
482    pub wrapped: bool,
483}
484
485/// The source of a number.
486#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
487#[try_clone(copy)]
488#[non_exhaustive]
489pub enum NumberSource {
490    /// The number is from the source text (and need to be parsed while it's
491    /// being resolved).
492    Text(NumberText),
493    /// The number is synthetic, and stored in the specified slot.
494    Synthetic(SyntheticId),
495}
496
497/// The source of an item that implements Copy.
498#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
499#[try_clone(copy)]
500#[non_exhaustive]
501#[try_clone(bound = {T: TryClone})]
502pub enum CopySource<T>
503where
504    T: Copy,
505{
506    /// The item is from the source text (and need to be parsed while it's being
507    /// resolved).
508    Text(SourceId),
509    /// The char is inlined in the ast.
510    Inline(T),
511}
512
513/// Configuration of a text number.
514#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
515#[try_clone(copy)]
516#[non_exhaustive]
517pub struct NumberText {
518    /// The source of the text.
519    pub source_id: SourceId,
520    /// Indicates if it's a fractional number.
521    ///
522    /// A number is a fractional number if it contains a decimal point.
523    pub is_fractional: bool,
524    /// The number literal kind.
525    pub base: NumberBase,
526    /// The number part of the parsed number.
527    pub number: Span,
528    /// The suffix.
529    pub suffix: Span,
530}
531
532/// A delimiter, `{`, `{`, or `[`.
533#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
534#[try_clone(copy)]
535#[non_exhaustive]
536pub enum Delimiter {
537    /// A parenthesis delimiter `(` and `)`.
538    Parenthesis,
539    /// A brace delimiter `{` and `}`.
540    Brace,
541    /// A bracket delimiter `[` and `]`.
542    Bracket,
543    /// An empty group delimiter.
544    Empty,
545}
546
547impl Delimiter {
548    /// The character used as an open delimiter.
549    pub(crate) fn open(self) -> &'static str {
550        match self {
551            Self::Parenthesis => "(",
552            Self::Brace => "{",
553            Self::Bracket => "[",
554            Self::Empty => "",
555        }
556    }
557
558    /// The character used as a close delimiter.
559    pub(crate) fn close(self) -> &'static str {
560        match self {
561            Self::Parenthesis => ")",
562            Self::Brace => "}",
563            Self::Bracket => "]",
564            Self::Empty => "",
565        }
566    }
567}
rune/ast/token.rs

rune/ast/
token.rs
