rune/ast/
token.rs

Help
1use core::ascii;
2use core::fmt;
3use core::ops::Neg;
4
5use crate::ast::prelude::*;
6use crate::ast::{Kind, Span, Spanned};
7use crate::compile;
8use crate::macros::{MacroContext, SyntheticId, ToTokens, TokenStream};
9use crate::parse::{Expectation, IntoExpectation, Parse, Parser, Peek};
10use crate::SourceId;
11
12/// A single token encountered during parsing.
13#[derive(TryClone, Clone, Copy, PartialEq, Eq, Hash)]
14#[try_clone(copy)]
15#[non_exhaustive]
16pub struct Token {
17    /// The span of the token.
18    pub span: Span,
19    /// The kind of the token.
20    pub kind: Kind,
21}
22
23impl Token {
24    /// Format the current token to a formatter.
25    pub(crate) fn token_fmt(&self, cx: &MacroContext, f: &mut fmt::Formatter<'_>) -> fmt::Result {
26        match &self.kind {
27            Kind::Eof | Kind::Error => {
28                // NB: marker tokens can't be formatted.
29                return Err(fmt::Error);
30            }
31            Kind::Ident(s) => {
32                let literal = cx.literal_source(*s, self.span).ok_or(fmt::Error)?;
33                write!(f, "{}", literal)?;
34            }
35            Kind::Label(s) => {
36                let literal = cx.literal_source(*s, self.span).ok_or(fmt::Error)?;
37                write!(f, "'{}", literal)?;
38            }
39            Kind::Byte(s) => match s {
40                CopySource::Text(source_id) => {
41                    let s = cx
42                        .idx
43                        .q
44                        .sources
45                        .source(*source_id, self.span)
46                        .ok_or(fmt::Error)?;
47                    write!(f, "{}", s)?;
48                }
49                CopySource::Inline(b) => {
50                    write!(f, "{:?}", b)?;
51                }
52            },
53            Kind::ByteStr(s) => match s {
54                StrSource::Text(text) => {
55                    let span = if text.wrapped {
56                        self.span.narrow(1u32)
57                    } else {
58                        self.span
59                    };
60
61                    let s = cx
62                        .idx
63                        .q
64                        .sources
65                        .source(text.source_id, span)
66                        .ok_or(fmt::Error)?;
67
68                    write!(f, "b\"{}\"", s)?;
69                }
70                StrSource::Synthetic(id) => {
71                    let b = cx.idx.q.storage.get_byte_string(*id).ok_or(fmt::Error)?;
72                    write!(f, "{}", FormatBytes(b))?;
73                }
74            },
75            Kind::Str(s) => match s {
76                StrSource::Text(text) => {
77                    let span = if text.wrapped {
78                        self.span.narrow(1u32)
79                    } else {
80                        self.span
81                    };
82
83                    let s = cx
84                        .idx
85                        .q
86                        .sources
87                        .source(text.source_id, span)
88                        .ok_or(fmt::Error)?;
89                    write!(f, "\"{}\"", s)?;
90                }
91                StrSource::Synthetic(id) => {
92                    let s = cx.idx.q.storage.get_string(*id).ok_or(fmt::Error)?;
93                    write!(f, "{:?}", s)?;
94                }
95            },
96            Kind::Char(s) => match s {
97                CopySource::Text(source_id) => {
98                    let s = cx
99                        .idx
100                        .q
101                        .sources
102                        .source(*source_id, self.span)
103                        .ok_or(fmt::Error)?;
104                    write!(f, "{}", s)?;
105                }
106                CopySource::Inline(c) => {
107                    write!(f, "{:?}", c)?;
108                }
109            },
110            Kind::Number(s) => match s {
111                NumberSource::Text(text) => {
112                    let s = cx
113                        .idx
114                        .q
115                        .sources
116                        .source(text.source_id, self.span)
117                        .ok_or(fmt::Error)?;
118                    write!(f, "{}", s)?;
119                }
120                NumberSource::Synthetic(id) => {
121                    let n = cx.idx.q.storage.get_number(*id).ok_or(fmt::Error)?;
122                    write!(f, "{}", n)?;
123                }
124            },
125            other => {
126                let s = other.as_literal_str().ok_or(fmt::Error)?;
127                write!(f, "{}", s)?;
128            }
129        }
130
131        return Ok(());
132
133        struct FormatBytes<'a>(&'a [u8]);
134
135        impl fmt::Display for FormatBytes<'_> {
136            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137                write!(f, "b\"")?;
138
139                for b in bytes_escape_default(self.0) {
140                    write!(f, "{}", b as char)?;
141                }
142
143                write!(f, "\"")?;
144                Ok(())
145            }
146        }
147
148        fn bytes_escape_default(bytes: &[u8]) -> impl Iterator<Item = u8> + '_ {
149            bytes.iter().copied().flat_map(ascii::escape_default)
150        }
151    }
152}
153
154impl fmt::Debug for Token {
155    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
156        write!(f, "{:?}@{:?}", self.kind, self.span)
157    }
158}
159
160impl Parse for Token {
161    fn parse(p: &mut Parser<'_>) -> compile::Result<Self> {
162        p.next()
163    }
164}
165
166impl Peek for Token {
167    fn peek(p: &mut super::prelude::Peeker<'_>) -> bool {
168        !p.is_eof()
169    }
170}
171
172impl ToTokens for Token {
173    fn to_tokens(
174        &self,
175        _: &mut MacroContext<'_, '_, '_>,
176        stream: &mut TokenStream,
177    ) -> alloc::Result<()> {
178        stream.push(*self)
179    }
180}
181
182impl Spanned for Token {
183    fn span(&self) -> Span {
184        self.span
185    }
186}
187
188impl IntoExpectation for Token {
189    fn into_expectation(self) -> Expectation {
190        self.kind.into_expectation()
191    }
192}
193
194/// The value of a number literal.
195#[derive(Debug, TryClone)]
196#[non_exhaustive]
197pub enum NumberValue {
198    /// A float literal number.
199    Float(f64),
200    /// An integer literal number.
201    Integer(#[try_clone(with = num::BigInt::clone)] num::BigInt),
202}
203
204/// The literal size of a number.
205///
206/// If this comes from a `u8` or `i8` suffix it would be `S8`.
207#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
208#[try_clone(copy)]
209#[non_exhaustive]
210pub enum NumberSize {
211    /// A 8-bit sized integer.
212    S8,
213    /// A 16-bit sized integer.
214    S16,
215    /// A 32-bit sized integer.
216    S32,
217    /// A 64-bit sized integer.
218    S64,
219}
220
221impl NumberSize {
222    pub(crate) fn signed_in(&self, value: i64) -> bool {
223        self.signed_min() <= value && value <= self.signed_max()
224    }
225
226    pub(crate) fn unsigned_in(&self, value: u64) -> bool {
227        self.unsigned_min() <= value && value <= self.unsigned_max()
228    }
229
230    pub(crate) fn signed_min(&self) -> i64 {
231        match self {
232            Self::S8 => i8::MIN as i64,
233            Self::S16 => i16::MIN as i64,
234            Self::S32 => i32::MIN as i64,
235            Self::S64 => i64::MIN,
236        }
237    }
238
239    pub(crate) fn signed_max(&self) -> i64 {
240        match self {
241            Self::S8 => i8::MAX as i64,
242            Self::S16 => i16::MAX as i64,
243            Self::S32 => i32::MAX as i64,
244            Self::S64 => i64::MAX,
245        }
246    }
247
248    pub(crate) fn unsigned_min(&self) -> u64 {
249        match self {
250            Self::S8 => u8::MIN as u64,
251            Self::S16 => u16::MIN as u64,
252            Self::S32 => u32::MIN as u64,
253            Self::S64 => u64::MIN,
254        }
255    }
256
257    pub(crate) fn unsigned_max(&self) -> u64 {
258        match self {
259            Self::S8 => u8::MAX as u64,
260            Self::S16 => u16::MAX as u64,
261            Self::S32 => u32::MAX as u64,
262            Self::S64 => u64::MAX,
263        }
264    }
265}
266
267/// The suffix of a number.
268#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
269#[try_clone(copy)]
270#[non_exhaustive]
271pub enum NumberSuffix {
272    /// The `u64` suffix.
273    Unsigned(Span, NumberSize),
274    /// The `i64` suffix.
275    Signed(Span, NumberSize),
276    /// The `f64` suffix.
277    Float(Span),
278}
279
280/// A resolved number literal.
281#[derive(Debug, TryClone)]
282#[non_exhaustive]
283pub struct Number {
284    /// The parsed number value.
285    pub value: NumberValue,
286    /// The parsed number suffix.
287    pub suffix: Option<NumberSuffix>,
288}
289
290impl Number {
291    /// Convert into a 32-bit unsigned number.
292    pub(crate) fn as_u32(&self, neg: bool) -> Option<u32> {
293        self.as_primitive(neg, num::ToPrimitive::to_u32)
294    }
295
296    /// Convert into usize.
297    pub(crate) fn as_usize(&self, neg: bool) -> Option<usize> {
298        self.as_primitive(neg, num::ToPrimitive::to_usize)
299    }
300
301    /// Try to convert number into a tuple index.
302    pub(crate) fn as_tuple_index(&self) -> Option<usize> {
303        use num::ToPrimitive;
304
305        match &self.value {
306            NumberValue::Integer(n) => n.to_usize(),
307            _ => None,
308        }
309    }
310
311    fn as_primitive<T>(&self, neg: bool, to: impl FnOnce(&num::BigInt) -> Option<T>) -> Option<T> {
312        let NumberValue::Integer(number) = &self.value else {
313            return None;
314        };
315
316        let mut number = number;
317        let negated;
318
319        if neg {
320            negated = number.clone().neg();
321            number = &negated;
322        }
323
324        to(number)
325    }
326}
327
328macro_rules! impl_from_int {
329    ($($ty:ty),*) => {
330        $(
331            impl From<$ty> for Number {
332                #[inline]
333                fn from(value: $ty) -> Self {
334                    Self {
335                        value: NumberValue::Integer(num::BigInt::from(value)),
336                        suffix: None,
337                    }
338                }
339            }
340        )*
341    };
342}
343
344impl_from_int!(usize, isize, i8, u8, i16, u16, i32, u32, i64, u64, i128, u128);
345
346impl From<f32> for Number {
347    #[inline]
348    fn from(value: f32) -> Self {
349        Self {
350            value: NumberValue::Float(value as f64),
351            suffix: None,
352        }
353    }
354}
355
356impl From<f64> for Number {
357    #[inline]
358    fn from(value: f64) -> Self {
359        Self {
360            value: NumberValue::Float(value),
361            suffix: None,
362        }
363    }
364}
365
366impl fmt::Display for Number {
367    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
368        match &self.value {
369            NumberValue::Float(n) => write!(f, "{}", n),
370            NumberValue::Integer(n) => write!(f, "{}", n),
371        }
372    }
373}
374
375/// The kind of a number literal.
376#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
377#[try_clone(copy)]
378#[non_exhaustive]
379pub enum NumberBase {
380    /// A decimal number literal, like `3.14`.
381    Decimal,
382    /// A hex literal, like `0xffff`.
383    Hex,
384    /// An octal literal, like `0o7711`.
385    Octal,
386    /// A binary literal, like `0b110011`.
387    Binary,
388}
389
390impl fmt::Display for NumberBase {
391    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
392        match *self {
393            Self::Decimal => write!(fmt, "decimal"),
394            Self::Hex => write!(fmt, "hex"),
395            Self::Octal => write!(fmt, "octal"),
396            Self::Binary => write!(fmt, "binary"),
397        }
398    }
399}
400
401/// A built-in identifiers that do not have a source.
402///
403/// This is necessary to synthesize identifiers in the lexer since there's not
404/// storage available, nor is the identifier reflected in the source.
405#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
406#[try_clone(copy)]
407#[non_exhaustive]
408pub enum BuiltIn {
409    /// `template`.
410    Template,
411    /// `formatspec`.
412    Format,
413    /// `builtin`.
414    BuiltIn,
415    /// `literal`.
416    Literal,
417    /// `doc`.
418    Doc,
419}
420
421impl BuiltIn {
422    /// Coerce into static string.
423    pub(crate) fn as_str(self) -> &'static str {
424        match self {
425            Self::Template => "template",
426            Self::Format => "formatspec",
427            Self::BuiltIn => "builtin",
428            Self::Literal => "literal",
429            Self::Doc => "doc",
430        }
431    }
432}
433
434impl fmt::Display for BuiltIn {
435    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
436        f.write_str(self.as_str())
437    }
438}
439
440/// The kind of the identifier.
441#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
442#[try_clone(copy)]
443#[non_exhaustive]
444pub enum LitSource {
445    /// The identifier is from the source text.
446    Text(SourceId),
447    /// The identifier is synthetic (generated in a macro).
448    Synthetic(SyntheticId),
449    /// Built-in strings.
450    BuiltIn(BuiltIn),
451}
452
453/// The source of the literal string. This need to be treated separately from
454/// [LitSource] because it might encompass special things like quoting and
455/// escaping.
456#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
457#[try_clone(copy)]
458#[non_exhaustive]
459pub enum StrSource {
460    /// The literal string source is from the source text.
461    Text(StrText),
462    /// The string source is synthetic (generated in a macro).
463    Synthetic(SyntheticId),
464}
465
466/// Configuration for a literal string.
467#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
468#[try_clone(copy)]
469#[non_exhaustive]
470pub struct StrText {
471    /// The source of the text.
472    pub source_id: SourceId,
473    /// Indicates if the string is escaped or not.
474    pub escaped: bool,
475    /// Indicated if the buffer is wrapped or not.
476    pub wrapped: bool,
477}
478
479/// The source of a number.
480#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
481#[try_clone(copy)]
482#[non_exhaustive]
483pub enum NumberSource {
484    /// The number is from the source text (and need to be parsed while it's
485    /// being resolved).
486    Text(NumberText),
487    /// The number is synthetic, and stored in the specified slot.
488    Synthetic(SyntheticId),
489}
490
491/// The source of an item that implements Copy.
492#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
493#[try_clone(copy)]
494#[non_exhaustive]
495#[try_clone(bound = {T: TryClone})]
496pub enum CopySource<T>
497where
498    T: Copy,
499{
500    /// The item is from the source text (and need to be parsed while it's being
501    /// resolved).
502    Text(SourceId),
503    /// The char is inlined in the ast.
504    Inline(T),
505}
506
507/// Configuration of a text number.
508#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
509#[try_clone(copy)]
510#[non_exhaustive]
511pub struct NumberText {
512    /// The source of the text.
513    pub source_id: SourceId,
514    /// Indicates if it's a decimal number.
515    pub is_fractional: bool,
516    /// The number literal kind.
517    pub base: NumberBase,
518    /// The number part of the parsed number.
519    pub number: Span,
520    /// The suffix.
521    pub suffix: Span,
522}
523
524/// A delimiter, `{`, `{`, or `[`.
525#[derive(Debug, TryClone, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
526#[try_clone(copy)]
527#[non_exhaustive]
528pub enum Delimiter {
529    /// A parenthesis delimiter `(` and `)`.
530    Parenthesis,
531    /// A brace delimiter `{` and `}`.
532    Brace,
533    /// A bracket delimiter `[` and `]`.
534    Bracket,
535    /// An empty group delimiter.
536    Empty,
537}
538
539impl Delimiter {
540    /// The character used as an open delimiter.
541    pub(crate) fn open(self) -> &'static str {
542        match self {
543            Self::Parenthesis => "(",
544            Self::Brace => "{",
545            Self::Bracket => "[",
546            Self::Empty => "",
547        }
548    }
549
550    /// The character used as a close delimiter.
551    pub(crate) fn close(self) -> &'static str {
552        match self {
553            Self::Parenthesis => ")",
554            Self::Brace => "}",
555            Self::Bracket => "]",
556            Self::Empty => "",
557        }
558    }
559}
rune/ast/token.rs

rune/ast/
token.rs
