pest/iterators/
tokens.rs

Help
1// pest. The Elegant Parser
2// Copyright (c) 2018 Dragoș Tiselice
3//
4// Licensed under the Apache License, Version 2.0
5// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
6// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7// option. All files in the project carrying such notice may not be copied,
8// modified, or distributed except according to those terms.
9
10use alloc::rc::Rc;
11use alloc::vec::Vec;
12use core::fmt;
13use core::str;
14
15use super::queueable_token::QueueableToken;
16use crate::position;
17use crate::token::Token;
18use crate::RuleType;
19
20/// An iterator over [`Token`]s. It is created by [`Pair::tokens`] and [`Pairs::tokens`].
21///
22/// [`Token`]: ../enum.Token.html
23/// [`Pair::tokens`]: struct.Pair.html#method.tokens
24/// [`Pairs::tokens`]: struct.Pairs.html#method.tokens
25#[derive(Clone)]
26pub struct Tokens<'i, R> {
27    queue: Rc<Vec<QueueableToken<'i, R>>>,
28    input: &'i str,
29    start: usize,
30    end: usize,
31}
32
33pub fn new<'i, R: RuleType>(
34    queue: Rc<Vec<QueueableToken<'i, R>>>,
35    input: &'i str,
36    start: usize,
37    end: usize,
38) -> Tokens<'i, R> {
39    if cfg!(debug_assertions) {
40        for tok in queue.iter() {
41            match *tok {
42                QueueableToken::Start { input_pos, .. } | QueueableToken::End { input_pos, .. } => {
43                    assert!(
44                        input.get(input_pos..).is_some(),
45                        "💥 INVALID `Tokens` CREATED 💥"
46                    )
47                }
48            }
49        }
50    }
51
52    Tokens {
53        queue,
54        input,
55        start,
56        end,
57    }
58}
59
60impl<'i, R: RuleType> Tokens<'i, R> {
61    fn create_token(&self, index: usize) -> Token<'i, R> {
62        match self.queue[index] {
63            QueueableToken::Start {
64                end_token_index,
65                input_pos,
66            } => {
67                let rule = match self.queue[end_token_index] {
68                    QueueableToken::End { rule, .. } => rule,
69                    _ => unreachable!(),
70                };
71
72                Token::Start {
73                    rule,
74                    pos: position::Position::new_internal(self.input, input_pos),
75                }
76            }
77            QueueableToken::End {
78                rule, input_pos, ..
79            } => Token::End {
80                rule,
81                pos: position::Position::new_internal(self.input, input_pos),
82            },
83        }
84    }
85}
86
87impl<'i, R: RuleType> ExactSizeIterator for Tokens<'i, R> {
88    fn len(&self) -> usize {
89        self.end - self.start
90    }
91}
92
93impl<'i, R: RuleType> Iterator for Tokens<'i, R> {
94    type Item = Token<'i, R>;
95
96    fn next(&mut self) -> Option<Self::Item> {
97        if self.start >= self.end {
98            return None;
99        }
100
101        let token = self.create_token(self.start);
102
103        self.start += 1;
104
105        Some(token)
106    }
107
108    fn size_hint(&self) -> (usize, Option<usize>) {
109        let len = <Self as ExactSizeIterator>::len(self);
110        (len, Some(len))
111    }
112}
113
114impl<'i, R: RuleType> DoubleEndedIterator for Tokens<'i, R> {
115    fn next_back(&mut self) -> Option<Self::Item> {
116        if self.end <= self.start {
117            return None;
118        }
119
120        let token = self.create_token(self.end - 1);
121
122        self.end -= 1;
123
124        Some(token)
125    }
126}
127
128impl<'i, R: RuleType> fmt::Debug for Tokens<'i, R> {
129    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130        f.debug_list().entries(self.clone()).finish()
131    }
132}
133
134#[cfg(test)]
135mod tests {
136    use super::super::super::macros::tests::*;
137    use super::super::super::Parser;
138    use super::Token;
139    use alloc::vec::Vec;
140
141    #[test]
142    fn double_ended_iter_for_tokens() {
143        let pairs = AbcParser::parse(Rule::a, "abcde").unwrap();
144        let mut tokens = pairs.clone().tokens().collect::<Vec<Token<'_, Rule>>>();
145        tokens.reverse();
146        let reverse_tokens = pairs.tokens().rev().collect::<Vec<Token<'_, Rule>>>();
147        assert_eq!(tokens, reverse_tokens);
148    }
149
150    #[test]
151    fn exact_size_iter_for_tokens() {
152        let tokens = AbcParser::parse(Rule::a, "abcde").unwrap().tokens();
153        assert_eq!(tokens.len(), tokens.count());
154
155        let tokens = AbcParser::parse(Rule::a, "我很漂亮e").unwrap().tokens();
156        assert_eq!(tokens.len(), tokens.count());
157
158        let tokens = AbcParser::parse(Rule::a, "abcde").unwrap().tokens().rev();
159        assert_eq!(tokens.len(), tokens.count());
160
161        let mut tokens = AbcParser::parse(Rule::a, "abcde").unwrap().tokens();
162        let tokens_len = tokens.len();
163        let _ = tokens.next().unwrap();
164        assert_eq!(tokens.count() + 1, tokens_len);
165    }
166}
pest/iterators/tokens.rs

pest/iterators/
tokens.rs
