Skip to main content

rune/
highlight.rs

1//! Split source text into classified spans for syntax highlighting.
2//!
3//! This uses the lexer in the mode the formatter does, so it keeps whitespace
4//! and comments and copes with source that does not compile.
5
6use crate::alloc::{self, Vec};
7use crate::ast::{Kind, Span};
8use crate::parse::Lexer;
9use crate::SourceId;
10
11/// The class of a [`Token`].
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13#[non_exhaustive]
14pub enum Class {
15    /// A keyword, including `true`, `false` and `self`.
16    Keyword,
17    /// An identifier or label.
18    Ident,
19    /// A number literal.
20    Number,
21    /// A string or byte string literal.
22    String,
23    /// A character literal.
24    Char,
25    /// A byte literal.
26    Byte,
27    /// A template literal.
28    Template,
29    /// A comment which is not a doc comment.
30    Comment,
31    /// A doc comment.
32    DocComment,
33    /// An attribute, from `#` to its closing bracket.
34    Attribute,
35    /// An identifier immediately followed by `!`.
36    Macro,
37    /// Delimiters and separators.
38    Punctuation,
39    /// Operators.
40    Operator,
41    /// Whitespace.
42    Whitespace,
43    /// Text which could not be lexed.
44    Error,
45}
46
47impl Class {
48    /// A stable lowercase name for the class.
49    pub fn as_str(self) -> &'static str {
50        match self {
51            Class::Keyword => "keyword",
52            Class::Ident => "ident",
53            Class::Number => "number",
54            Class::String => "string",
55            Class::Char => "char",
56            Class::Byte => "byte",
57            Class::Template => "template",
58            Class::Comment => "comment",
59            Class::DocComment => "doc-comment",
60            Class::Attribute => "attribute",
61            Class::Macro => "macro",
62            Class::Punctuation => "punctuation",
63            Class::Operator => "operator",
64            Class::Whitespace => "whitespace",
65            Class::Error => "error",
66        }
67    }
68}
69
70/// A classified span of source.
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72pub struct Token {
73    /// The class of the span.
74    pub class: Class,
75    /// The byte span in the source.
76    pub span: Span,
77}
78
79/// Classify all of `source`.
80///
81/// The tokens are in order and cover every byte exactly once. If the source
82/// stops lexing partway through, the rest of it is one [`Class::Error`] token.
83///
84/// # Examples
85///
86/// ```
87/// use rune::highlight::{highlight, Class};
88///
89/// let tokens = highlight("fn main() {}")?;
90/// assert_eq!(tokens[0].class, Class::Keyword);
91/// # Ok::<_, rune::alloc::Error>(())
92/// ```
93pub fn highlight(source: &str) -> alloc::Result<Vec<Token>> {
94    let mut lexer = Lexer::new(source, SourceId::new(0), true).without_processing();
95    let mut tokens = Vec::new();
96    let mut tags = Vec::new();
97    let mut pos = 0;
98
99    loop {
100        let token = match lexer.next() {
101            Ok(Some(token)) => token,
102            Ok(None) => break,
103            Err(..) => {
104                if pos < source.len() {
105                    tokens.try_push(Token {
106                        class: Class::Error,
107                        span: Span::new(pos, source.len()),
108                    })?;
109                    tags.try_push(Kind::Error)?;
110                }
111
112                pos = source.len();
113                break;
114            }
115        };
116
117        let start = token.span.start.into_usize().max(pos);
118        let end = token.span.end.into_usize().min(source.len());
119
120        if start > pos {
121            tokens.try_push(Token {
122                class: Class::Error,
123                span: Span::new(pos, start),
124            })?;
125            tags.try_push(Kind::Error)?;
126        }
127
128        if end <= start {
129            continue;
130        }
131
132        let class = classify(token.kind, source.get(start..end).unwrap_or_default());
133        tokens.try_push(Token {
134            class,
135            span: Span::new(start, end),
136        })?;
137        tags.try_push(token.kind)?;
138        pos = end;
139    }
140
141    if pos < source.len() {
142        tokens.try_push(Token {
143            class: Class::Error,
144            span: Span::new(pos, source.len()),
145        })?;
146        tags.try_push(Kind::Error)?;
147    }
148
149    refine(&mut tokens, &tags);
150    Ok(tokens)
151}
152
153/// Mark macro invocations and attributes, which need to look at neighbours.
154fn refine(tokens: &mut [Token], tags: &[Kind]) {
155    let mut i = 0;
156
157    while i < tokens.len() {
158        match tags[i] {
159            Kind::Ident(..) if matches!(tags.get(i + 1), Some(Kind::Bang)) => {
160                tokens[i].class = Class::Macro;
161            }
162            Kind::Pound => {
163                let mut open = i + 1;
164
165                if matches!(tags.get(open), Some(Kind::Bang)) {
166                    open += 1;
167                }
168
169                if matches!(
170                    tags.get(open),
171                    Some(Kind::Open(crate::ast::Delimiter::Bracket))
172                ) {
173                    let mut depth = 0usize;
174                    let mut end = open;
175
176                    while end < tags.len() {
177                        match tags[end] {
178                            Kind::Open(..) => depth += 1,
179                            Kind::Close(..) => {
180                                depth = depth.saturating_sub(1);
181
182                                if depth == 0 {
183                                    break;
184                                }
185                            }
186                            _ => (),
187                        }
188
189                        end += 1;
190                    }
191
192                    let end = end.min(tags.len() - 1);
193
194                    for token in &mut tokens[i..=end] {
195                        if !matches!(token.class, Class::Whitespace | Class::Comment) {
196                            token.class = Class::Attribute;
197                        }
198                    }
199
200                    i = end;
201                }
202            }
203            _ => (),
204        }
205
206        i += 1;
207    }
208}
209
210fn classify(kind: Kind, text: &str) -> Class {
211    match kind {
212        Kind::Whitespace => Class::Whitespace,
213        Kind::Comment | Kind::MultilineComment(..) => {
214            if is_doc_comment(text) {
215                Class::DocComment
216            } else {
217                Class::Comment
218            }
219        }
220        Kind::Shebang(..) => Class::Comment,
221        Kind::Error => Class::Error,
222        Kind::Ident(..) | Kind::Label(..) => Class::Ident,
223        Kind::Number(..) => Class::Number,
224        Kind::Str(..) | Kind::ByteStr(..) => Class::String,
225        Kind::Char(..) => Class::Char,
226        Kind::Byte(..) => Class::Byte,
227        Kind::TemplateString => Class::Template,
228        Kind::Abstract
229        | Kind::AlignOf
230        | Kind::As
231        | Kind::Async
232        | Kind::Await
233        | Kind::Become
234        | Kind::Break
235        | Kind::Const
236        | Kind::Continue
237        | Kind::Crate
238        | Kind::Default
239        | Kind::Do
240        | Kind::Else
241        | Kind::Enum
242        | Kind::Extern
243        | Kind::False
244        | Kind::Final
245        | Kind::Fn
246        | Kind::For
247        | Kind::If
248        | Kind::Impl
249        | Kind::In
250        | Kind::Is
251        | Kind::Let
252        | Kind::Loop
253        | Kind::Macro
254        | Kind::Match
255        | Kind::Mod
256        | Kind::Move
257        | Kind::Mut
258        | Kind::Not
259        | Kind::OffsetOf
260        | Kind::Override
261        | Kind::Priv
262        | Kind::Proc
263        | Kind::Pub
264        | Kind::Pure
265        | Kind::Ref
266        | Kind::Return
267        | Kind::Select
268        | Kind::SelfType
269        | Kind::SelfValue
270        | Kind::SizeOf
271        | Kind::Static
272        | Kind::Struct
273        | Kind::Super
274        | Kind::True
275        | Kind::TypeOf
276        | Kind::Unsafe
277        | Kind::Use
278        | Kind::Virtual
279        | Kind::While
280        | Kind::Yield => Class::Keyword,
281        Kind::Amp
282        | Kind::AmpAmp
283        | Kind::AmpEq
284        | Kind::Bang
285        | Kind::BangEq
286        | Kind::Caret
287        | Kind::CaretEq
288        | Kind::Dash
289        | Kind::DashEq
290        | Kind::Div
291        | Kind::DotDot
292        | Kind::DotDotEq
293        | Kind::Eq
294        | Kind::EqEq
295        | Kind::Gt
296        | Kind::GtEq
297        | Kind::GtGt
298        | Kind::GtGtEq
299        | Kind::Lt
300        | Kind::LtEq
301        | Kind::LtLt
302        | Kind::LtLtEq
303        | Kind::Perc
304        | Kind::PercEq
305        | Kind::Pipe
306        | Kind::PipeEq
307        | Kind::PipePipe
308        | Kind::Plus
309        | Kind::PlusEq
310        | Kind::SlashEq
311        | Kind::Star
312        | Kind::StarEq => Class::Operator,
313        _ => Class::Punctuation,
314    }
315}
316
317fn is_doc_comment(text: &str) -> bool {
318    if let Some(rest) = text.strip_prefix("//") {
319        return rest.starts_with("!") || (rest.starts_with('/') && !rest.starts_with("//"));
320    }
321
322    if let Some(rest) = text.strip_prefix("/*") {
323        return rest.starts_with('!')
324            || (rest.starts_with('*') && !rest.starts_with("**") && !rest.starts_with("*/"));
325    }
326
327    false
328}