rune/indexing/
indexer.rs

Help
1use rust_alloc::rc::Rc;
2
3use core::mem::replace;
4use core::num::NonZeroUsize;
5
6use crate::alloc::path::Path;
7use crate::alloc::prelude::*;
8use crate::alloc::{self, HashMap, VecDeque};
9use crate::ast::spanned;
10use crate::ast::{self, Span, Spanned};
11use crate::compile::attrs;
12use crate::compile::{
13    self, Doc, DynLocation, Error, ErrorKind, ItemId, ItemMeta, ModId, Visibility, WithSpan,
14};
15use crate::grammar::{Ignore, Node, Tree};
16use crate::macros::MacroCompiler;
17use crate::parse::{Parse, Parser, Resolve};
18use crate::query::{BuiltInFile, BuiltInFormat, BuiltInLine, BuiltInMacro, BuiltInTemplate, Query};
19use crate::runtime::{format, Call};
20use crate::worker::{LoadFileKind, Task};
21use crate::SourceId;
22
23use super::{Guard, Items, Layer, Scopes};
24
25/// Macros are only allowed to expand recursively into other macros 64 times.
26const MAX_MACRO_RECURSION: usize = 64;
27
28pub(crate) struct Indexer<'a, 'arena> {
29    /// Query engine.
30    pub(crate) q: Query<'a, 'arena>,
31    pub(crate) source_id: SourceId,
32    pub(crate) items: Items,
33    /// Helper to calculate details about an indexed scope.
34    pub(crate) scopes: Scopes,
35    /// The current item state.
36    pub(crate) item: IndexItem,
37    /// Indicates if indexer is nested privately inside of another item, and if
38    /// so, the descriptive span of its declaration.
39    ///
40    /// Private items are nested declarations inside of for example fn
41    /// declarations:
42    ///
43    /// ```text
44    /// pub fn public() {
45    ///     fn private() {
46    ///     }
47    /// }
48    /// ```
49    ///
50    /// Then, `nested_item` would point to the span of `pub fn public`.
51    pub(crate) nested_item: Option<Span>,
52    /// Depth of expression macro expansion that we're currently in.
53    pub(crate) macro_depth: usize,
54    /// The root URL that the indexed file originated from.
55    pub(crate) root: Option<&'a Path>,
56    /// Imports to process.
57    pub(crate) queue: Option<&'a mut VecDeque<Task>>,
58    /// Loaded modules.
59    pub(crate) loaded: Option<&'a mut HashMap<ModId, (SourceId, Span)>>,
60    /// The current tree being processed.
61    pub(crate) tree: &'a Rc<Tree>,
62}
63
64impl<'a> Ignore<'a> for Indexer<'_, '_> {
65    /// Report an error.
66    fn error(&mut self, error: Error) -> alloc::Result<()> {
67        self.q.diagnostics.error(self.source_id, error)
68    }
69
70    fn ignore(&mut self, _: Node<'a>) -> compile::Result<()> {
71        Ok(())
72    }
73}
74
75impl Indexer<'_, '_> {
76    /// Push an identifier item.
77    pub(super) fn push_id(&mut self) -> alloc::Result<Guard> {
78        let id = self.q.pool.next_id(self.item.id);
79        self.items.push_id(id)
80    }
81
82    /// Insert a new item at the current indexed location.
83    pub(crate) fn insert_new_item(
84        &mut self,
85        span: &dyn Spanned,
86        visibility: Visibility,
87        docs: &[Doc],
88    ) -> compile::Result<ItemMeta> {
89        self.q.insert_new_item(
90            &self.items,
91            self.item.module,
92            self.item.impl_item,
93            &DynLocation::new(self.source_id, span),
94            visibility,
95            docs,
96        )
97    }
98
99    /// Indicate that we've entered an expanded macro context, and ensure that
100    /// we don't blow past [`MAX_MACRO_RECURSION`].
101    ///
102    /// This is used when entering expressions which have been expanded from a
103    /// macro - cause those expression might in turn be macros themselves.
104    pub(super) fn enter_macro<S>(&mut self, span: &S) -> compile::Result<()>
105    where
106        S: Spanned,
107    {
108        self.macro_depth = self.macro_depth.wrapping_add(1);
109
110        if self.macro_depth >= MAX_MACRO_RECURSION {
111            return Err(compile::Error::new(
112                span,
113                ErrorKind::MaxMacroRecursion {
114                    depth: self.macro_depth,
115                    max: MAX_MACRO_RECURSION,
116                },
117            ));
118        }
119
120        Ok(())
121    }
122
123    /// Leave the last macro context.
124    pub(super) fn leave_macro(&mut self) {
125        self.macro_depth = self.macro_depth.wrapping_sub(1);
126    }
127
128    /// Try to expand an internal macro.
129    pub(super) fn try_expand_internal_macro(
130        &mut self,
131        p: &mut attrs::Parser,
132        ast: &mut ast::MacroCall,
133    ) -> compile::Result<bool> {
134        let Some((_, builtin)) =
135            p.try_parse::<attrs::BuiltIn>(resolve_context!(self.q), &ast.attributes)?
136        else {
137            return Ok(false);
138        };
139
140        let args = builtin.args(resolve_context!(self.q))?;
141
142        // NB: internal macros are
143        let Some(ident) = ast.path.try_as_ident() else {
144            return Err(compile::Error::new(
145                &ast.path,
146                ErrorKind::NoSuchBuiltInMacro {
147                    name: ast.path.resolve(resolve_context!(self.q))?,
148                },
149            ));
150        };
151
152        let ident = ident.resolve(resolve_context!(self.q))?;
153
154        let mut internal_macro = match ident {
155            "template" => self.expand_template_macro(ast, &args)?,
156            "format" => self.expand_format_macro(ast, &args)?,
157            "file" => self.expand_file_macro(ast)?,
158            "line" => self.expand_line_macro(ast)?,
159            _ => {
160                return Err(compile::Error::new(
161                    &ast.path,
162                    ErrorKind::NoSuchBuiltInMacro {
163                        name: ast.path.resolve(resolve_context!(self.q))?,
164                    },
165                ))
166            }
167        };
168
169        match &mut internal_macro {
170            BuiltInMacro::Template(template) => {
171                for e in &mut template.exprs {
172                    super::index::expr(self, e)?;
173                }
174            }
175            BuiltInMacro::Format(format) => {
176                super::index::expr(self, &mut format.value)?;
177            }
178
179            BuiltInMacro::Line(_) | BuiltInMacro::File(_) => { /* Nothing to index */ }
180        }
181
182        let id = self.q.insert_new_builtin_macro(internal_macro)?;
183        ast.id = Some(id);
184        Ok(true)
185    }
186
187    /// Expand the template macro.
188    fn expand_template_macro(
189        &mut self,
190        ast: &ast::MacroCall,
191        args: &attrs::BuiltInArgs,
192    ) -> compile::Result<BuiltInMacro> {
193        let mut p = Parser::from_token_stream(&ast.input, ast.span());
194        let mut exprs = Vec::new();
195
196        while !p.is_eof()? {
197            exprs.try_push(p.parse::<ast::Expr>()?)?;
198
199            if p.parse::<Option<T![,]>>()?.is_none() {
200                break;
201            }
202        }
203
204        p.eof()?;
205
206        Ok(BuiltInMacro::Template(BuiltInTemplate {
207            span: ast.span(),
208            from_literal: args.literal,
209            exprs,
210        }))
211    }
212
213    /// Expand the template macro.
214    fn expand_format_macro(
215        &mut self,
216        ast: &ast::MacroCall,
217        _: &attrs::BuiltInArgs,
218    ) -> compile::Result<BuiltInMacro> {
219        let mut p = Parser::from_token_stream(&ast.input, ast.span());
220
221        let value = p.parse::<ast::Expr>()?;
222
223        // parsed options
224        let mut fill = None;
225        let mut align = None;
226        let mut flags = None;
227        let mut width = None;
228        let mut precision = None;
229        let mut format_type = None;
230
231        while p.try_consume::<T![,]>()? && !p.is_eof()? {
232            let key = p.parse::<ast::Ident>()?;
233            let _ = p.parse::<T![=]>()?;
234
235            let k = key.resolve(resolve_context!(self.q))?;
236
237            match k {
238                "fill" => {
239                    if fill.is_some() {
240                        return Err(compile::Error::unsupported(
241                            key,
242                            "Multiple `format!(.., fill = ..)`",
243                        ));
244                    }
245
246                    let arg = p.parse::<ast::LitChar>()?;
247                    let f = arg.resolve(resolve_context!(self.q))?;
248
249                    fill = Some(f);
250                }
251                "align" => {
252                    if align.is_some() {
253                        return Err(compile::Error::unsupported(
254                            key,
255                            "Multiple `format!(.., align = ..)`",
256                        ));
257                    }
258
259                    let arg = p.parse::<ast::Ident>()?;
260                    let a = arg.resolve(resolve_context!(self.q))?;
261
262                    let Ok(a) = str::parse::<format::Alignment>(a) else {
263                        return Err(compile::Error::unsupported(
264                            key,
265                            "`format!(.., align = ..)`",
266                        ));
267                    };
268
269                    align = Some(a);
270                }
271                "flags" => {
272                    if flags.is_some() {
273                        return Err(compile::Error::unsupported(
274                            key,
275                            "Multiple `format!(.., flags = ..)`",
276                        ));
277                    }
278
279                    let arg = p.parse::<ast::LitNumber>()?;
280
281                    let Some(f) = arg.resolve(resolve_context!(self.q))?.as_u32(false) else {
282                        return Err(compile::Error::msg(arg, "Argument out-of-bounds"));
283                    };
284
285                    let f = format::Flags::from(f);
286                    flags = Some(f);
287                }
288                "width" => {
289                    if width.is_some() {
290                        return Err(compile::Error::unsupported(
291                            key,
292                            "Multiple `format!(.., width = ..)`",
293                        ));
294                    }
295
296                    let arg = p.parse::<ast::LitNumber>()?;
297
298                    let Some(f) = arg.resolve(resolve_context!(self.q))?.as_usize(false) else {
299                        return Err(compile::Error::msg(arg, "Argument out-of-bounds"));
300                    };
301
302                    width = NonZeroUsize::new(f);
303                }
304                "precision" => {
305                    if precision.is_some() {
306                        return Err(compile::Error::unsupported(
307                            key,
308                            "Multiple `format!(.., precision = ..)`",
309                        ));
310                    }
311
312                    let arg = p.parse::<ast::LitNumber>()?;
313
314                    let Some(f) = arg.resolve(resolve_context!(self.q))?.as_usize(false) else {
315                        return Err(compile::Error::msg(arg, "Argument out-of-bounds"));
316                    };
317
318                    precision = NonZeroUsize::new(f);
319                }
320                "type" => {
321                    if format_type.is_some() {
322                        return Err(compile::Error::unsupported(
323                            key,
324                            "Multiple `format!(.., type = ..)`",
325                        ));
326                    }
327
328                    let arg = p.parse::<ast::Ident>()?;
329                    let a = arg.resolve(resolve_context!(self.q))?;
330
331                    format_type = Some(match str::parse::<format::Type>(a) {
332                        Ok(format_type) => format_type,
333                        _ => {
334                            return Err(compile::Error::unsupported(
335                                key,
336                                "`format!(.., type = ..)`",
337                            ));
338                        }
339                    });
340                }
341                _ => {
342                    return Err(compile::Error::unsupported(key, "`format!(.., <key>)`"));
343                }
344            }
345        }
346
347        p.eof()?;
348
349        Ok(BuiltInMacro::Format(BuiltInFormat {
350            span: ast.span(),
351            fill,
352            align,
353            width,
354            precision,
355            flags,
356            format_type,
357            value,
358        }))
359    }
360
361    /// Expand a macro returning the current file
362    fn expand_file_macro(&mut self, ast: &ast::MacroCall) -> compile::Result<BuiltInMacro> {
363        let name = self.q.sources.name(self.source_id).ok_or_else(|| {
364            compile::Error::new(
365                ast,
366                ErrorKind::MissingSourceId {
367                    source_id: self.source_id,
368                },
369            )
370        })?;
371        let id = self.q.storage.insert_str(name)?;
372        let source = ast::StrSource::Synthetic(id);
373        let value = ast::Lit::Str(ast::LitStr {
374            span: ast.span(),
375            source,
376        });
377
378        Ok(BuiltInMacro::File(BuiltInFile { value }))
379    }
380
381    /// Expand a macro returning the current line for where the macro invocation begins
382    fn expand_line_macro(&mut self, ast: &ast::MacroCall) -> compile::Result<BuiltInMacro> {
383        let (l, _) = self
384            .q
385            .sources
386            .get(self.source_id)
387            .map(|s| s.pos_to_utf8_linecol(ast.open.span.start.into_usize()))
388            .unwrap_or_default();
389
390        // 1-indexed as that is what most editors will use
391        let id = self.q.storage.insert_number(l + 1)?;
392        let source = ast::NumberSource::Synthetic(id);
393
394        Ok(BuiltInMacro::Line(BuiltInLine {
395            value: ast::Lit::Number(ast::LitNumber {
396                span: ast.span(),
397                source,
398            }),
399        }))
400    }
401
402    /// Perform a macro expansion.
403    pub(super) fn expand_macro<T>(&mut self, ast: &mut ast::MacroCall) -> compile::Result<T>
404    where
405        T: Parse,
406    {
407        ast.path.id = self.item.id;
408
409        let item = self.q.item_for("macro", self.item.id).with_span(&ast)?;
410
411        let mut compiler = MacroCompiler {
412            item_meta: item,
413            idx: self,
414        };
415
416        compiler.eval_macro::<T>(ast)
417    }
418
419    /// Perform an attribute macro expansion.
420    pub(super) fn expand_attribute_macro<T>(
421        &mut self,
422        attr: &mut ast::Attribute,
423        item: &ast::Item,
424    ) -> compile::Result<Option<T>>
425    where
426        T: Parse,
427    {
428        attr.path.id = self.item.id;
429
430        let containing = self
431            .q
432            .item_for("attribute macro", self.item.id)
433            .with_span(&*attr)?;
434
435        let mut compiler = MacroCompiler {
436            item_meta: containing,
437            idx: self,
438        };
439
440        compiler.eval_attribute_macro::<T>(attr, item)
441    }
442
443    /// Handle a filesystem module.
444    pub(super) fn handle_file_mod(
445        &mut self,
446        ast: &mut ast::ItemMod,
447        docs: &[Doc],
448    ) -> compile::Result<()> {
449        let name = ast.name.resolve(resolve_context!(self.q))?;
450        let visibility = ast_to_visibility(&ast.visibility)?;
451        let guard = self.items.push_name(name.as_ref())?;
452
453        let (mod_item, mod_item_id) = self.q.insert_mod(
454            &self.items,
455            &DynLocation::new(self.source_id, spanned::from_fn(|| ast.name_span())),
456            self.item.module,
457            visibility,
458            docs,
459        )?;
460
461        self.items.pop(guard).with_span(&*ast)?;
462
463        ast.id = mod_item_id;
464
465        let Some(root) = &self.root else {
466            return Err(compile::Error::new(
467                &*ast,
468                ErrorKind::UnsupportedModuleSource,
469            ));
470        };
471
472        let source = self
473            .q
474            .source_loader
475            .load(root, self.q.pool.module_item(mod_item), &*ast)?;
476
477        if let Some(loaded) = self.loaded.as_mut() {
478            if let Some(_existing) = loaded.try_insert(mod_item, (self.source_id, ast.span()))? {
479                return Err(compile::Error::new(
480                    &*ast,
481                    ErrorKind::ModAlreadyLoaded {
482                        item: self.q.pool.module_item(mod_item).try_to_owned()?,
483                        #[cfg(feature = "emit")]
484                        existing: _existing,
485                    },
486                ));
487            }
488        }
489
490        let source_id = self.q.sources.insert(source)?;
491
492        self.q
493            .visitor
494            .visit_mod(&DynLocation::new(source_id, &*ast))
495            .with_span(&*ast)?;
496
497        if let Some(queue) = self.queue.as_mut() {
498            queue.try_push_back(Task::LoadFile {
499                kind: LoadFileKind::Module {
500                    root: self.root.map(|p| p.try_to_owned()).transpose()?,
501                },
502                source_id,
503                mod_item,
504                mod_item_id,
505            })?;
506        }
507
508        Ok(())
509    }
510}
511
512#[derive(Debug, Clone, Copy)]
513pub(crate) struct IndexItem {
514    /// The current module being indexed.
515    pub(crate) module: ModId,
516    /// Whether the item has been inserted or not.
517    pub(crate) id: ItemId,
518    /// Set if we are inside of an impl self.
519    pub(crate) impl_item: Option<ItemId>,
520}
521
522impl IndexItem {
523    pub(crate) fn new(module: ModId, id: ItemId) -> Self {
524        Self {
525            module,
526            id,
527            impl_item: None,
528        }
529    }
530
531    pub(crate) fn with_impl_item(module: ModId, id: ItemId, impl_item: ItemId) -> Self {
532        Self {
533            module,
534            id,
535            impl_item: Some(impl_item),
536        }
537    }
538
539    /// Replace item we're currently in.
540    #[tracing::instrument(skip(self), fields(self.module = ?self.module, self.id = ?self.id, self.impl_item = ?self.impl_item))]
541    pub(super) fn replace(&mut self, id: ItemId) -> IndexItem {
542        tracing::debug!("replacing item");
543
544        IndexItem {
545            module: self.module,
546            id: replace(&mut self.id, id),
547            impl_item: self.impl_item,
548        }
549    }
550
551    /// Replace module id.
552    pub(super) fn replace_module(&mut self, module: ModId, id: ItemId) -> IndexItem {
553        IndexItem {
554            module: replace(&mut self.module, module),
555            id: replace(&mut self.id, id),
556            impl_item: self.impl_item,
557        }
558    }
559}
560
561/// Construct visibility from ast.
562pub(super) fn ast_to_visibility(vis: &ast::Visibility) -> compile::Result<Visibility> {
563    let span = match vis {
564        ast::Visibility::Inherited => return Ok(Visibility::Inherited),
565        ast::Visibility::Public(..) => return Ok(Visibility::Public),
566        ast::Visibility::Crate(..) => return Ok(Visibility::Crate),
567        ast::Visibility::Super(..) => return Ok(Visibility::Super),
568        ast::Visibility::SelfValue(..) => return Ok(Visibility::SelfValue),
569        ast::Visibility::In(restrict) => restrict.span(),
570    };
571
572    Err(compile::Error::new(span, ErrorKind::UnsupportedVisibility))
573}
574
575/// Construct the calling convention based on the parameters.
576pub(super) fn validate_call(
577    is_const: bool,
578    is_async: bool,
579    layer: &Layer,
580) -> compile::Result<Option<Call>> {
581    for span in &layer.awaits {
582        if is_const {
583            return Err(compile::Error::new(span, ErrorKind::AwaitInConst));
584        }
585
586        if !is_async {
587            return Err(compile::Error::new(span, ErrorKind::AwaitOutsideAsync));
588        }
589    }
590
591    for span in &layer.yields {
592        if is_const {
593            return Err(compile::Error::new(span, ErrorKind::YieldInConst));
594        }
595    }
596
597    if is_const {
598        return Ok(None);
599    }
600
601    Ok(match (!layer.yields.is_empty(), is_async) {
602        (true, false) => Some(Call::Generator),
603        (false, false) => Some(Call::Immediate),
604        (true, true) => Some(Call::Stream),
605        (false, true) => Some(Call::Async),
606    })
607}
rune/indexing/indexer.rs

rune/indexing/
indexer.rs
