rune/worker/
mod.rs

Help
1//! Worker used by compiler.
2
3mod import;
4mod task;
5mod wildcard_import;
6
7use rust_alloc::rc::Rc;
8
9use crate::alloc::prelude::*;
10use crate::alloc::{self, HashMap, Vec, VecDeque};
11use crate::ast::{self, Kind, Span, Spanned};
12use crate::compile::{self, ItemId, ModId, WithSpan};
13use crate::grammar::{Node, Stream};
14use crate::indexing::{index, index2};
15use crate::macros::{MacroContext, TokenStream};
16use crate::parse::Resolve;
17use crate::query::{
18    BuiltInLiteral, BuiltInMacro2, DeferEntry, ExpandMacroBuiltin, ExpandedMacro,
19    GenericsParameters, ImplItem, ImplItemKind, Query, Used,
20};
21use crate::SourceId;
22
23pub(crate) use self::import::{Import, ImportState};
24pub(crate) use self::task::{LoadFileKind, Task};
25pub(crate) use self::wildcard_import::WildcardImport;
26
27pub(crate) struct Worker<'a, 'arena> {
28    /// Query engine.
29    pub(crate) q: Query<'a, 'arena>,
30    /// Files that have been loaded.
31    pub(crate) loaded: HashMap<ModId, (SourceId, Span)>,
32    /// Worker queue.
33    pub(crate) queue: VecDeque<Task>,
34}
35
36impl<'a, 'arena> Worker<'a, 'arena> {
37    /// Construct a new worker.
38    pub(crate) fn new(q: Query<'a, 'arena>) -> Self {
39        Self {
40            q,
41            loaded: HashMap::new(),
42            queue: VecDeque::new(),
43        }
44    }
45
46    /// Perform indexing in the worker.
47    #[tracing::instrument(skip_all)]
48    pub(crate) fn index(&mut self) -> alloc::Result<()> {
49        // NB: defer wildcard expansion until all other imports have been
50        // indexed.
51        let mut wildcard_imports = Vec::new();
52
53        while !self.queue.is_empty() {
54            // Prioritise processing the indexing queue. This ensures that files
55            // and imports are loaded which might be used by subsequent steps.
56            // We only advance wildcard imports and impl items once this is
57            // empty.
58            //
59            // Language semantics also ensures that once this queue is drained,
60            // every item which might affect the behavior of imports has been
61            // indexed.
62            while let Some(task) = self.queue.pop_front() {
63                match task {
64                    Task::LoadFile {
65                        kind,
66                        source_id,
67                        mod_item,
68                        mod_item_id,
69                    } => {
70                        let result = self.load_file(kind, source_id, mod_item, mod_item_id);
71
72                        if let Err(error) = result {
73                            self.q.diagnostics.error(source_id, error)?;
74                        }
75                    }
76                    Task::ExpandImport(import) => {
77                        tracing::trace!("expand import");
78
79                        let source_id = import.source_id;
80                        let queue = &mut self.queue;
81
82                        let result = import.process(&mut self.q, &mut |task| {
83                            queue.try_push_back(task)?;
84                            Ok(())
85                        });
86
87                        if let Err(error) = result {
88                            self.q.diagnostics.error(source_id, error)?;
89                        }
90                    }
91                    Task::ExpandWildcardImport(wildcard_import) => {
92                        tracing::trace!("expand wildcard import");
93
94                        let source_id = wildcard_import.location.source_id;
95
96                        if let Err(error) = wildcard_imports.try_push(wildcard_import) {
97                            self.q
98                                .diagnostics
99                                .error(source_id, compile::Error::from(error))?;
100                        }
101                    }
102                }
103            }
104
105            // Process discovered wildcard imports, since they might be used
106            // during impl items below.
107            for mut wildcard_import in wildcard_imports.drain(..) {
108                if let Err(error) = wildcard_import.process_local(&mut self.q) {
109                    self.q
110                        .diagnostics
111                        .error(wildcard_import.location.source_id, error)?;
112                }
113            }
114
115            // Expand impl items since they might be non-local. We need to look up the metadata associated with the item.
116            while let Some(entry) = self.q.next_defer_entry() {
117                match entry {
118                    DeferEntry::ImplItem(item) => {
119                        let source_id = item.location.source_id;
120
121                        if let Err(error) = self.impl_item(item) {
122                            self.q.diagnostics.error(source_id, error)?;
123                        }
124                    }
125                    DeferEntry::ExpandMacroBuiltin(this) => {
126                        let source_id = this.location.source_id;
127
128                        if let Err(error) = self.expand_macro_builtin(this) {
129                            self.q.diagnostics.error(source_id, error)?;
130                        }
131                    }
132                    DeferEntry::ExpandMacroCall(this) => {
133                        let source_id = this.location.source_id;
134
135                        if let Err(error) = self.expand_macro_call(this) {
136                            self.q.diagnostics.error(source_id, error)?;
137                        }
138                    }
139                }
140            }
141        }
142
143        Ok(())
144    }
145
146    #[tracing::instrument(skip_all)]
147    fn load_file(
148        &mut self,
149        kind: LoadFileKind,
150        source_id: SourceId,
151        mod_item: ModId,
152        mod_item_id: ItemId,
153    ) -> compile::Result<()> {
154        let Some(source) = self.q.sources.get(source_id) else {
155            self.q
156                .diagnostics
157                .internal(source_id, "Missing queued source by id")?;
158            return Ok(());
159        };
160
161        let (root, is_module) = match kind {
162            LoadFileKind::Root => (source.path().map(|p| p.try_to_owned()).transpose()?, false),
163            LoadFileKind::Module { root } => (root, true),
164        };
165
166        macro_rules! indexer {
167            ($tree:expr) => {{
168                let item = self.q.pool.module_item(mod_item);
169                let items = $crate::indexing::Items::new(item)?;
170
171                tracing::trace!(?item, "load file: {}", item);
172
173                $crate::indexing::Indexer {
174                    q: self.q.borrow(),
175                    root: root.as_deref(),
176                    source_id,
177                    items,
178                    scopes: $crate::indexing::Scopes::new()?,
179                    item: $crate::indexing::IndexItem::new(mod_item, mod_item_id),
180                    nested_item: None,
181                    macro_depth: 0,
182                    loaded: Some(&mut self.loaded),
183                    queue: Some(&mut self.queue),
184                    tree: $tree,
185                }
186            }};
187        }
188
189        let as_function_body = self.q.options.function_body && !is_module;
190
191        #[allow(clippy::collapsible_else_if)]
192        if self.q.options.v2 {
193            let tree = crate::grammar::text(source_id, source.as_str()).root()?;
194
195            let tree = Rc::new(tree);
196
197            #[cfg(feature = "std")]
198            if self.q.options.print_tree {
199                tree.print_with_source(
200                    &Span::empty(),
201                    format_args!("Loading file (source: {source_id})"),
202                    source.as_str(),
203                )?;
204            }
205
206            if as_function_body {
207                let mut idx = indexer!(&tree);
208
209                tree.parse_all(|p: &mut crate::grammar::Stream| index2::bare(&mut idx, p))?;
210            } else {
211                let mut idx = indexer!(&tree);
212                tree.parse_all(|p| index2::file(&mut idx, p))?;
213            }
214        } else {
215            if as_function_body {
216                let ast =
217                    crate::parse::parse_all::<ast::EmptyBlock>(source.as_str(), source_id, true)?;
218
219                let span = Span::new(0, source.len());
220
221                let empty = Rc::default();
222                let mut idx = indexer!(&empty);
223
224                index::empty_block_fn(&mut idx, ast, &span)?;
225            } else {
226                let mut ast =
227                    crate::parse::parse_all::<ast::File>(source.as_str(), source_id, true)?;
228
229                let empty = Rc::default();
230                let mut idx = indexer!(&empty);
231                index::file(&mut idx, &mut ast)?;
232            }
233        }
234
235        Ok(())
236    }
237
238    #[tracing::instrument(skip_all)]
239    fn impl_item(&mut self, this: ImplItem) -> compile::Result<()> {
240        macro_rules! indexer {
241            ($tree:expr, $named:expr, $meta:expr) => {{
242                let items =
243                    $crate::indexing::Items::new({ self.q.pool.item($meta.item_meta.item) })?;
244
245                $crate::indexing::Indexer {
246                    q: self.q.borrow(),
247                    root: this.root.as_deref(),
248                    source_id: this.location.source_id,
249                    items,
250                    scopes: $crate::indexing::Scopes::new()?,
251                    item: $crate::indexing::IndexItem::with_impl_item(
252                        $named.module,
253                        $named.item,
254                        $meta.item_meta.item,
255                    ),
256                    nested_item: this.nested_item,
257                    macro_depth: this.macro_depth,
258                    loaded: Some(&mut self.loaded),
259                    queue: Some(&mut self.queue),
260                    tree: $tree,
261                }
262            }};
263        }
264
265        // When converting a path, we conservatively deny `Self` impl
266        // since that is what Rust does, and at some point in the future
267        // we might introduce bounds which would not be communicated
268        // through `Self`.
269        match this.kind {
270            ImplItemKind::Ast { path, functions } => {
271                let named = self
272                    .q
273                    .convert_path_with(&path, true, Used::Used, Used::Unused)?;
274
275                if let Some((spanned, _)) = named.parameters.into_iter().flatten().next() {
276                    return Err(compile::Error::new(
277                        spanned.span(),
278                        compile::ErrorKind::UnsupportedGenerics,
279                    ));
280                }
281
282                tracing::trace!(item = ?self.q.pool.item(named.item), "next impl item entry");
283
284                let meta = self.q.lookup_meta(
285                    &this.location,
286                    named.item,
287                    GenericsParameters::default(),
288                )?;
289
290                let empty = Rc::default();
291                let mut idx = indexer!(&empty, named, meta);
292
293                for f in functions {
294                    index::item_fn(&mut idx, f)?;
295                }
296            }
297            ImplItemKind::Node { path, functions } => {
298                let named =
299                    path.parse(|p| self.q.convert_path2_with(p, true, Used::Used, Used::Unused))?;
300
301                if let Some(spanned) = named.parameters.into_iter().flatten().next() {
302                    return Err(compile::Error::new(
303                        spanned.span(),
304                        compile::ErrorKind::UnsupportedGenerics,
305                    ));
306                }
307
308                tracing::trace!(item = ?self.q.pool.item(named.item), "next impl item entry");
309
310                let meta = self.q.lookup_meta(
311                    &this.location,
312                    named.item,
313                    GenericsParameters::default(),
314                )?;
315
316                let mut idx = indexer!(path.tree(), named, meta);
317
318                for (id, attrs) in functions {
319                    path.parse_id(id, |p| index2::item(&mut idx, p, attrs))?;
320                }
321            }
322        }
323
324        Ok(())
325    }
326
327    #[tracing::instrument(skip_all)]
328    fn expand_macro_builtin(&mut self, mut this: ExpandMacroBuiltin) -> compile::Result<()> {
329        let (name, stream) = this.node.parse(|p| {
330            let Some([ident]) = p.pump()?.nodes::<1>() else {
331                return Err(compile::Error::msg(&*p, "missing macro identifier"));
332            };
333
334            let ident = ident.ast::<ast::Ident>()?;
335            let name = ident.resolve(resolve_context!(self.q))?;
336
337            p.expect(K![!])?;
338
339            let close = match p.peek() {
340                K!['{'] => K!['}'],
341                K!['('] => K![')'],
342                token => {
343                    return Err(compile::Error::msg(
344                        p.peek_span(),
345                        try_format!("expected `{{` or `(`, found {token}"),
346                    ));
347                }
348            };
349
350            p.pump()?;
351            let stream = p.expect(Kind::TokenStream)?;
352            p.expect(close)?;
353
354            Ok((name, stream))
355        })?;
356
357        let expanded = match name {
358            "file" => stream.parse(|p| self.expand_file_macro(&this, p))?,
359            "line" => stream.parse(|p| self.expand_line_macro(&this, p))?,
360            "format" => self.expand_format_macro(&this, stream)?,
361            "template" => {
362                let literal = this.literal.take();
363                self.expand_template_macro(&this, literal, stream)?
364            }
365            name => {
366                return Err(compile::Error::msg(
367                    &this.node,
368                    try_format!("no internal macro named `{name}`"),
369                ));
370            }
371        };
372
373        let id = this.finish()?;
374
375        self.q
376            .insert_expanded_macro(id, ExpandedMacro::Builtin(expanded))?;
377        Ok(())
378    }
379
380    #[tracing::instrument(skip_all)]
381    fn expand_file_macro(
382        &mut self,
383        this: &ExpandMacroBuiltin,
384        p: &mut Stream<'_>,
385    ) -> compile::Result<BuiltInMacro2> {
386        let name = self
387            .q
388            .sources
389            .name(this.location.source_id)
390            .ok_or_else(|| {
391                compile::Error::new(
392                    &*p,
393                    compile::ErrorKind::MissingSourceId {
394                        source_id: this.location.source_id,
395                    },
396                )
397            })?;
398
399        let id = self.q.storage.insert_str(name)?;
400
401        let value = ast::LitStr {
402            span: p.span(),
403            source: ast::StrSource::Synthetic(id),
404        };
405
406        Ok(BuiltInMacro2::File(value))
407    }
408
409    fn expand_line_macro(
410        &mut self,
411        this: &ExpandMacroBuiltin,
412        p: &mut Stream<'_>,
413    ) -> compile::Result<BuiltInMacro2> {
414        let (l, _) = self
415            .q
416            .sources
417            .get(this.location.source_id)
418            .map(|s| s.pos_to_utf8_linecol(p.span().start.into_usize()))
419            .unwrap_or_default();
420
421        Ok(BuiltInMacro2::Line(l + 1))
422    }
423
424    fn expand_format_macro(
425        &mut self,
426        this: &ExpandMacroBuiltin,
427        stream: Node<'_>,
428    ) -> compile::Result<BuiltInMacro2> {
429        let tree = crate::grammar::node(stream).format()?;
430        let tree = Rc::new(tree);
431
432        let items = crate::indexing::Items::new(self.q.pool.item(this.item.id))?;
433
434        let mut idx = crate::indexing::Indexer {
435            q: self.q.borrow(),
436            root: this.root.as_deref(),
437            source_id: this.location.source_id,
438            items,
439            scopes: crate::indexing::Scopes::new()?,
440            item: this.item,
441            nested_item: None,
442            macro_depth: this.macro_depth + 1,
443            loaded: Some(&mut self.loaded),
444            queue: Some(&mut self.queue),
445            tree: &tree,
446        };
447
448        tree.parse_all(|p| index2::any(&mut idx, p))?;
449
450        #[cfg(feature = "std")]
451        if self.q.options.print_tree {
452            tree.print(
453                &this.node,
454                format_args!("Expanded format!() macro {}", this.id),
455            )?;
456        }
457
458        Ok(BuiltInMacro2::Format(tree))
459    }
460
461    fn expand_template_macro(
462        &mut self,
463        this: &ExpandMacroBuiltin,
464        literal: BuiltInLiteral,
465        stream: Node<'_>,
466    ) -> compile::Result<BuiltInMacro2> {
467        let tree = crate::grammar::node(stream).exprs(K![,])?;
468        let tree = Rc::new(tree);
469
470        let items = crate::indexing::Items::new(self.q.pool.item(this.item.id))?;
471
472        let mut idx = crate::indexing::Indexer {
473            q: self.q.borrow(),
474            root: this.root.as_deref(),
475            source_id: this.location.source_id,
476            items,
477            scopes: crate::indexing::Scopes::new()?,
478            item: this.item,
479            nested_item: None,
480            macro_depth: this.macro_depth + 1,
481            loaded: Some(&mut self.loaded),
482            queue: Some(&mut self.queue),
483            tree: &tree,
484        };
485
486        tree.parse_all(|p| index2::any(&mut idx, p))?;
487
488        #[cfg(feature = "std")]
489        if self.q.options.print_tree {
490            tree.print(
491                &this.node,
492                format_args!("Expanded template!() macro {}", this.id),
493            )?;
494        }
495
496        Ok(BuiltInMacro2::Template(tree, literal))
497    }
498
499    #[tracing::instrument(skip_all)]
500    fn expand_macro_call(&mut self, this: ExpandMacroBuiltin) -> compile::Result<()> {
501        if this.macro_depth >= self.q.options.max_macro_depth {
502            return Err(compile::Error::new(
503                this.node.span(),
504                compile::ErrorKind::MaxMacroRecursion {
505                    depth: this.macro_depth,
506                    max: self.q.options.max_macro_depth,
507                },
508            ));
509        }
510
511        let item_meta = self
512            .q
513            .item_for("macro call", this.item.id)
514            .with_span(&this.node)?;
515
516        let (named, stream) = this.node.parse(|p| {
517            let named = p
518                .pump()?
519                .parse(|p| self.q.convert_path2_with(p, true, Used::Used, Used::Unused))?;
520
521            p.expect(K![!])?;
522
523            let close = match p.peek() {
524                K!['{'] => K!['}'],
525                K!['('] => K![')'],
526                token => {
527                    return Err(compile::Error::msg(
528                        p.peek_span(),
529                        try_format!("expected `{{` or `(`, found {token}"),
530                    ));
531                }
532            };
533
534            p.pump()?;
535            let stream = p.expect(Kind::TokenStream)?;
536            p.expect(close)?;
537
538            Ok((named, stream))
539        })?;
540
541        if let Some(spanned) = named.parameters.into_iter().flatten().next() {
542            return Err(compile::Error::new(
543                spanned.span(),
544                compile::ErrorKind::UnsupportedGenerics,
545            ));
546        }
547
548        let hash = self.q.pool.item_type_hash(named.item);
549
550        let Some(handler) = self.q.context.lookup_macro(hash) else {
551            return Err(compile::Error::new(
552                &this.node,
553                compile::ErrorKind::MissingMacro {
554                    item: self.q.pool.item(named.item).try_to_owned()?,
555                },
556            ));
557        };
558
559        let items = crate::indexing::Items::new(self.q.pool.item(this.item.id))?;
560
561        let mut idx = crate::indexing::Indexer {
562            q: self.q.borrow(),
563            root: this.root.as_deref(),
564            source_id: this.location.source_id,
565            items,
566            scopes: crate::indexing::Scopes::new()?,
567            item: this.item,
568            nested_item: None,
569            macro_depth: this.macro_depth + 1,
570            loaded: Some(&mut self.loaded),
571            queue: Some(&mut self.queue),
572            tree: this.node.tree(),
573        };
574
575        let mut input_stream = TokenStream::new();
576
577        for node in stream
578            .children()
579            .flat_map(|c| c.walk())
580            .filter(|n| n.is_empty())
581        {
582            input_stream.push(node.token())?;
583        }
584
585        let output_stream = {
586            let mut macro_context = MacroContext {
587                macro_span: this.node.span(),
588                input_span: stream.span(),
589                item_meta,
590                idx: &mut idx,
591            };
592
593            handler(&mut macro_context, &input_stream)?
594        };
595
596        let inner_tree = crate::grammar::token_stream(&output_stream).root()?;
597
598        let tree = Rc::new(inner_tree);
599        idx.tree = &tree;
600        tree.parse_all(|p| index2::any(&mut idx, p))?;
601
602        #[cfg(feature = "std")]
603        if self.q.options.print_tree {
604            tree.print(
605                &this.node,
606                format_args!("Expanded macro {} from output stream", this.id),
607            )?;
608        }
609
610        let id = this.finish()?;
611        self.q
612            .insert_expanded_macro(id, ExpandedMacro::Tree(tree))?;
613        Ok(())
614    }
615}
616
617#[derive(Debug, Clone, Copy)]
618pub(crate) enum ImportKind {
619    /// The import is in-place.
620    Local,
621    /// The import is deferred.
622    Global,
623}
rune/worker/mod.rs

rune/worker/
mod.rs
