rune/cli/
tests.rs

Help
1use std::fmt;
2use std::io::Write;
3use std::mem::take;
4use std::slice;
5use std::sync::Arc;
6use std::time::Instant;
7
8use anyhow::{Context, Result};
9
10use crate::alloc::fmt::TryWrite;
11use crate::alloc::prelude::*;
12use crate::cli::naming::Naming;
13use crate::cli::visitor;
14use crate::cli::{
15    AssetKind, Color, CommandBase, Config, Entry, EntryPoint, ExitCode, Io, Options, SharedFlags,
16    Stream,
17};
18use crate::compile::FileSourceLoader;
19use crate::doc::{TestKind, TestParams};
20use crate::modules::capture_io::CaptureIo;
21use crate::runtime::{Repr, Value, Vm, VmError, VmResult};
22use crate::{Diagnostics, Hash, Item, ItemBuf, Source, Sources, TypeHash, Unit};
23
24mod cli {
25    use std::string::String;
26    use std::vec::Vec;
27
28    use clap::Parser;
29
30    #[derive(Parser, Debug, Clone)]
31    #[command(rename_all = "kebab-case")]
32    pub struct Flags {
33        /// Exit with a non-zero exit-code even for warnings
34        #[arg(long)]
35        pub warnings_are_errors: bool,
36        /// Display one character per test instead of one line
37        #[arg(long, short = 'q')]
38        pub quiet: bool,
39        /// Break on the first test failed.
40        #[arg(long)]
41        pub fail_fast: bool,
42        /// Skip building dynamic lib tests from entrypoints. This means only
43        /// tests found in runtime contexts will be run.
44        #[arg(long)]
45        pub skip_lib_tests: bool,
46        /// Filter tests by name.
47        pub filters: Vec<String>,
48    }
49}
50
51pub(super) use cli::Flags;
52
53impl CommandBase for Flags {
54    #[inline]
55    fn is_debug(&self) -> bool {
56        true
57    }
58
59    #[inline]
60    fn is_workspace(&self, kind: AssetKind) -> bool {
61        matches!(kind, AssetKind::Test)
62    }
63
64    #[inline]
65    fn describe(&self) -> &str {
66        "Testing"
67    }
68
69    #[inline]
70    fn propagate(&mut self, c: &mut Config, _: &mut SharedFlags) {
71        c.test = true;
72    }
73}
74
75enum BatchKind {
76    LibTests,
77    DocTests,
78    ContextDocTests,
79}
80
81impl fmt::Display for BatchKind {
82    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
83        match self {
84            Self::LibTests => write!(f, "lib tests"),
85            Self::DocTests => write!(f, "doc tests"),
86            Self::ContextDocTests => write!(f, "context doc tests"),
87        }
88    }
89}
90
91struct Batch<'a> {
92    kind: BatchKind,
93    entry: Option<EntryPoint<'a>>,
94    cases: Vec<TestCase>,
95}
96
97/// Run all tests that can be found.
98pub(super) async fn run<'p, I>(
99    io: &mut Io<'_>,
100    c: &Config,
101    flags: &Flags,
102    shared: &SharedFlags,
103    options: &Options,
104    entry: &mut Entry<'_>,
105    entries: I,
106) -> anyhow::Result<ExitCode>
107where
108    I: IntoIterator<Item = EntryPoint<'p>>,
109{
110    let start = Instant::now();
111
112    let mut executed = 0usize;
113    let mut skipped = 0usize;
114    let mut build_errors = 0usize;
115    let mut skipped_entries = 0usize;
116    let mut collected = Vec::new();
117
118    let capture = crate::modules::capture_io::CaptureIo::new();
119    let context = shared.context(entry, c, Some(&capture))?;
120
121    let mut batches = Vec::new();
122    let mut naming = Naming::default();
123    let mut name = String::new();
124
125    let mut filter = |item: &Item| -> Result<bool> {
126        if flags.filters.is_empty() {
127            return Ok(false);
128        }
129
130        name.clear();
131
132        write!(name, "{item}")?;
133
134        if !flags.filters.iter().any(|f| name.contains(f.as_str())) {
135            return Ok(true);
136        }
137
138        Ok(false)
139    };
140
141    for e in entries {
142        if flags.skip_lib_tests {
143            continue;
144        }
145
146        let mut options = options.clone();
147
148        if e.is_argument() {
149            options.function_body = true;
150        }
151
152        let item = naming.item(&e)?;
153
154        let mut sources = Sources::new();
155
156        let source = match Source::from_path(e.path()) {
157            Ok(source) => source,
158            Err(error) => return Err(error).context(e.path().display().try_to_string()?),
159        };
160
161        sources.insert(source)?;
162
163        let mut diagnostics = if shared.warnings || flags.warnings_are_errors {
164            Diagnostics::new()
165        } else {
166            Diagnostics::without_warnings()
167        };
168
169        let mut doc_visitor = crate::doc::Visitor::new(&item)?;
170        let mut functions = visitor::FunctionVisitor::new(visitor::Attribute::Test);
171        let mut source_loader = FileSourceLoader::new();
172
173        let unit = crate::prepare(&mut sources)
174            .with_context(&context)
175            .with_diagnostics(&mut diagnostics)
176            .with_options(&options)
177            .with_visitor(&mut doc_visitor)?
178            .with_visitor(&mut functions)?
179            .with_source_loader(&mut source_loader)
180            .build();
181
182        if diagnostics.has_error() || flags.warnings_are_errors && diagnostics.has_warning() {
183            build_errors = build_errors.wrapping_add(1);
184            collected.try_push((diagnostics, sources))?;
185            continue;
186        }
187
188        diagnostics.emit(&mut io.stdout.lock(), &sources)?;
189
190        let unit = Arc::new(unit?);
191        let sources = Arc::new(sources);
192
193        let mut cases = Vec::new();
194
195        for (hash, item) in functions.into_functions() {
196            let filtered = filter(&item)?;
197
198            cases.try_push(TestCase::new(
199                hash,
200                item,
201                TestKind::Free,
202                unit.clone(),
203                sources.clone(),
204                TestParams::default(),
205                filtered,
206            ))?;
207        }
208
209        batches.try_push(Batch {
210            kind: BatchKind::LibTests,
211            entry: Some(e.try_clone()?),
212            cases,
213        })?;
214
215        let mut artifacts = crate::doc::Artifacts::without_assets();
216        crate::doc::build("root", &mut artifacts, None, slice::from_ref(&doc_visitor))?;
217
218        if !c.filtered {
219            let cases = populate_doc_tests(
220                io,
221                artifacts,
222                shared,
223                flags,
224                &options,
225                &context,
226                &mut build_errors,
227                &mut skipped_entries,
228                &mut collected,
229                &mut filter,
230            )?;
231
232            batches.try_push(Batch {
233                kind: BatchKind::DocTests,
234                entry: Some(e),
235                cases,
236            })?;
237        }
238    }
239
240    let mut artifacts = crate::doc::Artifacts::without_assets();
241    crate::doc::build("root", &mut artifacts, Some(&context), &[])?;
242
243    if !c.filtered {
244        let cases = populate_doc_tests(
245            io,
246            artifacts,
247            shared,
248            flags,
249            options,
250            &context,
251            &mut build_errors,
252            &mut skipped_entries,
253            &mut collected,
254            &mut filter,
255        )?;
256
257        batches.try_push(Batch {
258            kind: BatchKind::ContextDocTests,
259            entry: None,
260            cases,
261        })?;
262    }
263
264    let runtime = Arc::new(context.runtime()?);
265    let mut failed = Vec::new();
266
267    for batch in batches {
268        if batch.cases.is_empty() {
269            continue;
270        }
271
272        let all_ignored = batch
273            .cases
274            .iter()
275            .all(|case| case.filtered || case.params.no_run);
276
277        let mut section = None;
278
279        if shared.verbose {
280            if all_ignored {
281                section = Some(("Ignoring", Color::Ignore));
282            } else {
283                section = Some(("Running", Color::Highlight));
284            }
285        }
286
287        if let Some((title, color)) = section {
288            let mut section = io.section(title, Stream::Stdout, color)?;
289
290            if !flags.quiet && !all_ignored {
291                section.append(format_args!(" {} {}", batch.cases.len(), batch.kind))?;
292
293                if let Some(entry) = batch.entry {
294                    section.append(format_args!(" from {entry}"))?;
295                }
296            }
297
298            section.close()?;
299        }
300
301        for mut case in batch.cases {
302            if case.filtered {
303                skipped = skipped.wrapping_add(1);
304                continue;
305            }
306
307            if case.params.no_run {
308                continue;
309            }
310
311            let mut vm = Vm::new(runtime.clone(), case.unit.clone());
312            case.execute(&mut vm, &capture).await?;
313            executed = executed.wrapping_add(1);
314
315            if case.outcome.is_ok() {
316                if flags.quiet {
317                    write!(io.stdout, ".")?;
318                } else {
319                    case.emit(io)?;
320                }
321
322                continue;
323            }
324
325            if flags.quiet {
326                write!(io.stdout, "f")?;
327            }
328
329            failed.try_push(case)?;
330
331            if flags.fail_fast {
332                break;
333            }
334        }
335    }
336
337    if flags.quiet {
338        writeln!(io.stdout)?;
339    }
340
341    let failures = failed.len();
342
343    for (diagnostics, sources) in collected {
344        diagnostics.emit(&mut io.stdout.lock(), &sources)?;
345    }
346
347    for case in failed {
348        case.emit(io)?;
349    }
350
351    let elapsed = start.elapsed();
352
353    let mut section = io.section("Executed", Stream::Stdout, Color::Highlight)?;
354
355    section.append(format_args!(" {executed} tests"))?;
356
357    let any = failures > 0 || build_errors > 0 || skipped > 0 || skipped_entries > 0;
358
359    if any {
360        section.append(" with")?;
361
362        let mut first = true;
363
364        let mut emit = |color: Color, count: usize, singular: &str, plural: &str| {
365            if count == 0 {
366                return Ok::<_, anyhow::Error>(());
367            }
368
369            if !take(&mut first) {
370                section.append(", ")?;
371            } else {
372                section.append(" ")?;
373            }
374
375            let what = if count == 1 { singular } else { plural };
376
377            section.append(format_args!("{count} "))?;
378            section.append_with(what, color)?;
379            Ok::<_, anyhow::Error>(())
380        };
381
382        emit(Color::Error, failures, "failure", "failures")?;
383        emit(Color::Error, build_errors, "build error", "build errors")?;
384        emit(Color::Ignore, skipped, "filtered", "filtered")?;
385        emit(
386            Color::Ignore,
387            skipped_entries,
388            "filtered entries",
389            "filtered entries",
390        )?;
391    }
392
393    writeln!(io.stdout, " in {:.3} seconds", elapsed.as_secs_f64())?;
394
395    if build_errors == 0 && failures == 0 {
396        Ok(ExitCode::Success)
397    } else {
398        Ok(ExitCode::Failure)
399    }
400}
401
402fn populate_doc_tests(
403    io: &mut Io,
404    artifacts: crate::doc::Artifacts,
405    shared: &SharedFlags,
406    flags: &Flags,
407    options: &Options,
408    context: &crate::Context,
409    build_errors: &mut usize,
410    skipped_entries: &mut usize,
411    collected: &mut Vec<(Diagnostics, Sources)>,
412    filter: &mut dyn FnMut(&Item) -> Result<bool>,
413) -> Result<Vec<TestCase>> {
414    let mut cases = Vec::new();
415
416    for test in artifacts.tests() {
417        if !options.test_std && test.item.as_crate() == Some("std") || test.params.ignore {
418            continue;
419        }
420
421        let is_filtered = filter(&test.item)?;
422
423        if is_filtered {
424            *skipped_entries = skipped_entries.wrapping_add(1);
425            continue;
426        }
427
428        let mut sources = Sources::new();
429
430        let source = Source::new(test.item.try_to_string()?, &test.content)?;
431        sources.insert(source)?;
432
433        let mut diagnostics = if shared.warnings || flags.warnings_are_errors {
434            Diagnostics::new()
435        } else {
436            Diagnostics::without_warnings()
437        };
438
439        let mut source_loader = FileSourceLoader::new();
440
441        let mut options = options.clone();
442        options.function_body = true;
443
444        let unit = crate::prepare(&mut sources)
445            .with_context(context)
446            .with_diagnostics(&mut diagnostics)
447            .with_options(&options)
448            .with_source_loader(&mut source_loader)
449            .build();
450
451        if diagnostics.has_error() || flags.warnings_are_errors && diagnostics.has_warning() {
452            *build_errors = build_errors.wrapping_add(1);
453            collected.try_push((diagnostics, sources))?;
454            continue;
455        }
456
457        diagnostics.emit(&mut io.stdout.lock(), &sources)?;
458
459        if !test.params.no_run {
460            let unit = Arc::new(unit?);
461            let sources = Arc::new(sources);
462
463            cases.try_push(TestCase::new(
464                Hash::EMPTY,
465                test.item.try_clone()?,
466                test.kind,
467                unit.clone(),
468                sources.clone(),
469                test.params,
470                is_filtered,
471            ))?;
472        }
473    }
474
475    Ok(cases)
476}
477
478#[derive(Debug)]
479enum Outcome {
480    Ok,
481    Panic(VmError),
482    ExpectedPanic,
483    None,
484    Err(Value),
485}
486
487impl Outcome {
488    fn is_ok(&self) -> bool {
489        matches!(self, Outcome::Ok)
490    }
491}
492
493struct TestCase {
494    hash: Hash,
495    item: ItemBuf,
496    kind: TestKind,
497    unit: Arc<Unit>,
498    sources: Arc<Sources>,
499    params: TestParams,
500    outcome: Outcome,
501    output: Vec<u8>,
502    filtered: bool,
503}
504
505impl TestCase {
506    fn new(
507        hash: Hash,
508        item: ItemBuf,
509        kind: TestKind,
510        unit: Arc<Unit>,
511        sources: Arc<Sources>,
512        params: TestParams,
513        filtered: bool,
514    ) -> Self {
515        Self {
516            hash,
517            item,
518            kind,
519            unit,
520            sources,
521            params,
522            outcome: Outcome::Ok,
523            output: Vec::new(),
524            filtered,
525        }
526    }
527
528    async fn execute(&mut self, vm: &mut Vm, capture_io: &CaptureIo) -> Result<()> {
529        let result = match vm.execute(self.hash, ()) {
530            Ok(mut execution) => execution.async_complete().await,
531            Err(err) => VmResult::Err(err),
532        };
533
534        capture_io.drain_into(&mut self.output)?;
535
536        self.outcome = match result {
537            VmResult::Ok(v) => match v.as_ref() {
538                Repr::Any(value) => match value.type_hash() {
539                    Result::<Value, Value>::HASH => {
540                        let result = value.borrow_ref::<Result<Value, Value>>()?;
541
542                        match &*result {
543                            Ok(..) => Outcome::Ok,
544                            Err(error) => Outcome::Err(error.clone()),
545                        }
546                    }
547                    Option::<Value>::HASH => {
548                        let option = value.borrow_ref::<Option<Value>>()?;
549
550                        match &*option {
551                            Some(..) => Outcome::Ok,
552                            None => Outcome::None,
553                        }
554                    }
555                    _ => Outcome::Ok,
556                },
557                _ => Outcome::Ok,
558            },
559            VmResult::Err(e) => Outcome::Panic(e),
560        };
561
562        if self.params.should_panic {
563            if matches!(self.outcome, Outcome::Panic(..)) {
564                self.outcome = Outcome::Ok;
565            } else {
566                self.outcome = Outcome::ExpectedPanic;
567            }
568        }
569
570        Ok(())
571    }
572
573    fn emit(self, io: &mut Io<'_>) -> Result<()> {
574        let mut section = io.section("Test", Stream::Stdout, Color::Highlight)?;
575
576        match self.kind {
577            TestKind::Free => {
578                section.append(format_args!(" {}: ", self.item))?;
579            }
580            TestKind::Protocol(protocol) => {
581                section.append(format_args!(" {}", self.item))?;
582                section.append_with(format_args!(" {}: ", protocol.name), Color::Important)?;
583            }
584        }
585
586        let mut emitted = None;
587
588        match &self.outcome {
589            Outcome::Panic(error) => {
590                section.error("errored")?;
591                emitted = Some(error);
592            }
593            Outcome::ExpectedPanic => {
594                section.error("expected panic because of `should_panic`, but ran without issue")?;
595            }
596            Outcome::Err(error) => {
597                section.error("err: ")?;
598                section.append(format_args!("{error:?}"))?;
599            }
600            Outcome::None => {
601                section.error("returned none")?;
602            }
603            Outcome::Ok => {
604                section.passed("ok")?;
605            }
606        }
607
608        section.close()?;
609
610        if let Some(error) = emitted {
611            error.emit(io.stdout, &self.sources)?;
612        }
613
614        if !self.outcome.is_ok() && !self.output.is_empty() {
615            writeln!(io.stdout, "-- output --")?;
616            io.stdout.write_all(&self.output)?;
617            writeln!(io.stdout, "-- end of output --")?;
618        }
619
620        Ok(())
621    }
622}
rune/cli/tests.rs

rune/cli/
tests.rs
