mirrors/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-04-21 20:15:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

LibTest: Add the RANDOMIZED_TEST_CASE macro and its main loop

Browse source 
Tests defined like

RANDOMIZED_TEST_CASE(test_name)
{
    GEN(dice, Gen::unsigned_int(1,6));
    EXPECT(dice >= 1 && dice <= 6);
}

will be run many times (100x by default, can be overriden with
MAX_GENERATED_VALUES_PER_TEST), each time generating different random
values, and if any of the test runs fails, we'll shrink the generated
values and report the final minimal ones to the user.
This commit is contained in:
Martin Janiczek 2023-10-24 01:46:52 +02:00 • committed by Andrew Kaster
parent 30f73221fd
commit 2782334152
Notes: sideshowbarker 2024-07-17 04:01:41 +09:00
1 changed files with 95 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
Userland/Libraries/LibTest/TestCase.h
View file

@ -8,16 +8,36 @@
#pragma once
#include <LibTest/Macros.h> // intentionally first -- we redefine VERIFY and friends in here
#include <LibTest/Randomized/RandomnessSource.h>
#include <LibTest/Randomized/Shrink.h>
#include <AK/DeprecatedString.h>
#include <AK/Function.h>
#include <AK/NonnullRefPtr.h>
#include <AK/RefCounted.h>
#ifndef MAX_GENERATED_VALUES_PER_TEST
# define MAX_GENERATED_VALUES_PER_TEST 100
#endif
#ifndef MAX_GEN_ATTEMPTS_PER_VALUE
# define MAX_GEN_ATTEMPTS_PER_VALUE 15
#endif
namespace Test {
using TestFunction = Function<void()>;
inline void run_with_randomness_source(Randomized::RandomnessSource source, TestFunction const& test_function)
{
set_randomness_source(move(source));
set_current_test_result(TestResult::NotRun);
test_function();
if (current_test_result() == TestResult::NotRun) {
set_current_test_result(TestResult::Passed);
}
}
class TestCase : public RefCounted<TestCase> {
public:
TestCase(DeprecatedString const& name, TestFunction&& fn, bool is_benchmark)
@ -31,6 +51,60 @@ public:
DeprecatedString const& name() const { return m_name; }
TestFunction const& func() const { return m_function; }
static NonnullRefPtr<TestCase> randomized(DeprecatedString const& name, TestFunction&& test_function)
{
using namespace Randomized;
TestFunction test_case_function = [test_function = move(test_function)]() {
for (u32 i = 0; i < MAX_GENERATED_VALUES_PER_TEST; ++i) {
bool generated_successfully = false;
u8 gen_attempt;
for (gen_attempt = 0; gen_attempt < MAX_GEN_ATTEMPTS_PER_VALUE && !generated_successfully; ++gen_attempt) {
// We're going to run the test function many times, so let's turn off the reporting until we finish.
disable_reporting();
set_current_test_result(TestResult::NotRun);
run_with_randomness_source(RandomnessSource::live(), test_function);
switch (current_test_result()) {
case TestResult::NotRun:
VERIFY_NOT_REACHED();
break;
case TestResult::Passed: {
generated_successfully = true;
break;
}
case TestResult::Failed: {
generated_successfully = true;
RandomRun first_failure = randomness_source().run();
RandomRun best_failure = shrink(first_failure, test_function);
// Run one last time with reporting on, so that the user can see the minimal failure
enable_reporting();
run_with_randomness_source(RandomnessSource::recorded(best_failure), test_function);
return;
}
case TestResult::Rejected:
break;
case TestResult::Overrun:
break;
default:
VERIFY_NOT_REACHED();
break;
}
}
enable_reporting();
if (!generated_successfully) {
// The loop above got to the full MAX_GEN_ATTEMPTS_PER_VALUE and gave up.
// Run one last time with reporting on, so that the user gets the REJECTED message.
RandomRun last_failure = randomness_source().run();
run_with_randomness_source(RandomnessSource::recorded(last_failure), test_function);
return;
}
}
// MAX_GENERATED_VALUES_PER_TEST values generated, all passed the test.
};
return make_ref_counted<TestCase>(name, move(test_case_function), false);
}
private:
DeprecatedString m_name;
TestFunction m_function;
@ -53,6 +127,8 @@ void set_suite_setup_function(Function<void()> setup);
static struct __setup_type __setup_type; \
static void __setup()
// Unit test
#define __TESTCASE_FUNC(x) __test_##x
#define __TESTCASE_TYPE(x) __TestCase_##x
@ -68,6 +144,8 @@ void set_suite_setup_function(Function<void()> setup);
static struct __TESTCASE_TYPE(x) __TESTCASE_TYPE(x); \
static void __TESTCASE_FUNC(x)()
// Benchmark
#define __BENCHMARK_FUNC(x) __benchmark_##x
#define __BENCHMARK_TYPE(x) __BenchmarkCase_##x
@ -83,6 +161,23 @@ void set_suite_setup_function(Function<void()> setup);
static struct __BENCHMARK_TYPE(x) __BENCHMARK_TYPE(x); \
static void __BENCHMARK_FUNC(x)()
// Randomized test
#define __RANDOMIZED_TEST_FUNC(x) __randomized_test_##x
#define __RANDOMIZED_TEST_TYPE(x) __RandomizedTestCase_##x
#define RANDOMIZED_TEST_CASE(x) \
static void __RANDOMIZED_TEST_FUNC(x)(); \
struct __RANDOMIZED_TEST_TYPE(x) { \
__RANDOMIZED_TEST_TYPE(x) \
() \
{ \
add_test_case_to_suite(::Test::TestCase::randomized(#x, __RANDOMIZED_TEST_FUNC(x))); \
} \
}; \
static struct __RANDOMIZED_TEST_TYPE(x) __RANDOMIZED_TEST_TYPE(x); \
static void __RANDOMIZED_TEST_FUNC(x)()
// This allows us to print the generated locals in the test after a failure is fully shrunk.
#define GEN(identifier, value) \
auto identifier = (value); \