ci: improve test failure detection in parallel tests
All checks were successful
CI Code / Check spelling (pull_request) Successful in 18s
CI Code / Code Coverage (pull_request) Has been skipped
CI Code / Check coding style (pull_request) Successful in 30s
CI Code / Linux (ubuntu) (pull_request) Successful in 18m10s
CI Code / Linux (debian) (pull_request) Successful in 18m23s
CI Code / Linux (arch) (pull_request) Successful in 20m2s
All checks were successful
CI Code / Check spelling (pull_request) Successful in 18s
CI Code / Code Coverage (pull_request) Has been skipped
CI Code / Check coding style (pull_request) Successful in 30s
CI Code / Linux (ubuntu) (pull_request) Successful in 18m10s
CI Code / Linux (debian) (pull_request) Successful in 18m23s
CI Code / Linux (arch) (pull_request) Successful in 20m2s
1. ci-build.sh: Add verify_test_failure_detection() function - Tests that single test failures are properly detected - Tests that parallel test failures are properly detected - Runs at CI start to catch framework issues early 2. Makefile.am: Fix check-functional-parallel exit code handling - OLD: used 'wait' without checking individual exit codes - NEW: capture each PID and check exit code with 'wait $pid' - On failure: print the log file and set failed flag - Exit with error if any group failed This ensures CI correctly fails when: - Any unit test fails - Any of the 4 functional test groups fails - Any parallel build configuration fails
This commit is contained in:
24
Makefile.am
24
Makefile.am
@@ -311,17 +311,19 @@ tests_functionaltests_functionaltests_LDADD = -lcmocka -lstabber @FORKPTY_LIB@
|
||||
check-functional-parallel: tests/functionaltests/functionaltests
|
||||
@echo "Running functional tests in parallel (4 groups)..."
|
||||
@mkdir -p $(builddir)/test-logs
|
||||
@( \
|
||||
./tests/functionaltests/functionaltests 1 > $(builddir)/test-logs/group1.log 2>&1 & \
|
||||
./tests/functionaltests/functionaltests 2 > $(builddir)/test-logs/group2.log 2>&1 & \
|
||||
./tests/functionaltests/functionaltests 3 > $(builddir)/test-logs/group3.log 2>&1 & \
|
||||
./tests/functionaltests/functionaltests 4 > $(builddir)/test-logs/group4.log 2>&1 & \
|
||||
wait \
|
||||
) && ( \
|
||||
echo "=== Test Results ==="; \
|
||||
grep -E 'PASSED|FAILED|Running' $(builddir)/test-logs/group*.log; \
|
||||
! grep -q FAILED $(builddir)/test-logs/group*.log \
|
||||
)
|
||||
@failed=0; \
|
||||
./tests/functionaltests/functionaltests 1 > $(builddir)/test-logs/group1.log 2>&1 & pid1=$$!; \
|
||||
./tests/functionaltests/functionaltests 2 > $(builddir)/test-logs/group2.log 2>&1 & pid2=$$!; \
|
||||
./tests/functionaltests/functionaltests 3 > $(builddir)/test-logs/group3.log 2>&1 & pid3=$$!; \
|
||||
./tests/functionaltests/functionaltests 4 > $(builddir)/test-logs/group4.log 2>&1 & pid4=$$!; \
|
||||
wait $$pid1 || { echo "Group 1 FAILED (exit $$?)"; cat $(builddir)/test-logs/group1.log; failed=1; }; \
|
||||
wait $$pid2 || { echo "Group 2 FAILED (exit $$?)"; cat $(builddir)/test-logs/group2.log; failed=1; }; \
|
||||
wait $$pid3 || { echo "Group 3 FAILED (exit $$?)"; cat $(builddir)/test-logs/group3.log; failed=1; }; \
|
||||
wait $$pid4 || { echo "Group 4 FAILED (exit $$?)"; cat $(builddir)/test-logs/group4.log; failed=1; }; \
|
||||
echo "=== Test Results Summary ==="; \
|
||||
grep -E 'PASSED|FAILED|Running' $(builddir)/test-logs/group*.log || true; \
|
||||
if [ $$failed -ne 0 ]; then echo "FUNCTIONAL TESTS FAILED"; exit 1; fi; \
|
||||
echo "All functional test groups passed!"
|
||||
endif
|
||||
endif
|
||||
|
||||
|
||||
76
ci-build.sh
76
ci-build.sh
@@ -23,6 +23,79 @@ error_handler()
|
||||
|
||||
trap error_handler ERR
|
||||
|
||||
# Verify that test failures are properly detected
|
||||
# This is a meta-test: it runs a deliberately failing test
|
||||
# and checks that the test framework reports the failure correctly
|
||||
verify_test_failure_detection()
|
||||
{
|
||||
echo
|
||||
echo "==> Verifying test failure detection..."
|
||||
|
||||
# Create a simple failing test
|
||||
cat > /tmp/test_must_fail.c << 'EOF'
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <setjmp.h>
|
||||
#include <cmocka.h>
|
||||
|
||||
static void test_that_must_fail(void **state) {
|
||||
(void)state;
|
||||
assert_true(0); // This MUST fail
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
const struct CMUnitTest tests[] = {
|
||||
cmocka_unit_test(test_that_must_fail),
|
||||
};
|
||||
return cmocka_run_group_tests(tests, NULL, NULL);
|
||||
}
|
||||
EOF
|
||||
|
||||
# Compile the failing test
|
||||
if ! gcc -o /tmp/test_must_fail /tmp/test_must_fail.c -lcmocka 2>/dev/null; then
|
||||
echo "Warning: Could not compile test failure verification (cmocka not available?)"
|
||||
echo "Skipping test failure detection verification"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Test 1: Single failing test detection
|
||||
echo " Testing single test failure detection..."
|
||||
if /tmp/test_must_fail > /tmp/test_must_fail.log 2>&1; then
|
||||
echo "ERROR: Test that should fail returned success (exit code 0)"
|
||||
echo "This means the test framework is NOT detecting failures correctly!"
|
||||
echo "--- Test output ---"
|
||||
cat /tmp/test_must_fail.log
|
||||
echo "--- End output ---"
|
||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log
|
||||
exit 1
|
||||
fi
|
||||
echo " ✓ Single test failure correctly detected"
|
||||
|
||||
# Test 2: Parallel failure detection (simulates check-functional-parallel)
|
||||
echo " Testing parallel test failure detection..."
|
||||
failed=0
|
||||
/tmp/test_must_fail > /tmp/p1.log 2>&1 & pid1=$!
|
||||
true > /tmp/p2.log 2>&1 & pid2=$! # This passes
|
||||
/tmp/test_must_fail > /tmp/p3.log 2>&1 & pid3=$!
|
||||
true > /tmp/p4.log 2>&1 & pid4=$! # This passes
|
||||
|
||||
wait $pid1 || failed=$((failed + 1))
|
||||
wait $pid2 || failed=$((failed + 1))
|
||||
wait $pid3 || failed=$((failed + 1))
|
||||
wait $pid4 || failed=$((failed + 1))
|
||||
|
||||
if [ $failed -ne 2 ]; then
|
||||
echo "ERROR: Expected 2 failures in parallel tests, got $failed"
|
||||
echo "Parallel failure detection is broken!"
|
||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
|
||||
exit 1
|
||||
fi
|
||||
echo " ✓ Parallel test failures correctly detected (2 of 4 failed as expected)"
|
||||
|
||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
|
||||
echo "✓ Test failure detection verified"
|
||||
}
|
||||
|
||||
num_cores()
|
||||
{
|
||||
# Check for cores, for systems with:
|
||||
@@ -34,6 +107,9 @@ num_cores()
|
||||
|| getconf _NPROCESSORS_ONLN 2>/dev/null
|
||||
}
|
||||
|
||||
# Run test failure detection verification first
|
||||
verify_test_failure_detection
|
||||
|
||||
./bootstrap.sh
|
||||
|
||||
tests=()
|
||||
|
||||
Reference in New Issue
Block a user