ci: improve test failure detection in parallel tests
All checks were successful
CI Code / Check spelling (pull_request) Successful in 18s
CI Code / Code Coverage (pull_request) Has been skipped
CI Code / Check coding style (pull_request) Successful in 30s
CI Code / Linux (ubuntu) (pull_request) Successful in 18m10s
CI Code / Linux (debian) (pull_request) Successful in 18m23s
CI Code / Linux (arch) (pull_request) Successful in 20m2s

1. ci-build.sh: Add verify_test_failure_detection() function
   - Tests that single test failures are properly detected
   - Tests that parallel test failures are properly detected
   - Runs at CI start to catch framework issues early

2. Makefile.am: Fix check-functional-parallel exit code handling
   - OLD: used 'wait' without checking individual exit codes
   - NEW: capture each PID and check exit code with 'wait $pid'
   - On failure: print the log file and set failed flag
   - Exit with error if any group failed

This ensures CI correctly fails when:
- Any unit test fails
- Any of the 4 functional test groups fails
- Any parallel build configuration fails
This commit is contained in:
2026-01-17 18:03:07 +03:00
parent f41888b7a5
commit e40b89ad92
2 changed files with 89 additions and 11 deletions

View File

@@ -311,17 +311,19 @@ tests_functionaltests_functionaltests_LDADD = -lcmocka -lstabber @FORKPTY_LIB@
check-functional-parallel: tests/functionaltests/functionaltests
@echo "Running functional tests in parallel (4 groups)..."
@mkdir -p $(builddir)/test-logs
@( \
./tests/functionaltests/functionaltests 1 > $(builddir)/test-logs/group1.log 2>&1 & \
./tests/functionaltests/functionaltests 2 > $(builddir)/test-logs/group2.log 2>&1 & \
./tests/functionaltests/functionaltests 3 > $(builddir)/test-logs/group3.log 2>&1 & \
./tests/functionaltests/functionaltests 4 > $(builddir)/test-logs/group4.log 2>&1 & \
wait \
) && ( \
echo "=== Test Results ==="; \
grep -E 'PASSED|FAILED|Running' $(builddir)/test-logs/group*.log; \
! grep -q FAILED $(builddir)/test-logs/group*.log \
)
@failed=0; \
./tests/functionaltests/functionaltests 1 > $(builddir)/test-logs/group1.log 2>&1 & pid1=$$!; \
./tests/functionaltests/functionaltests 2 > $(builddir)/test-logs/group2.log 2>&1 & pid2=$$!; \
./tests/functionaltests/functionaltests 3 > $(builddir)/test-logs/group3.log 2>&1 & pid3=$$!; \
./tests/functionaltests/functionaltests 4 > $(builddir)/test-logs/group4.log 2>&1 & pid4=$$!; \
wait $$pid1 || { echo "Group 1 FAILED (exit $$?)"; cat $(builddir)/test-logs/group1.log; failed=1; }; \
wait $$pid2 || { echo "Group 2 FAILED (exit $$?)"; cat $(builddir)/test-logs/group2.log; failed=1; }; \
wait $$pid3 || { echo "Group 3 FAILED (exit $$?)"; cat $(builddir)/test-logs/group3.log; failed=1; }; \
wait $$pid4 || { echo "Group 4 FAILED (exit $$?)"; cat $(builddir)/test-logs/group4.log; failed=1; }; \
echo "=== Test Results Summary ==="; \
grep -E 'PASSED|FAILED|Running' $(builddir)/test-logs/group*.log || true; \
if [ $$failed -ne 0 ]; then echo "FUNCTIONAL TESTS FAILED"; exit 1; fi; \
echo "All functional test groups passed!"
endif
endif

View File

@@ -23,6 +23,79 @@ error_handler()
trap error_handler ERR
# Verify that test failures are properly detected
# This is a meta-test: it runs a deliberately failing test
# and checks that the test framework reports the failure correctly
verify_test_failure_detection()
{
echo
echo "==> Verifying test failure detection..."
# Create a simple failing test
cat > /tmp/test_must_fail.c << 'EOF'
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
static void test_that_must_fail(void **state) {
(void)state;
assert_true(0); // This MUST fail
}
int main(void) {
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_that_must_fail),
};
return cmocka_run_group_tests(tests, NULL, NULL);
}
EOF
# Compile the failing test
if ! gcc -o /tmp/test_must_fail /tmp/test_must_fail.c -lcmocka 2>/dev/null; then
echo "Warning: Could not compile test failure verification (cmocka not available?)"
echo "Skipping test failure detection verification"
return 0
fi
# Test 1: Single failing test detection
echo " Testing single test failure detection..."
if /tmp/test_must_fail > /tmp/test_must_fail.log 2>&1; then
echo "ERROR: Test that should fail returned success (exit code 0)"
echo "This means the test framework is NOT detecting failures correctly!"
echo "--- Test output ---"
cat /tmp/test_must_fail.log
echo "--- End output ---"
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log
exit 1
fi
echo " ✓ Single test failure correctly detected"
# Test 2: Parallel failure detection (simulates check-functional-parallel)
echo " Testing parallel test failure detection..."
failed=0
/tmp/test_must_fail > /tmp/p1.log 2>&1 & pid1=$!
true > /tmp/p2.log 2>&1 & pid2=$! # This passes
/tmp/test_must_fail > /tmp/p3.log 2>&1 & pid3=$!
true > /tmp/p4.log 2>&1 & pid4=$! # This passes
wait $pid1 || failed=$((failed + 1))
wait $pid2 || failed=$((failed + 1))
wait $pid3 || failed=$((failed + 1))
wait $pid4 || failed=$((failed + 1))
if [ $failed -ne 2 ]; then
echo "ERROR: Expected 2 failures in parallel tests, got $failed"
echo "Parallel failure detection is broken!"
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
exit 1
fi
echo " ✓ Parallel test failures correctly detected (2 of 4 failed as expected)"
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
echo "✓ Test failure detection verified"
}
num_cores()
{
# Check for cores, for systems with:
@@ -34,6 +107,9 @@ num_cores()
|| getconf _NPROCESSORS_ONLN 2>/dev/null
}
# Run test failure detection verification first
verify_test_failure_detection
./bootstrap.sh
tests=()