From e40b89ad92852fb5adfc260cad93cff7ca18b290 Mon Sep 17 00:00:00 2001 From: "jabber.developer2" Date: Sat, 17 Jan 2026 18:03:07 +0300 Subject: [PATCH] ci: improve test failure detection in parallel tests 1. ci-build.sh: Add verify_test_failure_detection() function - Tests that single test failures are properly detected - Tests that parallel test failures are properly detected - Runs at CI start to catch framework issues early 2. Makefile.am: Fix check-functional-parallel exit code handling - OLD: used 'wait' without checking individual exit codes - NEW: capture each PID and check exit code with 'wait $pid' - On failure: print the log file and set failed flag - Exit with error if any group failed This ensures CI correctly fails when: - Any unit test fails - Any of the 4 functional test groups fails - Any parallel build configuration fails --- Makefile.am | 24 +++++++++-------- ci-build.sh | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9d66ddf6..b8c8c706 100644 --- a/Makefile.am +++ b/Makefile.am @@ -311,17 +311,19 @@ tests_functionaltests_functionaltests_LDADD = -lcmocka -lstabber @FORKPTY_LIB@ check-functional-parallel: tests/functionaltests/functionaltests @echo "Running functional tests in parallel (4 groups)..." @mkdir -p $(builddir)/test-logs - @( \ - ./tests/functionaltests/functionaltests 1 > $(builddir)/test-logs/group1.log 2>&1 & \ - ./tests/functionaltests/functionaltests 2 > $(builddir)/test-logs/group2.log 2>&1 & \ - ./tests/functionaltests/functionaltests 3 > $(builddir)/test-logs/group3.log 2>&1 & \ - ./tests/functionaltests/functionaltests 4 > $(builddir)/test-logs/group4.log 2>&1 & \ - wait \ - ) && ( \ - echo "=== Test Results ==="; \ - grep -E 'PASSED|FAILED|Running' $(builddir)/test-logs/group*.log; \ - ! grep -q FAILED $(builddir)/test-logs/group*.log \ - ) + @failed=0; \ + ./tests/functionaltests/functionaltests 1 > $(builddir)/test-logs/group1.log 2>&1 & pid1=$$!; \ + ./tests/functionaltests/functionaltests 2 > $(builddir)/test-logs/group2.log 2>&1 & pid2=$$!; \ + ./tests/functionaltests/functionaltests 3 > $(builddir)/test-logs/group3.log 2>&1 & pid3=$$!; \ + ./tests/functionaltests/functionaltests 4 > $(builddir)/test-logs/group4.log 2>&1 & pid4=$$!; \ + wait $$pid1 || { echo "Group 1 FAILED (exit $$?)"; cat $(builddir)/test-logs/group1.log; failed=1; }; \ + wait $$pid2 || { echo "Group 2 FAILED (exit $$?)"; cat $(builddir)/test-logs/group2.log; failed=1; }; \ + wait $$pid3 || { echo "Group 3 FAILED (exit $$?)"; cat $(builddir)/test-logs/group3.log; failed=1; }; \ + wait $$pid4 || { echo "Group 4 FAILED (exit $$?)"; cat $(builddir)/test-logs/group4.log; failed=1; }; \ + echo "=== Test Results Summary ==="; \ + grep -E 'PASSED|FAILED|Running' $(builddir)/test-logs/group*.log || true; \ + if [ $$failed -ne 0 ]; then echo "FUNCTIONAL TESTS FAILED"; exit 1; fi; \ + echo "All functional test groups passed!" endif endif diff --git a/ci-build.sh b/ci-build.sh index 99407308..f98bedfa 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -23,6 +23,79 @@ error_handler() trap error_handler ERR +# Verify that test failures are properly detected +# This is a meta-test: it runs a deliberately failing test +# and checks that the test framework reports the failure correctly +verify_test_failure_detection() +{ + echo + echo "==> Verifying test failure detection..." + + # Create a simple failing test + cat > /tmp/test_must_fail.c << 'EOF' +#include +#include +#include +#include + +static void test_that_must_fail(void **state) { + (void)state; + assert_true(0); // This MUST fail +} + +int main(void) { + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_that_must_fail), + }; + return cmocka_run_group_tests(tests, NULL, NULL); +} +EOF + + # Compile the failing test + if ! gcc -o /tmp/test_must_fail /tmp/test_must_fail.c -lcmocka 2>/dev/null; then + echo "Warning: Could not compile test failure verification (cmocka not available?)" + echo "Skipping test failure detection verification" + return 0 + fi + + # Test 1: Single failing test detection + echo " Testing single test failure detection..." + if /tmp/test_must_fail > /tmp/test_must_fail.log 2>&1; then + echo "ERROR: Test that should fail returned success (exit code 0)" + echo "This means the test framework is NOT detecting failures correctly!" + echo "--- Test output ---" + cat /tmp/test_must_fail.log + echo "--- End output ---" + rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log + exit 1 + fi + echo " ✓ Single test failure correctly detected" + + # Test 2: Parallel failure detection (simulates check-functional-parallel) + echo " Testing parallel test failure detection..." + failed=0 + /tmp/test_must_fail > /tmp/p1.log 2>&1 & pid1=$! + true > /tmp/p2.log 2>&1 & pid2=$! # This passes + /tmp/test_must_fail > /tmp/p3.log 2>&1 & pid3=$! + true > /tmp/p4.log 2>&1 & pid4=$! # This passes + + wait $pid1 || failed=$((failed + 1)) + wait $pid2 || failed=$((failed + 1)) + wait $pid3 || failed=$((failed + 1)) + wait $pid4 || failed=$((failed + 1)) + + if [ $failed -ne 2 ]; then + echo "ERROR: Expected 2 failures in parallel tests, got $failed" + echo "Parallel failure detection is broken!" + rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log + exit 1 + fi + echo " ✓ Parallel test failures correctly detected (2 of 4 failed as expected)" + + rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log + echo "✓ Test failure detection verified" +} + num_cores() { # Check for cores, for systems with: @@ -34,6 +107,9 @@ num_cores() || getconf _NPROCESSORS_ONLN 2>/dev/null } +# Run test failure detection verification first +verify_test_failure_detection + ./bootstrap.sh tests=()