Improve Functional Test Stability and CI Reliability #79

Manually merged
jabber.developer merged 7 commits from fix/test-CI-stability into master 2026-02-02 16:47:30 +00:00
8 changed files with 365 additions and 80 deletions
Showing only changes of commit cf86e60af6 - Show all commits

View File

@@ -91,12 +91,15 @@ set -e
This will run the same tests that the CI runs and refuse the push if it fails. This will run the same tests that the CI runs and refuse the push if it fails.
The CI script runs 4 parallel builds with different configurations, each with Valgrind and functional tests. The CI script runs 4 parallel builds with different configurations, each with Valgrind and functional tests.
Build 1 also collects code coverage (lines, functions, branches). Use `./ci-build.sh --coverage-only` to run only build 1 with coverage collection.
Output shows test results per build: Output shows test results per build:
``` ```
✓ Build 1 passed (unit: 437/0, func: 69/0, cov: L:27.5% F:36.2% B:18.1%, 5m39s) ✓ BUILD 1 PASSED
✓ Build 2 passed (unit: 398/0, func: 69/0, 5m00s) Unit tests: 437 passed, 0 failed
Functional tests: 69 passed, 0 failed
Coverage: Lines: 27.5% | Functions: 36.2% | Branches: 18.1%
Duration: 5m39s
``` ```
Note that it will run on the actual content of the repository directory and not Note that it will run on the actual content of the repository directory and not

View File

@@ -23,6 +23,55 @@ error_handler()
trap error_handler ERR trap error_handler ERR
# =============================================================================
# Constants
# =============================================================================
# Regex patterns for parsing test output
readonly CMOCKA_PASSED_PATTERN='^\[ PASSED \] [0-9]+ test'
readonly CMOCKA_FAILED_PATTERN='^\[ FAILED \] [0-9]+ test'
# Coverage extraction patterns (matches both Docker and CI paths)
readonly COVERAGE_PATTERNS='*/profanity/src/* */src/src/*'
# =============================================================================
# Helper Functions
# =============================================================================
# Parse STATS line from build log and set global variables
# Usage: parse_build_stats <log_file>
# Sets: STAT_UNIT_P, STAT_UNIT_F, STAT_FUNC_P, STAT_FUNC_F,
jabber.developer marked this conversation as resolved
Review

Is it required? It needs to be updated each time the set variables of the method are updated, yet it does not provide additional data. It would make more sense to explain what each variable means next to their initiation place.

Is it required? It needs to be updated each time the set variables of the method are updated, yet it does not provide additional data. It would make more sense to explain what each variable means next to their initiation place.

Refactored, moved to separate function

Refactored, moved to separate function
# STAT_COV_LINES, STAT_COV_FUNCS, STAT_COV_BRANCHES, STAT_TIME
parse_build_stats() {
local log_file="$1"
local stats_line
stats_line=$(grep "^STATS:" "$log_file" 2>/dev/null | tail -1)
STAT_UNIT_P=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2)
STAT_UNIT_F=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2)
STAT_FUNC_P=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2)
STAT_FUNC_F=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2)
STAT_COV_LINES=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2)
STAT_COV_FUNCS=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2)
STAT_COV_BRANCHES=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2)
STAT_TIME=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2)
: "${STAT_UNIT_P:=0}"
: "${STAT_UNIT_F:=0}"
: "${STAT_FUNC_P:=0}"
: "${STAT_FUNC_F:=0}"
}
# Extract test count from log file
# Usage: extract_test_count <log_file> <pattern>
extract_test_count() {
grep -E "$2" "$1" 2>/dev/null | grep -oE "[0-9]+" | head -1
}
# =============================================================================
# Test Verification
# =============================================================================
# Verify that test failures are properly detected # Verify that test failures are properly detected
# This is a meta-test: it runs a deliberately failing test # This is a meta-test: it runs a deliberately failing test
# and checks that the test framework reports the failure correctly # and checks that the test framework reports the failure correctly
@@ -246,10 +295,9 @@ build_and_test() {
fi fi
fi fi
# Extract unit test counts from cmocka output # Extract unit test counts from cmocka output
unit_passed=$(grep -E "^\[ PASSED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1) unit_passed=$(extract_test_count unit-tests-output.log "$CMOCKA_PASSED_PATTERN")
unit_failed=$(grep -E "^\[ FAILED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1) unit_failed=$(extract_test_count unit-tests-output.log "$CMOCKA_FAILED_PATTERN")
[ -z "$unit_passed" ] && unit_passed=0 : "${unit_passed:=0}" "${unit_failed:=0}"
[ -z "$unit_failed" ] && unit_failed=0
echo "UNIT_TESTS: passed=$unit_passed failed=$unit_failed" echo "UNIT_TESTS: passed=$unit_passed failed=$unit_failed"
# Set build index for port allocation: build 1 uses ports 5230-5233, # Set build index for port allocation: build 1 uses ports 5230-5233,
@@ -264,9 +312,9 @@ build_and_test() {
echo "=== Functional test results ===" echo "=== Functional test results ==="
for glog in ./test-logs/group*.log; do for glog in ./test-logs/group*.log; do
if [ -f "$glog" ]; then if [ -f "$glog" ]; then
cnt=$(grep -E "^\[ PASSED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1) cnt=$(extract_test_count "$glog" "$CMOCKA_PASSED_PATTERN")
[ -n "$cnt" ] && func_passed=$((func_passed + cnt)) [ -n "$cnt" ] && func_passed=$((func_passed + cnt))
cnt=$(grep -E "^\[ FAILED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1) cnt=$(extract_test_count "$glog" "$CMOCKA_FAILED_PATTERN")
[ -n "$cnt" ] && func_failed=$((func_failed + cnt)) [ -n "$cnt" ] && func_failed=$((func_failed + cnt))
fi fi
done done
@@ -281,8 +329,8 @@ build_and_test() {
lcov --capture --directory . --output-file coverage-full.info \ lcov --capture --directory . --output-file coverage-full.info \
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true --rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
# Extract only production code from src/ directory, exclude tests # Extract only production code from src/ directory, exclude tests
# Pattern matches both Docker (/src/src/*) and CI (/usr/src/profanity/src/*) # shellcheck disable=SC2086
lcov --extract coverage-full.info '*/profanity/src/*' '*/src/src/*' \ lcov --extract coverage-full.info $COVERAGE_PATTERNS \
--output-file coverage.info \ --output-file coverage.info \
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true --rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
if [ -f coverage.info ] && [ -s coverage.info ]; then if [ -f coverage.info ] && [ -s coverage.info ]; then
@@ -374,28 +422,15 @@ for i in "${!pids[@]}"; do
idx=$((i + 1)) idx=$((i + 1))
if wait "${pids[$i]}"; then if wait "${pids[$i]}"; then
if [ -f "build-$idx.log" ]; then if [ -f "build-$idx.log" ]; then
stats_line=$(grep "^STATS:" "build-$idx.log" | tail -1) parse_build_stats "build-$idx.log"
unit_p=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2)
unit_f=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2)
func_p=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2)
func_f=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2)
cov_lines=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2)
cov_funcs=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2)
cov_branches=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2)
time=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2)
unit_p=${unit_p:-0}
unit_f=${unit_f:-0}
func_p=${func_p:-0}
func_f=${func_f:-0}
echo "✓ BUILD $idx PASSED" echo "✓ BUILD $idx PASSED"
echo " Unit tests: $unit_p passed, $unit_f failed" echo " Unit tests: $STAT_UNIT_P passed, $STAT_UNIT_F failed"
echo " Functional tests: $func_p passed, $func_f failed" echo " Functional tests: $STAT_FUNC_P passed, $STAT_FUNC_F failed"
if [ "$cov_lines" != "n/a" ] && [ -n "$cov_lines" ]; then if [ "$STAT_COV_LINES" != "n/a" ] && [ -n "$STAT_COV_LINES" ]; then
echo " Coverage: Lines: $cov_lines | Functions: $cov_funcs | Branches: $cov_branches" echo " Coverage: Lines: $STAT_COV_LINES | Functions: $STAT_COV_FUNCS | Branches: $STAT_COV_BRANCHES"
fi fi
echo " Duration: ${time:-?}" echo " Duration: ${STAT_TIME:-?}"
else else
echo "✓ Build $idx passed (no stats available)" echo "✓ Build $idx passed (no stats available)"
fi fi