Improve Functional Test Stability and CI Reliability #79

Manually merged
jabber.developer merged 7 commits from fix/test-CI-stability into master 2026-02-02 16:47:30 +00:00
8 changed files with 365 additions and 80 deletions
Showing only changes of commit cf86e60af6 - Show all commits

View File

@@ -91,12 +91,15 @@ set -e
This will run the same tests that the CI runs and refuse the push if it fails.
The CI script runs 4 parallel builds with different configurations, each with Valgrind and functional tests.
Build 1 also collects code coverage (lines, functions, branches).
Use `./ci-build.sh --coverage-only` to run only build 1 with coverage collection.
jabber.developer marked this conversation as resolved Outdated

Could you clarify that "build 1" is a standard build without flags?

Could you clarify that "build 1" is a standard build without flags?

Corrected

Corrected
Output shows test results per build:
```
✓ Build 1 passed (unit: 437/0, func: 69/0, cov: L:27.5% F:36.2% B:18.1%, 5m39s)
✓ Build 2 passed (unit: 398/0, func: 69/0, 5m00s)
✓ BUILD 1 PASSED
Unit tests: 437 passed, 0 failed
Functional tests: 69 passed, 0 failed
Coverage: Lines: 27.5% | Functions: 36.2% | Branches: 18.1%
Duration: 5m39s
```
Note that it will run on the actual content of the repository directory and not

View File

@@ -23,6 +23,55 @@ error_handler()
trap error_handler ERR
# =============================================================================
# Constants
# =============================================================================
# Regex patterns for parsing test output
readonly CMOCKA_PASSED_PATTERN='^\[ PASSED \] [0-9]+ test'
readonly CMOCKA_FAILED_PATTERN='^\[ FAILED \] [0-9]+ test'
# Coverage extraction patterns (matches both Docker and CI paths)
readonly COVERAGE_PATTERNS='*/profanity/src/* */src/src/*'
# =============================================================================
# Helper Functions
# =============================================================================
# Parse STATS line from build log and set global variables
# Usage: parse_build_stats <log_file>
# Sets: STAT_UNIT_P, STAT_UNIT_F, STAT_FUNC_P, STAT_FUNC_F,
jabber.developer marked this conversation as resolved
Review

Is it required? It needs to be updated each time the set variables of the method are updated, yet it does not provide additional data. It would make more sense to explain what each variable means next to their initiation place.

Is it required? It needs to be updated each time the set variables of the method are updated, yet it does not provide additional data. It would make more sense to explain what each variable means next to their initiation place.

Refactored, moved to separate function

Refactored, moved to separate function
# STAT_COV_LINES, STAT_COV_FUNCS, STAT_COV_BRANCHES, STAT_TIME
parse_build_stats() {
local log_file="$1"
local stats_line
stats_line=$(grep "^STATS:" "$log_file" 2>/dev/null | tail -1)
STAT_UNIT_P=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2)
STAT_UNIT_F=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2)
STAT_FUNC_P=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2)
STAT_FUNC_F=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2)
STAT_COV_LINES=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2)
STAT_COV_FUNCS=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2)
STAT_COV_BRANCHES=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2)
STAT_TIME=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2)
: "${STAT_UNIT_P:=0}"
: "${STAT_UNIT_F:=0}"
: "${STAT_FUNC_P:=0}"
: "${STAT_FUNC_F:=0}"
}
# Extract test count from log file
# Usage: extract_test_count <log_file> <pattern>
extract_test_count() {
grep -E "$2" "$1" 2>/dev/null | grep -oE "[0-9]+" | head -1
}
# =============================================================================
# Test Verification
# =============================================================================
# Verify that test failures are properly detected
# This is a meta-test: it runs a deliberately failing test
# and checks that the test framework reports the failure correctly
@@ -246,10 +295,9 @@ build_and_test() {
fi
fi
# Extract unit test counts from cmocka output
unit_passed=$(grep -E "^\[ PASSED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1)
unit_failed=$(grep -E "^\[ FAILED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1)
[ -z "$unit_passed" ] && unit_passed=0
[ -z "$unit_failed" ] && unit_failed=0
unit_passed=$(extract_test_count unit-tests-output.log "$CMOCKA_PASSED_PATTERN")
unit_failed=$(extract_test_count unit-tests-output.log "$CMOCKA_FAILED_PATTERN")
: "${unit_passed:=0}" "${unit_failed:=0}"
echo "UNIT_TESTS: passed=$unit_passed failed=$unit_failed"
# Set build index for port allocation: build 1 uses ports 5230-5233,
@@ -264,9 +312,9 @@ build_and_test() {
echo "=== Functional test results ==="
for glog in ./test-logs/group*.log; do
if [ -f "$glog" ]; then
cnt=$(grep -E "^\[ PASSED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1)
cnt=$(extract_test_count "$glog" "$CMOCKA_PASSED_PATTERN")
[ -n "$cnt" ] && func_passed=$((func_passed + cnt))
cnt=$(grep -E "^\[ FAILED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1)
cnt=$(extract_test_count "$glog" "$CMOCKA_FAILED_PATTERN")
[ -n "$cnt" ] && func_failed=$((func_failed + cnt))
fi
done
@@ -281,8 +329,8 @@ build_and_test() {
lcov --capture --directory . --output-file coverage-full.info \
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
# Extract only production code from src/ directory, exclude tests
# Pattern matches both Docker (/src/src/*) and CI (/usr/src/profanity/src/*)
lcov --extract coverage-full.info '*/profanity/src/*' '*/src/src/*' \
# shellcheck disable=SC2086
lcov --extract coverage-full.info $COVERAGE_PATTERNS \
--output-file coverage.info \
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
if [ -f coverage.info ] && [ -s coverage.info ]; then
@@ -374,28 +422,15 @@ for i in "${!pids[@]}"; do
idx=$((i + 1))
if wait "${pids[$i]}"; then
if [ -f "build-$idx.log" ]; then
stats_line=$(grep "^STATS:" "build-$idx.log" | tail -1)
unit_p=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2)
unit_f=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2)
func_p=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2)
func_f=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2)
cov_lines=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2)
cov_funcs=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2)
cov_branches=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2)
time=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2)
unit_p=${unit_p:-0}
unit_f=${unit_f:-0}
func_p=${func_p:-0}
func_f=${func_f:-0}
parse_build_stats "build-$idx.log"
echo "✓ BUILD $idx PASSED"
echo " Unit tests: $unit_p passed, $unit_f failed"
echo " Functional tests: $func_p passed, $func_f failed"
if [ "$cov_lines" != "n/a" ] && [ -n "$cov_lines" ]; then
echo " Coverage: Lines: $cov_lines | Functions: $cov_funcs | Branches: $cov_branches"
echo " Unit tests: $STAT_UNIT_P passed, $STAT_UNIT_F failed"
echo " Functional tests: $STAT_FUNC_P passed, $STAT_FUNC_F failed"
if [ "$STAT_COV_LINES" != "n/a" ] && [ -n "$STAT_COV_LINES" ]; then
echo " Coverage: Lines: $STAT_COV_LINES | Functions: $STAT_COV_FUNCS | Branches: $STAT_COV_BRANCHES"
fi
echo " Duration: ${time:-?}"
echo " Duration: ${STAT_TIME:-?}"
else
echo "✓ Build $idx passed (no stats available)"
fi