Improve Functional Test Stability and CI Reliability #79
@@ -91,12 +91,15 @@ set -e
|
||||
|
||||
This will run the same tests that the CI runs and refuse the push if it fails.
|
||||
The CI script runs 4 parallel builds with different configurations, each with Valgrind and functional tests.
|
||||
Build 1 also collects code coverage (lines, functions, branches).
|
||||
Use `./ci-build.sh --coverage-only` to run only build 1 with coverage collection.
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
|
||||
|
||||
Output shows test results per build:
|
||||
```
|
||||
✓ Build 1 passed (unit: 437/0, func: 69/0, cov: L:27.5% F:36.2% B:18.1%, 5m39s)
|
||||
✓ Build 2 passed (unit: 398/0, func: 69/0, 5m00s)
|
||||
✓ BUILD 1 PASSED
|
||||
Unit tests: 437 passed, 0 failed
|
||||
Functional tests: 69 passed, 0 failed
|
||||
Coverage: Lines: 27.5% | Functions: 36.2% | Branches: 18.1%
|
||||
Duration: 5m39s
|
||||
```
|
||||
|
||||
Note that it will run on the actual content of the repository directory and not
|
||||
|
||||
89
ci-build.sh
89
ci-build.sh
@@ -23,6 +23,55 @@ error_handler()
|
||||
|
||||
trap error_handler ERR
|
||||
|
||||
# =============================================================================
|
||||
# Constants
|
||||
# =============================================================================
|
||||
|
||||
# Regex patterns for parsing test output
|
||||
readonly CMOCKA_PASSED_PATTERN='^\[ PASSED \] [0-9]+ test'
|
||||
readonly CMOCKA_FAILED_PATTERN='^\[ FAILED \] [0-9]+ test'
|
||||
|
||||
# Coverage extraction patterns (matches both Docker and CI paths)
|
||||
readonly COVERAGE_PATTERNS='*/profanity/src/* */src/src/*'
|
||||
|
||||
# =============================================================================
|
||||
# Helper Functions
|
||||
# =============================================================================
|
||||
|
||||
# Parse STATS line from build log and set global variables
|
||||
# Usage: parse_build_stats <log_file>
|
||||
# Sets: STAT_UNIT_P, STAT_UNIT_F, STAT_FUNC_P, STAT_FUNC_F,
|
||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
Is it required? It needs to be updated each time the set variables of the method are updated, yet it does not provide additional data. It would make more sense to explain what each variable means next to their initiation place. Is it required? It needs to be updated each time the set variables of the method are updated, yet it does not provide additional data. It would make more sense to explain what each variable means next to their initiation place.
jabber.developer2
commented
Refactored, moved to separate function Refactored, moved to separate function
|
||||
# STAT_COV_LINES, STAT_COV_FUNCS, STAT_COV_BRANCHES, STAT_TIME
|
||||
parse_build_stats() {
|
||||
local log_file="$1"
|
||||
local stats_line
|
||||
stats_line=$(grep "^STATS:" "$log_file" 2>/dev/null | tail -1)
|
||||
|
||||
STAT_UNIT_P=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2)
|
||||
STAT_UNIT_F=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2)
|
||||
STAT_FUNC_P=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2)
|
||||
STAT_FUNC_F=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2)
|
||||
STAT_COV_LINES=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2)
|
||||
STAT_COV_FUNCS=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2)
|
||||
STAT_COV_BRANCHES=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2)
|
||||
STAT_TIME=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2)
|
||||
|
||||
: "${STAT_UNIT_P:=0}"
|
||||
: "${STAT_UNIT_F:=0}"
|
||||
: "${STAT_FUNC_P:=0}"
|
||||
: "${STAT_FUNC_F:=0}"
|
||||
}
|
||||
|
||||
# Extract test count from log file
|
||||
# Usage: extract_test_count <log_file> <pattern>
|
||||
extract_test_count() {
|
||||
grep -E "$2" "$1" 2>/dev/null | grep -oE "[0-9]+" | head -1
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Test Verification
|
||||
# =============================================================================
|
||||
|
||||
# Verify that test failures are properly detected
|
||||
# This is a meta-test: it runs a deliberately failing test
|
||||
# and checks that the test framework reports the failure correctly
|
||||
@@ -246,10 +295,9 @@ build_and_test() {
|
||||
fi
|
||||
fi
|
||||
# Extract unit test counts from cmocka output
|
||||
unit_passed=$(grep -E "^\[ PASSED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1)
|
||||
unit_failed=$(grep -E "^\[ FAILED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1)
|
||||
[ -z "$unit_passed" ] && unit_passed=0
|
||||
[ -z "$unit_failed" ] && unit_failed=0
|
||||
unit_passed=$(extract_test_count unit-tests-output.log "$CMOCKA_PASSED_PATTERN")
|
||||
unit_failed=$(extract_test_count unit-tests-output.log "$CMOCKA_FAILED_PATTERN")
|
||||
: "${unit_passed:=0}" "${unit_failed:=0}"
|
||||
echo "UNIT_TESTS: passed=$unit_passed failed=$unit_failed"
|
||||
|
||||
# Set build index for port allocation: build 1 uses ports 5230-5233,
|
||||
@@ -264,9 +312,9 @@ build_and_test() {
|
||||
echo "=== Functional test results ==="
|
||||
for glog in ./test-logs/group*.log; do
|
||||
if [ -f "$glog" ]; then
|
||||
cnt=$(grep -E "^\[ PASSED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1)
|
||||
cnt=$(extract_test_count "$glog" "$CMOCKA_PASSED_PATTERN")
|
||||
[ -n "$cnt" ] && func_passed=$((func_passed + cnt))
|
||||
cnt=$(grep -E "^\[ FAILED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1)
|
||||
cnt=$(extract_test_count "$glog" "$CMOCKA_FAILED_PATTERN")
|
||||
[ -n "$cnt" ] && func_failed=$((func_failed + cnt))
|
||||
fi
|
||||
done
|
||||
@@ -281,8 +329,8 @@ build_and_test() {
|
||||
lcov --capture --directory . --output-file coverage-full.info \
|
||||
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
|
||||
# Extract only production code from src/ directory, exclude tests
|
||||
# Pattern matches both Docker (/src/src/*) and CI (/usr/src/profanity/src/*)
|
||||
lcov --extract coverage-full.info '*/profanity/src/*' '*/src/src/*' \
|
||||
# shellcheck disable=SC2086
|
||||
lcov --extract coverage-full.info $COVERAGE_PATTERNS \
|
||||
--output-file coverage.info \
|
||||
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
|
||||
if [ -f coverage.info ] && [ -s coverage.info ]; then
|
||||
@@ -374,28 +422,15 @@ for i in "${!pids[@]}"; do
|
||||
idx=$((i + 1))
|
||||
if wait "${pids[$i]}"; then
|
||||
if [ -f "build-$idx.log" ]; then
|
||||
stats_line=$(grep "^STATS:" "build-$idx.log" | tail -1)
|
||||
unit_p=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2)
|
||||
unit_f=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2)
|
||||
func_p=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2)
|
||||
func_f=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2)
|
||||
cov_lines=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2)
|
||||
cov_funcs=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2)
|
||||
cov_branches=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2)
|
||||
time=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2)
|
||||
|
||||
unit_p=${unit_p:-0}
|
||||
unit_f=${unit_f:-0}
|
||||
func_p=${func_p:-0}
|
||||
func_f=${func_f:-0}
|
||||
parse_build_stats "build-$idx.log"
|
||||
|
||||
echo "✓ BUILD $idx PASSED"
|
||||
echo " Unit tests: $unit_p passed, $unit_f failed"
|
||||
echo " Functional tests: $func_p passed, $func_f failed"
|
||||
if [ "$cov_lines" != "n/a" ] && [ -n "$cov_lines" ]; then
|
||||
echo " Coverage: Lines: $cov_lines | Functions: $cov_funcs | Branches: $cov_branches"
|
||||
echo " Unit tests: $STAT_UNIT_P passed, $STAT_UNIT_F failed"
|
||||
echo " Functional tests: $STAT_FUNC_P passed, $STAT_FUNC_F failed"
|
||||
if [ "$STAT_COV_LINES" != "n/a" ] && [ -n "$STAT_COV_LINES" ]; then
|
||||
echo " Coverage: Lines: $STAT_COV_LINES | Functions: $STAT_COV_FUNCS | Branches: $STAT_COV_BRANCHES"
|
||||
fi
|
||||
echo " Duration: ${time:-?}"
|
||||
echo " Duration: ${STAT_TIME:-?}"
|
||||
else
|
||||
echo "✓ Build $idx passed (no stats available)"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user
Could you clarify that "build 1" is a standard build without flags?
Corrected