Improve Functional Test Stability and CI Reliability #79
19
.github/workflows/ci-code.yml
vendored
19
.github/workflows/ci-code.yml
vendored
@@ -106,20 +106,5 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Build and run coverage
|
- name: Build and run coverage
|
||||||
run: |
|
run: |
|
||||||
docker build -f Dockerfile.debian -t profanity-cov .
|
docker build -f Dockerfile.arch -t profanity-cov .
|
||||||
docker run -v ${{ github.workspace }}/coverage:/coverage profanity-cov bash -c '
|
docker run profanity-cov ./ci-build.sh --coverage-only
|
||||||
./bootstrap.sh
|
|
||||||
./configure --enable-coverage --enable-otr --enable-pgp --enable-omemo --enable-plugins
|
|
||||||
make -j$(nproc)
|
|
||||||
make check || true
|
|
||||||
make check-functional-parallel || true
|
|
||||||
lcov --capture --directory . --output-file /coverage/coverage.info \
|
|
||||||
--rc branch_coverage=1 \
|
|
||||||
--ignore-errors inconsistent
|
|
||||||
lcov --remove /coverage/coverage.info \
|
|
||||||
"/usr/include/*" "*/tests/*" \
|
|
||||||
--output-file /coverage/coverage.info \
|
|
||||||
--rc branch_coverage=1 \
|
|
||||||
--ignore-errors inconsistent,empty,unused
|
|
||||||
lcov --summary /coverage/coverage.info
|
|
||||||
'
|
|
||||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -62,6 +62,8 @@ tests/unittests/unittests
|
|||||||
tests/unittests/unittests.log
|
tests/unittests/unittests.log
|
||||||
tests/unittests/unittests.trs
|
tests/unittests/unittests.trs
|
||||||
test-suite.log
|
test-suite.log
|
||||||
|
test-files/
|
||||||
|
test-logs/
|
||||||
|
|
||||||
# valgrind output
|
# valgrind output
|
||||||
profval*
|
profval*
|
||||||
@@ -108,3 +110,7 @@ breaks
|
|||||||
*.zip
|
*.zip
|
||||||
*.log*
|
*.log*
|
||||||
coverage/
|
coverage/
|
||||||
|
*.gcno
|
||||||
|
*.gcda
|
||||||
|
*.gcov
|
||||||
|
coverage.info
|
||||||
|
|||||||
@@ -90,6 +90,24 @@ set -e
|
|||||||
```
|
```
|
||||||
|
|
||||||
This will run the same tests that the CI runs and refuse the push if it fails.
|
This will run the same tests that the CI runs and refuse the push if it fails.
|
||||||
|
The CI script runs 4 parallel builds with different configurations:
|
||||||
|
- **Full** — all features enabled (+ coverage in `--coverage-only` mode)
|
||||||
|
- **Minimal** — all optional features disabled
|
||||||
|
- **NoEncrypt** — no encryption (OTR, PGP, OMEMO disabled)
|
||||||
|
- **Default** — default ./configure options
|
||||||
|
|
||||||
|
Each build runs Valgrind and functional tests on Linux.
|
||||||
|
Use `./ci-build.sh --coverage-only` to run only the Full build with coverage collection.
|
||||||
|
|
||||||
|
Output shows test results per build:
|
||||||
|
```
|
||||||
|
✓ Full PASSED
|
||||||
|
Unit tests: 437 passed, 0 failed
|
||||||
|
Functional tests: 69 passed, 0 failed
|
||||||
|
Coverage: Lines: 27.5% | Functions: 36.2% | Branches: 18.1%
|
||||||
|
Duration: 5m39s
|
||||||
|
```
|
||||||
|
|
||||||
Note that it will run on the actual content of the repository directory and not
|
Note that it will run on the actual content of the repository directory and not
|
||||||
what may have been staged/committed.
|
what may have been staged/committed.
|
||||||
|
|
||||||
|
|||||||
@@ -313,7 +313,7 @@ FUNC_TEST_GROUPS = 1 2 3 4
|
|||||||
|
|
||||||
check-functional-parallel: tests/functionaltests/functionaltests
|
check-functional-parallel: tests/functionaltests/functionaltests
|
||||||
@echo "Running functional tests in parallel ($(words $(FUNC_TEST_GROUPS)) groups)..."
|
@echo "Running functional tests in parallel ($(words $(FUNC_TEST_GROUPS)) groups)..."
|
||||||
@mkdir -p $(builddir)/test-logs
|
@mkdir -p $(builddir)/test-logs $(builddir)/test-files
|
||||||
@pids=""; \
|
@pids=""; \
|
||||||
for g in $(FUNC_TEST_GROUPS); do \
|
for g in $(FUNC_TEST_GROUPS); do \
|
||||||
./tests/functionaltests/functionaltests $$g > $(builddir)/test-logs/group$$g.log 2>&1 & \
|
./tests/functionaltests/functionaltests $$g > $(builddir)/test-logs/group$$g.log 2>&1 & \
|
||||||
@@ -372,7 +372,10 @@ clean-local:
|
|||||||
rm -f $(git_include) $(git_include).in
|
rm -f $(git_include) $(git_include).in
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: my-prof.supp
|
clean-functional-tests:
|
||||||
|
rm -rf $(builddir)/test-files $(builddir)/test-logs
|
||||||
|
|
||||||
|
.PHONY: my-prof.supp clean-functional-tests
|
||||||
my-prof.supp:
|
my-prof.supp:
|
||||||
@sed '/^# AUTO-GENERATED START/q' prof.supp > $@
|
@sed '/^# AUTO-GENERATED START/q' prof.supp > $@
|
||||||
@printf "\n\n# glib\n" >> $@
|
@printf "\n\n# glib\n" >> $@
|
||||||
@@ -388,7 +391,7 @@ check-unit: tests/unittests/unittests
|
|||||||
tests/unittests/unittests
|
tests/unittests/unittests
|
||||||
|
|
||||||
@VALGRIND_CHECK_RULES@
|
@VALGRIND_CHECK_RULES@
|
||||||
VALGRIND_SUPPRESSIONS_FILES=prof.supp
|
VALGRIND_SUPPRESSIONS_FILES=$(srcdir)/prof.supp
|
||||||
|
|
||||||
# Code coverage targets (requires --enable-coverage)
|
# Code coverage targets (requires --enable-coverage)
|
||||||
coverage-clean:
|
coverage-clean:
|
||||||
|
|||||||
321
ci-build.sh
321
ci-build.sh
@@ -15,14 +15,72 @@ error_handler()
|
|||||||
log_content ./test-suite.log
|
log_content ./test-suite.log
|
||||||
log_content ./test-suite-memcheck.log
|
log_content ./test-suite-memcheck.log
|
||||||
|
|
||||||
echo
|
echo >&2
|
||||||
echo "Error ${ERR_CODE} with command '${BASH_COMMAND}' on line ${BASH_LINENO[0]}. Exiting."
|
echo "Error ${ERR_CODE} with command '${BASH_COMMAND}' on line ${BASH_LINENO[0]}. Exiting." >&2
|
||||||
echo
|
echo >&2
|
||||||
exit ${ERR_CODE}
|
exit ${ERR_CODE}
|
||||||
}
|
}
|
||||||
|
|
||||||
trap error_handler ERR
|
trap error_handler ERR
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Constants
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
# Number of parallel build configurations
|
||||||
|
readonly TEST_BUILDS=4
|
||||||
|
|
||||||
|
# Human-readable names for each build configuration
|
||||||
|
readonly BUILD_NAMES=(
|
||||||
|
"Full" # 1. All features enabled
|
||||||
|
"Minimal" # 2. All optional features disabled
|
||||||
|
"NoEncrypt" # 3. No encryption (otr, pgp, omemo disabled)
|
||||||
|
"Default" # 4. Default ./configure options
|
||||||
|
)
|
||||||
|
|
||||||
|
# Regex patterns for parsing test output
|
||||||
|
readonly CMOCKA_PASSED_PATTERN='^\[ PASSED \] [0-9]+ test'
|
||||||
|
readonly CMOCKA_FAILED_PATTERN='^\[ FAILED \] [0-9]+ test'
|
||||||
|
jabber.developer marked this conversation as resolved
|
|||||||
|
|
||||||
|
# Coverage extraction patterns (matches both Docker and CI paths)
|
||||||
|
readonly COVERAGE_PATTERNS='*/profanity/src/* */src/src/*'
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Helper Functions
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
# Parse STATS line from build log and set global variables
|
||||||
|
# Usage: parse_build_stats <log_file>
|
||||||
|
parse_build_stats() {
|
||||||
|
local log_file="$1"
|
||||||
|
local stats_line
|
||||||
|
stats_line=$(grep "^STATS:" "$log_file" 2>/dev/null | tail -1)
|
||||||
|
|
||||||
|
STAT_UNIT_P=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2)
|
||||||
|
STAT_UNIT_F=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2)
|
||||||
|
STAT_FUNC_P=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2)
|
||||||
|
STAT_FUNC_F=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2)
|
||||||
|
STAT_COV_LINES=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2)
|
||||||
|
STAT_COV_FUNCS=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2)
|
||||||
|
STAT_COV_BRANCHES=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2)
|
||||||
|
STAT_TIME=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2)
|
||||||
|
|
||||||
|
: "${STAT_UNIT_P:=0}"
|
||||||
|
: "${STAT_UNIT_F:=0}"
|
||||||
|
: "${STAT_FUNC_P:=0}"
|
||||||
|
: "${STAT_FUNC_F:=0}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Extract test count from log file
|
||||||
|
# Usage: extract_test_count <log_file> <pattern>
|
||||||
|
extract_test_count() {
|
||||||
|
grep -E "$2" "$1" 2>/dev/null | grep -oE "[0-9]+" | head -1
|
||||||
|
}
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Test Verification
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
# Verify that test failures are properly detected
|
# Verify that test failures are properly detected
|
||||||
# This is a meta-test: it runs a deliberately failing test
|
# This is a meta-test: it runs a deliberately failing test
|
||||||
# and checks that the test framework reports the failure correctly
|
# and checks that the test framework reports the failure correctly
|
||||||
@@ -61,11 +119,11 @@ EOF
|
|||||||
# Test 1: Single failing test detection
|
# Test 1: Single failing test detection
|
||||||
echo " Testing single test failure detection..."
|
echo " Testing single test failure detection..."
|
||||||
if /tmp/test_must_fail > /tmp/test_must_fail.log 2>&1; then
|
if /tmp/test_must_fail > /tmp/test_must_fail.log 2>&1; then
|
||||||
echo "ERROR: Test that should fail returned success (exit code 0)"
|
echo "ERROR: Test that should fail returned success (exit code 0)" >&2
|
||||||
echo "This means the test framework is NOT detecting failures correctly!"
|
echo "This means the test framework is NOT detecting failures correctly!" >&2
|
||||||
echo "--- Test output ---"
|
echo "--- Test output ---" >&2
|
||||||
cat /tmp/test_must_fail.log
|
cat /tmp/test_must_fail.log >&2
|
||||||
echo "--- End output ---"
|
echo "--- End output ---" >&2
|
||||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log
|
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -85,8 +143,8 @@ EOF
|
|||||||
wait $pid4 || failed=$((failed + 1))
|
wait $pid4 || failed=$((failed + 1))
|
||||||
|
|
||||||
if [ $failed -ne 2 ]; then
|
if [ $failed -ne 2 ]; then
|
||||||
echo "ERROR: Expected 2 failures in parallel tests, got $failed"
|
echo "ERROR: Expected 2 failures in parallel tests, got $failed" >&2
|
||||||
echo "Parallel failure detection is broken!"
|
echo "Parallel failure detection is broken!" >&2
|
||||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
|
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -110,6 +168,17 @@ num_cores()
|
|||||||
# Run test failure detection verification first
|
# Run test failure detection verification first
|
||||||
verify_test_failure_detection
|
verify_test_failure_detection
|
||||||
|
|
||||||
|
# Parse arguments
|
||||||
|
COVERAGE_ONLY=no
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--coverage-only)
|
||||||
|
COVERAGE_ONLY=yes
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
./bootstrap.sh
|
./bootstrap.sh
|
||||||
|
|
||||||
tests=()
|
tests=()
|
||||||
@@ -180,30 +249,13 @@ case "$ARCH" in
|
|||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
case "$ARCH" in
|
|
||||||
linux*)
|
|
||||||
echo
|
|
||||||
echo "--> Building with ./configure -C ${tests[0]} --enable-valgrind $*"
|
|
||||||
echo
|
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
./configure -C ${tests[0]} --enable-valgrind $*
|
|
||||||
|
|
||||||
$MAKE CC="${CC}"
|
|
||||||
if grep '^ID=' /etc/os-release | grep -q -e debian; then
|
|
||||||
$MAKE check-valgrind
|
|
||||||
else
|
|
||||||
$MAKE check-valgrind || log_content ./test-suite-memcheck.log
|
|
||||||
fi
|
|
||||||
$MAKE distclean
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Function to build and test a single configuration
|
# Function to build and test a single configuration
|
||||||
build_and_test() {
|
build_and_test() {
|
||||||
local features="$1"
|
local features="$1"
|
||||||
local extra_args="$2"
|
local extra_args="$2"
|
||||||
local idx="$3"
|
local idx="$3"
|
||||||
|
local run_valgrind="$4"
|
||||||
|
local run_coverage="$5"
|
||||||
local build_dir="build-$idx"
|
local build_dir="build-$idx"
|
||||||
local log_file="build-$idx.log"
|
local log_file="build-$idx.log"
|
||||||
|
|
||||||
@@ -211,61 +263,210 @@ build_and_test() {
|
|||||||
echo "=== Build $idx started at $(date) ==="
|
echo "=== Build $idx started at $(date) ==="
|
||||||
echo "--> Building in $build_dir with ./configure -C $features $extra_args"
|
echo "--> Building in $build_dir with ./configure -C $features $extra_args"
|
||||||
|
|
||||||
|
local start_time=$SECONDS
|
||||||
|
|
||||||
mkdir -p "$build_dir"
|
mkdir -p "$build_dir"
|
||||||
cd "$build_dir"
|
cd "$build_dir"
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
../configure -C $features $extra_args
|
if ! ../configure -C $features $extra_args; then
|
||||||
|
echo "ERROR: configure failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
$MAKE CC="${CC}"
|
if ! $MAKE CC="${CC}"; then
|
||||||
$MAKE check-functional-parallel
|
echo "ERROR: make failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run unit tests
|
||||||
|
local unit_passed=0 unit_failed=0
|
||||||
|
if [ "$run_valgrind" = "yes" ]; then
|
||||||
|
echo "--> Running unit tests under Valgrind..."
|
||||||
|
# Build unit tests first
|
||||||
|
$MAKE tests/unittests/unittests
|
||||||
|
# Run valgrind directly to capture cmocka output
|
||||||
|
valgrind --error-exitcode=1 --leak-check=full \
|
||||||
|
--suppressions=../prof.supp \
|
||||||
|
tests/unittests/unittests 2>&1 | tee unit-tests-output.log
|
||||||
|
valgrind_exit=${PIPESTATUS[0]}
|
||||||
|
if [ $valgrind_exit -ne 0 ]; then
|
||||||
|
echo "ERROR: Valgrind unit tests failed (exit code $valgrind_exit)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "--> Running unit tests..."
|
||||||
|
$MAKE tests/unittests/unittests
|
||||||
|
tests/unittests/unittests 2>&1 | tee unit-tests-output.log
|
||||||
|
if [ ${PIPESTATUS[0]} -ne 0 ]; then
|
||||||
|
echo "ERROR: Unit tests failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
# Extract unit test counts from cmocka output
|
||||||
|
unit_passed=$(extract_test_count unit-tests-output.log "$CMOCKA_PASSED_PATTERN")
|
||||||
|
unit_failed=$(extract_test_count unit-tests-output.log "$CMOCKA_FAILED_PATTERN")
|
||||||
|
: "${unit_passed:=0}" "${unit_failed:=0}"
|
||||||
|
echo "UNIT_TESTS: passed=$unit_passed failed=$unit_failed"
|
||||||
|
|
||||||
|
# Set build index for port allocation: build 1 uses ports 5230-5233,
|
||||||
|
# build 2 uses 5234-5237, etc. This prevents port conflicts in parallel builds.
|
||||||
|
export PROF_BUILD_INDEX=$idx
|
||||||
|
local func_passed=0 func_failed=0
|
||||||
|
if ! $MAKE check-functional-parallel; then
|
||||||
|
echo "ERROR: functional tests failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# Extract functional test counts from group logs
|
||||||
|
echo "=== Functional test results ==="
|
||||||
|
for glog in ./test-logs/group*.log; do
|
||||||
|
if [ -f "$glog" ]; then
|
||||||
|
cnt=$(extract_test_count "$glog" "$CMOCKA_PASSED_PATTERN")
|
||||||
|
[ -n "$cnt" ] && func_passed=$((func_passed + cnt))
|
||||||
|
cnt=$(extract_test_count "$glog" "$CMOCKA_FAILED_PATTERN")
|
||||||
|
[ -n "$cnt" ] && func_failed=$((func_failed + cnt))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "FUNC_TESTS: passed=$func_passed failed=$func_failed"
|
||||||
|
|
||||||
|
# Collect coverage data if enabled (lines, functions, branches)
|
||||||
|
# Must be done BEFORE make clean which removes .gcda files
|
||||||
|
local cov_lines="n/a" cov_funcs="n/a" cov_branches="n/a"
|
||||||
|
if [ "$run_coverage" = "yes" ]; then
|
||||||
|
echo "--> Collecting coverage data..."
|
||||||
|
if command -v lcov >/dev/null 2>&1; then
|
||||||
|
lcov --capture --directory . --output-file coverage-full.info \
|
||||||
|
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
|
||||||
|
# Extract only production code from src/ directory, exclude tests
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
lcov --extract coverage-full.info $COVERAGE_PATTERNS \
|
||||||
|
--output-file coverage.info \
|
||||||
|
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true
|
||||||
|
if [ -f coverage.info ] && [ -s coverage.info ]; then
|
||||||
|
local summary
|
||||||
|
summary=$(lcov --summary coverage.info \
|
||||||
|
--rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true)
|
||||||
|
cov_lines=$(echo "$summary" | grep -E "lines\.*:" | grep -oE "[0-9]+\.[0-9]+%" | head -1)
|
||||||
|
cov_funcs=$(echo "$summary" | grep -E "functions\.*:" | grep -oE "[0-9]+\.[0-9]+%" | head -1)
|
||||||
|
cov_branches=$(echo "$summary" | grep -E "branches\.*:" | grep -oE "[0-9]+\.[0-9]+%" | head -1)
|
||||||
|
[ -z "$cov_lines" ] && cov_lines="n/a"
|
||||||
|
[ -z "$cov_funcs" ] && cov_funcs="n/a"
|
||||||
|
[ -z "$cov_branches" ] && cov_branches="n/a"
|
||||||
|
else
|
||||||
|
echo "WARNING: coverage.info is empty or not created"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "WARNING: lcov not found"
|
||||||
|
fi
|
||||||
|
echo "COVERAGE: lines=$cov_lines funcs=$cov_funcs branches=$cov_branches"
|
||||||
|
fi
|
||||||
|
|
||||||
./profanity -v
|
./profanity -v
|
||||||
|
|
||||||
|
# Save coverage.info before cleanup (for CI artifact)
|
||||||
|
# Only copy in CI environment to avoid leaving artifacts during local runs
|
||||||
|
if [ "$run_coverage" = "yes" ] && [ -f coverage.info ] && [ -n "$CI" ]; then
|
||||||
|
cp coverage.info ../coverage.info
|
||||||
|
echo "Coverage report saved to coverage.info"
|
||||||
|
fi
|
||||||
|
|
||||||
$MAKE clean
|
$MAKE clean
|
||||||
|
|
||||||
cd ..
|
cd ..
|
||||||
rm -rf "$build_dir"
|
rm -rf "$build_dir"
|
||||||
|
|
||||||
|
local elapsed=$((SECONDS - start_time))
|
||||||
|
local mins=$((elapsed / 60))
|
||||||
|
local secs=$((elapsed % 60))
|
||||||
|
|
||||||
echo "=== Build $idx completed at $(date) ==="
|
echo "=== Build $idx completed at $(date) ==="
|
||||||
|
echo "STATS: unit_passed=$unit_passed unit_failed=$unit_failed func_passed=$func_passed func_failed=$func_failed cov_lines=$cov_lines cov_funcs=$cov_funcs cov_branches=$cov_branches time=${mins}m${secs}s"
|
||||||
} > "$log_file" 2>&1
|
} > "$log_file" 2>&1
|
||||||
}
|
}
|
||||||
|
|
||||||
# Run all 4 configurations in parallel
|
# Run configurations
|
||||||
echo "Starting parallel builds..."
|
# Coverage enabled only for build 1 (Full) - it has most code paths
|
||||||
pids=()
|
echo
|
||||||
for idx in 1 2 3 4; do
|
echo "=== Start build ==="
|
||||||
if [ $idx -le ${#tests[@]} ]; then
|
echo
|
||||||
build_and_test "${tests[$((idx-1))]}" "$*" "$idx" &
|
|
||||||
pids+=("$!")
|
if [ "$COVERAGE_ONLY" = "yes" ]; then
|
||||||
echo "Started build $idx (PID: $!)"
|
echo "Running coverage-only mode (${BUILD_NAMES[0]} build)..."
|
||||||
fi
|
echo
|
||||||
done
|
run_valgrind="no"
|
||||||
|
run_coverage="yes"
|
||||||
|
extra_flags="--enable-coverage"
|
||||||
|
build_and_test "${tests[0]}" "$* $extra_flags" "1" "$run_valgrind" "$run_coverage" &
|
||||||
|
pids=("$!")
|
||||||
|
echo " → ${BUILD_NAMES[0]}: ${tests[0]} [+Coverage]"
|
||||||
|
else
|
||||||
|
echo "Starting $TEST_BUILDS parallel build configurations..."
|
||||||
|
echo
|
||||||
|
pids=()
|
||||||
|
for idx in $(seq 1 $TEST_BUILDS); do
|
||||||
|
if [ $idx -le ${#tests[@]} ]; then
|
||||||
|
# All builds run Valgrind on Linux
|
||||||
|
if [ "$ARCH" = "linux" ]; then
|
||||||
|
run_valgrind="yes"
|
||||||
|
extra_flags="--enable-valgrind"
|
||||||
|
else
|
||||||
|
run_valgrind="no"
|
||||||
|
extra_flags=""
|
||||||
|
fi
|
||||||
|
run_coverage="no"
|
||||||
|
build_and_test "${tests[$((idx-1))]}" "$* $extra_flags" "$idx" "$run_valgrind" "$run_coverage" &
|
||||||
|
pids+=("$!")
|
||||||
|
flags_desc=""
|
||||||
|
[ "$run_valgrind" = "yes" ] && flags_desc=" [+Valgrind]"
|
||||||
|
echo " → ${BUILD_NAMES[$((idx-1))]}: ${tests[$((idx-1))]}$flags_desc"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
|
||||||
# Wait for all builds and check exit codes
|
# Wait for all builds and check exit codes
|
||||||
failed=0
|
echo "Waiting for builds to complete..."
|
||||||
|
echo
|
||||||
|
failed_builds=()
|
||||||
for i in "${!pids[@]}"; do
|
for i in "${!pids[@]}"; do
|
||||||
idx=$((i + 1))
|
idx=$((i + 1))
|
||||||
if wait "${pids[$i]}"; then
|
if wait "${pids[$i]}"; then
|
||||||
echo "✓ Build $idx passed"
|
if [ -f "build-$idx.log" ]; then
|
||||||
|
parse_build_stats "build-$idx.log"
|
||||||
|
|
||||||
|
echo "✓ ${BUILD_NAMES[$i]} PASSED"
|
||||||
|
echo " Unit tests: $STAT_UNIT_P passed, $STAT_UNIT_F failed"
|
||||||
|
echo " Functional tests: $STAT_FUNC_P passed, $STAT_FUNC_F failed"
|
||||||
|
if [ "$STAT_COV_LINES" != "n/a" ] && [ -n "$STAT_COV_LINES" ]; then
|
||||||
|
echo " Coverage: Lines: $STAT_COV_LINES | Functions: $STAT_COV_FUNCS | Branches: $STAT_COV_BRANCHES"
|
||||||
|
fi
|
||||||
|
echo " Duration: ${STAT_TIME:-?}"
|
||||||
|
else
|
||||||
|
echo "✓ ${BUILD_NAMES[$i]} passed (no stats available)"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo "✗ Build $idx failed"
|
echo "✗ ${BUILD_NAMES[$i]} FAILED" >&2
|
||||||
echo "--- Log for build $idx ---"
|
failed_builds+=("$idx")
|
||||||
cat "build-$idx.log"
|
fi
|
||||||
echo "--- End log ---"
|
echo
|
||||||
failed=1
|
done
|
||||||
|
|
||||||
|
# Show failed builds full logs
|
||||||
|
for idx in "${failed_builds[@]}"; do
|
||||||
|
if [ -f "build-$idx.log" ]; then
|
||||||
|
echo "=== ${BUILD_NAMES[$((idx-1))]} FAILURE LOG ===" >&2
|
||||||
|
cat "build-$idx.log" >&2
|
||||||
|
echo >&2
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# Show all logs on success too
|
if [ ${#failed_builds[@]} -gt 0 ]; then
|
||||||
if [ $failed -eq 0 ]; then
|
echo "RESULT: FAILED (builds ${failed_builds[*]})" >&2
|
||||||
echo
|
|
||||||
echo "All builds passed!"
|
|
||||||
for idx in 1 2 3 4; do
|
|
||||||
if [ -f "build-$idx.log" ]; then
|
|
||||||
echo "--- Log for build $idx ---"
|
|
||||||
cat "build-$idx.log"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
else
|
|
||||||
exit 1
|
exit 1
|
||||||
|
else
|
||||||
|
if [ "$COVERAGE_ONLY" = "yes" ]; then
|
||||||
|
echo "RESULT: COVERAGE BUILD PASSED ✓"
|
||||||
|
else
|
||||||
|
echo "RESULT: ALL $TEST_BUILDS BUILDS PASSED ✓"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
23
prof.supp
23
prof.supp
@@ -8,6 +8,29 @@
|
|||||||
# * python suppressions file from https://github.com/python/cpython/blob/main/Misc/valgrind-python.supp
|
# * python suppressions file from https://github.com/python/cpython/blob/main/Misc/valgrind-python.supp
|
||||||
#
|
#
|
||||||
|
|
||||||
|
# ============================================
|
||||||
|
# glibc AVX2 optimizations (false positives)
|
||||||
|
# See: https://sourceware.org/bugzilla/show_bug.cgi?id=19796
|
||||||
|
# ============================================
|
||||||
|
|
||||||
|
{
|
||||||
|
glibc_wcpncpy_avx2
|
||||||
|
Memcheck:Addr32
|
||||||
|
fun:__wcpncpy_avx2
|
||||||
|
fun:wcsxfrm_l
|
||||||
|
fun:g_utf8_collate_key
|
||||||
|
...
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
glibc_wcsxfrm_avx2
|
||||||
|
Memcheck:Addr32
|
||||||
|
...
|
||||||
|
fun:wcsxfrm_l
|
||||||
|
fun:g_utf8_collate_key
|
||||||
|
...
|
||||||
|
}
|
||||||
|
|
||||||
# ============================================
|
# ============================================
|
||||||
# Functional tests suppressions (stabber/pthread)
|
# Functional tests suppressions (stabber/pthread)
|
||||||
# ============================================
|
# ============================================
|
||||||
|
|||||||
@@ -69,6 +69,12 @@ main(int argc, char* argv[])
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
char group_env[16];
|
||||||
|
snprintf(group_env, sizeof(group_env), "%d", group);
|
||||||
|
setenv("PROF_TEST_GROUP", group_env, 1);
|
||||||
|
|
||||||
|
fprintf(stderr, "[PROF_TEST] Starting functional tests, group=%d\n", group);
|
||||||
|
|
||||||
/* ============================================================
|
/* ============================================================
|
||||||
* GROUP 1: Connect, Ping, Rooms, Software
|
* GROUP 1: Connect, Ping, Rooms, Software
|
||||||
* Basic XMPP session establishment and server queries
|
* Basic XMPP session establishment and server queries
|
||||||
|
|||||||
@@ -17,6 +17,9 @@
|
|||||||
|
|
||||||
#include "proftest.h"
|
#include "proftest.h"
|
||||||
|
|
||||||
|
/* Number of parallel test groups for CI builds */
|
||||||
|
#define TEST_GROUPS 4
|
||||||
|
|
||||||
char *config_orig;
|
char *config_orig;
|
||||||
char *data_orig;
|
char *data_orig;
|
||||||
|
|
||||||
@@ -136,8 +139,14 @@ _create_logs_dir(void)
|
|||||||
void
|
void
|
||||||
_cleanup_dirs(void)
|
_cleanup_dirs(void)
|
||||||
{
|
{
|
||||||
|
const char *group_env = getenv("PROF_TEST_GROUP");
|
||||||
|
int group = group_env ? atoi(group_env) : 0;
|
||||||
|
int dir_id = (group >= 1 && group <= TEST_GROUPS) ? group : stub_port;
|
||||||
|
|
||||||
|
printf("[PROF_TEST] Cleaning up directories for group %d (dir_id %d)\n", group, dir_id);
|
||||||
|
|
||||||
char cmd[512];
|
char cmd[512];
|
||||||
snprintf(cmd, sizeof(cmd), "rm -rf ./tests/functionaltests/files/%d", stub_port);
|
snprintf(cmd, sizeof(cmd), "rm -rf ./test-files/%d", dir_id);
|
||||||
int res = system(cmd);
|
int res = system(cmd);
|
||||||
if (res == -1) {
|
if (res == -1) {
|
||||||
assert_true(FALSE);
|
assert_true(FALSE);
|
||||||
@@ -221,30 +230,74 @@ prof_start(void)
|
|||||||
/* Set non-blocking mode for reading */
|
/* Set non-blocking mode for reading */
|
||||||
int flags = fcntl(fd, F_GETFL, 0);
|
int flags = fcntl(fd, F_GETFL, 0);
|
||||||
fcntl(fd, F_SETFL, flags | O_NONBLOCK);
|
fcntl(fd, F_SETFL, flags | O_NONBLOCK);
|
||||||
|
|
||||||
|
/* Brief wait for process to initialize */
|
||||||
|
usleep(50000); /* 50ms */
|
||||||
}
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
init_prof_test(void **state)
|
init_prof_test(void **state)
|
||||||
{
|
{
|
||||||
|
/* Get test group from environment for static resource allocation */
|
||||||
|
const char *group_env = getenv("PROF_TEST_GROUP");
|
||||||
|
int group = group_env ? atoi(group_env) : 0;
|
||||||
|
|
||||||
|
/* Get build index for port offset (for parallel CI builds) */
|
||||||
|
const char *build_env = getenv("PROF_BUILD_INDEX");
|
||||||
|
int build_idx = build_env ? atoi(build_env) : 0;
|
||||||
|
|
||||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
Why 2 builds (0 and 1) share the same ports range? Why 2 builds (0 and 1) share the same ports range?
jabber.developer2
commented
Added explanation: builds 0/1 share ports because they're mutually exclusive (sequential mode or parallel). Added explanation: builds 0/1 share ports because they're mutually exclusive (sequential mode or parallel).
|
|||||||
|
/* Calculate port base: each build uses a different range of TEST_GROUPS ports.
|
||||||
|
* Build 0 (local/default): 5230-5233, Full: 5230-5233, Minimal: 5234-5237, etc.
|
||||||
|
* Build 0 and Full share the same range because build 0 is for local runs or sequential run (no parallel builds),
|
||||||
|
* while Full/Minimal/NoEncrypt/Default are used in CI where they run in parallel. */
|
||||||
|
int port_base = 5230 + ((build_idx > 0 ? build_idx - 1 : 0) * TEST_GROUPS);
|
||||||
|
|
||||||
|
/* Static resource allocation to avoid conflicts in parallel execution.
|
||||||
|
* Group 1-4: use static port assignment.
|
||||||
|
* Group 0 (all groups): use dynamic allocation as fallback. */
|
||||||
gboolean started = FALSE;
|
gboolean started = FALSE;
|
||||||
for (int p = 5230; p < 5250; ++p) {
|
|
||||||
int ret = stbbr_start(STBBR_LOGDEBUG, p, 0);
|
if (group >= 1 && group <= TEST_GROUPS) {
|
||||||
if (ret == 0) {
|
/* Static allocation: each group gets a dedicated port */
|
||||||
stub_port = p;
|
stub_port = port_base + group - 1;
|
||||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
It shouldn't be at the It shouldn't be at the `stderr`. It would make more sense to put it in the stdout, since it's informational/debug data.
jabber.developer2
commented
Fixed. Info messages now use printf() (stdout), stderr reserved for errors only. Fixed. Info messages now use printf() (stdout), stderr reserved for errors only.
|
|||||||
|
printf("[PROF_TEST] Build %d, Group %d: trying port %d\n", build_idx, group, stub_port);
|
||||||
|
|
||||||
|
if (stbbr_start(STBBR_LOGDEBUG, stub_port, 0) == 0) {
|
||||||
started = TRUE;
|
started = TRUE;
|
||||||
break;
|
printf("[PROF_TEST] Started stabber on port %d\n", stub_port);
|
||||||
|
} else {
|
||||||
|
printf("[PROF_TEST] Failed to start stabber on port %d\n", stub_port);
|
||||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
Should be in stderr. Please check analogous cases Should be in stderr. Please check analogous cases
jabber.developer2
commented
Corrected. Corrected.
|
|||||||
}
|
}
|
||||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
It shouldn't be at the It shouldn't be at the `stderr`. It would make more sense to put it in the stdout, since it's informational/debug data.
jabber.developer2
commented
Fixed. Info messages now use printf() (stdout), stderr reserved for errors only. Fixed. Info messages now use printf() (stdout), stderr reserved for errors only.
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
/* Fallback to dynamic allocation if static failed or group=0 */
|
||||||
if (!started) {
|
if (!started) {
|
||||||
assert_true(FALSE); // could not start stabber on any port in range
|
printf("[PROF_TEST] Using dynamic port allocation\n");
|
||||||
|
for (int p = port_base; p < port_base + 20; ++p) {
|
||||||
|
if (stbbr_start(STBBR_LOGDEBUG, p, 0) == 0) {
|
||||||
|
stub_port = p;
|
||||||
|
started = TRUE;
|
||||||
|
printf("[PROF_TEST] Started stabber on port %d\n", stub_port);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!started) {
|
||||||
|
fprintf(stderr, "[PROF_TEST] ERROR: could not start stabber on any port\n");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate unique XDG paths based on stub_port for parallel execution
|
/* Generate unique XDG paths based on group for parallel execution.
|
||||||
|
* Use ./test-files/ in current (build) directory for out-of-tree builds compatibility. */
|
||||||
|
int dir_id = (group >= 1 && group <= TEST_GROUPS) ? group : stub_port;
|
||||||
snprintf(xdg_config_home, sizeof(xdg_config_home),
|
snprintf(xdg_config_home, sizeof(xdg_config_home),
|
||||||
"./tests/functionaltests/files/%d/xdg_config_home", stub_port);
|
"./test-files/%d/xdg_config_home", dir_id);
|
||||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
same as above same as above
jabber.developer2
commented
Removed Removed
|
|||||||
snprintf(xdg_data_home, sizeof(xdg_data_home),
|
snprintf(xdg_data_home, sizeof(xdg_data_home),
|
||||||
"./tests/functionaltests/files/%d/xdg_data_home", stub_port);
|
"./test-files/%d/xdg_data_home", dir_id);
|
||||||
|
|
||||||
|
printf("[PROF_TEST] Group %d using directories: config=%s, data=%s\n",
|
||||||
|
group, xdg_config_home, xdg_data_home);
|
||||||
|
|
||||||
// Give stabber server thread time to start listening
|
// Give stabber server thread time to start listening
|
||||||
usleep(100000); // 100ms
|
usleep(100000); // 100ms
|
||||||
@@ -401,6 +454,16 @@ prof_output_regex(const char *pattern)
|
|||||||
usleep(50000); /* 50ms */
|
usleep(50000); /* 50ms */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Timeout reached - log diagnostic info */
|
||||||
|
fprintf(stderr, "Timeout waiting for regex '%s' after %d seconds. Last output:\n", pattern, expect_timeout);
|
||||||
|
size_t len = strlen(output_buffer);
|
||||||
|
if (len > 500) {
|
||||||
|
fprintf(stderr, "...%s", output_buffer + len - 500);
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "%s", output_buffer);
|
||||||
|
}
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
|
||||||
regfree(®ex);
|
regfree(®ex);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user
Is it required? It needs to be updated each time the set variables of the method are updated, yet it does not provide additional data. It would make more sense to explain what each variable means next to their initiation place.
Refactored, moved to separate function