diff --git a/.gitignore b/.gitignore index 59b5eab2..0559b357 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,8 @@ tests/unittests/unittests tests/unittests/unittests.log tests/unittests/unittests.trs test-suite.log +test-files/ +test-logs/ # valgrind output profval* @@ -108,3 +110,7 @@ breaks *.zip *.log* coverage/ +*.gcno +*.gcda +*.gcov +coverage.info diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0ad4248..4aadd667 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,6 +90,15 @@ set -e ``` This will run the same tests that the CI runs and refuse the push if it fails. +The CI script runs 4 parallel builds with different configurations, each with Valgrind and functional tests. +Build 1 also collects code coverage (lines, functions, branches). + +Output shows test results per build: +``` +✓ Build 1 passed (unit: 437/0, func: 69/0, cov: L:27.5% F:36.2% B:18.1%, 5m39s) +✓ Build 2 passed (unit: 398/0, func: 69/0, 5m00s) +``` + Note that it will run on the actual content of the repository directory and not what may have been staged/committed. diff --git a/Makefile.am b/Makefile.am index 89809594..1735d1ea 100644 --- a/Makefile.am +++ b/Makefile.am @@ -313,7 +313,7 @@ FUNC_TEST_GROUPS = 1 2 3 4 check-functional-parallel: tests/functionaltests/functionaltests @echo "Running functional tests in parallel ($(words $(FUNC_TEST_GROUPS)) groups)..." - @mkdir -p $(builddir)/test-logs + @mkdir -p $(builddir)/test-logs $(builddir)/test-files @pids=""; \ for g in $(FUNC_TEST_GROUPS); do \ ./tests/functionaltests/functionaltests $$g > $(builddir)/test-logs/group$$g.log 2>&1 & \ @@ -372,7 +372,10 @@ clean-local: rm -f $(git_include) $(git_include).in endif -.PHONY: my-prof.supp +clean-functional-tests: + rm -rf $(builddir)/test-files $(builddir)/test-logs + +.PHONY: my-prof.supp clean-functional-tests my-prof.supp: @sed '/^# AUTO-GENERATED START/q' prof.supp > $@ @printf "\n\n# glib\n" >> $@ @@ -388,7 +391,7 @@ check-unit: tests/unittests/unittests tests/unittests/unittests @VALGRIND_CHECK_RULES@ -VALGRIND_SUPPRESSIONS_FILES=prof.supp +VALGRIND_SUPPRESSIONS_FILES=$(srcdir)/prof.supp # Code coverage targets (requires --enable-coverage) coverage-clean: diff --git a/ci-build.sh b/ci-build.sh index f98bedfa..7efd1697 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -180,30 +180,13 @@ case "$ARCH" in ;; esac -case "$ARCH" in - linux*) - echo - echo "--> Building with ./configure -C ${tests[0]} --enable-valgrind $*" - echo - - # shellcheck disable=SC2086 - ./configure -C ${tests[0]} --enable-valgrind $* - - $MAKE CC="${CC}" - if grep '^ID=' /etc/os-release | grep -q -e debian; then - $MAKE check-valgrind - else - $MAKE check-valgrind || log_content ./test-suite-memcheck.log - fi - $MAKE distclean - ;; -esac - # Function to build and test a single configuration build_and_test() { local features="$1" local extra_args="$2" local idx="$3" + local run_valgrind="$4" + local run_coverage="$5" local build_dir="build-$idx" local log_file="build-$idx.log" @@ -211,14 +194,93 @@ build_and_test() { echo "=== Build $idx started at $(date) ===" echo "--> Building in $build_dir with ./configure -C $features $extra_args" + local start_time=$SECONDS + mkdir -p "$build_dir" cd "$build_dir" # shellcheck disable=SC2086 - ../configure -C $features $extra_args + if ! ../configure -C $features $extra_args; then + echo "ERROR: configure failed" + exit 1 + fi - $MAKE CC="${CC}" - $MAKE check-functional-parallel + if ! $MAKE CC="${CC}"; then + echo "ERROR: make failed" + exit 1 + fi + + # Run unit tests under Valgrind + local unit_passed=0 unit_failed=0 + if [ "$run_valgrind" = "yes" ]; then + echo "--> Running unit tests under Valgrind..." + # Build unit tests first + $MAKE tests/unittests/unittests + # Run valgrind directly to capture cmocka output + valgrind --error-exitcode=1 --leak-check=full \ + --suppressions=../prof.supp \ + tests/unittests/unittests 2>&1 | tee unit-tests-output.log + valgrind_exit=${PIPESTATUS[0]} + if [ $valgrind_exit -ne 0 ]; then + echo "ERROR: Valgrind unit tests failed (exit code $valgrind_exit)" + exit 1 + fi + # Extract unit test counts from cmocka output + unit_passed=$(grep -E "^\[ PASSED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1) + unit_failed=$(grep -E "^\[ FAILED \] [0-9]+ test" unit-tests-output.log | grep -oE "[0-9]+" | head -1) + [ -z "$unit_passed" ] && unit_passed=0 + [ -z "$unit_failed" ] && unit_failed=0 + echo "UNIT_TESTS: passed=$unit_passed failed=$unit_failed" + fi + + # Set build index for port allocation: build 1 uses ports 5230-5233, + # build 2 uses 5234-5237, etc. This prevents port conflicts in parallel builds. + export PROF_BUILD_INDEX=$idx + local func_passed=0 func_failed=0 + if ! $MAKE check-functional-parallel; then + echo "ERROR: functional tests failed" + exit 1 + fi + # Extract functional test counts from group logs + echo "=== Functional test results ===" + for glog in ./test-logs/group*.log; do + if [ -f "$glog" ]; then + cnt=$(grep -E "^\[ PASSED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1) + [ -n "$cnt" ] && func_passed=$((func_passed + cnt)) + cnt=$(grep -E "^\[ FAILED \] [0-9]+ test" "$glog" | grep -oE "[0-9]+" | head -1) + [ -n "$cnt" ] && func_failed=$((func_failed + cnt)) + fi + done + echo "FUNC_TESTS: passed=$func_passed failed=$func_failed" + + # Collect coverage data if enabled (lines, functions, branches) + # Must be done BEFORE make clean which removes .gcda files + local cov_lines="n/a" cov_funcs="n/a" cov_branches="n/a" + if [ "$run_coverage" = "yes" ]; then + echo "--> Collecting coverage data..." + if command -v lcov >/dev/null 2>&1; then + lcov --capture --directory . --output-file coverage.info \ + --rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true + lcov --remove coverage.info '/usr/*' '*/tests/*' --output-file coverage.info \ + --rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true + if [ -f coverage.info ] && [ -s coverage.info ]; then + local summary + summary=$(lcov --summary coverage.info \ + --rc lcov_branch_coverage=1 --ignore-errors inconsistent 2>&1 || true) + cov_lines=$(echo "$summary" | grep -E "lines\.*:" | grep -oE "[0-9]+\.[0-9]+%" | head -1) + cov_funcs=$(echo "$summary" | grep -E "functions\.*:" | grep -oE "[0-9]+\.[0-9]+%" | head -1) + cov_branches=$(echo "$summary" | grep -E "branches\.*:" | grep -oE "[0-9]+\.[0-9]+%" | head -1) + [ -z "$cov_lines" ] && cov_lines="n/a" + [ -z "$cov_funcs" ] && cov_funcs="n/a" + [ -z "$cov_branches" ] && cov_branches="n/a" + else + echo "WARNING: coverage.info is empty or not created" + fi + else + echo "WARNING: lcov not found" + fi + echo "COVERAGE: lines=$cov_lines funcs=$cov_funcs branches=$cov_branches" + fi ./profanity -v $MAKE clean @@ -226,46 +288,91 @@ build_and_test() { cd .. rm -rf "$build_dir" + local elapsed=$((SECONDS - start_time)) + local mins=$((elapsed / 60)) + local secs=$((elapsed % 60)) + echo "=== Build $idx completed at $(date) ===" + echo "STATS: unit_passed=$unit_passed unit_failed=$unit_failed func_passed=$func_passed func_failed=$func_failed cov_lines=$cov_lines cov_funcs=$cov_funcs cov_branches=$cov_branches time=${mins}m${secs}s" } > "$log_file" 2>&1 } # Run all 4 configurations in parallel +# Coverage enabled only for build 1 (Full) - it has most code paths echo "Starting parallel builds..." pids=() for idx in 1 2 3 4; do if [ $idx -le ${#tests[@]} ]; then - build_and_test "${tests[$((idx-1))]}" "$*" "$idx" & + # All builds run Valgrind on Linux + if [ "$ARCH" = "linux" ]; then + run_valgrind="yes" + if [ $idx -eq 1 ]; then + extra_flags="--enable-valgrind --enable-coverage" + run_coverage="yes" + else + extra_flags="--enable-valgrind" + run_coverage="no" + fi + else + extra_flags="" + run_valgrind="no" + run_coverage="no" + fi + build_and_test "${tests[$((idx-1))]}" "$* $extra_flags" "$idx" "$run_valgrind" "$run_coverage" & pids+=("$!") - echo "Started build $idx (PID: $!)" + flags_desc="" + [ "$run_valgrind" = "yes" ] && flags_desc="Valgrind" + [ "$run_coverage" = "yes" ] && flags_desc="${flags_desc:+$flags_desc+}Coverage" + [ -n "$flags_desc" ] && flags_desc=" [+$flags_desc]" + echo "Started build $idx (PID: $!)$flags_desc" fi done -# Wait for all builds and check exit codes -failed=0 -for i in "${!pids[@]}"; do +failed_builds=() +for i in \"${!pids[@]}\"; do idx=$((i + 1)) - if wait "${pids[$i]}"; then - echo "✓ Build $idx passed" + if wait \"${pids[$i]}\"; then + stats="" + if [ -f "build-$idx.log" ]; then + stats_line=$(grep "^STATS:" "build-$idx.log" | tail -1) + unit_p=$(echo "$stats_line" | grep -oE "unit_passed=[0-9]+" | cut -d= -f2) + unit_f=$(echo "$stats_line" | grep -oE "unit_failed=[0-9]+" | cut -d= -f2) + func_p=$(echo "$stats_line" | grep -oE "func_passed=[0-9]+" | cut -d= -f2) + func_f=$(echo "$stats_line" | grep -oE "func_failed=[0-9]+" | cut -d= -f2) + cov_lines=$(echo "$stats_line" | grep -oE "cov_lines=[0-9.]+%|cov_lines=n/a" | cut -d= -f2) + cov_funcs=$(echo "$stats_line" | grep -oE "cov_funcs=[0-9.]+%|cov_funcs=n/a" | cut -d= -f2) + cov_branches=$(echo "$stats_line" | grep -oE "cov_branches=[0-9.]+%|cov_branches=n/a" | cut -d= -f2) + time=$(echo "$stats_line" | grep -oE "time=[0-9]+m[0-9]+s" | cut -d= -f2) + + unit_str="${unit_p:-0}/${unit_f:-0}" + func_str="${func_p:-0}/${func_f:-0}" + + if [ "$cov_lines" != "n/a" ] && [ -n "$cov_lines" ]; then + stats=" (unit: $unit_str, func: $func_str, cov: L:$cov_lines F:$cov_funcs B:$cov_branches, ${time:-?})" + else + stats=" (unit: $unit_str, func: $func_str, ${time:-?})" + fi + fi + echo "✓ Build $idx passed$stats" else echo "✗ Build $idx failed" - echo "--- Log for build $idx ---" - cat "build-$idx.log" - echo "--- End log ---" - failed=1 + failed_builds+=("$idx") fi done -# Show all logs on success too -if [ $failed -eq 0 ]; then - echo - echo "All builds passed!" - for idx in 1 2 3 4; do - if [ -f "build-$idx.log" ]; then - echo "--- Log for build $idx ---" - cat "build-$idx.log" - fi - done -else +echo + +for idx in "${failed_builds[@]}"; do + if [ -f "build-$idx.log" ]; then + echo "=== Build $idx (FAILED) - full log ===" + cat "build-$idx.log" + echo + fi +done + +if [ ${#failed_builds[@]} -gt 0 ]; then + echo "FAILED: builds ${failed_builds[*]}" exit 1 +else + echo "All builds passed!" fi diff --git a/prof.supp b/prof.supp index 87aca009..0110477b 100644 --- a/prof.supp +++ b/prof.supp @@ -8,6 +8,29 @@ # * python suppressions file from https://github.com/python/cpython/blob/main/Misc/valgrind-python.supp # +# ============================================ +# glibc AVX2 optimizations (false positives) +# See: https://sourceware.org/bugzilla/show_bug.cgi?id=19796 +# ============================================ + +{ + glibc_wcpncpy_avx2 + Memcheck:Addr32 + fun:__wcpncpy_avx2 + fun:wcsxfrm_l + fun:g_utf8_collate_key + ... +} + +{ + glibc_wcsxfrm_avx2 + Memcheck:Addr32 + ... + fun:wcsxfrm_l + fun:g_utf8_collate_key + ... +} + # ============================================ # Functional tests suppressions (stabber/pthread) # ============================================ diff --git a/tests/functionaltests/functionaltests.c b/tests/functionaltests/functionaltests.c index e5f59d4e..cd89c58f 100644 --- a/tests/functionaltests/functionaltests.c +++ b/tests/functionaltests/functionaltests.c @@ -69,6 +69,12 @@ main(int argc, char* argv[]) } } + char group_env[16]; + snprintf(group_env, sizeof(group_env), "%d", group); + setenv("PROF_TEST_GROUP", group_env, 1); + + fprintf(stderr, "[PROF_TEST] Starting functional tests, group=%d\n", group); + /* ============================================================ * GROUP 1: Connect, Ping, Rooms, Software * Basic XMPP session establishment and server queries diff --git a/tests/functionaltests/proftest.c b/tests/functionaltests/proftest.c index 4ece33d6..15a353cf 100644 --- a/tests/functionaltests/proftest.c +++ b/tests/functionaltests/proftest.c @@ -136,8 +136,15 @@ _create_logs_dir(void) void _cleanup_dirs(void) { + /* Get test group from environment for consistent cleanup */ + const char *group_env = getenv("PROF_TEST_GROUP"); + int group = group_env ? atoi(group_env) : 0; + int dir_id = (group >= 1 && group <= 4) ? group : stub_port; + + fprintf(stderr, "[PROF_TEST] Cleaning up directories for group %d (dir_id %d)\\n\", group, dir_id); + char cmd[512]; - snprintf(cmd, sizeof(cmd), "rm -rf ./tests/functionaltests/files/%d", stub_port); + snprintf(cmd, sizeof(cmd), "rm -rf ./test-files/%d", dir_id); int res = system(cmd); if (res == -1) { assert_true(FALSE); @@ -221,30 +228,73 @@ prof_start(void) /* Set non-blocking mode for reading */ int flags = fcntl(fd, F_GETFL, 0); fcntl(fd, F_SETFL, flags | O_NONBLOCK); + + /* Brief wait for process to initialize */ + usleep(50000); /* 50ms */ } int init_prof_test(void **state) { + /* Get test group from environment for static resource allocation */ + const char *group_env = getenv("PROF_TEST_GROUP"); + int group = group_env ? atoi(group_env) : 0; + + /* Get build index for port offset (for parallel CI builds) */ + const char *build_env = getenv("PROF_BUILD_INDEX"); + int build_idx = build_env ? atoi(build_env) : 0; + + /* Calculate port base: each build uses a different range of 4 ports + * Build 0/1: 5230-5233, Build 2: 5234-5237, Build 3: 5238-5241, Build 4: 5242-5245 */ + int port_base = 5230 + ((build_idx > 0 ? build_idx - 1 : 0) * 4); + + /* Static resource allocation to avoid conflicts in parallel execution. + * Group 1-4: use static port assignment. + * Group 0 (all groups): use dynamic allocation as fallback. */ gboolean started = FALSE; - for (int p = 5230; p < 5250; ++p) { - int ret = stbbr_start(STBBR_LOGDEBUG, p, 0); - if (ret == 0) { - stub_port = p; + + if (group >= 1 && group <= 4) { + /* Static allocation: each group gets a dedicated port */ + stub_port = port_base + group - 1; + fprintf(stderr, "[PROF_TEST] Build %d, Group %d: trying port %d\n", build_idx, group, stub_port); + + if (stbbr_start(STBBR_LOGDEBUG, stub_port, 0) == 0) { started = TRUE; - break; + fprintf(stderr, "[PROF_TEST] Started stabber on port %d\n", stub_port); + } else { + fprintf(stderr, "[PROF_TEST] Failed to start stabber on port %d\n", stub_port); } } + + /* Fallback to dynamic allocation if static failed or group=0 */ if (!started) { - assert_true(FALSE); // could not start stabber on any port in range + fprintf(stderr, "[PROF_TEST] Using dynamic port allocation\n"); + for (int p = port_base; p < port_base + 20; ++p) { + if (stbbr_start(STBBR_LOGDEBUG, p, 0) == 0) { + stub_port = p; + started = TRUE; + fprintf(stderr, "[PROF_TEST] Started stabber on port %d\n", stub_port); + break; + } + } + } + + if (!started) { + fprintf(stderr, "[PROF_TEST] ERROR: could not start stabber on any port\n"); + assert_true(FALSE); return -1; } - // Generate unique XDG paths based on stub_port for parallel execution + // Generate unique XDG paths based on group for parallel execution + // Use ./test-files/ in current (build) directory for out-of-tree builds compatibility + int dir_id = (group >= 1 && group <= 4) ? group : stub_port; snprintf(xdg_config_home, sizeof(xdg_config_home), - "./tests/functionaltests/files/%d/xdg_config_home", stub_port); + "./test-files/%d/xdg_config_home", dir_id); snprintf(xdg_data_home, sizeof(xdg_data_home), - "./tests/functionaltests/files/%d/xdg_data_home", stub_port); + "./test-files/%d/xdg_data_home", dir_id); + + fprintf(stderr, "[PROF_TEST] Group %d using directories: config=%s, data=%s\n", + group, xdg_config_home, xdg_data_home); // Give stabber server thread time to start listening usleep(100000); // 100ms @@ -398,9 +448,19 @@ prof_output_regex(const char *pattern) return 1; } - usleep(50000); /* 50ms */ + usleep(50000); } + /* Timeout reached - log diagnostic info */ + fprintf(stderr, "Timeout waiting for regex '%s' after %d seconds. Last output:\n", pattern, expect_timeout); + size_t len = strlen(output_buffer); + if (len > 500) { + fprintf(stderr, "...%s", output_buffer + len - 500); + } else { + fprintf(stderr, "%s", output_buffer); + } + fprintf(stderr, "\n"); + regfree(®ex); return 0; }