Test execution on CI optimization #72
25
.github/workflows/ci-code.yml
vendored
25
.github/workflows/ci-code.yml
vendored
@@ -98,3 +98,28 @@ jobs:
|
||||
- name: Check spelling
|
||||
run: |
|
||||
codespell
|
||||
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
name: Code Coverage
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build and run coverage
|
||||
run: |
|
||||
docker build -f Dockerfile.debian -t profanity-cov .
|
||||
docker run -v ${{ github.workspace }}/coverage:/coverage profanity-cov bash -c '
|
||||
./bootstrap.sh
|
||||
./configure --enable-coverage --enable-otr --enable-pgp --enable-omemo --enable-plugins
|
||||
make -j$(nproc)
|
||||
make check || true
|
||||
make check-functional-parallel || true
|
||||
lcov --capture --directory . --output-file /coverage/coverage.info \
|
||||
--rc branch_coverage=1 \
|
||||
--ignore-errors inconsistent
|
||||
lcov --remove /coverage/coverage.info \
|
||||
"/usr/include/*" "*/tests/*" \
|
||||
--output-file /coverage/coverage.info \
|
||||
--rc branch_coverage=1 \
|
||||
--ignore-errors inconsistent,empty,unused
|
||||
lcov --summary /coverage/coverage.info
|
||||
'
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -107,3 +107,4 @@ breaks
|
||||
*.tar.*
|
||||
*.zip
|
||||
*.log*
|
||||
coverage/
|
||||
|
||||
@@ -147,9 +147,28 @@ The functional test suite uses [stabber](https://git.jabber.space/devs/stabber)
|
||||
Functional tests require stabber to be installed. Once installed, tests run as part of `make check`:
|
||||
|
||||
```bash
|
||||
make check
|
||||
make check # Run all tests (unit + functional)
|
||||
make check-functional-parallel # Run functional tests in parallel (~3x faster)
|
||||
./tests/functionaltests/functionaltests # Run all functional tests sequentially
|
||||
./tests/functionaltests/functionaltests 1 # Run specific group (1-4)
|
||||
```
|
||||
|
||||
#### Test groups
|
||||
|
||||
Tests are organized into 4 groups for parallel execution:
|
||||
|
||||
| Group | Description |
|
||||
|-------|-------------|
|
||||
| 1 | Connect, Ping, Rooms, Software |
|
||||
| 2 | Message, Receipts, Roster, Chat Session |
|
||||
| 3 | Presence, Disconnect |
|
||||
| 4 | MUC, Carbons |
|
||||
|
||||
To add a new group:
|
||||
1. Define the test array in `functionaltests.c`
|
||||
2. Add entry to `groups[]` array
|
||||
3. Update `FUNC_TEST_GROUPS` in `Makefile.am`
|
||||
|
||||
#### Writing functional tests
|
||||
|
||||
Use content-based stubbing with stabber:
|
||||
@@ -168,4 +187,5 @@ Guidelines:
|
||||
2. Use `stbbr_send(xml)` for presence, message, and other push-style stanzas.
|
||||
3. Keep assertions tolerant of ordering when possible; use `prof_output_regex()` for flexible matching.
|
||||
4. If timing issues appear, use `prof_timeout()` around critical expectations and reset afterwards.
|
||||
5. When adding new tests, place them in the appropriate group based on functionality.
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
FROM archlinux
|
||||
|
||||
ENV TERM=xterm
|
||||
ENV CC="ccache gcc"
|
||||
|
||||
RUN pacman -Syu --noconfirm
|
||||
# reflector is optional - if it fails due to network issues, continue with default mirrorlist
|
||||
@@ -12,9 +13,11 @@ RUN pacman -S --needed --noconfirm \
|
||||
autoconf-archive \
|
||||
automake \
|
||||
base-devel \
|
||||
ccache \
|
||||
check \
|
||||
cmake \
|
||||
cmocka \
|
||||
lcov \
|
||||
curl \
|
||||
debuginfod \
|
||||
doxygen \
|
||||
@@ -61,12 +64,12 @@ USER root
|
||||
RUN pacman -U --noconfirm libstrophe-git/libstrophe-git-*.pkg.tar.zst
|
||||
|
||||
WORKDIR /usr/src
|
||||
RUN git clone -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
|
||||
WORKDIR /usr/src/stabber
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr --disable-dependency-tracking
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
WORKDIR /usr/src/profanity
|
||||
|
||||
@@ -3,13 +3,16 @@ FROM debian:testing
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive"
|
||||
ENV TERM=xterm
|
||||
ENV CC="ccache gcc"
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
autoconf \
|
||||
autoconf-archive \
|
||||
automake \
|
||||
ccache \
|
||||
gcc \
|
||||
git \
|
||||
lcov \
|
||||
libcmocka-dev \
|
||||
libcurl3-dev \
|
||||
libgcrypt-dev \
|
||||
@@ -37,19 +40,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN mkdir -p /usr/src/{stabber,libstrophe,profanity}
|
||||
WORKDIR /usr/src
|
||||
|
||||
RUN git clone -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
RUN git clone -c http.sslverify=false https://github.com/strophe/libstrophe
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://github.com/strophe/libstrophe
|
||||
|
||||
WORKDIR /usr/src/stabber
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr --disable-dependency-tracking
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
WORKDIR /usr/src/libstrophe
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
WORKDIR /usr/src/profanity
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# Build the latest Fedora image
|
||||
FROM fedora:latest
|
||||
|
||||
ENV TERM=xterm
|
||||
ENV CC="ccache gcc"
|
||||
|
||||
# libmicrohttpd - for stabber
|
||||
# glibc-locale - to have en_US locale
|
||||
RUN dnf install -y \
|
||||
@@ -8,8 +11,10 @@ RUN dnf install -y \
|
||||
autoconf-archive \
|
||||
automake \
|
||||
awk \
|
||||
ccache \
|
||||
gcc \
|
||||
git \
|
||||
lcov \
|
||||
glib2-devel \
|
||||
glibc-all-langpacks \
|
||||
gtk2-devel \
|
||||
@@ -45,20 +50,20 @@ ENV TERM=xterm
|
||||
RUN mkdir -p /usr/src
|
||||
WORKDIR /usr/src
|
||||
|
||||
RUN git clone -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
WORKDIR /usr/src/stabber
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr --disable-dependency-tracking
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
WORKDIR /usr/src
|
||||
RUN mkdir -p /usr/src/libstrophe
|
||||
RUN git clone -c http.sslverify=false https://github.com/strophe/libstrophe
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://github.com/strophe/libstrophe
|
||||
WORKDIR /usr/src/libstrophe
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
RUN mkdir -p /usr/src/profanity
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
# Build the latest openSUSE Tumbleweed image
|
||||
FROM opensuse/tumbleweed
|
||||
|
||||
ENV TERM=xterm
|
||||
ENV CC="ccache gcc"
|
||||
|
||||
# libmicrohttpd - for stabber
|
||||
# glibc-locale - to have en_US locale
|
||||
RUN zypper --non-interactive in --no-recommends \
|
||||
autoconf \
|
||||
autoconf-archive \
|
||||
automake \
|
||||
ccache \
|
||||
gcc \
|
||||
git \
|
||||
lcov \
|
||||
glib2-devel \
|
||||
glibc-locale \
|
||||
gtk2-devel \
|
||||
@@ -44,11 +49,11 @@ ENV TERM=xterm
|
||||
RUN mkdir -p /usr/src
|
||||
WORKDIR /usr/src
|
||||
|
||||
RUN git clone -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
WORKDIR /usr/src/stabber
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr --disable-dependency-tracking
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
RUN mkdir -p /usr/src/profanity
|
||||
|
||||
@@ -2,13 +2,16 @@ FROM ubuntu:latest
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive"
|
||||
ENV TERM=xterm
|
||||
ENV CC="ccache gcc"
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
autoconf \
|
||||
autoconf-archive \
|
||||
automake \
|
||||
ccache \
|
||||
gcc \
|
||||
git \
|
||||
lcov \
|
||||
libcmocka-dev \
|
||||
libcurl3-dev \
|
||||
libgcrypt-dev \
|
||||
@@ -36,19 +39,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN mkdir -p /usr/src/{stabber,libstrophe,profanity}
|
||||
WORKDIR /usr/src
|
||||
|
||||
RUN git clone -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
RUN git clone -c http.sslverify=false https://github.com/strophe/libstrophe
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://git.jabber.space/devs/stabber
|
||||
RUN git clone --depth 1 -c http.sslverify=false https://github.com/strophe/libstrophe
|
||||
|
||||
WORKDIR /usr/src/stabber
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr --disable-dependency-tracking
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
WORKDIR /usr/src/libstrophe
|
||||
RUN ./bootstrap.sh
|
||||
RUN ./configure --prefix=/usr
|
||||
RUN make
|
||||
RUN make -j$(nproc)
|
||||
RUN make install
|
||||
|
||||
WORKDIR /usr/src/profanity
|
||||
|
||||
41
Makefile.am
41
Makefile.am
@@ -286,7 +286,7 @@ endif
|
||||
|
||||
TESTS = tests/unittests/unittests
|
||||
check_PROGRAMS = tests/unittests/unittests
|
||||
tests_unittests_unittests_CPPFLAGS = -Itests/
|
||||
tests_unittests_unittests_CPPFLAGS = -I$(srcdir)/tests
|
||||
tests_unittests_unittests_SOURCES = $(unittest_sources)
|
||||
tests_unittests_unittests_LDADD = -lcmocka
|
||||
|
||||
@@ -302,9 +302,32 @@ if HAVE_FORKPTY
|
||||
TESTS += tests/functionaltests/functionaltests
|
||||
check_PROGRAMS += tests/functionaltests/functionaltests
|
||||
tests_functionaltests_functionaltests_SOURCES = $(functionaltest_sources)
|
||||
tests_functionaltests_functionaltests_CPPFLAGS = -Itests/
|
||||
tests_functionaltests_functionaltests_CPPFLAGS = -I$(srcdir)/tests
|
||||
tests_functionaltests_functionaltests_CFLAGS = $(AM_CFLAGS)
|
||||
tests_functionaltests_functionaltests_LDADD = -lcmocka -lstabber @FORKPTY_LIB@
|
||||
|
||||
# Parallel functional tests target (~3x faster than sequential)
|
||||
# Usage: make check-functional-parallel
|
||||
# To add more groups: increase FUNC_TEST_GROUPS and add group in functionaltests.c
|
||||
FUNC_TEST_GROUPS = 1 2 3 4
|
||||
|
||||
check-functional-parallel: tests/functionaltests/functionaltests
|
||||
@echo "Running functional tests in parallel ($(words $(FUNC_TEST_GROUPS)) groups)..."
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
|
||||
@mkdir -p $(builddir)/test-logs
|
||||
@pids=""; \
|
||||
for g in $(FUNC_TEST_GROUPS); do \
|
||||
./tests/functionaltests/functionaltests $$g > $(builddir)/test-logs/group$$g.log 2>&1 & \
|
||||
pids="$$pids $$!"; \
|
||||
done; \
|
||||
failed=0; i=1; \
|
||||
for pid in $$pids; do \
|
||||
wait $$pid || { echo "Group $$i FAILED"; cat $(builddir)/test-logs/group$$i.log; failed=1; }; \
|
||||
i=$$((i + 1)); \
|
||||
done; \
|
||||
echo "=== Test Results Summary ==="; \
|
||||
grep -E 'PASSED|FAILED|Running' $(builddir)/test-logs/group*.log || true; \
|
||||
if [ $$failed -ne 0 ]; then echo "FUNCTIONAL TESTS FAILED"; exit 1; fi; \
|
||||
echo "All functional test groups passed!"
|
||||
endif
|
||||
endif
|
||||
|
||||
@@ -367,6 +390,20 @@ check-unit: tests/unittests/unittests
|
||||
@VALGRIND_CHECK_RULES@
|
||||
VALGRIND_SUPPRESSIONS_FILES=prof.supp
|
||||
|
||||
# Code coverage targets (requires --enable-coverage)
|
||||
coverage-clean:
|
||||
find . -name '*.gcda' -delete
|
||||
find . -name '*.gcno' -delete
|
||||
rm -rf coverage-html coverage.info
|
||||
|
||||
coverage-report: check
|
||||
lcov --capture --directory . --output-file coverage.info --ignore-errors inconsistent
|
||||
lcov --remove coverage.info '/usr/*' '*/tests/*' --output-file coverage.info --ignore-errors inconsistent
|
||||
genhtml coverage.info --output-directory coverage-html
|
||||
@echo "Coverage report generated in coverage-html/index.html"
|
||||
|
||||
.PHONY: coverage-clean coverage-report
|
||||
|
||||
format: $(all_c_sources)
|
||||
clang-format -i $(all_c_sources)
|
||||
|
||||
|
||||
175
ci-build.sh
175
ci-build.sh
@@ -23,6 +23,79 @@ error_handler()
|
||||
|
||||
trap error_handler ERR
|
||||
|
||||
# Verify that test failures are properly detected
|
||||
# This is a meta-test: it runs a deliberately failing test
|
||||
# and checks that the test framework reports the failure correctly
|
||||
verify_test_failure_detection()
|
||||
{
|
||||
echo
|
||||
echo "==> Verifying test failure detection..."
|
||||
|
||||
# Create a simple failing test
|
||||
cat > /tmp/test_must_fail.c << 'EOF'
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <setjmp.h>
|
||||
#include <cmocka.h>
|
||||
|
||||
static void test_that_must_fail(void **state) {
|
||||
(void)state;
|
||||
assert_true(0); // This MUST fail
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
const struct CMUnitTest tests[] = {
|
||||
cmocka_unit_test(test_that_must_fail),
|
||||
};
|
||||
return cmocka_run_group_tests(tests, NULL, NULL);
|
||||
}
|
||||
EOF
|
||||
|
||||
# Compile the failing test
|
||||
if ! gcc -o /tmp/test_must_fail /tmp/test_must_fail.c -lcmocka 2>/dev/null; then
|
||||
echo "Warning: Could not compile test failure verification (cmocka not available?)"
|
||||
echo "Skipping test failure detection verification"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Test 1: Single failing test detection
|
||||
echo " Testing single test failure detection..."
|
||||
if /tmp/test_must_fail > /tmp/test_must_fail.log 2>&1; then
|
||||
echo "ERROR: Test that should fail returned success (exit code 0)"
|
||||
echo "This means the test framework is NOT detecting failures correctly!"
|
||||
echo "--- Test output ---"
|
||||
cat /tmp/test_must_fail.log
|
||||
echo "--- End output ---"
|
||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log
|
||||
exit 1
|
||||
fi
|
||||
echo " ✓ Single test failure correctly detected"
|
||||
|
||||
# Test 2: Parallel failure detection (simulates check-functional-parallel)
|
||||
echo " Testing parallel test failure detection..."
|
||||
failed=0
|
||||
/tmp/test_must_fail > /tmp/p1.log 2>&1 & pid1=$!
|
||||
true > /tmp/p2.log 2>&1 & pid2=$! # This passes
|
||||
/tmp/test_must_fail > /tmp/p3.log 2>&1 & pid3=$!
|
||||
true > /tmp/p4.log 2>&1 & pid4=$! # This passes
|
||||
|
||||
wait $pid1 || failed=$((failed + 1))
|
||||
wait $pid2 || failed=$((failed + 1))
|
||||
wait $pid3 || failed=$((failed + 1))
|
||||
wait $pid4 || failed=$((failed + 1))
|
||||
|
||||
if [ $failed -ne 2 ]; then
|
||||
echo "ERROR: Expected 2 failures in parallel tests, got $failed"
|
||||
echo "Parallel failure detection is broken!"
|
||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
|
||||
exit 1
|
||||
fi
|
||||
echo " ✓ Parallel test failures correctly detected (2 of 4 failed as expected)"
|
||||
|
||||
rm -f /tmp/test_must_fail /tmp/test_must_fail.c /tmp/test_must_fail.log /tmp/p?.log
|
||||
echo "✓ Test failure detection verified"
|
||||
}
|
||||
|
||||
num_cores()
|
||||
{
|
||||
# Check for cores, for systems with:
|
||||
@@ -34,6 +107,9 @@ num_cores()
|
||||
|| getconf _NPROCESSORS_ONLN 2>/dev/null
|
||||
}
|
||||
|
||||
# Run test failure detection verification first
|
||||
verify_test_failure_detection
|
||||
|
||||
./bootstrap.sh
|
||||
|
||||
tests=()
|
||||
@@ -44,7 +120,7 @@ ARCH="$(uname | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
case "$ARCH" in
|
||||
linux*)
|
||||
# Reduced set of configurations for faster CI
|
||||
# 4 configurations for parallel CI
|
||||
tests=(
|
||||
# 1. Full build (all features enabled)
|
||||
"--enable-notifications --enable-icons-and-clipboard --enable-otr --enable-pgp
|
||||
@@ -56,15 +132,13 @@ case "$ARCH" in
|
||||
--disable-python-plugins --without-xscreensaver --disable-omemo-qrcode --disable-gdk-pixbuf"
|
||||
# 3. No encryption (disable otr, pgp, omemo)
|
||||
"--disable-pgp --disable-otr --disable-omemo --disable-omemo-qrcode"
|
||||
# 4. No plugins
|
||||
"--disable-plugins --disable-c-plugins --disable-python-plugins"
|
||||
# 5. Default configuration
|
||||
# 4. Default configuration
|
||||
""
|
||||
)
|
||||
source /etc/profile.d/debuginfod.sh 2>/dev/null || true
|
||||
;;
|
||||
darwin*)
|
||||
# Reduced set of configurations for faster CI
|
||||
# 4 configurations for parallel CI
|
||||
tests=(
|
||||
# 1. Full build (all features enabled)
|
||||
"--enable-notifications --enable-icons-and-clipboard --enable-otr --enable-pgp
|
||||
@@ -76,9 +150,7 @@ case "$ARCH" in
|
||||
--disable-python-plugins"
|
||||
# 3. No encryption (disable otr, pgp, omemo)
|
||||
"--disable-pgp --disable-otr --disable-omemo"
|
||||
# 4. No plugins
|
||||
"--disable-plugins --disable-c-plugins --disable-python-plugins"
|
||||
# 5. Default configuration
|
||||
# 4. Default configuration
|
||||
""
|
||||
)
|
||||
;;
|
||||
@@ -90,7 +162,7 @@ case "$ARCH" in
|
||||
# src/event/server_events.c:1454:19: error: universal character names are only valid in C++ and C99
|
||||
CC="egcc -std=gnu99 -fexec-charset=UTF-8"
|
||||
|
||||
# Reduced set of configurations for faster CI
|
||||
# 4 configurations for parallel CI
|
||||
tests=(
|
||||
# 1. Full build (all features enabled)
|
||||
"--enable-notifications --enable-icons-and-clipboard --enable-otr --enable-pgp
|
||||
@@ -102,9 +174,7 @@ case "$ARCH" in
|
||||
--disable-python-plugins"
|
||||
# 3. No encryption (disable otr, pgp, omemo)
|
||||
"--disable-pgp --disable-otr --disable-omemo"
|
||||
# 4. No plugins
|
||||
"--disable-plugins --disable-c-plugins --disable-python-plugins"
|
||||
# 5. Default configuration
|
||||
# 4. Default configuration
|
||||
""
|
||||
)
|
||||
;;
|
||||
@@ -113,11 +183,11 @@ esac
|
||||
case "$ARCH" in
|
||||
linux*)
|
||||
echo
|
||||
echo "--> Building with ./configure ${tests[0]} --enable-valgrind $*"
|
||||
echo "--> Building with ./configure -C ${tests[0]} --enable-valgrind $*"
|
||||
echo
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
./configure ${tests[0]} --enable-valgrind $*
|
||||
./configure -C ${tests[0]} --enable-valgrind $*
|
||||
|
||||
$MAKE CC="${CC}"
|
||||
if grep '^ID=' /etc/os-release | grep -q -e debian; then
|
||||
@@ -129,18 +199,73 @@ case "$ARCH" in
|
||||
;;
|
||||
esac
|
||||
|
||||
for features in "${tests[@]}"
|
||||
do
|
||||
echo
|
||||
echo "--> Building with ./configure ${features} $*"
|
||||
echo
|
||||
# Function to build and test a single configuration
|
||||
build_and_test() {
|
||||
local features="$1"
|
||||
local extra_args="$2"
|
||||
local idx="$3"
|
||||
local build_dir="build-$idx"
|
||||
local log_file="build-$idx.log"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
./configure $features $*
|
||||
{
|
||||
echo "=== Build $idx started at $(date) ==="
|
||||
echo "--> Building in $build_dir with ./configure -C $features $extra_args"
|
||||
|
||||
$MAKE CC="${CC}"
|
||||
$MAKE check
|
||||
mkdir -p "$build_dir"
|
||||
cd "$build_dir"
|
||||
|
||||
./profanity -v
|
||||
$MAKE clean
|
||||
# shellcheck disable=SC2086
|
||||
../configure -C $features $extra_args
|
||||
|
||||
$MAKE CC="${CC}"
|
||||
$MAKE check-functional-parallel
|
||||
|
||||
./profanity -v
|
||||
$MAKE clean
|
||||
|
||||
cd ..
|
||||
rm -rf "$build_dir"
|
||||
|
||||
echo "=== Build $idx completed at $(date) ==="
|
||||
} > "$log_file" 2>&1
|
||||
}
|
||||
|
||||
# Run all 4 configurations in parallel
|
||||
echo "Starting parallel builds..."
|
||||
pids=()
|
||||
for idx in 1 2 3 4; do
|
||||
if [ $idx -le ${#tests[@]} ]; then
|
||||
build_and_test "${tests[$((idx-1))]}" "$*" "$idx" &
|
||||
pids+=("$!")
|
||||
echo "Started build $idx (PID: $!)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for all builds and check exit codes
|
||||
failed=0
|
||||
for i in "${!pids[@]}"; do
|
||||
idx=$((i + 1))
|
||||
if wait "${pids[$i]}"; then
|
||||
echo "✓ Build $idx passed"
|
||||
else
|
||||
echo "✗ Build $idx failed"
|
||||
echo "--- Log for build $idx ---"
|
||||
cat "build-$idx.log"
|
||||
echo "--- End log ---"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Show all logs on success too
|
||||
if [ $failed -eq 0 ]; then
|
||||
echo
|
||||
echo "All builds passed!"
|
||||
for idx in 1 2 3 4; do
|
||||
if [ -f "build-$idx.log" ]; then
|
||||
echo "--- Log for build $idx ---"
|
||||
cat "build-$idx.log"
|
||||
fi
|
||||
done
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -69,6 +69,8 @@ AC_ARG_ENABLE([gdk-pixbuf],
|
||||
[AS_HELP_STRING([--enable-gdk-pixbuf], [enable GDK Pixbuf support to scale avatars before uploading])])
|
||||
AC_ARG_ENABLE([omemo-qrcode],
|
||||
[AS_HELP_STRING([--enable-omemo-qrcode], [enable ability to display omemo qr code])])
|
||||
AC_ARG_ENABLE([coverage],
|
||||
[AS_HELP_STRING([--enable-coverage], [enable code coverage analysis])])
|
||||
|
||||
m4_include([m4/ax_valgrind_check.m4])
|
||||
AX_VALGRIND_DFLT([drd], [off])
|
||||
@@ -386,6 +388,11 @@ AC_SUBST([FORKPTY_LIB])
|
||||
AM_CFLAGS="$AM_CFLAGS -Wall -Wno-deprecated-declarations -std=gnu99 -ggdb3"
|
||||
AM_LDFLAGS="$AM_LDFLAGS -export-dynamic"
|
||||
|
||||
AS_IF([test "x$enable_coverage" = xyes],
|
||||
[AM_CFLAGS="$AM_CFLAGS --coverage -O0"
|
||||
AM_LDFLAGS="$AM_LDFLAGS --coverage"
|
||||
AC_MSG_NOTICE([Code coverage analysis enabled])])
|
||||
|
||||
AS_IF([test "x$PACKAGE_STATUS" = xdevelopment],
|
||||
[AM_CFLAGS="$AM_CFLAGS -Wunused -Werror"])
|
||||
AS_IF([test "x$PLATFORM" = xosx],
|
||||
|
||||
@@ -14,21 +14,31 @@
|
||||
* flaky tests caused by leftover state. The overhead is acceptable since
|
||||
* functional tests run less frequently than unit tests.
|
||||
*
|
||||
* Tests are organized into groups for better maintainability:
|
||||
* Group 1: Connection, Ping, Rooms, Presence
|
||||
* Group 2: Messages, Receipts, Roster management
|
||||
* Group 3: MUC (Multi-User Chat) functionality
|
||||
* Group 4: Carbons, Chat sessions, Software version, Disconnect
|
||||
* Tests are organized into groups for better maintainability and parallel execution:
|
||||
* Group 1: Connect, Ping, Rooms, Software
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Please remove counts (such as 17 tests) from the commentary. It's highly likely that we will miss to update those and we'll have outdated docs. Updating them each time we add/remove test is going to be painful as well. Please remove counts (such as 17 tests) from the commentary. It's highly likely that we will miss to update those and we'll have outdated docs. Updating them each time we add/remove test is going to be painful as well.
jabber.developer2
commented
Corrected Corrected
|
||||
* Group 2: Message, Receipts, Roster, Chat Session
|
||||
* Group 3: Presence, Disconnect
|
||||
* Group 4: MUC, Carbons
|
||||
*
|
||||
* Parallel execution:
|
||||
* ./functionaltests - run all tests sequentially
|
||||
* ./functionaltests N - run group N only (N = 1..num_groups)
|
||||
*
|
||||
* For parallel execution, run multiple groups simultaneously:
|
||||
* ./functionaltests 1 & ./functionaltests 2 & ./functionaltests 3 & ./functionaltests 4 & wait
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <string.h>
|
||||
#include "prof_cmocka.h"
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "proftest.h"
|
||||
#include "test_connect.h"
|
||||
#include "test_ping.h"
|
||||
@@ -49,13 +59,21 @@
|
||||
int
|
||||
main(int argc, char* argv[])
|
||||
{
|
||||
const struct CMUnitTest all_tests[] = {
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 1: Connect, Ping, Rooms, Presence
|
||||
* Basic XMPP session establishment and presence management
|
||||
* ============================================================ */
|
||||
int group = 0; /* 0 = all groups */
|
||||
if (argc > 1) {
|
||||
group = atoi(argv[1]);
|
||||
if (group < 1 || group > 4) {
|
||||
fprintf(stderr, "Usage: %s [group]\n", argv[0]);
|
||||
fprintf(stderr, " group: 1-4 to run specific group, or omit for all\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 1: Connect, Ping, Rooms, Software
|
||||
* Basic XMPP session establishment and server queries
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Previous style (here and later) feels better for readability. Previous style (here and later) feels better for readability.
jabber.developer2
commented
Corrected Corrected
|
||||
* ============================================================ */
|
||||
const struct CMUnitTest group1_tests[] = {
|
||||
/* Connection tests - verify login, roster, bookmarks */
|
||||
PROF_FUNC_TEST(connect_jid_requests_roster),
|
||||
PROF_FUNC_TEST(connect_jid_sends_presence_after_receiving_roster),
|
||||
@@ -73,28 +91,20 @@ main(int argc, char* argv[])
|
||||
/* Room discovery - XEP-0045 */
|
||||
PROF_FUNC_TEST(rooms_query),
|
||||
|
||||
/* Presence tests - online/away/xa/dnd/chat status */
|
||||
PROF_FUNC_TEST(presence_online),
|
||||
PROF_FUNC_TEST(presence_online_with_message),
|
||||
PROF_FUNC_TEST(presence_away),
|
||||
PROF_FUNC_TEST(presence_away_with_message),
|
||||
PROF_FUNC_TEST(presence_xa),
|
||||
PROF_FUNC_TEST(presence_xa_with_message),
|
||||
PROF_FUNC_TEST(presence_dnd),
|
||||
PROF_FUNC_TEST(presence_dnd_with_message),
|
||||
PROF_FUNC_TEST(presence_chat),
|
||||
PROF_FUNC_TEST(presence_chat_with_message),
|
||||
PROF_FUNC_TEST(presence_set_priority),
|
||||
PROF_FUNC_TEST(presence_includes_priority),
|
||||
PROF_FUNC_TEST(presence_keeps_status),
|
||||
PROF_FUNC_TEST(presence_received),
|
||||
PROF_FUNC_TEST(presence_missing_resource_defaults),
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 2: Message, Receipts, Roster
|
||||
* Core messaging and contact management
|
||||
* ============================================================ */
|
||||
/* Software Version - XEP-0092 */
|
||||
PROF_FUNC_TEST(send_software_version_request),
|
||||
PROF_FUNC_TEST(display_software_version_result),
|
||||
PROF_FUNC_TEST(shows_message_when_software_version_error),
|
||||
PROF_FUNC_TEST(display_software_version_result_when_from_domainpart),
|
||||
PROF_FUNC_TEST(show_message_in_chat_window_when_no_resource),
|
||||
PROF_FUNC_TEST(display_software_version_result_in_chat),
|
||||
};
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 2: Message, Receipts, Roster, Chat Session
|
||||
* Core messaging and contact management
|
||||
* ============================================================ */
|
||||
const struct CMUnitTest group2_tests[] = {
|
||||
/* Basic message send/receive */
|
||||
PROF_FUNC_TEST(message_send),
|
||||
PROF_FUNC_TEST(message_receive_console),
|
||||
@@ -112,21 +122,54 @@ main(int argc, char* argv[])
|
||||
PROF_FUNC_TEST(sends_remove_item_nick),
|
||||
PROF_FUNC_TEST(sends_nick_change),
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 3: MUC (Multi-User Chat)
|
||||
* XEP-0045 conference room functionality
|
||||
* ============================================================ */
|
||||
/* Chat session management - bare/full JID routing */
|
||||
PROF_FUNC_TEST(sends_message_to_barejid_when_contact_offline),
|
||||
PROF_FUNC_TEST(sends_message_to_barejid_when_contact_online),
|
||||
PROF_FUNC_TEST(sends_message_to_fulljid_when_received_from_fulljid),
|
||||
PROF_FUNC_TEST(sends_subsequent_messages_to_fulljid),
|
||||
PROF_FUNC_TEST(resets_to_barejid_after_presence_received),
|
||||
PROF_FUNC_TEST(new_session_when_message_received_from_different_fulljid),
|
||||
};
|
||||
|
||||
/* Room join with various options */
|
||||
/* ============================================================
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Same as earlier: style and counts Same as earlier: style and counts
jabber.developer2
commented
Corrected Corrected
|
||||
* GROUP 3: Presence, Disconnect
|
||||
* Online/away/xa/dnd/chat status management
|
||||
* ============================================================ */
|
||||
const struct CMUnitTest group3_tests[] = {
|
||||
PROF_FUNC_TEST(presence_online),
|
||||
PROF_FUNC_TEST(presence_online_with_message),
|
||||
PROF_FUNC_TEST(presence_away),
|
||||
PROF_FUNC_TEST(presence_away_with_message),
|
||||
PROF_FUNC_TEST(presence_xa),
|
||||
PROF_FUNC_TEST(presence_xa_with_message),
|
||||
PROF_FUNC_TEST(presence_dnd),
|
||||
PROF_FUNC_TEST(presence_dnd_with_message),
|
||||
PROF_FUNC_TEST(presence_chat),
|
||||
PROF_FUNC_TEST(presence_chat_with_message),
|
||||
PROF_FUNC_TEST(presence_set_priority),
|
||||
PROF_FUNC_TEST(presence_includes_priority),
|
||||
PROF_FUNC_TEST(presence_keeps_status),
|
||||
PROF_FUNC_TEST(presence_received),
|
||||
PROF_FUNC_TEST(presence_missing_resource_defaults),
|
||||
|
||||
/* Disconnect - clean session termination */
|
||||
PROF_FUNC_TEST(disconnect_ends_session),
|
||||
};
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 4: MUC, Carbons
|
||||
* Multi-user chat and message synchronization
|
||||
* ============================================================ */
|
||||
const struct CMUnitTest group4_tests[] = {
|
||||
/* MUC room join with various options - XEP-0045 */
|
||||
PROF_FUNC_TEST(sends_room_join),
|
||||
PROF_FUNC_TEST(sends_room_join_with_nick),
|
||||
PROF_FUNC_TEST(sends_room_join_with_password),
|
||||
PROF_FUNC_TEST(sends_room_join_with_nick_and_password),
|
||||
|
||||
/* Room information display */
|
||||
/* MUC room information display */
|
||||
PROF_FUNC_TEST(shows_role_and_affiliation_on_join),
|
||||
PROF_FUNC_TEST(shows_subject_on_join),
|
||||
// PROF_FUNC_TEST(shows_history_message), // temporarily disabled due to timing issues in CI
|
||||
PROF_FUNC_TEST(shows_occupant_join),
|
||||
|
||||
/* MUC messaging */
|
||||
@@ -134,16 +177,11 @@ main(int argc, char* argv[])
|
||||
PROF_FUNC_TEST(shows_me_message_from_occupant),
|
||||
PROF_FUNC_TEST(shows_me_message_from_self),
|
||||
|
||||
/* Console notification settings for MUC */
|
||||
/* MUC console notification settings */
|
||||
PROF_FUNC_TEST(shows_all_messages_in_console_when_window_not_focussed),
|
||||
PROF_FUNC_TEST(shows_first_message_in_console_when_window_not_focussed),
|
||||
PROF_FUNC_TEST(shows_no_message_in_console_when_window_not_focussed),
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 4: Carbons, Chat Session, Software, Disconnect
|
||||
* Message synchronization and session management
|
||||
* ============================================================ */
|
||||
|
||||
/* Message Carbons - XEP-0280 (message sync across devices) */
|
||||
PROF_FUNC_TEST(send_enable_carbons),
|
||||
PROF_FUNC_TEST(connect_with_carbons_enabled),
|
||||
@@ -151,26 +189,34 @@ main(int argc, char* argv[])
|
||||
PROF_FUNC_TEST(receive_carbon),
|
||||
PROF_FUNC_TEST(receive_self_carbon),
|
||||
PROF_FUNC_TEST(receive_private_carbon),
|
||||
|
||||
/* Chat session management - bare/full JID routing */
|
||||
PROF_FUNC_TEST(sends_message_to_barejid_when_contact_offline),
|
||||
PROF_FUNC_TEST(sends_message_to_barejid_when_contact_online),
|
||||
PROF_FUNC_TEST(sends_message_to_fulljid_when_received_from_fulljid),
|
||||
PROF_FUNC_TEST(sends_subsequent_messages_to_fulljid),
|
||||
PROF_FUNC_TEST(resets_to_barejid_after_presence_received),
|
||||
PROF_FUNC_TEST(new_session_when_message_received_from_different_fulljid),
|
||||
|
||||
/* Software Version - XEP-0092 */
|
||||
PROF_FUNC_TEST(send_software_version_request),
|
||||
PROF_FUNC_TEST(display_software_version_result),
|
||||
PROF_FUNC_TEST(shows_message_when_software_version_error),
|
||||
PROF_FUNC_TEST(display_software_version_result_when_from_domainpart),
|
||||
PROF_FUNC_TEST(show_message_in_chat_window_when_no_resource),
|
||||
PROF_FUNC_TEST(display_software_version_result_in_chat),
|
||||
|
||||
/* Disconnect - clean session termination */
|
||||
PROF_FUNC_TEST(disconnect_ends_session),
|
||||
};
|
||||
|
||||
return cmocka_run_group_tests(all_tests, NULL, NULL);
|
||||
/* Test group registry for easy extension */
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Do we plan to increase count of groups? If so, maybe it would make sense to make a different structure to store group names and their content and then loop over it? Do we plan to increase count of groups? If so, maybe it would make sense to make a different structure to store group names and their content and then loop over it?
jabber.developer2
commented
Corrected Corrected
|
||||
struct {
|
||||
const char* name;
|
||||
const struct CMUnitTest* tests;
|
||||
size_t count;
|
||||
} groups[] = {
|
||||
{ "Group 1: Connect/Ping/Rooms/Software", group1_tests, ARRAY_SIZE(group1_tests) },
|
||||
{ "Group 2: Message/Receipts/Roster/Session", group2_tests, ARRAY_SIZE(group2_tests) },
|
||||
{ "Group 3: Presence/Disconnect", group3_tests, ARRAY_SIZE(group3_tests) },
|
||||
{ "Group 4: MUC/Carbons", group4_tests, ARRAY_SIZE(group4_tests) },
|
||||
};
|
||||
const int num_groups = ARRAY_SIZE(groups);
|
||||
|
||||
int result = 0;
|
||||
|
||||
if (group > 0 && group <= num_groups) {
|
||||
/* Run specific group */
|
||||
result = _cmocka_run_group_tests(groups[group - 1].name, groups[group - 1].tests,
|
||||
groups[group - 1].count, NULL, NULL);
|
||||
} else {
|
||||
/* Run all groups sequentially */
|
||||
for (int i = 0; i < num_groups; i++) {
|
||||
result |= _cmocka_run_group_tests(groups[i].name, groups[i].tests,
|
||||
groups[i].count, NULL, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -24,6 +24,13 @@ int fd = 0;
|
||||
int stub_port = 5230;
|
||||
pid_t child_pid = 0;
|
||||
|
||||
/*
|
||||
* Dynamic XDG paths based on stub_port for parallel test execution.
|
||||
* Each test instance gets unique directories to avoid file conflicts.
|
||||
*/
|
||||
char xdg_config_home[256];
|
||||
char xdg_data_home[256];
|
||||
|
||||
/*
|
||||
* Buffer for accumulating output from profanity.
|
||||
* 64KB is sufficient for typical test output while keeping memory usage
|
||||
@@ -77,7 +84,7 @@ _mkdir_recursive(const char *dir)
|
||||
void
|
||||
_create_config_dir(void)
|
||||
{
|
||||
GString *profanity_dir = g_string_new(XDG_CONFIG_HOME);
|
||||
GString *profanity_dir = g_string_new(xdg_config_home);
|
||||
g_string_append(profanity_dir, "/profanity");
|
||||
|
||||
if (!_mkdir_recursive(profanity_dir->str)) {
|
||||
@@ -90,7 +97,7 @@ _create_config_dir(void)
|
||||
void
|
||||
_create_data_dir(void)
|
||||
{
|
||||
GString *profanity_dir = g_string_new(XDG_DATA_HOME);
|
||||
GString *profanity_dir = g_string_new(xdg_data_home);
|
||||
g_string_append(profanity_dir, "/profanity");
|
||||
|
||||
if (!_mkdir_recursive(profanity_dir->str)) {
|
||||
@@ -103,7 +110,7 @@ _create_data_dir(void)
|
||||
void
|
||||
_create_chatlogs_dir(void)
|
||||
{
|
||||
GString *chatlogs_dir = g_string_new(XDG_DATA_HOME);
|
||||
GString *chatlogs_dir = g_string_new(xdg_data_home);
|
||||
g_string_append(chatlogs_dir, "/profanity/chatlogs");
|
||||
|
||||
if (!_mkdir_recursive(chatlogs_dir->str)) {
|
||||
@@ -116,7 +123,7 @@ _create_chatlogs_dir(void)
|
||||
void
|
||||
_create_logs_dir(void)
|
||||
{
|
||||
GString *logs_dir = g_string_new(XDG_DATA_HOME);
|
||||
GString *logs_dir = g_string_new(xdg_data_home);
|
||||
g_string_append(logs_dir, "/profanity/logs");
|
||||
|
||||
if (!_mkdir_recursive(logs_dir->str)) {
|
||||
@@ -129,7 +136,9 @@ _create_logs_dir(void)
|
||||
void
|
||||
_cleanup_dirs(void)
|
||||
{
|
||||
int res = system("rm -rf ./tests/functionaltests/files");
|
||||
char cmd[512];
|
||||
snprintf(cmd, sizeof(cmd), "rm -rf ./tests/functionaltests/files/%d", stub_port);
|
||||
int res = system(cmd);
|
||||
if (res == -1) {
|
||||
assert_true(FALSE);
|
||||
}
|
||||
@@ -231,14 +240,20 @@ init_prof_test(void **state)
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Generate unique XDG paths based on stub_port for parallel execution
|
||||
snprintf(xdg_config_home, sizeof(xdg_config_home),
|
||||
"./tests/functionaltests/files/%d/xdg_config_home", stub_port);
|
||||
snprintf(xdg_data_home, sizeof(xdg_data_home),
|
||||
"./tests/functionaltests/files/%d/xdg_data_home", stub_port);
|
||||
|
||||
// Give stabber server thread time to start listening
|
||||
usleep(100000); // 100ms
|
||||
|
||||
config_orig = getenv("XDG_CONFIG_HOME");
|
||||
data_orig = getenv("XDG_DATA_HOME");
|
||||
|
||||
setenv("XDG_CONFIG_HOME", XDG_CONFIG_HOME, 1);
|
||||
setenv("XDG_DATA_HOME", XDG_DATA_HOME, 1);
|
||||
setenv("XDG_CONFIG_HOME", xdg_config_home, 1);
|
||||
setenv("XDG_DATA_HOME", xdg_data_home, 1);
|
||||
|
||||
_cleanup_dirs();
|
||||
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
#ifndef __H_PROFTEST
|
||||
#define __H_PROFTEST
|
||||
|
||||
#define XDG_CONFIG_HOME "./tests/functionaltests/files/xdg_config_home"
|
||||
#define XDG_DATA_HOME "./tests/functionaltests/files/xdg_data_home"
|
||||
/*
|
||||
* XDG paths are dynamic and generated per-test based on stub_port.
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Adjust comment so it reflects current state of things, not the change that has been made (E.g. "XDG paths are used for..."). Adjust comment so it reflects current state of things, not the change that has been made (E.g. "XDG paths are used for...").
jabber.developer2
commented
Corrected Corrected
|
||||
* Each test instance uses unique directories (./tests/functionaltests/files/{port}/xdg_*)
|
||||
* to allow parallel test execution without file conflicts.
|
||||
*/
|
||||
extern char xdg_config_home[256];
|
||||
extern char xdg_data_home[256];
|
||||
|
||||
extern int stub_port;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user
Can we make it a loop for extensibility?
Corrected