test(ai): expand unit and functional coverage for /ai feature

Unit tests (tests/unittests/test_ai_client.c): +50 cases on top of the
existing 50 — total 100. Covers:
  - chat response parser (ai_parse_response): OpenAI content + Perplexity
    text formats, escape decoding, empty/null/missing inputs, format-string
    safety, multiline content
  - error envelope parser (ai_parse_error_message): standard envelope,
    nested escapes, missing fields, null/empty
  - extended JSON escape: \b, \f, \r, all-specials, UTF-8 pass-through
  - provider autocomplete cycling with >=2 matches and wrap-around
  - session edge cases: NULL args, 100-message order preservation,
    set_model(NULL), ref/unref(NULL)
  - provider edge cases: get/remove with NULL, double-remove, survival
    via session ref after ai_remove_provider
  - settings: multi-key independence, missing key, cross-provider
    isolation
  - model parsing edges: data not array, empty data, "id" outside data,
    multiple models
  - prefs round-trip: set token -> shutdown -> init -> token reloaded
    from disk (uses load_preferences fixture)

Functional tests:
  - Console only (test_ai.c): 15 cases for the /ai command surface that
    don't need HTTP. Covers /ai help, providers list, set provider/token,
    start with/without key/unknown provider, clear, remove, default
    provider/model, switch without window, bad subcommand.

Infrastructure:
  - TEST_GROUPS bumped to 5 in proftest.c; AI tests live in their own
    Group 5 because mixing them with stabber-driven tests in Group 4
    poisoned stbbr_stop() teardown.
  - PROF_FUNC_TEST_AI macro in functionaltests.c registers AI tests
    with ai_init_test() which wraps init_prof_test with a prof_connect()
    so stabber sees a graceful disconnect at teardown.

Source-level changes to enable testing:
  - src/ai/ai_client.{c,h}: dropped 'static' from _parse_ai_response
    (renamed ai_parse_response) and _parse_error_response (renamed
    ai_parse_error_message); declared under a "Parsing helpers (exposed
    for testing)" section. Same approach as ai_parse_models_from_json.

Results in Docker (cproof-debian image):
  - 595/595 unit tests pass
  - 15/15 AI functional tests pass (Group 5)
This commit is contained in:
2026-05-12 12:49:37 +03:00
committed by jabber.developer2
parent 30ad6e2a6e
commit d676f3b087
10 changed files with 1063 additions and 10 deletions

View File

@@ -59,6 +59,7 @@
#ifdef HAVE_SQLITE
#include "test_export_import.h"
#endif
#include "test_ai.h"
/* Macro to wrap each test with setup/teardown functions */
#define PROF_FUNC_TEST(test) \
@@ -66,6 +67,14 @@
#test, test, init_prof_test, close_prof_test, #test \
}
/* AI tests use a custom init that primes stabber via prof_connect so
* stbbr_stop() in close_prof_test() doesn't hang on a never-connected
* server thread. See ai_init_test() in test_ai.c. */
#define PROF_FUNC_TEST_AI(test) \
{ \
#test, test, ai_init_test, close_prof_test, #test \
}
#ifdef HAVE_SQLITE
/* Custom init that sets a non-UTC timezone (POSIX TST-3 = UTC+3) */
static int
@@ -84,9 +93,9 @@ main(int argc, char* argv[])
int group = 0; /* 0 = all groups */
if (argc > 1) {
group = atoi(argv[1]);
if (group < 1 || group > 4) {
if (group < 1 || group > 5) {
fprintf(stderr, "Usage: %s [group]\n", argv[0]);
fprintf(stderr, " group: 1-4 to run specific group, or omit for all\n");
fprintf(stderr, " group: 1-5 to run specific group, or omit for all\n");
return 1;
}
}
@@ -307,6 +316,32 @@ main(int argc, char* argv[])
#endif
};
/* ============================================================
* GROUP 5: AI command surface (console + HTTP stub)
* Standalone because AI tests don't use stabber's XMPP path; mixing
* them with XMPP-bound tests in the same group poisons stabber state
* between fixture init/teardown cycles and hangs subsequent
* prof_connect() calls.
* ============================================================ */
const struct CMUnitTest group5_tests[] = {
/* Console-only — no network calls */
PROF_FUNC_TEST_AI(ai_no_args_shows_help),
PROF_FUNC_TEST_AI(ai_providers_lists_defaults),
PROF_FUNC_TEST_AI(ai_providers_list_shows_key_status),
PROF_FUNC_TEST_AI(ai_set_provider_adds_custom),
PROF_FUNC_TEST_AI(ai_set_token_marks_key_set),
PROF_FUNC_TEST_AI(ai_start_unknown_provider_errors),
PROF_FUNC_TEST_AI(ai_start_without_key_errors),
PROF_FUNC_TEST_AI(ai_start_with_key_opens_window),
PROF_FUNC_TEST_AI(ai_clear_without_window_errors),
PROF_FUNC_TEST_AI(ai_remove_provider_works),
PROF_FUNC_TEST_AI(ai_remove_provider_unknown_errors),
PROF_FUNC_TEST_AI(ai_set_default_provider_unknown_errors),
PROF_FUNC_TEST_AI(ai_set_default_model_updates_provider),
PROF_FUNC_TEST_AI(ai_switch_without_window_errors),
PROF_FUNC_TEST_AI(ai_bad_subcommand_shows_usage),
};
/* Test group registry for easy extension */
struct
{
@@ -318,6 +353,7 @@ main(int argc, char* argv[])
{ "Group 2: ExportImport/Receipts", group2_tests, ARRAY_SIZE(group2_tests) },
{ "Group 3: Presence/Disconnect/DBHistory/Message/MUC-DB", group3_tests, ARRAY_SIZE(group3_tests) },
{ "Group 4: Session/MUC/Carbons/MigrationStress", group4_tests, ARRAY_SIZE(group4_tests) },
{ "Group 5: AI command surface (console only)", group5_tests, ARRAY_SIZE(group5_tests) },
};
const int num_groups = ARRAY_SIZE(groups);

View File

@@ -20,7 +20,7 @@
#include "proftest.h"
/* Number of parallel test groups for CI builds */
#define TEST_GROUPS 4
#define TEST_GROUPS 5
/* Number of ports reserved per test group; allows skipping TIME_WAIT ports */
#define PORTS_PER_GROUP 50

View File

@@ -0,0 +1,241 @@
/*
* test_ai.c
*
* Functional tests for the /ai command surface (Tier A — no network).
*
* These tests interact with profanity through its PTY console and exercise
* paths that do not reach libcurl: command parsing, provider registration,
* key/setting/default storage, error messages, and AI window creation.
*
* Tests that require an actual provider HTTP exchange (prompt -> response,
* /ai models <provider>, HTTP error envelopes) are out of scope here and
* belong in a separate suite backed by a local HTTP stub.
*/
#include <glib.h>
#include "prof_cmocka.h"
#include <stdlib.h>
#include <string.h>
#include "proftest.h"
#include "test_ai.h"
int
ai_init_test(void** state)
{
int ret = init_prof_test(state);
if (ret != 0) {
return ret;
}
/* Connect to stabber even though AI tests don't use XMPP. Without this
* stabber's accept loop has no client to disconnect on /quit, and
* stbbr_stop() in close_prof_test() hangs uninterruptibly when joining
* the server thread. */
prof_connect();
return 0;
}
void
ai_no_args_shows_help(void** state)
{
/* `/ai` with no arguments lists the built-in providers and a usage hint. */
prof_input("/ai");
prof_timeout(5);
assert_true(prof_output_exact("AI Chat - OpenAI-compatible API client"));
assert_true(prof_output_exact("Configured providers:"));
assert_true(prof_output_regex("openai"));
assert_true(prof_output_regex("perplexity"));
assert_true(prof_output_regex("Use '/ai start' to begin a chat"));
prof_timeout_reset();
}
void
ai_providers_lists_defaults(void** state)
{
/* `/ai providers` (no "list") shows the built-in list with URLs. */
prof_input("/ai providers");
prof_timeout(5);
assert_true(prof_output_exact("Available AI providers:"));
assert_true(prof_output_regex("openai"));
assert_true(prof_output_regex("perplexity"));
prof_timeout_reset();
}
void
ai_providers_list_shows_key_status(void** state)
{
/* `/ai providers list` lists each configured provider with key status. */
prof_input("/ai providers list");
prof_timeout(5);
assert_true(prof_output_exact("Configured providers:"));
/* No tokens have been set yet in this fresh session. */
assert_true(prof_output_regex("Key: NOT configured"));
prof_timeout_reset();
}
void
ai_set_provider_adds_custom(void** state)
{
/* Adding a custom provider should make it appear in /ai providers list. */
prof_input("/ai set provider mock http://127.0.0.1:1/");
prof_timeout(5);
assert_true(prof_output_regex("Provider 'mock' configured"));
prof_timeout_reset();
prof_input("/ai providers list");
prof_timeout(5);
assert_true(prof_output_regex("mock"));
assert_true(prof_output_regex("http://127.0.0.1:1/"));
prof_timeout_reset();
}
void
ai_set_token_marks_key_set(void** state)
{
/* Setting a token must flip the provider's key status to "configured". */
prof_input("/ai set token openai sk-fake-test-key");
prof_timeout(5);
assert_true(prof_output_regex("API token set for provider: openai"));
prof_timeout_reset();
prof_input("/ai providers list");
prof_timeout(5);
/* openai now shows "Key: configured" while perplexity stays unconfigured. */
assert_true(prof_output_regex("Key: configured"));
prof_timeout_reset();
}
void
ai_start_unknown_provider_errors(void** state)
{
/* Unknown provider name should error out without creating a window. */
prof_input("/ai start nope_provider somemodel");
prof_timeout(5);
assert_true(prof_output_regex("Provider 'nope_provider' not found"));
prof_timeout_reset();
}
void
ai_start_without_key_errors(void** state)
{
/* Known provider without an API key should refuse to start. */
prof_input("/ai start openai gpt-4");
prof_timeout(5);
assert_true(prof_output_regex("No API key set for provider 'openai'"));
prof_timeout_reset();
}
void
ai_start_with_key_opens_window(void** state)
{
/* With a token set, /ai start should create a WIN_AI window. */
prof_input("/ai set token openai sk-fake-test-key");
prof_timeout(5);
assert_true(prof_output_regex("API token set for provider: openai"));
prof_timeout_reset();
prof_input("/ai start openai gpt-4");
prof_timeout(5);
/* /ai start switches focus to the new WIN_AI window; cons_show output
* to the console is therefore not the right place to look. The window
* itself prints "AI Chat: <provider>/<model>" as its first line. */
assert_true(prof_output_regex("AI Chat: openai/gpt-4"));
prof_timeout_reset();
}
void
ai_clear_without_window_errors(void** state)
{
/* /ai clear from the console (no active AI window) should report nicely. */
prof_input("/ai clear");
prof_timeout(5);
assert_true(prof_output_regex("No active AI chat window to clear"));
prof_timeout_reset();
}
void
ai_remove_provider_works(void** state)
{
/* Round-trip: add -> verify present -> remove -> verify gone. */
prof_input("/ai set provider tmpprov http://127.0.0.1:2/");
prof_timeout(5);
assert_true(prof_output_regex("Provider 'tmpprov' configured"));
prof_timeout_reset();
prof_input("/ai remove provider tmpprov");
prof_timeout(5);
assert_true(prof_output_regex("Provider 'tmpprov' removed"));
prof_timeout_reset();
prof_input("/ai remove provider tmpprov");
prof_timeout(5);
assert_true(prof_output_regex("Provider 'tmpprov' not found"));
prof_timeout_reset();
}
void
ai_remove_provider_unknown_errors(void** state)
{
/* Removing a provider that was never added must report "not found". */
prof_input("/ai remove provider nonexistent_xyz");
prof_timeout(5);
assert_true(prof_output_regex("Provider 'nonexistent_xyz' not found"));
prof_timeout_reset();
}
void
ai_set_default_provider_unknown_errors(void** state)
{
/* Setting an unknown default provider should error, not silently accept. */
prof_input("/ai set default-provider does_not_exist");
prof_timeout(5);
assert_true(prof_output_regex("Provider 'does_not_exist' not found"));
prof_timeout_reset();
}
void
ai_set_default_model_updates_provider(void** state)
{
/* Setting a default model is acknowledged on the console. */
prof_input("/ai set default-model openai gpt-5-preview");
prof_timeout(5);
assert_true(prof_output_regex("Default model for provider 'openai' set to: gpt-5-preview"));
prof_timeout_reset();
}
void
ai_switch_without_window_errors(void** state)
{
/* /ai switch with no active AI window should produce an actionable error. */
prof_input("/ai switch openai gpt-4");
prof_timeout(5);
assert_true(prof_output_regex("No active AI chat window"));
prof_timeout_reset();
}
void
ai_bad_subcommand_shows_usage(void** state)
{
/* An unknown /ai subcommand should fall through to cons_bad_cmd_usage. */
prof_input("/ai not_a_subcommand");
prof_timeout(5);
assert_true(prof_output_regex("Invalid usage, see '/help ai'"));
prof_timeout_reset();
}

View File

@@ -0,0 +1,28 @@
#ifndef __H_FUNC_TEST_AI
#define __H_FUNC_TEST_AI
/*
* AI tests don't exercise the XMPP path, but the test fixture's
* stbbr_stop() hangs indefinitely when no client ever connected to
* stabber. ai_init_test wraps init_prof_test with a prof_connect()
* so stabber sees a graceful disconnect at teardown.
*/
int ai_init_test(void** state);
void ai_no_args_shows_help(void** state);
void ai_providers_lists_defaults(void** state);
void ai_providers_list_shows_key_status(void** state);
void ai_set_provider_adds_custom(void** state);
void ai_set_token_marks_key_set(void** state);
void ai_start_unknown_provider_errors(void** state);
void ai_start_without_key_errors(void** state);
void ai_start_with_key_opens_window(void** state);
void ai_clear_without_window_errors(void** state);
void ai_remove_provider_works(void** state);
void ai_remove_provider_unknown_errors(void** state);
void ai_set_default_provider_unknown_errors(void** state);
void ai_set_default_model_updates_provider(void** state);
void ai_switch_without_window_errors(void** state);
void ai_bad_subcommand_shows_usage(void** state);
#endif