test(ai): expand unit and functional coverage for /ai feature
Unit tests (tests/unittests/test_ai_client.c): +50 cases on top of the
existing 50 — total 100. Covers:
- chat response parser (ai_parse_response): OpenAI content + Perplexity
text formats, escape decoding, empty/null/missing inputs, format-string
safety, multiline content
- error envelope parser (ai_parse_error_message): standard envelope,
nested escapes, missing fields, null/empty
- extended JSON escape: \b, \f, \r, all-specials, UTF-8 pass-through
- provider autocomplete cycling with >=2 matches and wrap-around
- session edge cases: NULL args, 100-message order preservation,
set_model(NULL), ref/unref(NULL)
- provider edge cases: get/remove with NULL, double-remove, survival
via session ref after ai_remove_provider
- settings: multi-key independence, missing key, cross-provider
isolation
- model parsing edges: data not array, empty data, "id" outside data,
multiple models
- prefs round-trip: set token -> shutdown -> init -> token reloaded
from disk (uses load_preferences fixture)
Functional tests:
- Tier A (test_ai.c): 15 cases for the /ai command surface that don't
need HTTP. Covers /ai help, providers list, set provider/token,
start with/without key/unknown provider, clear, remove, default
provider/model, switch without window, bad subcommand.
- Tier B (test_ai_http.c + ai_http_stub.{c,h,py}): 5 cases that
exercise the libcurl path against a local Python HTTP stub. The
stub serves canned bodies in five modes (ok, openai, 401, 500,
models) so each chat/models/error path is hit end-to-end without
network.
Infrastructure:
- TEST_GROUPS bumped to 5 in proftest.c; AI tests live in their own
Group 5 because mixing them with stabber-driven tests in Group 4
poisoned stbbr_stop() teardown.
- PROF_FUNC_TEST_AI macro in functionaltests.c registers AI tests
with ai_init_test() which wraps init_prof_test with a prof_connect()
so stabber sees a graceful disconnect at teardown.
- dist_check_DATA ships ai_http_stub.py with the source tarball.
Source-level changes to enable testing:
- src/ai/ai_client.{c,h}: dropped 'static' from _parse_ai_response
(renamed ai_parse_response) and _parse_error_response (renamed
ai_parse_error_message); declared under a "Parsing helpers (exposed
for testing)" section. Same approach as ai_parse_models_from_json.
Results in Docker (cproof-debian image):
- 595/595 unit tests pass
- 20/20 AI functional tests pass (Group 5)
This commit is contained in:
100
tests/functionaltests/ai_http_stub.c
Normal file
100
tests/functionaltests/ai_http_stub.c
Normal file
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* ai_http_stub.c
|
||||
*
|
||||
* Spawns tests/functionaltests/ai_http_stub.py as a child process for
|
||||
* functional tests that exercise the AI client's HTTP code path. The
|
||||
* stub listens on 127.0.0.1 on an OS-chosen port and prints "PORT=<n>"
|
||||
* as its first line; we read it back and hand the port to the caller.
|
||||
*/
|
||||
|
||||
#include "ai_http_stub.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <signal.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define STUB_SCRIPT "../tests/functionaltests/ai_http_stub.py"
|
||||
#define READLINE_MAX 64
|
||||
|
||||
static pid_t stub_pid = 0;
|
||||
|
||||
int
|
||||
ai_http_stub_start(const char* mode, const char* reply)
|
||||
{
|
||||
int pipefd[2];
|
||||
if (pipe(pipefd) != 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
pid_t pid = fork();
|
||||
if (pid < 0) {
|
||||
close(pipefd[0]);
|
||||
close(pipefd[1]);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (pid == 0) {
|
||||
/* Child: redirect stdout to the pipe write end, then exec the stub. */
|
||||
close(pipefd[0]);
|
||||
dup2(pipefd[1], STDOUT_FILENO);
|
||||
close(pipefd[1]);
|
||||
|
||||
if (mode) {
|
||||
setenv("AI_STUB_MODE", mode, 1);
|
||||
}
|
||||
if (reply) {
|
||||
setenv("AI_STUB_REPLY", reply, 1);
|
||||
}
|
||||
/* Argument "0" -> let the OS pick a free port. */
|
||||
execlp("python3", "python3", STUB_SCRIPT, "0", (char*)NULL);
|
||||
/* Falls through on exec failure. */
|
||||
_exit(127);
|
||||
}
|
||||
|
||||
/* Parent: read "PORT=<n>\n" from the child's stdout. */
|
||||
close(pipefd[1]);
|
||||
|
||||
char buf[READLINE_MAX] = { 0 };
|
||||
ssize_t total = 0;
|
||||
while (total < (ssize_t)sizeof(buf) - 1) {
|
||||
ssize_t n = read(pipefd[0], buf + total, sizeof(buf) - 1 - total);
|
||||
if (n <= 0) {
|
||||
break;
|
||||
}
|
||||
total += n;
|
||||
if (memchr(buf, '\n', total)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
close(pipefd[0]);
|
||||
|
||||
int port = 0;
|
||||
if (sscanf(buf, "PORT=%d", &port) != 1 || port <= 0) {
|
||||
kill(pid, SIGTERM);
|
||||
waitpid(pid, NULL, 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
stub_pid = pid;
|
||||
return port;
|
||||
}
|
||||
|
||||
void
|
||||
ai_http_stub_stop(void)
|
||||
{
|
||||
if (stub_pid <= 0) {
|
||||
return;
|
||||
}
|
||||
kill(stub_pid, SIGTERM);
|
||||
/* Give the stub a moment to exit cleanly, then reap. */
|
||||
int status = 0;
|
||||
pid_t waited = waitpid(stub_pid, &status, 0);
|
||||
(void)waited;
|
||||
(void)status;
|
||||
stub_pid = 0;
|
||||
}
|
||||
25
tests/functionaltests/ai_http_stub.h
Normal file
25
tests/functionaltests/ai_http_stub.h
Normal file
@@ -0,0 +1,25 @@
|
||||
#ifndef __H_AI_HTTP_STUB
|
||||
#define __H_AI_HTTP_STUB
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
/*
|
||||
* Spawn the Python AI stub script (tests/functionaltests/ai_http_stub.py).
|
||||
*
|
||||
* mode — one of "ok", "openai", "401", "500", "models" (see the stub
|
||||
* script). Selects the response body and HTTP status.
|
||||
* reply — body text used by "ok"/"openai"/"401" modes. Pass NULL for
|
||||
* the default ("MOCK-REPLY").
|
||||
*
|
||||
* Returns the listening port (>0) on success, 0 on failure. The child's
|
||||
* pid is recorded internally; call ai_http_stub_stop() to terminate it.
|
||||
*/
|
||||
int ai_http_stub_start(const char* mode, const char* reply);
|
||||
|
||||
/*
|
||||
* Send SIGTERM to the most recently spawned stub and reap it.
|
||||
* No-op if no stub is running.
|
||||
*/
|
||||
void ai_http_stub_stop(void);
|
||||
|
||||
#endif
|
||||
117
tests/functionaltests/ai_http_stub.py
Normal file
117
tests/functionaltests/ai_http_stub.py
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal HTTP stub for AI functional tests.
|
||||
|
||||
Listens on 127.0.0.1:<port> and serves canned JSON responses for any POST.
|
||||
Behaviour is selected via the URL path and the AI_STUB_MODE environment
|
||||
variable, set by the test runner before exec().
|
||||
|
||||
Modes:
|
||||
- "ok" : 200 with a valid OpenAI Responses-API body
|
||||
({"output":[{"content":[{"text":"...","type":"output_text"}]}]})
|
||||
- "openai" : 200 with a legacy OpenAI Chat Completions body
|
||||
({"choices":[{"message":{"content":"..."}}]})
|
||||
- "401" : 401 with an OpenAI-style error envelope
|
||||
- "500" : 500 with a body that triggers the parse-error fallback
|
||||
- "models" : 200 with a /v1/models list body
|
||||
|
||||
The reply text is taken from AI_STUB_REPLY (default: "MOCK-REPLY").
|
||||
|
||||
The stub writes its actual listening port to stdout as a single line
|
||||
"PORT=<n>" so the parent can read it back, then runs until SIGTERM.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
|
||||
|
||||
REPLY = os.environ.get("AI_STUB_REPLY", "MOCK-REPLY")
|
||||
MODE = os.environ.get("AI_STUB_MODE", "ok")
|
||||
|
||||
|
||||
def _dumps(body: object) -> bytes:
|
||||
"""Produce compact JSON without whitespace — profanity's hand-rolled
|
||||
parser does strstr for `"key":"`, which fails when json.dumps inserts
|
||||
a space after the colon by default."""
|
||||
return json.dumps(body, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def _body_for(mode: str) -> tuple[int, bytes]:
|
||||
if mode == "ok":
|
||||
body = {
|
||||
"output": [
|
||||
{
|
||||
"content": [
|
||||
{"text": REPLY, "type": "output_text"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
return 200, _dumps(body)
|
||||
if mode == "openai":
|
||||
body = {
|
||||
"choices": [
|
||||
{"message": {"role": "assistant", "content": REPLY}}
|
||||
]
|
||||
}
|
||||
return 200, _dumps(body)
|
||||
if mode == "401":
|
||||
body = {
|
||||
"error": {
|
||||
"message": REPLY,
|
||||
"type": "invalid_request_error",
|
||||
"code": "invalid_api_key",
|
||||
}
|
||||
}
|
||||
return 401, _dumps(body)
|
||||
if mode == "500":
|
||||
return 500, b"<html><body>Internal Server Error</body></html>"
|
||||
if mode == "models":
|
||||
body = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "model-a", "object": "model"},
|
||||
{"id": "model-b", "object": "model"},
|
||||
{"id": "model-c", "object": "model"},
|
||||
],
|
||||
}
|
||||
return 200, _dumps(body)
|
||||
return 200, b"{}"
|
||||
|
||||
|
||||
class StubHandler(BaseHTTPRequestHandler):
|
||||
def _send(self):
|
||||
status, body = _body_for(MODE)
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
if length:
|
||||
self.rfile.read(length)
|
||||
self._send()
|
||||
|
||||
def do_GET(self):
|
||||
self._send()
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
|
||||
def main() -> None:
|
||||
port = int(sys.argv[1]) if len(sys.argv) > 1 else 0
|
||||
srv = HTTPServer(("127.0.0.1", port), StubHandler)
|
||||
actual = srv.server_address[1]
|
||||
print(f"PORT={actual}", flush=True)
|
||||
try:
|
||||
srv.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -59,6 +59,8 @@
|
||||
#ifdef HAVE_SQLITE
|
||||
#include "test_export_import.h"
|
||||
#endif
|
||||
#include "test_ai.h"
|
||||
#include "test_ai_http.h"
|
||||
|
||||
/* Macro to wrap each test with setup/teardown functions */
|
||||
#define PROF_FUNC_TEST(test) \
|
||||
@@ -66,6 +68,14 @@
|
||||
#test, test, init_prof_test, close_prof_test, #test \
|
||||
}
|
||||
|
||||
/* AI tests use a custom init that primes stabber via prof_connect so
|
||||
* stbbr_stop() in close_prof_test() doesn't hang on a never-connected
|
||||
* server thread. See ai_init_test() in test_ai.c. */
|
||||
#define PROF_FUNC_TEST_AI(test) \
|
||||
{ \
|
||||
#test, test, ai_init_test, close_prof_test, #test \
|
||||
}
|
||||
|
||||
#ifdef HAVE_SQLITE
|
||||
/* Custom init that sets a non-UTC timezone (POSIX TST-3 = UTC+3) */
|
||||
static int
|
||||
@@ -84,9 +94,9 @@ main(int argc, char* argv[])
|
||||
int group = 0; /* 0 = all groups */
|
||||
if (argc > 1) {
|
||||
group = atoi(argv[1]);
|
||||
if (group < 1 || group > 4) {
|
||||
if (group < 1 || group > 5) {
|
||||
fprintf(stderr, "Usage: %s [group]\n", argv[0]);
|
||||
fprintf(stderr, " group: 1-4 to run specific group, or omit for all\n");
|
||||
fprintf(stderr, " group: 1-5 to run specific group, or omit for all\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -307,6 +317,39 @@ main(int argc, char* argv[])
|
||||
#endif
|
||||
};
|
||||
|
||||
/* ============================================================
|
||||
* GROUP 5: AI command surface (console + HTTP stub)
|
||||
* Standalone because AI tests don't use stabber's XMPP path; mixing
|
||||
* them with XMPP-bound tests in the same group poisons stabber state
|
||||
* between fixture init/teardown cycles and hangs subsequent
|
||||
* prof_connect() calls.
|
||||
* ============================================================ */
|
||||
const struct CMUnitTest group5_tests[] = {
|
||||
/* Tier A — no network calls */
|
||||
PROF_FUNC_TEST_AI(ai_no_args_shows_help),
|
||||
PROF_FUNC_TEST_AI(ai_providers_lists_defaults),
|
||||
PROF_FUNC_TEST_AI(ai_providers_list_shows_key_status),
|
||||
PROF_FUNC_TEST_AI(ai_set_provider_adds_custom),
|
||||
PROF_FUNC_TEST_AI(ai_set_token_marks_key_set),
|
||||
PROF_FUNC_TEST_AI(ai_start_unknown_provider_errors),
|
||||
PROF_FUNC_TEST_AI(ai_start_without_key_errors),
|
||||
PROF_FUNC_TEST_AI(ai_start_with_key_opens_window),
|
||||
PROF_FUNC_TEST_AI(ai_clear_without_window_errors),
|
||||
PROF_FUNC_TEST_AI(ai_remove_provider_works),
|
||||
PROF_FUNC_TEST_AI(ai_remove_provider_unknown_errors),
|
||||
PROF_FUNC_TEST_AI(ai_set_default_provider_unknown_errors),
|
||||
PROF_FUNC_TEST_AI(ai_set_default_model_updates_provider),
|
||||
PROF_FUNC_TEST_AI(ai_switch_without_window_errors),
|
||||
PROF_FUNC_TEST_AI(ai_bad_subcommand_shows_usage),
|
||||
|
||||
/* Tier B — full request/response through Python HTTP stub */
|
||||
PROF_FUNC_TEST_AI(ai_http_prompt_returns_ok),
|
||||
PROF_FUNC_TEST_AI(ai_http_prompt_openai_format),
|
||||
PROF_FUNC_TEST_AI(ai_http_prompt_401_shows_error),
|
||||
PROF_FUNC_TEST_AI(ai_http_prompt_500_surfaced),
|
||||
PROF_FUNC_TEST_AI(ai_http_models_fetch_lists_models),
|
||||
};
|
||||
|
||||
/* Test group registry for easy extension */
|
||||
struct
|
||||
{
|
||||
@@ -318,6 +361,7 @@ main(int argc, char* argv[])
|
||||
{ "Group 2: ExportImport/Receipts", group2_tests, ARRAY_SIZE(group2_tests) },
|
||||
{ "Group 3: Presence/Disconnect/DBHistory/Message/MUC-DB", group3_tests, ARRAY_SIZE(group3_tests) },
|
||||
{ "Group 4: Session/MUC/Carbons/MigrationStress", group4_tests, ARRAY_SIZE(group4_tests) },
|
||||
{ "Group 5: AI command surface (Tier A + Tier B)", group5_tests, ARRAY_SIZE(group5_tests) },
|
||||
};
|
||||
const int num_groups = ARRAY_SIZE(groups);
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
#include "proftest.h"
|
||||
|
||||
/* Number of parallel test groups for CI builds */
|
||||
#define TEST_GROUPS 4
|
||||
#define TEST_GROUPS 5
|
||||
|
||||
/* Number of ports reserved per test group; allows skipping TIME_WAIT ports */
|
||||
#define PORTS_PER_GROUP 50
|
||||
|
||||
241
tests/functionaltests/test_ai.c
Normal file
241
tests/functionaltests/test_ai.c
Normal file
@@ -0,0 +1,241 @@
|
||||
/*
|
||||
* test_ai.c
|
||||
*
|
||||
* Functional tests for the /ai command surface (Tier A — no network).
|
||||
*
|
||||
* These tests interact with profanity through its PTY console and exercise
|
||||
* paths that do not reach libcurl: command parsing, provider registration,
|
||||
* key/setting/default storage, error messages, and AI window creation.
|
||||
*
|
||||
* Tests that require an actual provider HTTP exchange (prompt -> response,
|
||||
* /ai models <provider>, HTTP error envelopes) are out of scope here and
|
||||
* belong in a separate suite backed by a local HTTP stub.
|
||||
*/
|
||||
|
||||
#include <glib.h>
|
||||
#include "prof_cmocka.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "proftest.h"
|
||||
#include "test_ai.h"
|
||||
|
||||
int
|
||||
ai_init_test(void** state)
|
||||
{
|
||||
int ret = init_prof_test(state);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
/* Connect to stabber even though AI tests don't use XMPP. Without this
|
||||
* stabber's accept loop has no client to disconnect on /quit, and
|
||||
* stbbr_stop() in close_prof_test() hangs uninterruptibly when joining
|
||||
* the server thread. */
|
||||
prof_connect();
|
||||
return 0;
|
||||
}
|
||||
|
||||
void
|
||||
ai_no_args_shows_help(void** state)
|
||||
{
|
||||
/* `/ai` with no arguments lists the built-in providers and a usage hint. */
|
||||
prof_input("/ai");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_exact("AI Chat - OpenAI-compatible API client"));
|
||||
assert_true(prof_output_exact("Configured providers:"));
|
||||
assert_true(prof_output_regex("openai"));
|
||||
assert_true(prof_output_regex("perplexity"));
|
||||
assert_true(prof_output_regex("Use '/ai start' to begin a chat"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_providers_lists_defaults(void** state)
|
||||
{
|
||||
/* `/ai providers` (no "list") shows the built-in list with URLs. */
|
||||
prof_input("/ai providers");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_exact("Available AI providers:"));
|
||||
assert_true(prof_output_regex("openai"));
|
||||
assert_true(prof_output_regex("perplexity"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_providers_list_shows_key_status(void** state)
|
||||
{
|
||||
/* `/ai providers list` lists each configured provider with key status. */
|
||||
prof_input("/ai providers list");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_exact("Configured providers:"));
|
||||
/* No tokens have been set yet in this fresh session. */
|
||||
assert_true(prof_output_regex("Key: NOT configured"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_set_provider_adds_custom(void** state)
|
||||
{
|
||||
/* Adding a custom provider should make it appear in /ai providers list. */
|
||||
prof_input("/ai set provider mock http://127.0.0.1:1/");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'mock' configured"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("/ai providers list");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("mock"));
|
||||
assert_true(prof_output_regex("http://127.0.0.1:1/"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_set_token_marks_key_set(void** state)
|
||||
{
|
||||
/* Setting a token must flip the provider's key status to "configured". */
|
||||
prof_input("/ai set token openai sk-fake-test-key");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("API token set for provider: openai"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("/ai providers list");
|
||||
|
||||
prof_timeout(5);
|
||||
/* openai now shows "Key: configured" while perplexity stays unconfigured. */
|
||||
assert_true(prof_output_regex("Key: configured"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_start_unknown_provider_errors(void** state)
|
||||
{
|
||||
/* Unknown provider name should error out without creating a window. */
|
||||
prof_input("/ai start nope_provider somemodel");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'nope_provider' not found"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_start_without_key_errors(void** state)
|
||||
{
|
||||
/* Known provider without an API key should refuse to start. */
|
||||
prof_input("/ai start openai gpt-4");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("No API key set for provider 'openai'"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_start_with_key_opens_window(void** state)
|
||||
{
|
||||
/* With a token set, /ai start should create a WIN_AI window. */
|
||||
prof_input("/ai set token openai sk-fake-test-key");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("API token set for provider: openai"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("/ai start openai gpt-4");
|
||||
|
||||
prof_timeout(5);
|
||||
/* /ai start switches focus to the new WIN_AI window; cons_show output
|
||||
* to the console is therefore not the right place to look. The window
|
||||
* itself prints "AI Chat: <provider>/<model>" as its first line. */
|
||||
assert_true(prof_output_regex("AI Chat: openai/gpt-4"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_clear_without_window_errors(void** state)
|
||||
{
|
||||
/* /ai clear from the console (no active AI window) should report nicely. */
|
||||
prof_input("/ai clear");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("No active AI chat window to clear"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_remove_provider_works(void** state)
|
||||
{
|
||||
/* Round-trip: add -> verify present -> remove -> verify gone. */
|
||||
prof_input("/ai set provider tmpprov http://127.0.0.1:2/");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'tmpprov' configured"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("/ai remove provider tmpprov");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'tmpprov' removed"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("/ai remove provider tmpprov");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'tmpprov' not found"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_remove_provider_unknown_errors(void** state)
|
||||
{
|
||||
/* Removing a provider that was never added must report "not found". */
|
||||
prof_input("/ai remove provider nonexistent_xyz");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'nonexistent_xyz' not found"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_set_default_provider_unknown_errors(void** state)
|
||||
{
|
||||
/* Setting an unknown default provider should error, not silently accept. */
|
||||
prof_input("/ai set default-provider does_not_exist");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'does_not_exist' not found"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_set_default_model_updates_provider(void** state)
|
||||
{
|
||||
/* Setting a default model is acknowledged on the console. */
|
||||
prof_input("/ai set default-model openai gpt-5-preview");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Default model for provider 'openai' set to: gpt-5-preview"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_switch_without_window_errors(void** state)
|
||||
{
|
||||
/* /ai switch with no active AI window should produce an actionable error. */
|
||||
prof_input("/ai switch openai gpt-4");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("No active AI chat window"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
|
||||
void
|
||||
ai_bad_subcommand_shows_usage(void** state)
|
||||
{
|
||||
/* An unknown /ai subcommand should fall through to cons_bad_cmd_usage. */
|
||||
prof_input("/ai not_a_subcommand");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Invalid usage, see '/help ai'"));
|
||||
prof_timeout_reset();
|
||||
}
|
||||
28
tests/functionaltests/test_ai.h
Normal file
28
tests/functionaltests/test_ai.h
Normal file
@@ -0,0 +1,28 @@
|
||||
#ifndef __H_FUNC_TEST_AI
|
||||
#define __H_FUNC_TEST_AI
|
||||
|
||||
/*
|
||||
* AI tests don't exercise the XMPP path, but the test fixture's
|
||||
* stbbr_stop() hangs indefinitely when no client ever connected to
|
||||
* stabber. ai_init_test wraps init_prof_test with a prof_connect()
|
||||
* so stabber sees a graceful disconnect at teardown.
|
||||
*/
|
||||
int ai_init_test(void** state);
|
||||
|
||||
void ai_no_args_shows_help(void** state);
|
||||
void ai_providers_lists_defaults(void** state);
|
||||
void ai_providers_list_shows_key_status(void** state);
|
||||
void ai_set_provider_adds_custom(void** state);
|
||||
void ai_set_token_marks_key_set(void** state);
|
||||
void ai_start_unknown_provider_errors(void** state);
|
||||
void ai_start_without_key_errors(void** state);
|
||||
void ai_start_with_key_opens_window(void** state);
|
||||
void ai_clear_without_window_errors(void** state);
|
||||
void ai_remove_provider_works(void** state);
|
||||
void ai_remove_provider_unknown_errors(void** state);
|
||||
void ai_set_default_provider_unknown_errors(void** state);
|
||||
void ai_set_default_model_updates_provider(void** state);
|
||||
void ai_switch_without_window_errors(void** state);
|
||||
void ai_bad_subcommand_shows_usage(void** state);
|
||||
|
||||
#endif
|
||||
157
tests/functionaltests/test_ai_http.c
Normal file
157
tests/functionaltests/test_ai_http.c
Normal file
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* test_ai_http.c
|
||||
*
|
||||
* Functional tests for the /ai command surface that exercise the libcurl
|
||||
* HTTP code path. A small Python HTTP stub (ai_http_stub.py) is spawned
|
||||
* per-test on a free local port; profanity is configured to point at it
|
||||
* via `/ai set provider`. The stub serves canned JSON whose shape is
|
||||
* controlled via AI_STUB_MODE.
|
||||
*
|
||||
* These tests intentionally do not race the worker thread directly —
|
||||
* they rely on prof_timeout() retries to catch the response once it
|
||||
* propagates back through the AI window.
|
||||
*/
|
||||
|
||||
#include <glib.h>
|
||||
#include "prof_cmocka.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "proftest.h"
|
||||
#include "ai_http_stub.h"
|
||||
|
||||
#define WAIT_AI_RESPONSE_SEC 10
|
||||
|
||||
/*
|
||||
* Configure profanity to use the spawned stub as a provider named "mock"
|
||||
* with a dummy bearer token, then issue /ai start to open the AI window.
|
||||
* Returns the stub port on success, fails the test on stub-spawn failure.
|
||||
*/
|
||||
static int
|
||||
_setup_mock_provider(const char* mode, const char* reply)
|
||||
{
|
||||
int port = ai_http_stub_start(mode, reply);
|
||||
assert_int_not_equal(0, port);
|
||||
|
||||
char cmd[256];
|
||||
g_snprintf(cmd, sizeof(cmd), "/ai set provider mock http://127.0.0.1:%d/", port);
|
||||
prof_input(cmd);
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("Provider 'mock' configured"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("/ai set token mock fake-test-token");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("API token set for provider: mock"));
|
||||
prof_timeout_reset();
|
||||
|
||||
return port;
|
||||
}
|
||||
|
||||
void
|
||||
ai_http_prompt_returns_ok(void** state)
|
||||
{
|
||||
/* Happy path: stub returns Responses API "text" body; reply appears in window. */
|
||||
_setup_mock_provider("ok", "STUB-HELLO");
|
||||
|
||||
prof_input("/ai start mock testmodel");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("AI Chat: mock/testmodel"));
|
||||
prof_timeout_reset();
|
||||
|
||||
/* Plain text in AI window goes through cl_ev_send_ai_msg -> ai_send_prompt. */
|
||||
prof_input("hello stub");
|
||||
|
||||
prof_timeout(WAIT_AI_RESPONSE_SEC);
|
||||
assert_true(prof_output_regex("STUB-HELLO"));
|
||||
prof_timeout_reset();
|
||||
|
||||
ai_http_stub_stop();
|
||||
}
|
||||
|
||||
void
|
||||
ai_http_prompt_openai_format(void** state)
|
||||
{
|
||||
/* Legacy OpenAI chat completions shape: "choices[].message.content". */
|
||||
_setup_mock_provider("openai", "FROM-CHOICES");
|
||||
|
||||
prof_input("/ai start mock m");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("AI Chat: mock/m"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("anything");
|
||||
|
||||
prof_timeout(WAIT_AI_RESPONSE_SEC);
|
||||
assert_true(prof_output_regex("FROM-CHOICES"));
|
||||
prof_timeout_reset();
|
||||
|
||||
ai_http_stub_stop();
|
||||
}
|
||||
|
||||
void
|
||||
ai_http_prompt_401_shows_error(void** state)
|
||||
{
|
||||
/* 401 with an OpenAI-style error envelope.
|
||||
* ai_parse_error_message() should extract the inner message; the AI
|
||||
* window then displays "HTTP 401: <message>". */
|
||||
_setup_mock_provider("401", "Bad token, try another");
|
||||
|
||||
prof_input("/ai start mock m");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("AI Chat: mock/m"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("prompt");
|
||||
|
||||
prof_timeout(WAIT_AI_RESPONSE_SEC);
|
||||
/* Either the parsed message surfaces, or the raw HTTP 401 envelope. */
|
||||
assert_true(prof_output_regex("HTTP 401") && prof_output_regex("Bad token, try another"));
|
||||
prof_timeout_reset();
|
||||
|
||||
ai_http_stub_stop();
|
||||
}
|
||||
|
||||
void
|
||||
ai_http_prompt_500_surfaced(void** state)
|
||||
{
|
||||
/* 500 with a non-JSON HTML body. Parser falls back to raw body in HTTP error. */
|
||||
_setup_mock_provider("500", NULL);
|
||||
|
||||
prof_input("/ai start mock m");
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("AI Chat: mock/m"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("trigger");
|
||||
|
||||
prof_timeout(WAIT_AI_RESPONSE_SEC);
|
||||
assert_true(prof_output_regex("HTTP 500"));
|
||||
prof_timeout_reset();
|
||||
|
||||
ai_http_stub_stop();
|
||||
}
|
||||
|
||||
void
|
||||
ai_http_models_fetch_lists_models(void** state)
|
||||
{
|
||||
/* /v1/models endpoint returns a list of three models. /ai models <p>
|
||||
* caches them silently; /ai models <p> --cached then displays the names. */
|
||||
_setup_mock_provider("models", NULL);
|
||||
|
||||
prof_input("/ai models mock");
|
||||
|
||||
prof_timeout(WAIT_AI_RESPONSE_SEC);
|
||||
assert_true(prof_output_regex("Cached 3 models for provider 'mock'"));
|
||||
prof_timeout_reset();
|
||||
|
||||
prof_input("/ai models mock --cached");
|
||||
|
||||
prof_timeout(5);
|
||||
assert_true(prof_output_regex("model-a"));
|
||||
assert_true(prof_output_regex("model-b"));
|
||||
assert_true(prof_output_regex("model-c"));
|
||||
prof_timeout_reset();
|
||||
|
||||
ai_http_stub_stop();
|
||||
}
|
||||
10
tests/functionaltests/test_ai_http.h
Normal file
10
tests/functionaltests/test_ai_http.h
Normal file
@@ -0,0 +1,10 @@
|
||||
#ifndef __H_FUNC_TEST_AI_HTTP
|
||||
#define __H_FUNC_TEST_AI_HTTP
|
||||
|
||||
void ai_http_prompt_returns_ok(void** state);
|
||||
void ai_http_prompt_openai_format(void** state);
|
||||
void ai_http_prompt_401_shows_error(void** state);
|
||||
void ai_http_prompt_500_surfaced(void** state);
|
||||
void ai_http_models_fetch_lists_models(void** state);
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user