feat: complete field parity, harden export/import, add tests

Export/Import improvements: - Replace pagination with direct SQL query (db_sqlite_get_all_chat) - Wrap import in SQL transaction with rollback on error - Add fsync before fclose in export for data safety - Sort merged output by timestamp with secondary key (stanza_id, from_jid) - Export archive_id and marked_read from SQLite (lossless migration) - Add progress indication every 500 messages during write/import - Expand dedup key body prefix from 64 to 256 chars - Fix g_slist_append O(n²) → g_slist_prepend + g_slist_reverse O(n) Field parity (to_jid, to_resource, marked_read): - Add fields to ff_parsed_line_t struct - Write/parse to:|to_res:|read: metadata tags in flatfile format - Pass to_resource through _ff_add_message and all callers - Add marked_read to ProfMessage struct with -1 default (unset) - Preserve fields across export/import round-trips Tests (19 new: 11 unit + 8 functional): - Unit: to_jid_and_marked_read, bracket_in_stanza_id, backslash_in_resource, mucpm_type, all_enc_types, crlf_handling, to_jid_special_chars, multiple_lines, parsed_line_free_null_safe, no_space_rejected, unclosed_bracket - Functional: export_idempotent_no_duplicates, export_lmc_correction_survives, switch_preserves_old_backend_data, export_all_contacts, import_double_dedup, verify_after_export, switch_backends_independent_messages, export_empty_contact - Rebalance test groups: move Chat Session from Group 3 to Group 4 (25/33/30/27 instead of 25/33/36/21) - Remove hardcoded test counts from group comments Man page: - Document /history switch sqlite|flatfile
2026-03-09 12:14:41 +03:00
parent d5d67592ab
commit e91e8403b0
16 changed files with 1037 additions and 146 deletions
--- a/tests/functionaltests/functionaltests.c
+++ b/tests/functionaltests/functionaltests.c
@@ -16,9 +16,9 @@
 *
 * Tests are organized into groups for better maintainability and parallel execution:
 *   Group 1: Connect, Ping, Rooms, Software, Last Activity, Autoping
- *   Group 2: Message, Receipts, Roster, DB History (+2 with SQLite)
- *   Group 3: Chat Session, Presence, Disconnect, Disco
- *   Group 4: MUC, Carbons
+ *   Group 2: Message, Receipts, Roster, DB History (+export/import with SQLite)
+ *   Group 3: Presence, Disconnect, Disco
+ *   Group 4: Chat Session, MUC, Carbons
 *
 * Parallel execution:
 *   ./functionaltests        - run all tests sequentially
@@ -85,7 +85,6 @@ main(int argc, char* argv[])
    /* ============================================================
     * GROUP 1: Connect, Ping, Rooms, Software, Last Activity
     * Basic XMPP session establishment and server queries
-     * (19 tests)
     * ============================================================ */
    const struct CMUnitTest group1_tests[] = {
        /* Connection tests - verify login, roster, bookmarks */
@@ -131,13 +130,20 @@ main(int argc, char* argv[])
    /* ============================================================
     * GROUP 2: Message, Receipts, Roster, DB History
     * Core messaging and contact management
-     * (25 tests with SQLite, 23 without)
     * ============================================================ */
    const struct CMUnitTest group2_tests[] = {
 #ifdef HAVE_SQLITE
        /* Export/Import — cross-backend migration (SQLite ↔ flat-file) */
        PROF_FUNC_TEST(export_sqlite_to_flatfile),
        PROF_FUNC_TEST(import_flatfile_to_sqlite),
+        PROF_FUNC_TEST(export_idempotent_no_duplicates),
+        PROF_FUNC_TEST(export_lmc_correction_survives),
+        PROF_FUNC_TEST(switch_preserves_old_backend_data),
+        PROF_FUNC_TEST(export_all_contacts),
+        PROF_FUNC_TEST(import_double_dedup),
+        PROF_FUNC_TEST(verify_after_export),
+        PROF_FUNC_TEST(switch_backends_independent_messages),
+        PROF_FUNC_TEST(export_empty_contact),
 #endif

        /* Basic message send/receive */
@@ -173,19 +179,10 @@ main(int argc, char* argv[])
    };

    /* ============================================================
-     * GROUP 3: Chat Session, Presence, Disconnect
-     * Session routing, status management, clean teardown
-     * (22 tests)
+     * GROUP 3: Presence, Disconnect, Disco
+     * Status management, clean teardown, service discovery
     * ============================================================ */
    const struct CMUnitTest group3_tests[] = {
-        /* Chat session management - bare/full JID routing */
-        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_offline),
-        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_online),
-        PROF_FUNC_TEST(sends_message_to_fulljid_when_received_from_fulljid),
-        PROF_FUNC_TEST(sends_subsequent_messages_to_fulljid),
-        PROF_FUNC_TEST(resets_to_barejid_after_presence_received),
-        PROF_FUNC_TEST(new_session_when_message_received_from_different_fulljid),
-
        /* Presence - online/away/xa/dnd/chat status management */
        PROF_FUNC_TEST(presence_online),
        PROF_FUNC_TEST(presence_online_with_message),
@@ -224,11 +221,18 @@ main(int argc, char* argv[])
    };

    /* ============================================================
-     * GROUP 4: MUC, Carbons
-     * Multi-user chat and message synchronization
-     * (21 tests)
+     * GROUP 4: Chat Session, MUC, Carbons
+     * Session routing, multi-user chat, message synchronization
     * ============================================================ */
    const struct CMUnitTest group4_tests[] = {
+        /* Chat session management - bare/full JID routing */
+        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_offline),
+        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_online),
+        PROF_FUNC_TEST(sends_message_to_fulljid_when_received_from_fulljid),
+        PROF_FUNC_TEST(sends_subsequent_messages_to_fulljid),
+        PROF_FUNC_TEST(resets_to_barejid_after_presence_received),
+        PROF_FUNC_TEST(new_session_when_message_received_from_different_fulljid),
+
        /* MUC room join with various options - XEP-0045 */
        PROF_FUNC_TEST(sends_room_join),
        PROF_FUNC_TEST(sends_room_join_with_nick),
@@ -271,8 +275,8 @@ main(int argc, char* argv[])
    } groups[] = {
        { "Group 1: Connect/Ping/Rooms/Software/LastActivity/Autoping", group1_tests, ARRAY_SIZE(group1_tests) },
        { "Group 2: Message/Receipts/Roster/DBHistory", group2_tests, ARRAY_SIZE(group2_tests) },
-        { "Group 3: Session/Presence/Disconnect/Disco", group3_tests, ARRAY_SIZE(group3_tests) },
-        { "Group 4: MUC/Carbons", group4_tests, ARRAY_SIZE(group4_tests) },
+        { "Group 3: Presence/Disconnect/Disco", group3_tests, ARRAY_SIZE(group3_tests) },
+        { "Group 4: Session/MUC/Carbons", group4_tests, ARRAY_SIZE(group4_tests) },
    };
    const int num_groups = ARRAY_SIZE(groups);

--- a/tests/functionaltests/test_export_import.c
+++ b/tests/functionaltests/test_export_import.c
@@ -112,3 +112,390 @@ import_flatfile_to_sqlite(void** state)
    assert_true(prof_output_regex("Flatfile reply at 1445"));
    assert_true(prof_output_regex("Hello from flatfile import test"));
 }
+
+/*
+ * Test: re-export idempotency — export + import + re-export does not
+ * duplicate messages.
+ *
+ * Flow:
+ *   1. Connect (SQLite), send 2 messages, export to flatfile
+ *   2. Switch to flatfile, verify 2 messages
+ *   3. Switch back to SQLite, export again
+ *   4. Switch to flatfile, reopen — still exactly 2 messages
+ *      (the dedup logic in export must prevent duplication)
+ */
+void
+export_idempotent_no_duplicates(void** state)
+{
+    prof_connect();
+
+    /* Create 2 messages in SQLite */
+    prof_input("/msg buddy1@localhost idemp-msg-alpha");
+    assert_true(prof_output_regex("me: .+idemp-msg-alpha"));
+
+    stbbr_send(
+        "<message id='idemp1' to='stabber@localhost' from='buddy1@localhost/res' type='chat'>"
+        "<body>idemp-msg-beta</body>"
+        "</message>");
+    assert_true(prof_output_regex("Buddy1/res: .+idemp-msg-beta"));
+
+    prof_input("/close");
+
+    /* Export #1 */
+    prof_input("/history export buddy1@localhost");
+    assert_true(prof_output_regex("Export complete: 2 message\\(s\\) exported\\."));
+
+    /* Verify flatfile has the messages */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("idemp-msg-alpha"));
+    assert_true(prof_output_exact("idemp-msg-beta"));
+    prof_input("/close");
+
+    /* Switch back to SQLite and export again (re-export) */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    prof_input("/history export buddy1@localhost");
+    /* Dedup: 0 new messages exported (all already present in flatfile) */
+    assert_true(prof_output_regex("Export complete: 0 message\\(s\\) exported\\."));
+
+    /* Verify flatfile still has exactly 2, not 4 */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("idemp-msg-alpha"));
+    assert_true(prof_output_exact("idemp-msg-beta"));
+}
+
+/*
+ * Test: LMC (Last Message Correction, XEP-0308) survives export.
+ *
+ * A corrected message in SQLite should appear as corrected in the
+ * flat-file after export — the original body must be absent and the
+ * corrected body must be present.
+ */
+void
+export_lmc_correction_survives(void** state)
+{
+    prof_connect();
+
+    prof_input("/history on");
+    assert_true(prof_output_regex("Chat history enabled\\."));
+
+    /* Original message */
+    stbbr_send(
+        "<message id='lmc-exp-orig' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>lmc-export-before-fix</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy1/phone (win 2)"));
+
+    /* Correction replacing the original */
+    stbbr_send(
+        "<message id='lmc-exp-corr' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>lmc-export-after-fix</body>"
+        "<replace id='lmc-exp-orig' xmlns='urn:xmpp:message-correct:0'/>"
+        "</message>");
+
+    /* Sync barrier */
+    stbbr_send(
+        "<message id='lmc-exp-sync' to='stabber@localhost' "
+        "from='buddy2@localhost/phone' type='chat'>"
+        "<body>lmc-export-sync</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy2/phone (win 3)"));
+
+    prof_input("/close all");
+    assert_true(prof_output_exact("Closed 2 windows."));
+
+    /* Export to flatfile */
+    prof_input("/history export buddy1@localhost");
+    assert_true(prof_output_regex("Export complete: [0-9]+ message\\(s\\) exported\\."));
+
+    /* Switch to flatfile and verify correction survived */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("lmc-export-after-fix"));
+
+    /* Original should be gone (correction was applied in SQLite) */
+    prof_timeout(3);
+    assert_false(prof_output_exact("lmc-export-before-fix"));
+    prof_timeout_reset();
+}
+
+/*
+ * Test: /history switch preserves old backend data.
+ *
+ * Switching from SQLite to flatfile should not destroy SQLite data.
+ * Switching back should recover the original messages.
+ *
+ * Uses incoming messages received on the console so the body is never
+ * rendered to the terminal.  The negative check (flatfile is empty)
+ * is performed BEFORE the positive check (SQLite has the message)
+ * so that the body text has not yet entered the cumulative buffer.
+ */
+void
+switch_preserves_old_backend_data(void** state)
+{
+    prof_connect();
+
+    prof_input("/history on");
+    assert_true(prof_output_regex("Chat history enabled\\."));
+
+    /* Receive incoming on console — body NOT rendered to terminal */
+    stbbr_send(
+        "<message id='sw-pres1' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>switch-preserve-sqlite-42</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy1/phone (win 2)"));
+
+    /* Close window (destroy in-memory buffer) */
+    prof_input("/close 2");
+    assert_true(prof_output_exact("Closed window 2"));
+
+    /* Switch to flatfile BEFORE verifying SQLite — body is still
+     * absent from the cumulative terminal buffer at this point */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    /* Flatfile is empty — body must NOT appear (and it has never
+     * been rendered yet, so the cumulative buffer is clean) */
+    prof_input("/msg buddy1@localhost");
+    prof_timeout(3);
+    assert_false(prof_output_exact("switch-preserve-sqlite-42"));
+    prof_timeout_reset();
+    prof_input("/close");
+
+    /* Switch back to SQLite — original data should be intact */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    /* Now the body is rendered for the first time from DB history */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("switch-preserve-sqlite-42"));
+}
+
+/*
+ * Test: global export (no JID argument) exports all contacts.
+ *
+ * Creates messages for buddy1 and buddy2 in SQLite, then exports all.
+ * Both contacts' messages must appear in flatfile after switch.
+ */
+void
+export_all_contacts(void** state)
+{
+    prof_connect();
+
+    /* Messages to buddy1 */
+    prof_input("/msg buddy1@localhost global-export-buddy1-aaa");
+    assert_true(prof_output_regex("me: .+global-export-buddy1-aaa"));
+    prof_input("/close");
+
+    /* Messages to buddy2 */
+    prof_input("/msg buddy2@localhost global-export-buddy2-bbb");
+    assert_true(prof_output_regex("me: .+global-export-buddy2-bbb"));
+    prof_input("/close");
+
+    /* Export all (no JID) */
+    prof_input("/history export");
+    assert_true(prof_output_regex("Exporting all SQLite history to flat-file\\.\\.\\."));
+    assert_true(prof_output_regex("Export complete: [0-9]+ message\\(s\\) exported\\."));
+
+    /* Switch to flatfile */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    /* Verify buddy1 */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("global-export-buddy1-aaa"));
+    prof_input("/close");
+
+    /* Verify buddy2 */
+    prof_input("/msg buddy2@localhost");
+    assert_true(prof_output_exact("global-export-buddy2-bbb"));
+}
+
+/*
+ * Test: double import deduplication.
+ *
+ * Importing the same flatfile twice should not create duplicate messages
+ * in SQLite.
+ *
+ * Flow:
+ *   1. Create messages in flatfile
+ *   2. Switch to SQLite, import, verify count
+ *   3. Import again — count should be 0 (all deduplicated)
+ *   4. Verify only one copy of each message in history
+ */
+void
+import_double_dedup(void** state)
+{
+    prof_connect();
+
+    /* Create messages in flatfile */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    prof_input("/msg buddy1@localhost dedup-import-msg-one");
+    assert_true(prof_output_regex("me: .+dedup-import-msg-one"));
+
+    stbbr_send(
+        "<message id='dedup1' to='stabber@localhost' from='buddy1@localhost/res' type='chat'>"
+        "<body>dedup-import-msg-two</body>"
+        "</message>");
+    assert_true(prof_output_regex("Buddy1/res: .+dedup-import-msg-two"));
+    prof_input("/close");
+
+    /* Switch to SQLite and import #1 */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    prof_input("/history import buddy1@localhost");
+    assert_true(prof_output_regex("Import complete: 2 message\\(s\\) imported\\."));
+
+    /* Import #2 — should be all duplicates */
+    prof_input("/history import buddy1@localhost");
+    assert_true(prof_output_regex("Import complete: 0 message\\(s\\) imported\\."));
+
+    /* Verify messages exist and are not doubled */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("dedup-import-msg-one"));
+    assert_true(prof_output_exact("dedup-import-msg-two"));
+}
+
+/*
+ * Test: /history verify on data that went through export/import.
+ *
+ * After migrating SQLite → flatfile via export, /history verify should
+ * complete without errors on the resulting flat-files.
+ */
+void
+verify_after_export(void** state)
+{
+    prof_connect();
+
+    /* Create some messages in SQLite */
+    prof_input("/msg buddy1@localhost verify-after-export-msg");
+    assert_true(prof_output_regex("me: .+verify-after-export-msg"));
+
+    stbbr_send(
+        "<message id='ver-exp1' to='stabber@localhost' from='buddy1@localhost/res' type='chat'>"
+        "<body>verify-export-reply</body>"
+        "</message>");
+    assert_true(prof_output_regex("Buddy1/res: .+verify-export-reply"));
+    prof_input("/close");
+
+    /* Export to flatfile */
+    prof_input("/history export buddy1@localhost");
+    assert_true(prof_output_regex("Export complete: [0-9]+ message\\(s\\) exported\\."));
+
+    /* Switch to flatfile and verify integrity */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    prof_input("/history verify buddy1@localhost");
+    /* Accept "no issues found" or "0 error(s)" — minor info notes are OK */
+    assert_true(prof_output_regex("Verification complete"));
+}
+
+/*
+ * Test: each backend maintains its own independent messages.
+ *
+ * After switching backends, new messages go to the new backend only.
+ * Switching back reveals the original backend's messages unchanged.
+ *
+ * Uses incoming messages on the console (body never rendered) and two
+ * different contacts (buddy1 for SQLite, buddy2 for flatfile).
+ *
+ * All negative (assert_false) checks are done BEFORE any positive
+ * check renders a body, because prof_output_exact searches the
+ * cumulative terminal buffer.
+ */
+void
+switch_backends_independent_messages(void** state)
+{
+    prof_connect();
+
+    prof_input("/history on");
+    assert_true(prof_output_regex("Chat history enabled\\."));
+
+    /* Phase 1: receive buddy1 msg while on SQLite — body NOT rendered */
+    stbbr_send(
+        "<message id='indep-sq1' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>indep-sqlite-msg-7k</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy1/phone (win 2)"));
+    prof_input("/close 2");
+    assert_true(prof_output_exact("Closed window 2"));
+
+    /* Phase 2: switch to flatfile, receive buddy2 msg */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    stbbr_send(
+        "<message id='indep-ff1' to='stabber@localhost' "
+        "from='buddy2@localhost/laptop' type='chat'>"
+        "<body>indep-flatfile-msg-9m</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy2/laptop (win 2)"));
+    prof_input("/close 2");
+    assert_true(prof_output_exact("Closed window 2"));
+
+    /* --- Negative checks first (neither body has been rendered yet) --- */
+
+    /* buddy1 on flatfile — should be empty (received on SQLite) */
+    prof_input("/msg buddy1@localhost");
+    prof_timeout(3);
+    assert_false(prof_output_exact("indep-sqlite-msg-7k"));
+    prof_timeout_reset();
+    prof_input("/close");
+
+    /* Switch to SQLite for buddy2 negative check */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    /* buddy2 on SQLite — should be empty (received on flatfile) */
+    prof_input("/msg buddy2@localhost");
+    prof_timeout(3);
+    assert_false(prof_output_exact("indep-flatfile-msg-9m"));
+    prof_timeout_reset();
+    prof_input("/close");
+
+    /* --- Positive checks (bodies rendered for the first time) --- */
+
+    /* buddy1 on SQLite — should have the sqlite msg */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("indep-sqlite-msg-7k"));
+    prof_input("/close");
+
+    /* Switch to flatfile for buddy2 positive check */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    /* buddy2 on flatfile — should have the flatfile msg */
+    prof_input("/msg buddy2@localhost");
+    assert_true(prof_output_exact("indep-flatfile-msg-9m"));
+}
+
+/*
+ * Test: export of a contact with no messages doesn't crash and reports 0.
+ *
+ * buddy2 has no message history — export should succeed gracefully
+ * with 0 messages exported.
+ */
+void
+export_empty_contact(void** state)
+{
+    prof_connect();
+
+    /* buddy2 has never exchanged messages */
+    prof_input("/history export buddy2@localhost");
+    assert_true(prof_output_regex("Export complete: 0 message\\(s\\) exported\\."));
+}
--- a/tests/functionaltests/test_export_import.h
+++ b/tests/functionaltests/test_export_import.h
@@ -1,2 +1,10 @@
 void export_sqlite_to_flatfile(void** state);
 void import_flatfile_to_sqlite(void** state);
+void export_idempotent_no_duplicates(void** state);
+void export_lmc_correction_survives(void** state);
+void switch_preserves_old_backend_data(void** state);
+void export_all_contacts(void** state);
+void import_double_dedup(void** state);
+void verify_after_export(void** state);
+void switch_backends_independent_messages(void** state);
+void export_empty_contact(void** state);