feat: complete field parity, harden export/import, add tests

Export/Import improvements: - Replace pagination with direct SQL query (db_sqlite_get_all_chat) - Wrap import in SQL transaction with rollback on error - Add fsync before fclose in export for data safety - Sort merged output by timestamp with secondary key (stanza_id, from_jid) - Export archive_id and marked_read from SQLite (lossless migration) - Add progress indication every 500 messages during write/import - Expand dedup key body prefix from 64 to 256 chars - Fix g_slist_append O(n²) → g_slist_prepend + g_slist_reverse O(n) Field parity (to_jid, to_resource, marked_read): - Add fields to ff_parsed_line_t struct - Write/parse to:|to_res:|read: metadata tags in flatfile format - Pass to_resource through _ff_add_message and all callers - Add marked_read to ProfMessage struct with -1 default (unset) - Preserve fields across export/import round-trips Tests (19 new: 11 unit + 8 functional): - Unit: to_jid_and_marked_read, bracket_in_stanza_id, backslash_in_resource, mucpm_type, all_enc_types, crlf_handling, to_jid_special_chars, multiple_lines, parsed_line_free_null_safe, no_space_rejected, unclosed_bracket - Functional: export_idempotent_no_duplicates, export_lmc_correction_survives, switch_preserves_old_backend_data, export_all_contacts, import_double_dedup, verify_after_export, switch_backends_independent_messages, export_empty_contact - Rebalance test groups: move Chat Session from Group 3 to Group 4 (25/33/30/27 instead of 25/33/36/21) - Remove hardcoded test counts from group comments Man page: - Document /history switch sqlite|flatfile
2026-03-09 12:14:41 +03:00
parent 9183ec7c2a
commit fa2b21152f
16 changed files with 1037 additions and 146 deletions
--- a/tests/functionaltests/functionaltests.c
+++ b/tests/functionaltests/functionaltests.c
@@ -16,9 +16,9 @@
 *
 * Tests are organized into groups for better maintainability and parallel execution:
 *   Group 1: Connect, Ping, Rooms, Software, Last Activity, Autoping
- *   Group 2: Message, Receipts, Roster, DB History (+2 with SQLite)
- *   Group 3: Chat Session, Presence, Disconnect, Disco
- *   Group 4: MUC, Carbons
+ *   Group 2: Message, Receipts, Roster, DB History (+export/import with SQLite)
+ *   Group 3: Presence, Disconnect, Disco
+ *   Group 4: Chat Session, MUC, Carbons
 *
 * Parallel execution:
 *   ./functionaltests        - run all tests sequentially
@@ -85,7 +85,6 @@ main(int argc, char* argv[])
    /* ============================================================
     * GROUP 1: Connect, Ping, Rooms, Software, Last Activity
     * Basic XMPP session establishment and server queries
-     * (19 tests)
     * ============================================================ */
    const struct CMUnitTest group1_tests[] = {
        /* Connection tests - verify login, roster, bookmarks */
@@ -131,13 +130,20 @@ main(int argc, char* argv[])
    /* ============================================================
     * GROUP 2: Message, Receipts, Roster, DB History
     * Core messaging and contact management
-     * (25 tests with SQLite, 23 without)
     * ============================================================ */
    const struct CMUnitTest group2_tests[] = {
 #ifdef HAVE_SQLITE
        /* Export/Import — cross-backend migration (SQLite ↔ flat-file) */
        PROF_FUNC_TEST(export_sqlite_to_flatfile),
        PROF_FUNC_TEST(import_flatfile_to_sqlite),
+        PROF_FUNC_TEST(export_idempotent_no_duplicates),
+        PROF_FUNC_TEST(export_lmc_correction_survives),
+        PROF_FUNC_TEST(switch_preserves_old_backend_data),
+        PROF_FUNC_TEST(export_all_contacts),
+        PROF_FUNC_TEST(import_double_dedup),
+        PROF_FUNC_TEST(verify_after_export),
+        PROF_FUNC_TEST(switch_backends_independent_messages),
+        PROF_FUNC_TEST(export_empty_contact),
 #endif

        /* Basic message send/receive */
@@ -173,19 +179,10 @@ main(int argc, char* argv[])
    };

    /* ============================================================
-     * GROUP 3: Chat Session, Presence, Disconnect
-     * Session routing, status management, clean teardown
-     * (22 tests)
+     * GROUP 3: Presence, Disconnect, Disco
+     * Status management, clean teardown, service discovery
     * ============================================================ */
    const struct CMUnitTest group3_tests[] = {
-        /* Chat session management - bare/full JID routing */
-        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_offline),
-        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_online),
-        PROF_FUNC_TEST(sends_message_to_fulljid_when_received_from_fulljid),
-        PROF_FUNC_TEST(sends_subsequent_messages_to_fulljid),
-        PROF_FUNC_TEST(resets_to_barejid_after_presence_received),
-        PROF_FUNC_TEST(new_session_when_message_received_from_different_fulljid),
-
        /* Presence - online/away/xa/dnd/chat status management */
        PROF_FUNC_TEST(presence_online),
        PROF_FUNC_TEST(presence_online_with_message),
@@ -224,11 +221,18 @@ main(int argc, char* argv[])
    };

    /* ============================================================
-     * GROUP 4: MUC, Carbons
-     * Multi-user chat and message synchronization
-     * (21 tests)
+     * GROUP 4: Chat Session, MUC, Carbons
+     * Session routing, multi-user chat, message synchronization
     * ============================================================ */
    const struct CMUnitTest group4_tests[] = {
+        /* Chat session management - bare/full JID routing */
+        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_offline),
+        PROF_FUNC_TEST(sends_message_to_barejid_when_contact_online),
+        PROF_FUNC_TEST(sends_message_to_fulljid_when_received_from_fulljid),
+        PROF_FUNC_TEST(sends_subsequent_messages_to_fulljid),
+        PROF_FUNC_TEST(resets_to_barejid_after_presence_received),
+        PROF_FUNC_TEST(new_session_when_message_received_from_different_fulljid),
+
        /* MUC room join with various options - XEP-0045 */
        PROF_FUNC_TEST(sends_room_join),
        PROF_FUNC_TEST(sends_room_join_with_nick),
@@ -271,8 +275,8 @@ main(int argc, char* argv[])
    } groups[] = {
        { "Group 1: Connect/Ping/Rooms/Software/LastActivity/Autoping", group1_tests, ARRAY_SIZE(group1_tests) },
        { "Group 2: Message/Receipts/Roster/DBHistory", group2_tests, ARRAY_SIZE(group2_tests) },
-        { "Group 3: Session/Presence/Disconnect/Disco", group3_tests, ARRAY_SIZE(group3_tests) },
-        { "Group 4: MUC/Carbons", group4_tests, ARRAY_SIZE(group4_tests) },
+        { "Group 3: Presence/Disconnect/Disco", group3_tests, ARRAY_SIZE(group3_tests) },
+        { "Group 4: Session/MUC/Carbons", group4_tests, ARRAY_SIZE(group4_tests) },
    };
    const int num_groups = ARRAY_SIZE(groups);

--- a/tests/functionaltests/test_export_import.c
+++ b/tests/functionaltests/test_export_import.c
@@ -112,3 +112,390 @@ import_flatfile_to_sqlite(void** state)
    assert_true(prof_output_regex("Flatfile reply at 1445"));
    assert_true(prof_output_regex("Hello from flatfile import test"));
 }
+
+/*
+ * Test: re-export idempotency — export + import + re-export does not
+ * duplicate messages.
+ *
+ * Flow:
+ *   1. Connect (SQLite), send 2 messages, export to flatfile
+ *   2. Switch to flatfile, verify 2 messages
+ *   3. Switch back to SQLite, export again
+ *   4. Switch to flatfile, reopen — still exactly 2 messages
+ *      (the dedup logic in export must prevent duplication)
+ */
+void
+export_idempotent_no_duplicates(void** state)
+{
+    prof_connect();
+
+    /* Create 2 messages in SQLite */
+    prof_input("/msg buddy1@localhost idemp-msg-alpha");
+    assert_true(prof_output_regex("me: .+idemp-msg-alpha"));
+
+    stbbr_send(
+        "<message id='idemp1' to='stabber@localhost' from='buddy1@localhost/res' type='chat'>"
+        "<body>idemp-msg-beta</body>"
+        "</message>");
+    assert_true(prof_output_regex("Buddy1/res: .+idemp-msg-beta"));
+
+    prof_input("/close");
+
+    /* Export #1 */
+    prof_input("/history export buddy1@localhost");
+    assert_true(prof_output_regex("Export complete: 2 message\\(s\\) exported\\."));
+
+    /* Verify flatfile has the messages */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("idemp-msg-alpha"));
+    assert_true(prof_output_exact("idemp-msg-beta"));
+    prof_input("/close");
+
+    /* Switch back to SQLite and export again (re-export) */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    prof_input("/history export buddy1@localhost");
+    /* Dedup: 0 new messages exported (all already present in flatfile) */
+    assert_true(prof_output_regex("Export complete: 0 message\\(s\\) exported\\."));
+
+    /* Verify flatfile still has exactly 2, not 4 */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("idemp-msg-alpha"));
+    assert_true(prof_output_exact("idemp-msg-beta"));
+}
+
+/*
+ * Test: LMC (Last Message Correction, XEP-0308) survives export.
+ *
+ * A corrected message in SQLite should appear as corrected in the
+ * flat-file after export — the original body must be absent and the
+ * corrected body must be present.
+ */
+void
+export_lmc_correction_survives(void** state)
+{
+    prof_connect();
+
+    prof_input("/history on");
+    assert_true(prof_output_regex("Chat history enabled\\."));
+
+    /* Original message */
+    stbbr_send(
+        "<message id='lmc-exp-orig' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>lmc-export-before-fix</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy1/phone (win 2)"));
+
+    /* Correction replacing the original */
+    stbbr_send(
+        "<message id='lmc-exp-corr' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>lmc-export-after-fix</body>"
+        "<replace id='lmc-exp-orig' xmlns='urn:xmpp:message-correct:0'/>"
+        "</message>");
+
+    /* Sync barrier */
+    stbbr_send(
+        "<message id='lmc-exp-sync' to='stabber@localhost' "
+        "from='buddy2@localhost/phone' type='chat'>"
+        "<body>lmc-export-sync</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy2/phone (win 3)"));
+
+    prof_input("/close all");
+    assert_true(prof_output_exact("Closed 2 windows."));
+
+    /* Export to flatfile */
+    prof_input("/history export buddy1@localhost");
+    assert_true(prof_output_regex("Export complete: [0-9]+ message\\(s\\) exported\\."));
+
+    /* Switch to flatfile and verify correction survived */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("lmc-export-after-fix"));
+
+    /* Original should be gone (correction was applied in SQLite) */
+    prof_timeout(3);
+    assert_false(prof_output_exact("lmc-export-before-fix"));
+    prof_timeout_reset();
+}
+
+/*
+ * Test: /history switch preserves old backend data.
+ *
+ * Switching from SQLite to flatfile should not destroy SQLite data.
+ * Switching back should recover the original messages.
+ *
+ * Uses incoming messages received on the console so the body is never
+ * rendered to the terminal.  The negative check (flatfile is empty)
+ * is performed BEFORE the positive check (SQLite has the message)
+ * so that the body text has not yet entered the cumulative buffer.
+ */
+void
+switch_preserves_old_backend_data(void** state)
+{
+    prof_connect();
+
+    prof_input("/history on");
+    assert_true(prof_output_regex("Chat history enabled\\."));
+
+    /* Receive incoming on console — body NOT rendered to terminal */
+    stbbr_send(
+        "<message id='sw-pres1' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>switch-preserve-sqlite-42</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy1/phone (win 2)"));
+
+    /* Close window (destroy in-memory buffer) */
+    prof_input("/close 2");
+    assert_true(prof_output_exact("Closed window 2"));
+
+    /* Switch to flatfile BEFORE verifying SQLite — body is still
+     * absent from the cumulative terminal buffer at this point */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    /* Flatfile is empty — body must NOT appear (and it has never
+     * been rendered yet, so the cumulative buffer is clean) */
+    prof_input("/msg buddy1@localhost");
+    prof_timeout(3);
+    assert_false(prof_output_exact("switch-preserve-sqlite-42"));
+    prof_timeout_reset();
+    prof_input("/close");
+
+    /* Switch back to SQLite — original data should be intact */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    /* Now the body is rendered for the first time from DB history */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("switch-preserve-sqlite-42"));
+}
+
+/*
+ * Test: global export (no JID argument) exports all contacts.
+ *
+ * Creates messages for buddy1 and buddy2 in SQLite, then exports all.
+ * Both contacts' messages must appear in flatfile after switch.
+ */
+void
+export_all_contacts(void** state)
+{
+    prof_connect();
+
+    /* Messages to buddy1 */
+    prof_input("/msg buddy1@localhost global-export-buddy1-aaa");
+    assert_true(prof_output_regex("me: .+global-export-buddy1-aaa"));
+    prof_input("/close");
+
+    /* Messages to buddy2 */
+    prof_input("/msg buddy2@localhost global-export-buddy2-bbb");
+    assert_true(prof_output_regex("me: .+global-export-buddy2-bbb"));
+    prof_input("/close");
+
+    /* Export all (no JID) */
+    prof_input("/history export");
+    assert_true(prof_output_regex("Exporting all SQLite history to flat-file\\.\\.\\."));
+    assert_true(prof_output_regex("Export complete: [0-9]+ message\\(s\\) exported\\."));
+
+    /* Switch to flatfile */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    /* Verify buddy1 */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("global-export-buddy1-aaa"));
+    prof_input("/close");
+
+    /* Verify buddy2 */
+    prof_input("/msg buddy2@localhost");
+    assert_true(prof_output_exact("global-export-buddy2-bbb"));
+}
+
+/*
+ * Test: double import deduplication.
+ *
+ * Importing the same flatfile twice should not create duplicate messages
+ * in SQLite.
+ *
+ * Flow:
+ *   1. Create messages in flatfile
+ *   2. Switch to SQLite, import, verify count
+ *   3. Import again — count should be 0 (all deduplicated)
+ *   4. Verify only one copy of each message in history
+ */
+void
+import_double_dedup(void** state)
+{
+    prof_connect();
+
+    /* Create messages in flatfile */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    prof_input("/msg buddy1@localhost dedup-import-msg-one");
+    assert_true(prof_output_regex("me: .+dedup-import-msg-one"));
+
+    stbbr_send(
+        "<message id='dedup1' to='stabber@localhost' from='buddy1@localhost/res' type='chat'>"
+        "<body>dedup-import-msg-two</body>"
+        "</message>");
+    assert_true(prof_output_regex("Buddy1/res: .+dedup-import-msg-two"));
+    prof_input("/close");
+
+    /* Switch to SQLite and import #1 */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    prof_input("/history import buddy1@localhost");
+    assert_true(prof_output_regex("Import complete: 2 message\\(s\\) imported\\."));
+
+    /* Import #2 — should be all duplicates */
+    prof_input("/history import buddy1@localhost");
+    assert_true(prof_output_regex("Import complete: 0 message\\(s\\) imported\\."));
+
+    /* Verify messages exist and are not doubled */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("dedup-import-msg-one"));
+    assert_true(prof_output_exact("dedup-import-msg-two"));
+}
+
+/*
+ * Test: /history verify on data that went through export/import.
+ *
+ * After migrating SQLite → flatfile via export, /history verify should
+ * complete without errors on the resulting flat-files.
+ */
+void
+verify_after_export(void** state)
+{
+    prof_connect();
+
+    /* Create some messages in SQLite */
+    prof_input("/msg buddy1@localhost verify-after-export-msg");
+    assert_true(prof_output_regex("me: .+verify-after-export-msg"));
+
+    stbbr_send(
+        "<message id='ver-exp1' to='stabber@localhost' from='buddy1@localhost/res' type='chat'>"
+        "<body>verify-export-reply</body>"
+        "</message>");
+    assert_true(prof_output_regex("Buddy1/res: .+verify-export-reply"));
+    prof_input("/close");
+
+    /* Export to flatfile */
+    prof_input("/history export buddy1@localhost");
+    assert_true(prof_output_regex("Export complete: [0-9]+ message\\(s\\) exported\\."));
+
+    /* Switch to flatfile and verify integrity */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    prof_input("/history verify buddy1@localhost");
+    /* Accept "no issues found" or "0 error(s)" — minor info notes are OK */
+    assert_true(prof_output_regex("Verification complete"));
+}
+
+/*
+ * Test: each backend maintains its own independent messages.
+ *
+ * After switching backends, new messages go to the new backend only.
+ * Switching back reveals the original backend's messages unchanged.
+ *
+ * Uses incoming messages on the console (body never rendered) and two
+ * different contacts (buddy1 for SQLite, buddy2 for flatfile).
+ *
+ * All negative (assert_false) checks are done BEFORE any positive
+ * check renders a body, because prof_output_exact searches the
+ * cumulative terminal buffer.
+ */
+void
+switch_backends_independent_messages(void** state)
+{
+    prof_connect();
+
+    prof_input("/history on");
+    assert_true(prof_output_regex("Chat history enabled\\."));
+
+    /* Phase 1: receive buddy1 msg while on SQLite — body NOT rendered */
+    stbbr_send(
+        "<message id='indep-sq1' to='stabber@localhost' "
+        "from='buddy1@localhost/phone' type='chat'>"
+        "<body>indep-sqlite-msg-7k</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy1/phone (win 2)"));
+    prof_input("/close 2");
+    assert_true(prof_output_exact("Closed window 2"));
+
+    /* Phase 2: switch to flatfile, receive buddy2 msg */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    stbbr_send(
+        "<message id='indep-ff1' to='stabber@localhost' "
+        "from='buddy2@localhost/laptop' type='chat'>"
+        "<body>indep-flatfile-msg-9m</body>"
+        "</message>");
+    assert_true(prof_output_exact("<< chat message: Buddy2/laptop (win 2)"));
+    prof_input("/close 2");
+    assert_true(prof_output_exact("Closed window 2"));
+
+    /* --- Negative checks first (neither body has been rendered yet) --- */
+
+    /* buddy1 on flatfile — should be empty (received on SQLite) */
+    prof_input("/msg buddy1@localhost");
+    prof_timeout(3);
+    assert_false(prof_output_exact("indep-sqlite-msg-7k"));
+    prof_timeout_reset();
+    prof_input("/close");
+
+    /* Switch to SQLite for buddy2 negative check */
+    prof_input("/history switch sqlite");
+    assert_true(prof_output_regex("Database backend switched to 'sqlite'\\."));
+
+    /* buddy2 on SQLite — should be empty (received on flatfile) */
+    prof_input("/msg buddy2@localhost");
+    prof_timeout(3);
+    assert_false(prof_output_exact("indep-flatfile-msg-9m"));
+    prof_timeout_reset();
+    prof_input("/close");
+
+    /* --- Positive checks (bodies rendered for the first time) --- */
+
+    /* buddy1 on SQLite — should have the sqlite msg */
+    prof_input("/msg buddy1@localhost");
+    assert_true(prof_output_exact("indep-sqlite-msg-7k"));
+    prof_input("/close");
+
+    /* Switch to flatfile for buddy2 positive check */
+    prof_input("/history switch flatfile");
+    assert_true(prof_output_regex("Database backend switched to 'flatfile'\\."));
+
+    /* buddy2 on flatfile — should have the flatfile msg */
+    prof_input("/msg buddy2@localhost");
+    assert_true(prof_output_exact("indep-flatfile-msg-9m"));
+}
+
+/*
+ * Test: export of a contact with no messages doesn't crash and reports 0.
+ *
+ * buddy2 has no message history — export should succeed gracefully
+ * with 0 messages exported.
+ */
+void
+export_empty_contact(void** state)
+{
+    prof_connect();
+
+    /* buddy2 has never exchanged messages */
+    prof_input("/history export buddy2@localhost");
+    assert_true(prof_output_regex("Export complete: 0 message\\(s\\) exported\\."));
+}
--- a/tests/functionaltests/test_export_import.h
+++ b/tests/functionaltests/test_export_import.h
@@ -1,2 +1,10 @@
 void export_sqlite_to_flatfile(void** state);
 void import_flatfile_to_sqlite(void** state);
+void export_idempotent_no_duplicates(void** state);
+void export_lmc_correction_survives(void** state);
+void switch_preserves_old_backend_data(void** state);
+void export_all_contacts(void** state);
+void import_double_dedup(void** state);
+void verify_after_export(void** state);
+void switch_backends_independent_messages(void** state);
+void export_empty_contact(void** state);
--- a/tests/unittests/database/stub_database.c
+++ b/tests/unittests/database/stub_database.c
@@ -88,6 +88,23 @@ db_sqlite_list_contacts(void)
 {
    return NULL;
 }
+GSList*
+db_sqlite_get_all_chat(const gchar* const contact_barejid)
+{
+    return NULL;
+}
+void
+db_sqlite_begin_transaction(void)
+{
+}
+void
+db_sqlite_end_transaction(void)
+{
+}
+void
+db_sqlite_rollback_transaction(void)
+{
+}
 int
 log_database_export_to_flatfile(const gchar* const contact_jid)
 {
--- a/tests/unittests/test_database_export.c
+++ b/tests/unittests/test_database_export.c
@@ -28,7 +28,9 @@
 static ff_parsed_line_t*
 _roundtrip(const char* timestamp, const char* type, const char* enc,
           const char* stanza_id, const char* archive_id, const char* replace_id,
-           const char* from_jid, const char* from_resource, const char* message)
+           const char* from_jid, const char* from_resource,
+           const char* to_jid, const char* to_resource, int marked_read,
+           const char* message)
 {
    char tmppath[] = "/tmp/proftest_rt_XXXXXX";
    int fd = mkstemp(tmppath);
@@ -39,7 +41,9 @@ _roundtrip(const char* timestamp, const char* type, const char* enc,

    ff_write_line(fp, timestamp, type, enc,
                  stanza_id, archive_id, replace_id,
-                  from_jid, from_resource, message);
+                  from_jid, from_resource,
+                  to_jid, to_resource, marked_read,
+                  message);
    fclose(fp);

    /* Read & parse */
@@ -70,7 +74,9 @@ test_ff_roundtrip_simple_chat(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-06-15T12:30:00+00:00", "chat", "none",
        NULL, NULL, NULL,
-        "alice@example.com", NULL, "Hello, world!");
+        "alice@example.com", NULL,
+        NULL, NULL, -1,
+        "Hello, world!");

    assert_non_null(pl);
    assert_string_equal(pl->timestamp_str, "2025-06-15T12:30:00+00:00");
@@ -90,7 +96,9 @@ test_ff_roundtrip_with_all_metadata(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-06-15T12:30:00+00:00", "chat", "omemo",
        "sid-abc-123", "aid-xyz-789", "corrects-old-id",
-        "bob@example.com", "phone", "Encrypted message.");
+        "bob@example.com", "phone",
+        NULL, NULL, -1,
+        "Encrypted message.");

    assert_non_null(pl);
    assert_string_equal(pl->stanza_id, "sid-abc-123");
@@ -109,7 +117,9 @@ test_ff_roundtrip_with_resource(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-01-01T00:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "user@jabber.org", "Profanity.abc123", "hi");
+        "user@jabber.org", "Profanity.abc123",
+        NULL, NULL, -1,
+        "hi");

    assert_non_null(pl);
    assert_string_equal(pl->from_jid, "user@jabber.org");
@@ -124,7 +134,9 @@ test_ff_roundtrip_newline_in_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "alice@x.com", NULL, "line1\nline2\nline3");
+        "alice@x.com", NULL,
+        NULL, NULL, -1,
+        "line1\nline2\nline3");

    assert_non_null(pl);
    assert_string_equal(pl->message, "line1\nline2\nline3");
@@ -138,7 +150,9 @@ test_ff_roundtrip_pipe_in_stanza_id(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        "id|with|pipes", "aid|also|has|pipes", NULL,
-        "a@b.com", NULL, "test");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "test");

    assert_non_null(pl);
    assert_string_equal(pl->stanza_id, "id|with|pipes");
@@ -152,7 +166,9 @@ test_ff_roundtrip_backslash_in_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", NULL, "path\\to\\file C:\\Users\\test");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "path\\to\\file C:\\Users\\test");

    assert_non_null(pl);
    assert_string_equal(pl->message, "path\\to\\file C:\\Users\\test");
@@ -165,7 +181,9 @@ test_ff_roundtrip_unicode_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", NULL, "Привет мир 🌍 日本語テスト");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "Привет мир 🌍 日本語テスト");

    assert_non_null(pl);
    assert_string_equal(pl->message, "Привет мир 🌍 日本語テスト");
@@ -178,7 +196,9 @@ test_ff_roundtrip_empty_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", NULL, "");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "");

    assert_non_null(pl);
    assert_string_equal(pl->message, "");
@@ -192,7 +212,9 @@ test_ff_roundtrip_colonspace_in_resource(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", "res: with: colons", "msg");
+        "a@b.com", "res: with: colons",
+        NULL, NULL, -1,
+        "msg");

    assert_non_null(pl);
    assert_string_equal(pl->from_jid, "a@b.com");
@@ -207,7 +229,9 @@ test_ff_roundtrip_muc_type(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "muc", "none",
        NULL, NULL, NULL,
-        "room@conference.x.com", "nick", "hello room");
+        "room@conference.x.com", "nick",
+        NULL, NULL, -1,
+        "hello room");

    assert_non_null(pl);
    assert_string_equal(pl->type, "muc");
@@ -222,7 +246,9 @@ test_ff_roundtrip_omemo_enc(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "omemo",
        "sid-1", NULL, NULL,
-        "a@b.com", NULL, "secret");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "secret");

    assert_non_null(pl);
    assert_string_equal(pl->enc, "omemo");
@@ -236,7 +262,9 @@ test_ff_roundtrip_replace_id(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        "new-id", NULL, "old-id-to-correct",
-        "a@b.com", NULL, "corrected text");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "corrected text");

    assert_non_null(pl);
    assert_string_equal(pl->stanza_id, "new-id");
@@ -245,6 +273,55 @@ test_ff_roundtrip_replace_id(void** state)
    ff_parsed_line_free(pl);
 }

+void
+test_ff_roundtrip_to_jid_and_marked_read(void** state)
+{
+    /* to_jid, to_resource, and marked_read round-trip */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-06-15T12:30:00+00:00", "chat", "none",
+        "sid-1", NULL, NULL,
+        "alice@example.com", "phone",
+        "bob@example.com", "laptop", 1,
+        "Hello Bob!");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->from_jid, "alice@example.com");
+    assert_string_equal(pl->from_resource, "phone");
+    assert_string_equal(pl->to_jid, "bob@example.com");
+    assert_string_equal(pl->to_resource, "laptop");
+    assert_int_equal(pl->marked_read, 1);
+    assert_string_equal(pl->message, "Hello Bob!");
+    ff_parsed_line_free(pl);
+
+    /* marked_read = 0 (unread) */
+    pl = _roundtrip(
+        "2025-06-15T12:31:00+00:00", "chat", "none",
+        NULL, NULL, NULL,
+        "bob@example.com", NULL,
+        "alice@example.com", NULL, 0,
+        "Reply");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->to_jid, "alice@example.com");
+    assert_null(pl->to_resource);
+    assert_int_equal(pl->marked_read, 0);
+    ff_parsed_line_free(pl);
+
+    /* marked_read = -1 (unset) — should NOT appear in output */
+    pl = _roundtrip(
+        "2025-06-15T12:32:00+00:00", "chat", "none",
+        NULL, NULL, NULL,
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "no read flag");
+
+    assert_non_null(pl);
+    assert_null(pl->to_jid);
+    assert_null(pl->to_resource);
+    assert_int_equal(pl->marked_read, -1);
+    ff_parsed_line_free(pl);
+}
+
 /* ================================================================
 * Escape / unescape symmetry tests
 * ================================================================ */
@@ -442,3 +519,203 @@ test_ff_enc_str_roundtrip(void** state)
    assert_int_equal(PROF_MSG_ENC_OX, ff_get_message_enc_type(ff_get_message_enc_str(PROF_MSG_ENC_OX)));
    assert_int_equal(PROF_MSG_ENC_OMEMO, ff_get_message_enc_type(ff_get_message_enc_str(PROF_MSG_ENC_OMEMO)));
 }
+
+/* ================================================================
+ * Additional round-trip tests
+ * ================================================================ */
+
+void
+test_ff_roundtrip_bracket_in_stanza_id(void** state)
+{
+    /* ']' in stanza_id must be escaped as \] to avoid breaking metadata parser */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "chat", "none",
+        "id]with]brackets", "aid]also]has]brackets", NULL,
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "test brackets");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->stanza_id, "id]with]brackets");
+    assert_string_equal(pl->archive_id, "aid]also]has]brackets");
+    assert_string_equal(pl->message, "test brackets");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_backslash_in_resource(void** state)
+{
+    /* backslash in from_resource must survive round-trip */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "chat", "none",
+        NULL, NULL, NULL,
+        "a@b.com", "res\\with\\backslash",
+        NULL, NULL, -1,
+        "msg");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->from_jid, "a@b.com");
+    assert_string_equal(pl->from_resource, "res\\with\\backslash");
+    assert_string_equal(pl->message, "msg");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_mucpm_type(void** state)
+{
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "mucpm", "none",
+        NULL, NULL, NULL,
+        "room@conference.x.com", "nick",
+        NULL, NULL, -1,
+        "private message");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->type, "mucpm");
+    assert_string_equal(pl->from_resource, "nick");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_all_enc_types(void** state)
+{
+    const char* enc_types[] = { "none", "otr", "pgp", "ox", "omemo" };
+    for (size_t i = 0; i < sizeof(enc_types) / sizeof(enc_types[0]); i++) {
+        ff_parsed_line_t* pl = _roundtrip(
+            "2025-03-01T10:00:00Z", "chat", enc_types[i],
+            NULL, NULL, NULL,
+            "a@b.com", NULL,
+            NULL, NULL, -1,
+            "msg");
+
+        assert_non_null(pl);
+        assert_string_equal(pl->enc, enc_types[i]);
+        ff_parsed_line_free(pl);
+    }
+}
+
+void
+test_ff_roundtrip_crlf_handling(void** state)
+{
+    /* Write a line, then read it with \r\n ending — parser should strip \r */
+    char tmppath[] = "/tmp/proftest_crlf_XXXXXX";
+    int fd = mkstemp(tmppath);
+    assert_true(fd >= 0);
+
+    FILE* fp = fdopen(fd, "w");
+    assert_non_null(fp);
+
+    /* Write a raw line with \r\n ending */
+    fprintf(fp, "2025-03-01T10:00:00Z [chat|none] a@b.com: hello\r\n");
+    fclose(fp);
+
+    fp = fopen(tmppath, "r");
+    assert_non_null(fp);
+
+    gboolean truncated = FALSE;
+    char* buf = ff_readline(fp, &truncated);
+    fclose(fp);
+    unlink(tmppath);
+
+    assert_non_null(buf);
+    ff_parsed_line_t* pl = ff_parse_line(buf);
+    free(buf);
+
+    assert_non_null(pl);
+    assert_string_equal(pl->from_jid, "a@b.com");
+    assert_string_equal(pl->message, "hello");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_to_jid_special_chars(void** state)
+{
+    /* to: and to_res: with pipe and bracket chars that need escaping */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "chat", "none",
+        NULL, NULL, NULL,
+        "a@b.com", NULL,
+        "to|user@c.com", "res]with|special", -1,
+        "msg with special to");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->to_jid, "to|user@c.com");
+    assert_string_equal(pl->to_resource, "res]with|special");
+    assert_string_equal(pl->message, "msg with special to");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_multiple_lines(void** state)
+{
+    /* Write several lines, read and parse them all sequentially */
+    char tmppath[] = "/tmp/proftest_multi_XXXXXX";
+    int fd = mkstemp(tmppath);
+    assert_true(fd >= 0);
+
+    FILE* fp = fdopen(fd, "w");
+    assert_non_null(fp);
+
+    ff_write_line(fp, "2025-01-01T00:00:01Z", "chat", "none",
+                  NULL, NULL, NULL, "a@b.com", NULL,
+                  NULL, NULL, -1, "first");
+    ff_write_line(fp, "2025-01-01T00:00:02Z", "chat", "none",
+                  NULL, NULL, NULL, "c@d.com", NULL,
+                  NULL, NULL, -1, "second");
+    ff_write_line(fp, "2025-01-01T00:00:03Z", "muc", "omemo",
+                  "sid-3", NULL, NULL, "room@conf.com", "nick",
+                  NULL, NULL, -1, "third");
+    fclose(fp);
+
+    fp = fopen(tmppath, "r");
+    assert_non_null(fp);
+
+    int count = 0;
+    const char* expected_msgs[] = { "first", "second", "third" };
+    const char* expected_from[] = { "a@b.com", "c@d.com", "room@conf.com" };
+
+    while (1) {
+        gboolean truncated = FALSE;
+        char* buf = ff_readline(fp, &truncated);
+        if (!buf)
+            break;
+        ff_parsed_line_t* pl = ff_parse_line(buf);
+        free(buf);
+        if (pl) {
+            assert_true(count < 3);
+            assert_string_equal(pl->message, expected_msgs[count]);
+            assert_string_equal(pl->from_jid, expected_from[count]);
+            ff_parsed_line_free(pl);
+            count++;
+        }
+    }
+    fclose(fp);
+    unlink(tmppath);
+
+    assert_int_equal(count, 3);
+}
+
+/* ================================================================
+ * Additional parser edge-case tests
+ * ================================================================ */
+
+void
+test_ff_parsed_line_free_null_safe(void** state)
+{
+    /* ff_parsed_line_free(NULL) must not crash */
+    ff_parsed_line_free(NULL);
+}
+
+void
+test_ff_parse_line_no_space_rejected(void** state)
+{
+    /* A line with no spaces at all cannot be parsed */
+    assert_null(ff_parse_line("noseparatoratall"));
+}
+
+void
+test_ff_parse_line_unclosed_bracket(void** state)
+{
+    /* Unclosed metadata bracket should return NULL */
+    assert_null(ff_parse_line("2025-03-01T10:00:00Z [chat|none a@b.com: msg"));
+}
--- a/tests/unittests/test_database_export.h
+++ b/tests/unittests/test_database_export.h
@@ -14,6 +14,7 @@ void test_ff_roundtrip_colonspace_in_resource(void** state);
 void test_ff_roundtrip_muc_type(void** state);
 void test_ff_roundtrip_omemo_enc(void** state);
 void test_ff_roundtrip_replace_id(void** state);
+void test_ff_roundtrip_to_jid_and_marked_read(void** state);

 /* escape / unescape symmetry */
 void test_ff_escape_unescape_message_identity(void** state);
@@ -37,3 +38,17 @@ void test_ff_parse_line_invalid_timestamp(void** state);
 /* type/enc conversion round-trip */
 void test_ff_type_str_roundtrip(void** state);
 void test_ff_enc_str_roundtrip(void** state);
+
+/* additional round-trip */
+void test_ff_roundtrip_bracket_in_stanza_id(void** state);
+void test_ff_roundtrip_backslash_in_resource(void** state);
+void test_ff_roundtrip_mucpm_type(void** state);
+void test_ff_roundtrip_all_enc_types(void** state);
+void test_ff_roundtrip_crlf_handling(void** state);
+void test_ff_roundtrip_to_jid_special_chars(void** state);
+void test_ff_roundtrip_multiple_lines(void** state);
+
+/* additional parser edge cases */
+void test_ff_parsed_line_free_null_safe(void** state);
+void test_ff_parse_line_no_space_rejected(void** state);
+void test_ff_parse_line_unclosed_bracket(void** state);
--- a/tests/unittests/unittests.c
+++ b/tests/unittests/unittests.c
@@ -671,6 +671,7 @@ main(int argc, char* argv[])
        cmocka_unit_test(test_ff_roundtrip_muc_type),
        cmocka_unit_test(test_ff_roundtrip_omemo_enc),
        cmocka_unit_test(test_ff_roundtrip_replace_id),
+        cmocka_unit_test(test_ff_roundtrip_to_jid_and_marked_read),

        // Escape / unescape
        cmocka_unit_test(test_ff_escape_unescape_message_identity),
@@ -694,6 +695,20 @@ main(int argc, char* argv[])
        // Type/enc round-trip
        cmocka_unit_test(test_ff_type_str_roundtrip),
        cmocka_unit_test(test_ff_enc_str_roundtrip),
+
+        // Additional round-trip
+        cmocka_unit_test(test_ff_roundtrip_bracket_in_stanza_id),
+        cmocka_unit_test(test_ff_roundtrip_backslash_in_resource),
+        cmocka_unit_test(test_ff_roundtrip_mucpm_type),
+        cmocka_unit_test(test_ff_roundtrip_all_enc_types),
+        cmocka_unit_test(test_ff_roundtrip_crlf_handling),
+        cmocka_unit_test(test_ff_roundtrip_to_jid_special_chars),
+        cmocka_unit_test(test_ff_roundtrip_multiple_lines),
+
+        // Additional parser edge cases
+        cmocka_unit_test(test_ff_parsed_line_free_null_safe),
+        cmocka_unit_test(test_ff_parse_line_no_space_rejected),
+        cmocka_unit_test(test_ff_parse_line_unclosed_bracket),
    };
    return cmocka_run_group_tests(all_tests, NULL, NULL);
 }