feat: complete field parity, harden export/import, add tests

Export/Import improvements: - Replace pagination with direct SQL query (db_sqlite_get_all_chat) - Wrap import in SQL transaction with rollback on error - Add fsync before fclose in export for data safety - Sort merged output by timestamp with secondary key (stanza_id, from_jid) - Export archive_id and marked_read from SQLite (lossless migration) - Add progress indication every 500 messages during write/import - Expand dedup key body prefix from 64 to 256 chars - Fix g_slist_append O(n²) → g_slist_prepend + g_slist_reverse O(n) Field parity (to_jid, to_resource, marked_read): - Add fields to ff_parsed_line_t struct - Write/parse to:|to_res:|read: metadata tags in flatfile format - Pass to_resource through _ff_add_message and all callers - Add marked_read to ProfMessage struct with -1 default (unset) - Preserve fields across export/import round-trips Tests (19 new: 11 unit + 8 functional): - Unit: to_jid_and_marked_read, bracket_in_stanza_id, backslash_in_resource, mucpm_type, all_enc_types, crlf_handling, to_jid_special_chars, multiple_lines, parsed_line_free_null_safe, no_space_rejected, unclosed_bracket - Functional: export_idempotent_no_duplicates, export_lmc_correction_survives, switch_preserves_old_backend_data, export_all_contacts, import_double_dedup, verify_after_export, switch_backends_independent_messages, export_empty_contact - Rebalance test groups: move Chat Session from Group 3 to Group 4 (25/33/30/27 instead of 25/33/36/21) - Remove hardcoded test counts from group comments Man page: - Document /history switch sqlite|flatfile
2026-03-09 12:14:41 +03:00
parent 9183ec7c2a
commit fa2b21152f
16 changed files with 1037 additions and 146 deletions
--- a/tests/unittests/test_database_export.c
+++ b/tests/unittests/test_database_export.c
@@ -28,7 +28,9 @@
 static ff_parsed_line_t*
 _roundtrip(const char* timestamp, const char* type, const char* enc,
           const char* stanza_id, const char* archive_id, const char* replace_id,
-           const char* from_jid, const char* from_resource, const char* message)
+           const char* from_jid, const char* from_resource,
+           const char* to_jid, const char* to_resource, int marked_read,
+           const char* message)
 {
    char tmppath[] = "/tmp/proftest_rt_XXXXXX";
    int fd = mkstemp(tmppath);
@@ -39,7 +41,9 @@ _roundtrip(const char* timestamp, const char* type, const char* enc,

    ff_write_line(fp, timestamp, type, enc,
                  stanza_id, archive_id, replace_id,
-                  from_jid, from_resource, message);
+                  from_jid, from_resource,
+                  to_jid, to_resource, marked_read,
+                  message);
    fclose(fp);

    /* Read & parse */
@@ -70,7 +74,9 @@ test_ff_roundtrip_simple_chat(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-06-15T12:30:00+00:00", "chat", "none",
        NULL, NULL, NULL,
-        "alice@example.com", NULL, "Hello, world!");
+        "alice@example.com", NULL,
+        NULL, NULL, -1,
+        "Hello, world!");

    assert_non_null(pl);
    assert_string_equal(pl->timestamp_str, "2025-06-15T12:30:00+00:00");
@@ -90,7 +96,9 @@ test_ff_roundtrip_with_all_metadata(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-06-15T12:30:00+00:00", "chat", "omemo",
        "sid-abc-123", "aid-xyz-789", "corrects-old-id",
-        "bob@example.com", "phone", "Encrypted message.");
+        "bob@example.com", "phone",
+        NULL, NULL, -1,
+        "Encrypted message.");

    assert_non_null(pl);
    assert_string_equal(pl->stanza_id, "sid-abc-123");
@@ -109,7 +117,9 @@ test_ff_roundtrip_with_resource(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-01-01T00:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "user@jabber.org", "Profanity.abc123", "hi");
+        "user@jabber.org", "Profanity.abc123",
+        NULL, NULL, -1,
+        "hi");

    assert_non_null(pl);
    assert_string_equal(pl->from_jid, "user@jabber.org");
@@ -124,7 +134,9 @@ test_ff_roundtrip_newline_in_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "alice@x.com", NULL, "line1\nline2\nline3");
+        "alice@x.com", NULL,
+        NULL, NULL, -1,
+        "line1\nline2\nline3");

    assert_non_null(pl);
    assert_string_equal(pl->message, "line1\nline2\nline3");
@@ -138,7 +150,9 @@ test_ff_roundtrip_pipe_in_stanza_id(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        "id|with|pipes", "aid|also|has|pipes", NULL,
-        "a@b.com", NULL, "test");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "test");

    assert_non_null(pl);
    assert_string_equal(pl->stanza_id, "id|with|pipes");
@@ -152,7 +166,9 @@ test_ff_roundtrip_backslash_in_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", NULL, "path\\to\\file C:\\Users\\test");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "path\\to\\file C:\\Users\\test");

    assert_non_null(pl);
    assert_string_equal(pl->message, "path\\to\\file C:\\Users\\test");
@@ -165,7 +181,9 @@ test_ff_roundtrip_unicode_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", NULL, "Привет мир 🌍 日本語テスト");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "Привет мир 🌍 日本語テスト");

    assert_non_null(pl);
    assert_string_equal(pl->message, "Привет мир 🌍 日本語テスト");
@@ -178,7 +196,9 @@ test_ff_roundtrip_empty_body(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", NULL, "");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "");

    assert_non_null(pl);
    assert_string_equal(pl->message, "");
@@ -192,7 +212,9 @@ test_ff_roundtrip_colonspace_in_resource(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        NULL, NULL, NULL,
-        "a@b.com", "res: with: colons", "msg");
+        "a@b.com", "res: with: colons",
+        NULL, NULL, -1,
+        "msg");

    assert_non_null(pl);
    assert_string_equal(pl->from_jid, "a@b.com");
@@ -207,7 +229,9 @@ test_ff_roundtrip_muc_type(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "muc", "none",
        NULL, NULL, NULL,
-        "room@conference.x.com", "nick", "hello room");
+        "room@conference.x.com", "nick",
+        NULL, NULL, -1,
+        "hello room");

    assert_non_null(pl);
    assert_string_equal(pl->type, "muc");
@@ -222,7 +246,9 @@ test_ff_roundtrip_omemo_enc(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "omemo",
        "sid-1", NULL, NULL,
-        "a@b.com", NULL, "secret");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "secret");

    assert_non_null(pl);
    assert_string_equal(pl->enc, "omemo");
@@ -236,7 +262,9 @@ test_ff_roundtrip_replace_id(void** state)
    ff_parsed_line_t* pl = _roundtrip(
        "2025-03-01T10:00:00Z", "chat", "none",
        "new-id", NULL, "old-id-to-correct",
-        "a@b.com", NULL, "corrected text");
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "corrected text");

    assert_non_null(pl);
    assert_string_equal(pl->stanza_id, "new-id");
@@ -245,6 +273,55 @@ test_ff_roundtrip_replace_id(void** state)
    ff_parsed_line_free(pl);
 }

+void
+test_ff_roundtrip_to_jid_and_marked_read(void** state)
+{
+    /* to_jid, to_resource, and marked_read round-trip */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-06-15T12:30:00+00:00", "chat", "none",
+        "sid-1", NULL, NULL,
+        "alice@example.com", "phone",
+        "bob@example.com", "laptop", 1,
+        "Hello Bob!");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->from_jid, "alice@example.com");
+    assert_string_equal(pl->from_resource, "phone");
+    assert_string_equal(pl->to_jid, "bob@example.com");
+    assert_string_equal(pl->to_resource, "laptop");
+    assert_int_equal(pl->marked_read, 1);
+    assert_string_equal(pl->message, "Hello Bob!");
+    ff_parsed_line_free(pl);
+
+    /* marked_read = 0 (unread) */
+    pl = _roundtrip(
+        "2025-06-15T12:31:00+00:00", "chat", "none",
+        NULL, NULL, NULL,
+        "bob@example.com", NULL,
+        "alice@example.com", NULL, 0,
+        "Reply");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->to_jid, "alice@example.com");
+    assert_null(pl->to_resource);
+    assert_int_equal(pl->marked_read, 0);
+    ff_parsed_line_free(pl);
+
+    /* marked_read = -1 (unset) — should NOT appear in output */
+    pl = _roundtrip(
+        "2025-06-15T12:32:00+00:00", "chat", "none",
+        NULL, NULL, NULL,
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "no read flag");
+
+    assert_non_null(pl);
+    assert_null(pl->to_jid);
+    assert_null(pl->to_resource);
+    assert_int_equal(pl->marked_read, -1);
+    ff_parsed_line_free(pl);
+}
+
 /* ================================================================
 * Escape / unescape symmetry tests
 * ================================================================ */
@@ -442,3 +519,203 @@ test_ff_enc_str_roundtrip(void** state)
    assert_int_equal(PROF_MSG_ENC_OX, ff_get_message_enc_type(ff_get_message_enc_str(PROF_MSG_ENC_OX)));
    assert_int_equal(PROF_MSG_ENC_OMEMO, ff_get_message_enc_type(ff_get_message_enc_str(PROF_MSG_ENC_OMEMO)));
 }
+
+/* ================================================================
+ * Additional round-trip tests
+ * ================================================================ */
+
+void
+test_ff_roundtrip_bracket_in_stanza_id(void** state)
+{
+    /* ']' in stanza_id must be escaped as \] to avoid breaking metadata parser */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "chat", "none",
+        "id]with]brackets", "aid]also]has]brackets", NULL,
+        "a@b.com", NULL,
+        NULL, NULL, -1,
+        "test brackets");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->stanza_id, "id]with]brackets");
+    assert_string_equal(pl->archive_id, "aid]also]has]brackets");
+    assert_string_equal(pl->message, "test brackets");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_backslash_in_resource(void** state)
+{
+    /* backslash in from_resource must survive round-trip */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "chat", "none",
+        NULL, NULL, NULL,
+        "a@b.com", "res\\with\\backslash",
+        NULL, NULL, -1,
+        "msg");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->from_jid, "a@b.com");
+    assert_string_equal(pl->from_resource, "res\\with\\backslash");
+    assert_string_equal(pl->message, "msg");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_mucpm_type(void** state)
+{
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "mucpm", "none",
+        NULL, NULL, NULL,
+        "room@conference.x.com", "nick",
+        NULL, NULL, -1,
+        "private message");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->type, "mucpm");
+    assert_string_equal(pl->from_resource, "nick");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_all_enc_types(void** state)
+{
+    const char* enc_types[] = { "none", "otr", "pgp", "ox", "omemo" };
+    for (size_t i = 0; i < sizeof(enc_types) / sizeof(enc_types[0]); i++) {
+        ff_parsed_line_t* pl = _roundtrip(
+            "2025-03-01T10:00:00Z", "chat", enc_types[i],
+            NULL, NULL, NULL,
+            "a@b.com", NULL,
+            NULL, NULL, -1,
+            "msg");
+
+        assert_non_null(pl);
+        assert_string_equal(pl->enc, enc_types[i]);
+        ff_parsed_line_free(pl);
+    }
+}
+
+void
+test_ff_roundtrip_crlf_handling(void** state)
+{
+    /* Write a line, then read it with \r\n ending — parser should strip \r */
+    char tmppath[] = "/tmp/proftest_crlf_XXXXXX";
+    int fd = mkstemp(tmppath);
+    assert_true(fd >= 0);
+
+    FILE* fp = fdopen(fd, "w");
+    assert_non_null(fp);
+
+    /* Write a raw line with \r\n ending */
+    fprintf(fp, "2025-03-01T10:00:00Z [chat|none] a@b.com: hello\r\n");
+    fclose(fp);
+
+    fp = fopen(tmppath, "r");
+    assert_non_null(fp);
+
+    gboolean truncated = FALSE;
+    char* buf = ff_readline(fp, &truncated);
+    fclose(fp);
+    unlink(tmppath);
+
+    assert_non_null(buf);
+    ff_parsed_line_t* pl = ff_parse_line(buf);
+    free(buf);
+
+    assert_non_null(pl);
+    assert_string_equal(pl->from_jid, "a@b.com");
+    assert_string_equal(pl->message, "hello");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_to_jid_special_chars(void** state)
+{
+    /* to: and to_res: with pipe and bracket chars that need escaping */
+    ff_parsed_line_t* pl = _roundtrip(
+        "2025-03-01T10:00:00Z", "chat", "none",
+        NULL, NULL, NULL,
+        "a@b.com", NULL,
+        "to|user@c.com", "res]with|special", -1,
+        "msg with special to");
+
+    assert_non_null(pl);
+    assert_string_equal(pl->to_jid, "to|user@c.com");
+    assert_string_equal(pl->to_resource, "res]with|special");
+    assert_string_equal(pl->message, "msg with special to");
+    ff_parsed_line_free(pl);
+}
+
+void
+test_ff_roundtrip_multiple_lines(void** state)
+{
+    /* Write several lines, read and parse them all sequentially */
+    char tmppath[] = "/tmp/proftest_multi_XXXXXX";
+    int fd = mkstemp(tmppath);
+    assert_true(fd >= 0);
+
+    FILE* fp = fdopen(fd, "w");
+    assert_non_null(fp);
+
+    ff_write_line(fp, "2025-01-01T00:00:01Z", "chat", "none",
+                  NULL, NULL, NULL, "a@b.com", NULL,
+                  NULL, NULL, -1, "first");
+    ff_write_line(fp, "2025-01-01T00:00:02Z", "chat", "none",
+                  NULL, NULL, NULL, "c@d.com", NULL,
+                  NULL, NULL, -1, "second");
+    ff_write_line(fp, "2025-01-01T00:00:03Z", "muc", "omemo",
+                  "sid-3", NULL, NULL, "room@conf.com", "nick",
+                  NULL, NULL, -1, "third");
+    fclose(fp);
+
+    fp = fopen(tmppath, "r");
+    assert_non_null(fp);
+
+    int count = 0;
+    const char* expected_msgs[] = { "first", "second", "third" };
+    const char* expected_from[] = { "a@b.com", "c@d.com", "room@conf.com" };
+
+    while (1) {
+        gboolean truncated = FALSE;
+        char* buf = ff_readline(fp, &truncated);
+        if (!buf)
+            break;
+        ff_parsed_line_t* pl = ff_parse_line(buf);
+        free(buf);
+        if (pl) {
+            assert_true(count < 3);
+            assert_string_equal(pl->message, expected_msgs[count]);
+            assert_string_equal(pl->from_jid, expected_from[count]);
+            ff_parsed_line_free(pl);
+            count++;
+        }
+    }
+    fclose(fp);
+    unlink(tmppath);
+
+    assert_int_equal(count, 3);
+}
+
+/* ================================================================
+ * Additional parser edge-case tests
+ * ================================================================ */
+
+void
+test_ff_parsed_line_free_null_safe(void** state)
+{
+    /* ff_parsed_line_free(NULL) must not crash */
+    ff_parsed_line_free(NULL);
+}
+
+void
+test_ff_parse_line_no_space_rejected(void** state)
+{
+    /* A line with no spaces at all cannot be parsed */
+    assert_null(ff_parse_line("noseparatoratall"));
+}
+
+void
+test_ff_parse_line_unclosed_bracket(void** state)
+{
+    /* Unclosed metadata bracket should return NULL */
+    assert_null(ff_parse_line("2025-03-01T10:00:00Z [chat|none a@b.com: msg"));
+}