perf: O(1) dedup/LMC cache, g_slist_append → prepend+reverse

- Add archive_ids hash set and stanza_senders hash map to ff_contact_state_t, populated during index build/extend via lightweight _ff_cache_line_ids() (no full parse overhead). - Replace O(n) file scans in _ff_has_archive_id() and _ff_find_original_sender() with O(1) hash table lookups. - Replace g_slist_append with g_slist_prepend + g_slist_reverse in _ff_apply_lmc, _flatfile_get_previous_chat, and ff_verify_integrity to avoid O(n²) list building. - Add db_sqlite_last_changes() declaration to database.h.
2026-03-12 10:14:51 +03:00
parent 06463b75b4
commit 71f9ab5007
6 changed files with 146 additions and 86 deletions
--- a/src/database_flatfile.c
+++ b/src/database_flatfile.c
@@ -67,6 +67,8 @@ ff_state_new(const char* filepath)
    ff_contact_state_t* state = g_malloc0(sizeof(ff_contact_state_t));
    state->filepath = g_strdup(filepath);
    state->cursor_offset = -1;
+    state->archive_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+    state->stanza_senders = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
    return state;
 }

@@ -77,9 +79,77 @@ ff_state_free(ff_contact_state_t* state)
        return;
    g_free(state->filepath);
    g_free(state->entries);
+    if (state->archive_ids)
+        g_hash_table_destroy(state->archive_ids);
+    if (state->stanza_senders)
+        g_hash_table_destroy(state->stanza_senders);
    g_free(state);
 }

+// Lightweight ID extraction from a raw log line for the dedup/LMC cache.
+// Avoids full ff_parse_line() overhead (no GDateTime, no message unescape).
+static void
+_ff_cache_line_ids(ff_contact_state_t* state, const char* line)
+{
+    // Find metadata section: first '[' to first unescaped ']'
+    const char* bracket = strchr(line, '[');
+    if (!bracket)
+        return;
+
+    const char* close = ff_find_unescaped_char(bracket + 1, ']');
+    if (!close)
+        return;
+
+    // Split metadata on unescaped '|'
+    char* meta_str = g_strndup(bracket + 1, close - bracket - 1);
+    char** parts = ff_split_meta(meta_str);
+    g_free(meta_str);
+
+    char* stanza_id = NULL;
+    char* archive_id = NULL;
+
+    if (parts) {
+        for (int i = 0; parts[i]; i++) {
+            if (g_str_has_prefix(parts[i], "id:") && !stanza_id) {
+                stanza_id = ff_unescape_meta_value(parts[i] + 3);
+            } else if (g_str_has_prefix(parts[i], "aid:") && !archive_id) {
+                archive_id = ff_unescape_meta_value(parts[i] + 4);
+            }
+        }
+        g_strfreev(parts);
+    }
+
+    // Extract from_jid (needed for stanza_senders map): after "] " before "/" or ": "
+    char* from_jid = NULL;
+    if (stanza_id && *(close + 1) == ' ') {
+        const char* sender_start = close + 2;
+        const char* colonspace = ff_find_unescaped_colonspace(sender_start);
+        if (colonspace) {
+            const char* jid_end = colonspace;
+            for (const char* p = sender_start; p < colonspace; p++) {
+                if (*p == '/') {
+                    jid_end = p;
+                    break;
+                }
+            }
+            from_jid = g_strndup(sender_start, jid_end - sender_start);
+        }
+    }
+
+    if (archive_id && archive_id[0] != '\0') {
+        g_hash_table_add(state->archive_ids, archive_id);
+    } else {
+        g_free(archive_id);
+    }
+
+    if (stanza_id && stanza_id[0] != '\0' && from_jid) {
+        g_hash_table_insert(state->stanza_senders, stanza_id, from_jid);
+    } else {
+        g_free(stanza_id);
+        g_free(from_jid);
+    }
+}
+
 static gboolean
 _ff_state_build(ff_contact_state_t* state)
 {
@@ -101,6 +171,10 @@ _ff_state_build(ff_contact_state_t* state)
    state->total_lines = 0;
    size_t msg_count = 0;

+    // Clear ID caches for full rebuild
+    g_hash_table_remove_all(state->archive_ids);
+    g_hash_table_remove_all(state->stanza_senders);
+
    while (1) {
        off_t pos = ftell(fp);
        gboolean truncated = FALSE;
@@ -119,6 +193,9 @@ _ff_state_build(ff_contact_state_t* state)

        state->total_lines++;

+        // Extract IDs for O(1) dedup/LMC cache
+        _ff_cache_line_ids(state, buf);
+
        if (msg_count % FF_INDEX_STEP == 0) {
            char* space = strchr(buf, ' ');
            if (space) {
@@ -203,6 +280,9 @@ _ff_state_extend(ff_contact_state_t* state)

        state->total_lines++;

+        // Extract IDs for O(1) dedup/LMC cache
+        _ff_cache_line_ids(state, buf);
+
        if (new_count % FF_INDEX_STEP == 0) {
            char* space = strchr(buf, ' ');
            if (space) {
@@ -469,79 +549,39 @@ _flatfile_close(void)
 // =========================================================================

 // =========================================================================
-// Incoming message validation helpers
+// Incoming message validation helpers (O(1) via ID cache)
 // =========================================================================

-// Search log file for a line containing aid:{archive_id}, return TRUE if found.
+// Check if archive_id already exists in the contact's log via cached set.
 static gboolean
-_ff_has_archive_id(const char* log_path, const char* archive_id)
+_ff_has_archive_id(const char* contact_barejid, const char* archive_id)
 {
-    if (!log_path || !archive_id || strlen(archive_id) == 0)
+    if (!contact_barejid || !archive_id || archive_id[0] == '\0')
        return FALSE;

-    FILE* fp = fopen(log_path, "r");
-    if (!fp)
+    ff_contact_state_t* state = _ff_get_state(contact_barejid);
+    if (!state)
        return FALSE;

-    auto_gchar gchar* needle = g_strdup_printf("aid:%s", archive_id);
-    gboolean found = FALSE;
-
-    while (!found) {
-        gboolean trunc = FALSE;
-        char* line = ff_readline(fp, &trunc);
-        if (!line)
-            break;
-        if (trunc) {
-            free(line);
-            break;
-        }
-        if (strstr(line, needle)) {
-            found = TRUE;
-        }
-        free(line);
-    }
-
-    fclose(fp);
-    return found;
+    ff_state_ensure_fresh(state);
+    return g_hash_table_contains(state->archive_ids, archive_id);
 }

-// Search log file for a line with stanza id matching replace_id, return the
-// from_jid of the original message (caller must g_free). Returns NULL if not found.
+// Find the from_jid of the original message with given stanza_id via cached map.
+// Caller must g_free the result.
 static char*
-_ff_find_original_sender(const char* log_path, const char* replace_id)
+_ff_find_original_sender(const char* contact_barejid, const char* replace_id)
 {
-    if (!log_path || !replace_id || strlen(replace_id) == 0)
+    if (!contact_barejid || !replace_id || replace_id[0] == '\0')
        return NULL;

-    FILE* fp = fopen(log_path, "r");
-    if (!fp)
+    ff_contact_state_t* state = _ff_get_state(contact_barejid);
+    if (!state)
        return NULL;

-    auto_gchar gchar* needle = g_strdup_printf("id:%s", replace_id);
-    char* sender = NULL;
-
-    while (!sender) {
-        gboolean trunc = FALSE;
-        char* line = ff_readline(fp, &trunc);
-        if (!line)
-            break;
-        if (trunc) {
-            free(line);
-            break;
-        }
-        if (strstr(line, needle)) {
-            ff_parsed_line_t* pl = ff_parse_line(line);
-            if (pl && pl->stanza_id && g_strcmp0(pl->stanza_id, replace_id) == 0) {
-                sender = g_strdup(pl->from_jid);
-            }
-            if (pl)
-                ff_parsed_line_free(pl);
-        }
-        free(line);
-    }
-
-    fclose(fp);
-    return sender;
+    ff_state_ensure_fresh(state);
+    const char* sender = g_hash_table_lookup(state->stanza_senders, replace_id);
+    return sender ? g_strdup(sender) : NULL;
 }

 static void
@@ -561,11 +601,9 @@ _flatfile_add_incoming(ProfMessage* message)
    } else {
        contact = message->from_jid->barejid;
    }
-    auto_gchar gchar* log_path = ff_get_log_path(contact);
-
    // MAM dedup: skip if this stanza-id already exists in the log (XEP-0313 replay)
-    if (message->stanzaid && !message->is_mam && log_path) {
-        if (_ff_has_archive_id(log_path, message->stanzaid)) {
+    if (message->stanzaid && !message->is_mam) {
+        if (_ff_has_archive_id(contact, message->stanzaid)) {
            log_error("flatfile: duplicate stanza-id '%s' from %s, skipping",
                      message->stanzaid, message->from_jid->barejid);
            cons_show_error("Got a message with duplicate (server-generated) stanza-id from %s.",
@@ -575,8 +613,8 @@ _flatfile_add_incoming(ProfMessage* message)
    }

    // LMC sender validation: verify the correction comes from the original sender
-    if (message->replace_id && log_path) {
-        auto_gchar gchar* original_sender = _ff_find_original_sender(log_path, message->replace_id);
+    if (message->replace_id) {
+        auto_gchar gchar* original_sender = _ff_find_original_sender(contact, message->replace_id);
        if (original_sender && g_strcmp0(original_sender, message->from_jid->barejid) != 0) {
            log_error("flatfile: LMC sender mismatch — corrected msg sender: %s, original: %s, replace-id: %s",
                      message->from_jid->barejid, original_sender, message->replace_id);
@@ -691,9 +729,11 @@ _ff_apply_lmc(GSList* parsed_lines, GSList** result)
        } else {
            msg = ff_parsed_to_profmessage(pl);
        }
-        *result = g_slist_append(*result, msg);
+        *result = g_slist_prepend(*result, msg);
    }

+    *result = g_slist_reverse(*result);
+
    g_hash_table_destroy(id_map);
    g_hash_table_destroy(corrections);
    g_hash_table_destroy(correction_map);
@@ -845,10 +885,12 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta
            continue;
        }

-        all_parsed = g_slist_append(all_parsed, pl);
+        all_parsed = g_slist_prepend(all_parsed, pl);
    }
    fclose(fp);

+    all_parsed = g_slist_reverse(all_parsed);
+
    if (start_dt)
        g_date_time_unref(start_dt);
    if (end_dt)