perf: O(1) dedup/LMC cache, g_slist_append → prepend+reverse

- Add archive_ids hash set and stanza_senders hash map to
  ff_contact_state_t, populated during index build/extend via
  lightweight _ff_cache_line_ids() (no full parse overhead).
- Replace O(n) file scans in _ff_has_archive_id() and
  _ff_find_original_sender() with O(1) hash table lookups.
- Replace g_slist_append with g_slist_prepend + g_slist_reverse
  in _ff_apply_lmc, _flatfile_get_previous_chat, and
  ff_verify_integrity to avoid O(n²) list building.
- Add db_sqlite_last_changes() declaration to database.h.

src/database.h

@@ -92,6 +92,7 @@ GSList* db_sqlite_get_all_chat(const gchar* const contact_barejid);
 void db_sqlite_begin_transaction(void);
 void db_sqlite_end_transaction(void);
 void db_sqlite_rollback_transaction(void);
+int db_sqlite_last_changes(void);
 #endif
 db_backend_t* db_backend_flatfile(void);
 

src/database_export.c

@@ -213,7 +213,6 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
     if (!g_flatfile_account_jid) {
         // Flatfile backend not initialized — init it temporarily
         // We need g_flatfile_account_jid for path construction
-        g_free(g_flatfile_account_jid);
         g_flatfile_account_jid = g_strdup(myjid->barejid);
     }
 
@@ -289,7 +288,9 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
 
         // Lock the temp file
         int fd = fileno(fp);
-        flock(fd, LOCK_EX);
+        if (flock(fd, LOCK_EX) != 0) {
+            log_warning("export: flock(%s) failed: %s", tmp_path, strerror(errno));
+        }
 
         fprintf(fp, "%s", FLATFILE_HEADER);
 
@@ -416,7 +417,6 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
     }
 
     if (!g_flatfile_account_jid) {
-        g_free(g_flatfile_account_jid);
         g_flatfile_account_jid = g_strdup(myjid->barejid);
     }
 
@@ -522,6 +522,13 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
 
             sqlite_be->add_incoming(msg);
             message_free(msg);
+
+            // Verify the row was actually inserted
+            if (db_sqlite_last_changes() < 1) {
+                log_error("import: insert failed for %s at %s", contact, ts);
+                import_ok = FALSE;
+                break;
+            }
             contact_imported++;
 
             if (contact_imported % 500 == 0) {

src/database_flatfile.c

@@ -67,6 +67,8 @@ ff_state_new(const char* filepath)
     ff_contact_state_t* state = g_malloc0(sizeof(ff_contact_state_t));
     state->filepath = g_strdup(filepath);
     state->cursor_offset = -1;
+    state->archive_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+    state->stanza_senders = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
     return state;
 }
 
@@ -77,9 +79,77 @@ ff_state_free(ff_contact_state_t* state)
         return;
     g_free(state->filepath);
     g_free(state->entries);
+    if (state->archive_ids)
+        g_hash_table_destroy(state->archive_ids);
+    if (state->stanza_senders)
+        g_hash_table_destroy(state->stanza_senders);
     g_free(state);
 }
 
+// Lightweight ID extraction from a raw log line for the dedup/LMC cache.
+// Avoids full ff_parse_line() overhead (no GDateTime, no message unescape).
+static void
+_ff_cache_line_ids(ff_contact_state_t* state, const char* line)
+{
+    // Find metadata section: first '[' to first unescaped ']'
+    const char* bracket = strchr(line, '[');
+    if (!bracket)
+        return;
+
+    const char* close = ff_find_unescaped_char(bracket + 1, ']');
+    if (!close)
+        return;
+
+    // Split metadata on unescaped '|'
+    char* meta_str = g_strndup(bracket + 1, close - bracket - 1);
+    char** parts = ff_split_meta(meta_str);
+    g_free(meta_str);
+
+    char* stanza_id = NULL;
+    char* archive_id = NULL;
+
+    if (parts) {
+        for (int i = 0; parts[i]; i++) {
+            if (g_str_has_prefix(parts[i], "id:") && !stanza_id) {
+                stanza_id = ff_unescape_meta_value(parts[i] + 3);
+            } else if (g_str_has_prefix(parts[i], "aid:") && !archive_id) {
+                archive_id = ff_unescape_meta_value(parts[i] + 4);
+            }
+        }
+        g_strfreev(parts);
+    }
+
+    // Extract from_jid (needed for stanza_senders map): after "] " before "/" or ": "
+    char* from_jid = NULL;
+    if (stanza_id && *(close + 1) == ' ') {
+        const char* sender_start = close + 2;
+        const char* colonspace = ff_find_unescaped_colonspace(sender_start);
+        if (colonspace) {
+            const char* jid_end = colonspace;
+            for (const char* p = sender_start; p < colonspace; p++) {
+                if (*p == '/') {
+                    jid_end = p;
+                    break;
+                }
+            }
+            from_jid = g_strndup(sender_start, jid_end - sender_start);
+        }
+    }
+
+    if (archive_id && archive_id[0] != '\0') {
+        g_hash_table_add(state->archive_ids, archive_id);
+    } else {
+        g_free(archive_id);
+    }
+
+    if (stanza_id && stanza_id[0] != '\0' && from_jid) {
+        g_hash_table_insert(state->stanza_senders, stanza_id, from_jid);
+    } else {
+        g_free(stanza_id);
+        g_free(from_jid);
+    }
+}
+
 static gboolean
 _ff_state_build(ff_contact_state_t* state)
 {
@@ -101,6 +171,10 @@ _ff_state_build(ff_contact_state_t* state)
     state->total_lines = 0;
     size_t msg_count = 0;
 
+    // Clear ID caches for full rebuild
+    g_hash_table_remove_all(state->archive_ids);
+    g_hash_table_remove_all(state->stanza_senders);
+
     while (1) {
         off_t pos = ftell(fp);
         gboolean truncated = FALSE;
@@ -119,6 +193,9 @@ _ff_state_build(ff_contact_state_t* state)
 
         state->total_lines++;
 
+        // Extract IDs for O(1) dedup/LMC cache
+        _ff_cache_line_ids(state, buf);
+
         if (msg_count % FF_INDEX_STEP == 0) {
             char* space = strchr(buf, ' ');
             if (space) {
@@ -203,6 +280,9 @@ _ff_state_extend(ff_contact_state_t* state)
 
         state->total_lines++;
 
+        // Extract IDs for O(1) dedup/LMC cache
+        _ff_cache_line_ids(state, buf);
+
         if (new_count % FF_INDEX_STEP == 0) {
             char* space = strchr(buf, ' ');
             if (space) {
@@ -469,79 +549,39 @@ _flatfile_close(void)
 // =========================================================================
 
 // =========================================================================
-// Incoming message validation helpers
+// Incoming message validation helpers (O(1) via ID cache)
 // =========================================================================
 
-// Search log file for a line containing aid:{archive_id}, return TRUE if found.
+// Check if archive_id already exists in the contact's log via cached set.
 static gboolean
-_ff_has_archive_id(const char* log_path, const char* archive_id)
+_ff_has_archive_id(const char* contact_barejid, const char* archive_id)
 {
-    if (!log_path || !archive_id || strlen(archive_id) == 0)
+    if (!contact_barejid || !archive_id || archive_id[0] == '\0')
         return FALSE;
 
-    FILE* fp = fopen(log_path, "r");
-    if (!fp)
+    ff_contact_state_t* state = _ff_get_state(contact_barejid);
+    if (!state)
         return FALSE;
 
-    auto_gchar gchar* needle = g_strdup_printf("aid:%s", archive_id);
-    gboolean found = FALSE;
-
-    while (!found) {
-        gboolean trunc = FALSE;
-        char* line = ff_readline(fp, &trunc);
-        if (!line)
-            break;
-        if (trunc) {
-            free(line);
-            break;
-        }
-        if (strstr(line, needle)) {
-            found = TRUE;
-        }
-        free(line);
-    }
-
-    fclose(fp);
-    return found;
+    ff_state_ensure_fresh(state);
+    return g_hash_table_contains(state->archive_ids, archive_id);
 }
 
-// Search log file for a line with stanza id matching replace_id, return the
-// from_jid of the original message (caller must g_free). Returns NULL if not found.
+// Find the from_jid of the original message with given stanza_id via cached map.
+// Caller must g_free the result.
 static char*
-_ff_find_original_sender(const char* log_path, const char* replace_id)
+_ff_find_original_sender(const char* contact_barejid, const char* replace_id)
 {
-    if (!log_path || !replace_id || strlen(replace_id) == 0)
+    if (!contact_barejid || !replace_id || replace_id[0] == '\0')
         return NULL;
 
-    FILE* fp = fopen(log_path, "r");
-    if (!fp)
+    ff_contact_state_t* state = _ff_get_state(contact_barejid);
+    if (!state)
         return NULL;
 
-    auto_gchar gchar* needle = g_strdup_printf("id:%s", replace_id);
-    char* sender = NULL;
-
-    while (!sender) {
-        gboolean trunc = FALSE;
-        char* line = ff_readline(fp, &trunc);
-        if (!line)
-            break;
-        if (trunc) {
-            free(line);
-            break;
-        }
-        if (strstr(line, needle)) {
-            ff_parsed_line_t* pl = ff_parse_line(line);
-            if (pl && pl->stanza_id && g_strcmp0(pl->stanza_id, replace_id) == 0) {
-                sender = g_strdup(pl->from_jid);
-            }
-            if (pl)
-                ff_parsed_line_free(pl);
-        }
-        free(line);
-    }
-
-    fclose(fp);
-    return sender;
+    ff_state_ensure_fresh(state);
+    const char* sender = g_hash_table_lookup(state->stanza_senders, replace_id);
+    return sender ? g_strdup(sender) : NULL;
 }
 
 static void
@@ -561,11 +601,9 @@ _flatfile_add_incoming(ProfMessage* message)
     } else {
         contact = message->from_jid->barejid;
     }
-    auto_gchar gchar* log_path = ff_get_log_path(contact);
-
     // MAM dedup: skip if this stanza-id already exists in the log (XEP-0313 replay)
-    if (message->stanzaid && !message->is_mam && log_path) {
-        if (_ff_has_archive_id(log_path, message->stanzaid)) {
+    if (message->stanzaid && !message->is_mam) {
+        if (_ff_has_archive_id(contact, message->stanzaid)) {
             log_error("flatfile: duplicate stanza-id '%s' from %s, skipping",
                       message->stanzaid, message->from_jid->barejid);
             cons_show_error("Got a message with duplicate (server-generated) stanza-id from %s.",
@@ -575,8 +613,8 @@ _flatfile_add_incoming(ProfMessage* message)
     }
 
     // LMC sender validation: verify the correction comes from the original sender
-    if (message->replace_id && log_path) {
-        auto_gchar gchar* original_sender = _ff_find_original_sender(log_path, message->replace_id);
+    if (message->replace_id) {
+        auto_gchar gchar* original_sender = _ff_find_original_sender(contact, message->replace_id);
         if (original_sender && g_strcmp0(original_sender, message->from_jid->barejid) != 0) {
             log_error("flatfile: LMC sender mismatch — corrected msg sender: %s, original: %s, replace-id: %s",
                       message->from_jid->barejid, original_sender, message->replace_id);
@@ -691,9 +729,11 @@ _ff_apply_lmc(GSList* parsed_lines, GSList** result)
         } else {
             msg = ff_parsed_to_profmessage(pl);
         }
-        *result = g_slist_append(*result, msg);
+        *result = g_slist_prepend(*result, msg);
     }
 
+    *result = g_slist_reverse(*result);
+
     g_hash_table_destroy(id_map);
     g_hash_table_destroy(corrections);
     g_hash_table_destroy(correction_map);
@@ -845,10 +885,12 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta
             continue;
         }
 
-        all_parsed = g_slist_append(all_parsed, pl);
+        all_parsed = g_slist_prepend(all_parsed, pl);
     }
     fclose(fp);
 
+    all_parsed = g_slist_reverse(all_parsed);
+
     if (start_dt)
         g_date_time_unref(start_dt);
     if (end_dt)

src/database_flatfile.h

@@ -94,6 +94,8 @@ typedef struct
     ff_file_stamp_t stamp;
     off_t cursor_offset;
     int bom_len;
+    GHashTable* archive_ids;    // set: archive_id -> NULL (MAM dedup, O(1))
+    GHashTable* stanza_senders; // map: stanza_id -> from_jid (LMC validation, O(1))
 } ff_contact_state_t;
 
 // State management

src/database_flatfile_verify.c

@@ -48,7 +48,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
         issue->file = g_strdup("N/A");
         issue->line = 0;
         issue->message = g_strdup("Flat-file backend not initialized");
-        issues = g_slist_append(issues, issue);
+        issues = g_slist_prepend(issues, issue);
         return issues;
     }
 
@@ -58,14 +58,14 @@ ff_verify_integrity(const gchar* const contact_barejid)
     if (contact_barejid) {
         auto_gchar gchar* cdir = ff_get_contact_dir(contact_barejid);
         if (cdir && g_file_test(cdir, G_FILE_TEST_IS_DIR)) {
-            contact_dirs = g_slist_append(contact_dirs, g_strdup(cdir));
+            contact_dirs = g_slist_prepend(contact_dirs, g_strdup(cdir));
         } else {
             integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
             issue->level = INTEGRITY_INFO;
             issue->file = g_strdup(contact_barejid);
             issue->line = 0;
             issue->message = g_strdup("No log files found for this contact");
-            issues = g_slist_append(issues, issue);
+            issues = g_slist_prepend(issues, issue);
             return issues;
         }
     } else {
@@ -80,7 +80,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
             while ((dname = g_dir_read_name(dir)) != NULL) {
                 char* full = g_strdup_printf("%s/%s", base_dir, dname);
                 if (g_file_test(full, G_FILE_TEST_IS_DIR)) {
-                    contact_dirs = g_slist_append(contact_dirs, full);
+                    contact_dirs = g_slist_prepend(contact_dirs, full);
                 } else {
                     g_free(full);
                 }
@@ -108,7 +108,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                 issue->file = g_strdup(basename);
                 issue->line = 0;
                 issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777);
-                issues = g_slist_append(issues, issue);
+                issues = g_slist_prepend(issues, issue);
             }
         }
 
@@ -126,7 +126,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
             issue->file = g_strdup(basename);
             issue->line = 0;
             issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary");
-            issues = g_slist_append(issues, issue);
+            issues = g_slist_prepend(issues, issue);
         } else {
             fseek(fp, 0, SEEK_SET);
         }
@@ -161,7 +161,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                 issue->file = g_strdup(basename);
                 issue->line = lineno;
                 issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf));
-                issues = g_slist_append(issues, issue);
+                issues = g_slist_prepend(issues, issue);
                 free(buf);
                 continue;
             }
@@ -174,7 +174,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                     issue->file = g_strdup(basename);
                     issue->line = lineno;
                     issue->message = g_strdup_printf("Contains control character 0x%02x", ch);
-                    issues = g_slist_append(issues, issue);
+                    issues = g_slist_prepend(issues, issue);
                     break;
                 }
             }
@@ -186,7 +186,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                 issue->file = g_strdup(basename);
                 issue->line = lineno;
                 issue->message = g_strdup("Unparsable line");
-                issues = g_slist_append(issues, issue);
+                issues = g_slist_prepend(issues, issue);
                 free(buf);
                 continue;
             }
@@ -202,7 +202,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                 issue->file = g_strdup(basename);
                 issue->line = lineno;
                 issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev);
-                issues = g_slist_append(issues, issue);
+                issues = g_slist_prepend(issues, issue);
             }
             if (prev_ts)
                 g_date_time_unref(prev_ts);
@@ -216,7 +216,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                     issue->file = g_strdup(basename);
                     issue->line = lineno;
                     issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id);
-                    issues = g_slist_append(issues, issue);
+                    issues = g_slist_prepend(issues, issue);
                 } else {
                     g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
                 }
@@ -229,7 +229,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                     issue->file = g_strdup(basename);
                     issue->line = lineno;
                     issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id);
-                    issues = g_slist_append(issues, issue);
+                    issues = g_slist_prepend(issues, issue);
                 } else {
                     g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno));
                 }
@@ -246,7 +246,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
             issue->file = g_strdup(basename);
             issue->line = 0;
             issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF");
-            issues = g_slist_append(issues, issue);
+            issues = g_slist_prepend(issues, issue);
         }
 
         if (is_empty) {
@@ -255,7 +255,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
             issue->file = g_strdup(basename);
             issue->line = 0;
             issue->message = g_strdup("File is empty (no message lines)");
-            issues = g_slist_append(issues, issue);
+            issues = g_slist_prepend(issues, issue);
         }
 
         // Second pass: check LMC references
@@ -290,7 +290,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                         issue->file = g_strdup(basename);
                         issue->line = lineno;
                         issue->message = g_strdup_printf("Broken correction reference: corrects:%s not found", pl->replace_id);
-                        issues = g_slist_append(issues, issue);
+                        issues = g_slist_prepend(issues, issue);
                     }
                 }
                 ff_parsed_line_free(pl);
@@ -305,5 +305,5 @@ ff_verify_integrity(const gchar* const contact_barejid)
     }
 
     g_slist_free_full(contact_dirs, g_free);
-    return issues;
+    return g_slist_reverse(issues);
 }

src/database_sqlite.c

@@ -860,6 +860,14 @@ db_sqlite_rollback_transaction(void)
     }
 }
 
+int
+db_sqlite_last_changes(void)
+{
+    if (!g_chatlog_database)
+        return 0;
+    return sqlite3_changes(g_chatlog_database);
+}
+
 GSList*
 db_sqlite_get_all_chat(const gchar* const contact_barejid)
 {
@@ -949,10 +957,10 @@ db_sqlite_list_contacts(void)
     while (sqlite3_step(stmt) == SQLITE_ROW) {
         const char* jid = (const char*)sqlite3_column_text(stmt, 0);
         if (jid && strlen(jid) > 0) {
-            contacts = g_slist_append(contacts, g_strdup(jid));
+            contacts = g_slist_prepend(contacts, g_strdup(jid));
         }
     }
     sqlite3_finalize(stmt);
 
-    return contacts;
+    return g_slist_reverse(contacts);
 }