From 1583b8de9cd96ffc09c4c1f90de125da645d303a Mon Sep 17 00:00:00 2001 From: "jabber.developer2" Date: Thu, 12 Mar 2026 10:14:51 +0300 Subject: [PATCH] =?UTF-8?q?perf:=20O(1)=20dedup/LMC=20cache,=20g=5Fslist?= =?UTF-8?q?=5Fappend=20=E2=86=92=20prepend+reverse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add archive_ids hash set and stanza_senders hash map to ff_contact_state_t, populated during index build/extend via lightweight _ff_cache_line_ids() (no full parse overhead). - Replace O(n) file scans in _ff_has_archive_id() and _ff_find_original_sender() with O(1) hash table lookups. - Replace g_slist_append with g_slist_prepend + g_slist_reverse in _ff_apply_lmc, _flatfile_get_previous_chat, and ff_verify_integrity to avoid O(n²) list building. - Add db_sqlite_last_changes() declaration to database.h. --- src/database.h | 1 + src/database_export.c | 13 ++- src/database_flatfile.c | 172 ++++++++++++++++++++------------- src/database_flatfile.h | 2 + src/database_flatfile_verify.c | 32 +++--- src/database_sqlite.c | 12 ++- 6 files changed, 146 insertions(+), 86 deletions(-) diff --git a/src/database.h b/src/database.h index 647fedbd..473d9f7a 100644 --- a/src/database.h +++ b/src/database.h @@ -92,6 +92,7 @@ GSList* db_sqlite_get_all_chat(const gchar* const contact_barejid); void db_sqlite_begin_transaction(void); void db_sqlite_end_transaction(void); void db_sqlite_rollback_transaction(void); +int db_sqlite_last_changes(void); #endif db_backend_t* db_backend_flatfile(void); diff --git a/src/database_export.c b/src/database_export.c index 5fe24c51..4ce20028 100644 --- a/src/database_export.c +++ b/src/database_export.c @@ -213,7 +213,6 @@ log_database_export_to_flatfile(const gchar* const contact_jid) if (!g_flatfile_account_jid) { // Flatfile backend not initialized — init it temporarily // We need g_flatfile_account_jid for path construction - g_free(g_flatfile_account_jid); g_flatfile_account_jid = g_strdup(myjid->barejid); } @@ -289,7 +288,9 @@ log_database_export_to_flatfile(const gchar* const contact_jid) // Lock the temp file int fd = fileno(fp); - flock(fd, LOCK_EX); + if (flock(fd, LOCK_EX) != 0) { + log_warning("export: flock(%s) failed: %s", tmp_path, strerror(errno)); + } fprintf(fp, "%s", FLATFILE_HEADER); @@ -416,7 +417,6 @@ log_database_import_from_flatfile(const gchar* const contact_jid) } if (!g_flatfile_account_jid) { - g_free(g_flatfile_account_jid); g_flatfile_account_jid = g_strdup(myjid->barejid); } @@ -522,6 +522,13 @@ log_database_import_from_flatfile(const gchar* const contact_jid) sqlite_be->add_incoming(msg); message_free(msg); + + // Verify the row was actually inserted + if (db_sqlite_last_changes() < 1) { + log_error("import: insert failed for %s at %s", contact, ts); + import_ok = FALSE; + break; + } contact_imported++; if (contact_imported % 500 == 0) { diff --git a/src/database_flatfile.c b/src/database_flatfile.c index f0875d90..4a1a0fc2 100644 --- a/src/database_flatfile.c +++ b/src/database_flatfile.c @@ -67,6 +67,8 @@ ff_state_new(const char* filepath) ff_contact_state_t* state = g_malloc0(sizeof(ff_contact_state_t)); state->filepath = g_strdup(filepath); state->cursor_offset = -1; + state->archive_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + state->stanza_senders = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); return state; } @@ -77,9 +79,77 @@ ff_state_free(ff_contact_state_t* state) return; g_free(state->filepath); g_free(state->entries); + if (state->archive_ids) + g_hash_table_destroy(state->archive_ids); + if (state->stanza_senders) + g_hash_table_destroy(state->stanza_senders); g_free(state); } +// Lightweight ID extraction from a raw log line for the dedup/LMC cache. +// Avoids full ff_parse_line() overhead (no GDateTime, no message unescape). +static void +_ff_cache_line_ids(ff_contact_state_t* state, const char* line) +{ + // Find metadata section: first '[' to first unescaped ']' + const char* bracket = strchr(line, '['); + if (!bracket) + return; + + const char* close = ff_find_unescaped_char(bracket + 1, ']'); + if (!close) + return; + + // Split metadata on unescaped '|' + char* meta_str = g_strndup(bracket + 1, close - bracket - 1); + char** parts = ff_split_meta(meta_str); + g_free(meta_str); + + char* stanza_id = NULL; + char* archive_id = NULL; + + if (parts) { + for (int i = 0; parts[i]; i++) { + if (g_str_has_prefix(parts[i], "id:") && !stanza_id) { + stanza_id = ff_unescape_meta_value(parts[i] + 3); + } else if (g_str_has_prefix(parts[i], "aid:") && !archive_id) { + archive_id = ff_unescape_meta_value(parts[i] + 4); + } + } + g_strfreev(parts); + } + + // Extract from_jid (needed for stanza_senders map): after "] " before "/" or ": " + char* from_jid = NULL; + if (stanza_id && *(close + 1) == ' ') { + const char* sender_start = close + 2; + const char* colonspace = ff_find_unescaped_colonspace(sender_start); + if (colonspace) { + const char* jid_end = colonspace; + for (const char* p = sender_start; p < colonspace; p++) { + if (*p == '/') { + jid_end = p; + break; + } + } + from_jid = g_strndup(sender_start, jid_end - sender_start); + } + } + + if (archive_id && archive_id[0] != '\0') { + g_hash_table_add(state->archive_ids, archive_id); + } else { + g_free(archive_id); + } + + if (stanza_id && stanza_id[0] != '\0' && from_jid) { + g_hash_table_insert(state->stanza_senders, stanza_id, from_jid); + } else { + g_free(stanza_id); + g_free(from_jid); + } +} + static gboolean _ff_state_build(ff_contact_state_t* state) { @@ -101,6 +171,10 @@ _ff_state_build(ff_contact_state_t* state) state->total_lines = 0; size_t msg_count = 0; + // Clear ID caches for full rebuild + g_hash_table_remove_all(state->archive_ids); + g_hash_table_remove_all(state->stanza_senders); + while (1) { off_t pos = ftell(fp); gboolean truncated = FALSE; @@ -119,6 +193,9 @@ _ff_state_build(ff_contact_state_t* state) state->total_lines++; + // Extract IDs for O(1) dedup/LMC cache + _ff_cache_line_ids(state, buf); + if (msg_count % FF_INDEX_STEP == 0) { char* space = strchr(buf, ' '); if (space) { @@ -203,6 +280,9 @@ _ff_state_extend(ff_contact_state_t* state) state->total_lines++; + // Extract IDs for O(1) dedup/LMC cache + _ff_cache_line_ids(state, buf); + if (new_count % FF_INDEX_STEP == 0) { char* space = strchr(buf, ' '); if (space) { @@ -469,79 +549,39 @@ _flatfile_close(void) // ========================================================================= // ========================================================================= -// Incoming message validation helpers +// Incoming message validation helpers (O(1) via ID cache) // ========================================================================= -// Search log file for a line containing aid:{archive_id}, return TRUE if found. +// Check if archive_id already exists in the contact's log via cached set. static gboolean -_ff_has_archive_id(const char* log_path, const char* archive_id) +_ff_has_archive_id(const char* contact_barejid, const char* archive_id) { - if (!log_path || !archive_id || strlen(archive_id) == 0) + if (!contact_barejid || !archive_id || archive_id[0] == '\0') return FALSE; - FILE* fp = fopen(log_path, "r"); - if (!fp) + ff_contact_state_t* state = _ff_get_state(contact_barejid); + if (!state) return FALSE; - auto_gchar gchar* needle = g_strdup_printf("aid:%s", archive_id); - gboolean found = FALSE; - - while (!found) { - gboolean trunc = FALSE; - char* line = ff_readline(fp, &trunc); - if (!line) - break; - if (trunc) { - free(line); - break; - } - if (strstr(line, needle)) { - found = TRUE; - } - free(line); - } - - fclose(fp); - return found; + ff_state_ensure_fresh(state); + return g_hash_table_contains(state->archive_ids, archive_id); } -// Search log file for a line with stanza id matching replace_id, return the -// from_jid of the original message (caller must g_free). Returns NULL if not found. +// Find the from_jid of the original message with given stanza_id via cached map. +// Caller must g_free the result. static char* -_ff_find_original_sender(const char* log_path, const char* replace_id) +_ff_find_original_sender(const char* contact_barejid, const char* replace_id) { - if (!log_path || !replace_id || strlen(replace_id) == 0) + if (!contact_barejid || !replace_id || replace_id[0] == '\0') return NULL; - FILE* fp = fopen(log_path, "r"); - if (!fp) + ff_contact_state_t* state = _ff_get_state(contact_barejid); + if (!state) return NULL; - auto_gchar gchar* needle = g_strdup_printf("id:%s", replace_id); - char* sender = NULL; - - while (!sender) { - gboolean trunc = FALSE; - char* line = ff_readline(fp, &trunc); - if (!line) - break; - if (trunc) { - free(line); - break; - } - if (strstr(line, needle)) { - ff_parsed_line_t* pl = ff_parse_line(line); - if (pl && pl->stanza_id && g_strcmp0(pl->stanza_id, replace_id) == 0) { - sender = g_strdup(pl->from_jid); - } - if (pl) - ff_parsed_line_free(pl); - } - free(line); - } - - fclose(fp); - return sender; + ff_state_ensure_fresh(state); + const char* sender = g_hash_table_lookup(state->stanza_senders, replace_id); + return sender ? g_strdup(sender) : NULL; } static void @@ -561,11 +601,9 @@ _flatfile_add_incoming(ProfMessage* message) } else { contact = message->from_jid->barejid; } - auto_gchar gchar* log_path = ff_get_log_path(contact); - // MAM dedup: skip if this stanza-id already exists in the log (XEP-0313 replay) - if (message->stanzaid && !message->is_mam && log_path) { - if (_ff_has_archive_id(log_path, message->stanzaid)) { + if (message->stanzaid && !message->is_mam) { + if (_ff_has_archive_id(contact, message->stanzaid)) { log_error("flatfile: duplicate stanza-id '%s' from %s, skipping", message->stanzaid, message->from_jid->barejid); cons_show_error("Got a message with duplicate (server-generated) stanza-id from %s.", @@ -575,8 +613,8 @@ _flatfile_add_incoming(ProfMessage* message) } // LMC sender validation: verify the correction comes from the original sender - if (message->replace_id && log_path) { - auto_gchar gchar* original_sender = _ff_find_original_sender(log_path, message->replace_id); + if (message->replace_id) { + auto_gchar gchar* original_sender = _ff_find_original_sender(contact, message->replace_id); if (original_sender && g_strcmp0(original_sender, message->from_jid->barejid) != 0) { log_error("flatfile: LMC sender mismatch — corrected msg sender: %s, original: %s, replace-id: %s", message->from_jid->barejid, original_sender, message->replace_id); @@ -691,9 +729,11 @@ _ff_apply_lmc(GSList* parsed_lines, GSList** result) } else { msg = ff_parsed_to_profmessage(pl); } - *result = g_slist_append(*result, msg); + *result = g_slist_prepend(*result, msg); } + *result = g_slist_reverse(*result); + g_hash_table_destroy(id_map); g_hash_table_destroy(corrections); g_hash_table_destroy(correction_map); @@ -845,10 +885,12 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta continue; } - all_parsed = g_slist_append(all_parsed, pl); + all_parsed = g_slist_prepend(all_parsed, pl); } fclose(fp); + all_parsed = g_slist_reverse(all_parsed); + if (start_dt) g_date_time_unref(start_dt); if (end_dt) diff --git a/src/database_flatfile.h b/src/database_flatfile.h index 720e7475..3b055ef2 100644 --- a/src/database_flatfile.h +++ b/src/database_flatfile.h @@ -94,6 +94,8 @@ typedef struct ff_file_stamp_t stamp; off_t cursor_offset; int bom_len; + GHashTable* archive_ids; // set: archive_id -> NULL (MAM dedup, O(1)) + GHashTable* stanza_senders; // map: stanza_id -> from_jid (LMC validation, O(1)) } ff_contact_state_t; // State management diff --git a/src/database_flatfile_verify.c b/src/database_flatfile_verify.c index bc304b4b..a3c1e6d6 100644 --- a/src/database_flatfile_verify.c +++ b/src/database_flatfile_verify.c @@ -48,7 +48,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup("N/A"); issue->line = 0; issue->message = g_strdup("Flat-file backend not initialized"); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); return issues; } @@ -58,14 +58,14 @@ ff_verify_integrity(const gchar* const contact_barejid) if (contact_barejid) { auto_gchar gchar* cdir = ff_get_contact_dir(contact_barejid); if (cdir && g_file_test(cdir, G_FILE_TEST_IS_DIR)) { - contact_dirs = g_slist_append(contact_dirs, g_strdup(cdir)); + contact_dirs = g_slist_prepend(contact_dirs, g_strdup(cdir)); } else { integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); issue->level = INTEGRITY_INFO; issue->file = g_strdup(contact_barejid); issue->line = 0; issue->message = g_strdup("No log files found for this contact"); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); return issues; } } else { @@ -80,7 +80,7 @@ ff_verify_integrity(const gchar* const contact_barejid) while ((dname = g_dir_read_name(dir)) != NULL) { char* full = g_strdup_printf("%s/%s", base_dir, dname); if (g_file_test(full, G_FILE_TEST_IS_DIR)) { - contact_dirs = g_slist_append(contact_dirs, full); + contact_dirs = g_slist_prepend(contact_dirs, full); } else { g_free(full); } @@ -108,7 +108,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = 0; issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } } @@ -126,7 +126,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = 0; issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary"); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } else { fseek(fp, 0, SEEK_SET); } @@ -161,7 +161,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf)); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); free(buf); continue; } @@ -174,7 +174,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup_printf("Contains control character 0x%02x", ch); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); break; } } @@ -186,7 +186,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup("Unparsable line"); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); free(buf); continue; } @@ -202,7 +202,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } if (prev_ts) g_date_time_unref(prev_ts); @@ -216,7 +216,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } else { g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno)); } @@ -229,7 +229,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } else { g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno)); } @@ -246,7 +246,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = 0; issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF"); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } if (is_empty) { @@ -255,7 +255,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = 0; issue->message = g_strdup("File is empty (no message lines)"); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } // Second pass: check LMC references @@ -290,7 +290,7 @@ ff_verify_integrity(const gchar* const contact_barejid) issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup_printf("Broken correction reference: corrects:%s not found", pl->replace_id); - issues = g_slist_append(issues, issue); + issues = g_slist_prepend(issues, issue); } } ff_parsed_line_free(pl); @@ -305,5 +305,5 @@ ff_verify_integrity(const gchar* const contact_barejid) } g_slist_free_full(contact_dirs, g_free); - return issues; + return g_slist_reverse(issues); } diff --git a/src/database_sqlite.c b/src/database_sqlite.c index ec3ceb8b..82d2b05c 100644 --- a/src/database_sqlite.c +++ b/src/database_sqlite.c @@ -860,6 +860,14 @@ db_sqlite_rollback_transaction(void) } } +int +db_sqlite_last_changes(void) +{ + if (!g_chatlog_database) + return 0; + return sqlite3_changes(g_chatlog_database); +} + GSList* db_sqlite_get_all_chat(const gchar* const contact_barejid) { @@ -949,10 +957,10 @@ db_sqlite_list_contacts(void) while (sqlite3_step(stmt) == SQLITE_ROW) { const char* jid = (const char*)sqlite3_column_text(stmt, 0); if (jid && strlen(jid) > 0) { - contacts = g_slist_append(contacts, g_strdup(jid)); + contacts = g_slist_prepend(contacts, g_strdup(jid)); } } sqlite3_finalize(stmt); - return contacts; + return g_slist_reverse(contacts); } \ No newline at end of file