devs/profanity
2
0
Fork 0
mirror of https://git.jabber.space/devs/cproof.git synced 2026-07-29 07:26:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: O(1) dedup/LMC cache, g_slist_append → prepend+reverse

Browse Source 
- Add archive_ids hash set and stanza_senders hash map to
  ff_contact_state_t, populated during index build/extend via
  lightweight _ff_cache_line_ids() (no full parse overhead).
- Replace O(n) file scans in _ff_has_archive_id() and
  _ff_find_original_sender() with O(1) hash table lookups.
- Replace g_slist_append with g_slist_prepend + g_slist_reverse
  in _ff_apply_lmc, _flatfile_get_previous_chat, and
  ff_verify_integrity to avoid O(n²) list building.
- Add db_sqlite_last_changes() declaration to database.h.
This commit is contained in:
jabber.developer2
2026-03-12 10:14:51 +03:00
parent 06463b75b4
commit 71f9ab5007
6 changed files with 146 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/database.h
View File

@@ -92,6 +92,7 @@ GSList* db_sqlite_get_all_chat(const gchar* const contact_barejid);
void db_sqlite_begin_transaction(void); void db_sqlite_begin_transaction(void);
void db_sqlite_end_transaction(void); void db_sqlite_end_transaction(void);
void db_sqlite_rollback_transaction(void); void db_sqlite_rollback_transaction(void);
int db_sqlite_last_changes(void);
#endif #endif
db_backend_t* db_backend_flatfile(void); db_backend_t* db_backend_flatfile(void);

13
src/database_export.c
View File

@@ -213,7 +213,6 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
if (!g_flatfile_account_jid) { if (!g_flatfile_account_jid) {
// Flatfile backend not initialized — init it temporarily // Flatfile backend not initialized — init it temporarily
// We need g_flatfile_account_jid for path construction // We need g_flatfile_account_jid for path construction
g_free(g_flatfile_account_jid);
g_flatfile_account_jid = g_strdup(myjid->barejid); g_flatfile_account_jid = g_strdup(myjid->barejid);
} }
@@ -289,7 +288,9 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
// Lock the temp file // Lock the temp file
int fd = fileno(fp); int fd = fileno(fp);
flock(fd, LOCK_EX); if (flock(fd, LOCK_EX) != 0) {
log_warning("export: flock(%s) failed: %s", tmp_path, strerror(errno));
}
fprintf(fp, "%s", FLATFILE_HEADER); fprintf(fp, "%s", FLATFILE_HEADER);
@@ -416,7 +417,6 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
} }
if (!g_flatfile_account_jid) { if (!g_flatfile_account_jid) {
g_free(g_flatfile_account_jid);
g_flatfile_account_jid = g_strdup(myjid->barejid); g_flatfile_account_jid = g_strdup(myjid->barejid);
} }
@@ -522,6 +522,13 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
sqlite_be->add_incoming(msg); sqlite_be->add_incoming(msg);
message_free(msg); message_free(msg);
// Verify the row was actually inserted
if (db_sqlite_last_changes() < 1) {
log_error("import: insert failed for %s at %s", contact, ts);
import_ok = FALSE;
break;
}
contact_imported++; contact_imported++;
if (contact_imported % 500 == 0) { if (contact_imported % 500 == 0) {

172
src/database_flatfile.c
View File

@@ -67,6 +67,8 @@ ff_state_new(const char* filepath)
ff_contact_state_t* state = g_malloc0(sizeof(ff_contact_state_t)); ff_contact_state_t* state = g_malloc0(sizeof(ff_contact_state_t));
state->filepath = g_strdup(filepath); state->filepath = g_strdup(filepath);
state->cursor_offset = -1; state->cursor_offset = -1;
state->archive_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
state->stanza_senders = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
return state; return state;
} }
@@ -77,9 +79,77 @@ ff_state_free(ff_contact_state_t* state)
return; return;
g_free(state->filepath); g_free(state->filepath);
g_free(state->entries); g_free(state->entries);
if (state->archive_ids)
g_hash_table_destroy(state->archive_ids);
if (state->stanza_senders)
g_hash_table_destroy(state->stanza_senders);
g_free(state); g_free(state);
} }
// Lightweight ID extraction from a raw log line for the dedup/LMC cache.
// Avoids full ff_parse_line() overhead (no GDateTime, no message unescape).
static void
_ff_cache_line_ids(ff_contact_state_t* state, const char* line)
{
// Find metadata section: first '[' to first unescaped ']'
const char* bracket = strchr(line, '[');
if (!bracket)
return;
const char* close = ff_find_unescaped_char(bracket + 1, ']');
if (!close)
return;
// Split metadata on unescaped '|'
char* meta_str = g_strndup(bracket + 1, close - bracket - 1);
char** parts = ff_split_meta(meta_str);
g_free(meta_str);
char* stanza_id = NULL;
char* archive_id = NULL;
if (parts) {
for (int i = 0; parts[i]; i++) {
if (g_str_has_prefix(parts[i], "id:") && !stanza_id) {
stanza_id = ff_unescape_meta_value(parts[i] + 3);
} else if (g_str_has_prefix(parts[i], "aid:") && !archive_id) {
archive_id = ff_unescape_meta_value(parts[i] + 4);
}
}
g_strfreev(parts);
}
// Extract from_jid (needed for stanza_senders map): after "] " before "/" or ": "
char* from_jid = NULL;
if (stanza_id && *(close + 1) == ' ') {
const char* sender_start = close + 2;
const char* colonspace = ff_find_unescaped_colonspace(sender_start);
if (colonspace) {
const char* jid_end = colonspace;
for (const char* p = sender_start; p < colonspace; p++) {
if (*p == '/') {
jid_end = p;
break;
}
}
from_jid = g_strndup(sender_start, jid_end - sender_start);
}
}
if (archive_id && archive_id[0] != '\0') {
g_hash_table_add(state->archive_ids, archive_id);
} else {
g_free(archive_id);
}
if (stanza_id && stanza_id[0] != '\0' && from_jid) {
g_hash_table_insert(state->stanza_senders, stanza_id, from_jid);
} else {
g_free(stanza_id);
g_free(from_jid);
}
}
static gboolean static gboolean
_ff_state_build(ff_contact_state_t* state) _ff_state_build(ff_contact_state_t* state)
{ {
@@ -101,6 +171,10 @@ _ff_state_build(ff_contact_state_t* state)
state->total_lines = 0; state->total_lines = 0;
size_t msg_count = 0; size_t msg_count = 0;
// Clear ID caches for full rebuild
g_hash_table_remove_all(state->archive_ids);
g_hash_table_remove_all(state->stanza_senders);
while (1) { while (1) {
off_t pos = ftell(fp); off_t pos = ftell(fp);
gboolean truncated = FALSE; gboolean truncated = FALSE;
@@ -119,6 +193,9 @@ _ff_state_build(ff_contact_state_t* state)
state->total_lines++; state->total_lines++;
// Extract IDs for O(1) dedup/LMC cache
_ff_cache_line_ids(state, buf);
if (msg_count % FF_INDEX_STEP == 0) { if (msg_count % FF_INDEX_STEP == 0) {
char* space = strchr(buf, ' '); char* space = strchr(buf, ' ');
if (space) { if (space) {
@@ -203,6 +280,9 @@ _ff_state_extend(ff_contact_state_t* state)
state->total_lines++; state->total_lines++;
// Extract IDs for O(1) dedup/LMC cache
_ff_cache_line_ids(state, buf);
if (new_count % FF_INDEX_STEP == 0) { if (new_count % FF_INDEX_STEP == 0) {
char* space = strchr(buf, ' '); char* space = strchr(buf, ' ');
if (space) { if (space) {
@@ -469,79 +549,39 @@ _flatfile_close(void)
// ========================================================================= // =========================================================================
// ========================================================================= // =========================================================================
// Incoming message validation helpers // Incoming message validation helpers (O(1) via ID cache)
// ========================================================================= // =========================================================================
// Search log file for a line containing aid:{archive_id}, return TRUE if found. // Check if archive_id already exists in the contact's log via cached set.
static gboolean static gboolean
_ff_has_archive_id(const char* log_path, const char* archive_id) _ff_has_archive_id(const char* contact_barejid, const char* archive_id)
{ {
if (!log_path || !archive_id || strlen(archive_id) == 0) if (!contact_barejid || !archive_id || archive_id[0] == '\0')
return FALSE; return FALSE;
FILE* fp = fopen(log_path, "r"); ff_contact_state_t* state = _ff_get_state(contact_barejid);
if (!fp) if (!state)
return FALSE; return FALSE;
auto_gchar gchar* needle = g_strdup_printf("aid:%s", archive_id); ff_state_ensure_fresh(state);
gboolean found = FALSE; return g_hash_table_contains(state->archive_ids, archive_id);
while (!found) {
gboolean trunc = FALSE;
char* line = ff_readline(fp, &trunc);
if (!line)
break;
if (trunc) {
free(line);
break;
}
if (strstr(line, needle)) {
found = TRUE;
}
free(line);
}
fclose(fp);
return found;
} }
// Search log file for a line with stanza id matching replace_id, return the // Find the from_jid of the original message with given stanza_id via cached map.
// from_jid of the original message (caller must g_free). Returns NULL if not found. // Caller must g_free the result.
static char* static char*
_ff_find_original_sender(const char* log_path, const char* replace_id) _ff_find_original_sender(const char* contact_barejid, const char* replace_id)
{ {
if (!log_path || !replace_id || strlen(replace_id) == 0) if (!contact_barejid || !replace_id || replace_id[0] == '\0')
return NULL; return NULL;
FILE* fp = fopen(log_path, "r"); ff_contact_state_t* state = _ff_get_state(contact_barejid);
if (!fp) if (!state)
return NULL; return NULL;
auto_gchar gchar* needle = g_strdup_printf("id:%s", replace_id); ff_state_ensure_fresh(state);
char* sender = NULL; const char* sender = g_hash_table_lookup(state->stanza_senders, replace_id);
return sender ? g_strdup(sender) : NULL;
while (!sender) {
gboolean trunc = FALSE;
char* line = ff_readline(fp, &trunc);
if (!line)
break;
if (trunc) {
free(line);
break;
}
if (strstr(line, needle)) {
ff_parsed_line_t* pl = ff_parse_line(line);
if (pl && pl->stanza_id && g_strcmp0(pl->stanza_id, replace_id) == 0) {
sender = g_strdup(pl->from_jid);
}
if (pl)
ff_parsed_line_free(pl);
}
free(line);
}
fclose(fp);
return sender;
} }
static void static void
@@ -561,11 +601,9 @@ _flatfile_add_incoming(ProfMessage* message)
} else { } else {
contact = message->from_jid->barejid; contact = message->from_jid->barejid;
} }
auto_gchar gchar* log_path = ff_get_log_path(contact);
// MAM dedup: skip if this stanza-id already exists in the log (XEP-0313 replay) // MAM dedup: skip if this stanza-id already exists in the log (XEP-0313 replay)
if (message->stanzaid && !message->is_mam && log_path) { if (message->stanzaid && !message->is_mam) {
if (_ff_has_archive_id(log_path, message->stanzaid)) { if (_ff_has_archive_id(contact, message->stanzaid)) {
log_error("flatfile: duplicate stanza-id '%s' from %s, skipping", log_error("flatfile: duplicate stanza-id '%s' from %s, skipping",
message->stanzaid, message->from_jid->barejid); message->stanzaid, message->from_jid->barejid);
cons_show_error("Got a message with duplicate (server-generated) stanza-id from %s.", cons_show_error("Got a message with duplicate (server-generated) stanza-id from %s.",
@@ -575,8 +613,8 @@ _flatfile_add_incoming(ProfMessage* message)
} }
// LMC sender validation: verify the correction comes from the original sender // LMC sender validation: verify the correction comes from the original sender
if (message->replace_id && log_path) { if (message->replace_id) {
auto_gchar gchar* original_sender = _ff_find_original_sender(log_path, message->replace_id); auto_gchar gchar* original_sender = _ff_find_original_sender(contact, message->replace_id);
if (original_sender && g_strcmp0(original_sender, message->from_jid->barejid) != 0) { if (original_sender && g_strcmp0(original_sender, message->from_jid->barejid) != 0) {
log_error("flatfile: LMC sender mismatch — corrected msg sender: %s, original: %s, replace-id: %s", log_error("flatfile: LMC sender mismatch — corrected msg sender: %s, original: %s, replace-id: %s",
message->from_jid->barejid, original_sender, message->replace_id); message->from_jid->barejid, original_sender, message->replace_id);
@@ -691,9 +729,11 @@ _ff_apply_lmc(GSList* parsed_lines, GSList** result)
} else { } else {
msg = ff_parsed_to_profmessage(pl); msg = ff_parsed_to_profmessage(pl);
} }
*result = g_slist_append(*result, msg); *result = g_slist_prepend(*result, msg);
} }
*result = g_slist_reverse(*result);
g_hash_table_destroy(id_map); g_hash_table_destroy(id_map);
g_hash_table_destroy(corrections); g_hash_table_destroy(corrections);
g_hash_table_destroy(correction_map); g_hash_table_destroy(correction_map);
@@ -845,10 +885,12 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta
continue; continue;
} }
all_parsed = g_slist_append(all_parsed, pl); all_parsed = g_slist_prepend(all_parsed, pl);
} }
fclose(fp); fclose(fp);
all_parsed = g_slist_reverse(all_parsed);
if (start_dt) if (start_dt)
g_date_time_unref(start_dt); g_date_time_unref(start_dt);
if (end_dt) if (end_dt)

2
src/database_flatfile.h
View File

@@ -94,6 +94,8 @@ typedef struct
ff_file_stamp_t stamp; ff_file_stamp_t stamp;
off_t cursor_offset; off_t cursor_offset;
int bom_len; int bom_len;
GHashTable* archive_ids; // set: archive_id -> NULL (MAM dedup, O(1))
GHashTable* stanza_senders; // map: stanza_id -> from_jid (LMC validation, O(1))
} ff_contact_state_t; } ff_contact_state_t;
// State management // State management

32
src/database_flatfile_verify.c
View File

@@ -48,7 +48,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup("N/A"); issue->file = g_strdup("N/A");
issue->line = 0; issue->line = 0;
issue->message = g_strdup("Flat-file backend not initialized"); issue->message = g_strdup("Flat-file backend not initialized");
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
return issues; return issues;
} }
@@ -58,14 +58,14 @@ ff_verify_integrity(const gchar* const contact_barejid)
if (contact_barejid) { if (contact_barejid) {
auto_gchar gchar* cdir = ff_get_contact_dir(contact_barejid); auto_gchar gchar* cdir = ff_get_contact_dir(contact_barejid);
if (cdir && g_file_test(cdir, G_FILE_TEST_IS_DIR)) { if (cdir && g_file_test(cdir, G_FILE_TEST_IS_DIR)) {
contact_dirs = g_slist_append(contact_dirs, g_strdup(cdir)); contact_dirs = g_slist_prepend(contact_dirs, g_strdup(cdir));
} else { } else {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_INFO; issue->level = INTEGRITY_INFO;
issue->file = g_strdup(contact_barejid); issue->file = g_strdup(contact_barejid);
issue->line = 0; issue->line = 0;
issue->message = g_strdup("No log files found for this contact"); issue->message = g_strdup("No log files found for this contact");
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
return issues; return issues;
} }
} else { } else {
@@ -80,7 +80,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
while ((dname = g_dir_read_name(dir)) != NULL) { while ((dname = g_dir_read_name(dir)) != NULL) {
char* full = g_strdup_printf("%s/%s", base_dir, dname); char* full = g_strdup_printf("%s/%s", base_dir, dname);
if (g_file_test(full, G_FILE_TEST_IS_DIR)) { if (g_file_test(full, G_FILE_TEST_IS_DIR)) {
contact_dirs = g_slist_append(contact_dirs, full); contact_dirs = g_slist_prepend(contact_dirs, full);
} else { } else {
g_free(full); g_free(full);
} }
@@ -108,7 +108,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = 0; issue->line = 0;
issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777); issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777);
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} }
} }
@@ -126,7 +126,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = 0; issue->line = 0;
issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary"); issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary");
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} else { } else {
fseek(fp, 0, SEEK_SET); fseek(fp, 0, SEEK_SET);
} }
@@ -161,7 +161,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = lineno; issue->line = lineno;
issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf)); issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf));
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
free(buf); free(buf);
continue; continue;
} }
@@ -174,7 +174,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = lineno; issue->line = lineno;
issue->message = g_strdup_printf("Contains control character 0x%02x", ch); issue->message = g_strdup_printf("Contains control character 0x%02x", ch);
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
break; break;
} }
} }
@@ -186,7 +186,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = lineno; issue->line = lineno;
issue->message = g_strdup("Unparsable line"); issue->message = g_strdup("Unparsable line");
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
free(buf); free(buf);
continue; continue;
} }
@@ -202,7 +202,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = lineno; issue->line = lineno;
issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev); issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev);
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} }
if (prev_ts) if (prev_ts)
g_date_time_unref(prev_ts); g_date_time_unref(prev_ts);
@@ -216,7 +216,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = lineno; issue->line = lineno;
issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id); issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id);
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} else { } else {
g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno)); g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
} }
@@ -229,7 +229,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = lineno; issue->line = lineno;
issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id); issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id);
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} else { } else {
g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno)); g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno));
} }
@@ -246,7 +246,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = 0; issue->line = 0;
issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF"); issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF");
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} }
if (is_empty) { if (is_empty) {
@@ -255,7 +255,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = 0; issue->line = 0;
issue->message = g_strdup("File is empty (no message lines)"); issue->message = g_strdup("File is empty (no message lines)");
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} }
// Second pass: check LMC references // Second pass: check LMC references
@@ -290,7 +290,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
issue->file = g_strdup(basename); issue->file = g_strdup(basename);
issue->line = lineno; issue->line = lineno;
issue->message = g_strdup_printf("Broken correction reference: corrects:%s not found", pl->replace_id); issue->message = g_strdup_printf("Broken correction reference: corrects:%s not found", pl->replace_id);
issues = g_slist_append(issues, issue); issues = g_slist_prepend(issues, issue);
} }
} }
ff_parsed_line_free(pl); ff_parsed_line_free(pl);
@@ -305,5 +305,5 @@ ff_verify_integrity(const gchar* const contact_barejid)
} }
g_slist_free_full(contact_dirs, g_free); g_slist_free_full(contact_dirs, g_free);
return issues; return g_slist_reverse(issues);
} }

12
src/database_sqlite.c
View File

@@ -860,6 +860,14 @@ db_sqlite_rollback_transaction(void)
} }
} }
int
db_sqlite_last_changes(void)
{
if (!g_chatlog_database)
return 0;
return sqlite3_changes(g_chatlog_database);
}
GSList* GSList*
db_sqlite_get_all_chat(const gchar* const contact_barejid) db_sqlite_get_all_chat(const gchar* const contact_barejid)
{ {
@@ -949,10 +957,10 @@ db_sqlite_list_contacts(void)
while (sqlite3_step(stmt) == SQLITE_ROW) { while (sqlite3_step(stmt) == SQLITE_ROW) {
const char* jid = (const char*)sqlite3_column_text(stmt, 0); const char* jid = (const char*)sqlite3_column_text(stmt, 0);
if (jid && strlen(jid) > 0) { if (jid && strlen(jid) > 0) {
contacts = g_slist_append(contacts, g_strdup(jid)); contacts = g_slist_prepend(contacts, g_strdup(jid));
} }
} }
sqlite3_finalize(stmt); sqlite3_finalize(stmt);
return contacts; return g_slist_reverse(contacts);
} }