From 7ba7e53291c643fb8998ff065036790a30d090a0 Mon Sep 17 00:00:00 2001 From: "jabber.developer2" Date: Sat, 28 Mar 2026 12:58:27 +0300 Subject: [PATCH] refactor: extract helpers for verify print, scan loop, and per-contact export - extract _show_integrity_issues() from cmd_history verify block - extract _ff_scan_lines() shared by _ff_state_build and _ff_state_extend - extract _export_one_contact() from log_database_export_to_flatfile loop --- src/command/cmd_funcs.c | 93 +++++----- src/database_export.c | 363 +++++++++++++++++++++------------------- src/database_flatfile.c | 80 ++++----- 3 files changed, 265 insertions(+), 271 deletions(-) diff --git a/src/command/cmd_funcs.c b/src/command/cmd_funcs.c index 58c506e9..442a4e99 100644 --- a/src/command/cmd_funcs.c +++ b/src/command/cmd_funcs.c @@ -6729,6 +6729,54 @@ cmd_logging(ProfWin* window, const char* const command, gchar** args) return TRUE; } +static void +_show_integrity_issues(GSList* issues) +{ + int errors = 0, warnings = 0, infos = 0; + for (GSList* l = issues; l; l = l->next) { + integrity_issue_t* issue = l->data; + const char* level_str; + switch (issue->level) { + case INTEGRITY_ERROR: + level_str = "ERROR"; + errors++; + break; + case INTEGRITY_WARNING: + level_str = "WARN"; + warnings++; + break; + case INTEGRITY_INFO: + level_str = "INFO"; + infos++; + break; + default: + level_str = "???"; + break; + } + if (issue->line > 0) { + if (issue->level == INTEGRITY_ERROR) { + cons_show_error("[%s] %s:%d — %s", level_str, issue->file, issue->line, issue->message); + } else { + cons_show("[%s] %s:%d — %s", level_str, issue->file, issue->line, issue->message); + } + } else { + if (issue->level == INTEGRITY_ERROR) { + cons_show_error("[%s] %s — %s", level_str, issue->file, issue->message); + } else { + cons_show("[%s] %s — %s", level_str, issue->file, issue->message); + } + } + } + + if (!issues) { + cons_show("Verification complete: no issues found."); + } else { + cons_show("Verification complete: %d error(s), %d warning(s), %d info(s).", + errors, warnings, infos); + g_slist_free_full(issues, (GDestroyNotify)integrity_issue_free); + } +} + gboolean cmd_history(ProfWin* window, const char* const command, gchar** args) { @@ -6773,50 +6821,7 @@ cmd_history(ProfWin* window, const char* const command, gchar** args) cons_show("Verifying history integrity..."); GSList* issues = log_database_verify_integrity(contact_jid); - - int errors = 0, warnings = 0, infos = 0; - for (GSList* l = issues; l; l = l->next) { - integrity_issue_t* issue = l->data; - const char* level_str; - switch (issue->level) { - case INTEGRITY_ERROR: - level_str = "ERROR"; - errors++; - break; - case INTEGRITY_WARNING: - level_str = "WARN"; - warnings++; - break; - case INTEGRITY_INFO: - level_str = "INFO"; - infos++; - break; - default: - level_str = "???"; - break; - } - if (issue->line > 0) { - if (issue->level == INTEGRITY_ERROR) { - cons_show_error("[%s] %s:%d — %s", level_str, issue->file, issue->line, issue->message); - } else { - cons_show("[%s] %s:%d — %s", level_str, issue->file, issue->line, issue->message); - } - } else { - if (issue->level == INTEGRITY_ERROR) { - cons_show_error("[%s] %s — %s", level_str, issue->file, issue->message); - } else { - cons_show("[%s] %s — %s", level_str, issue->file, issue->message); - } - } - } - - if (!issues) { - cons_show("Verification complete: no issues found."); - } else { - cons_show("Verification complete: %d error(s), %d warning(s), %d info(s).", - errors, warnings, infos); - g_slist_free_full(issues, (GDestroyNotify)integrity_issue_free); - } + _show_integrity_issues(issues); return TRUE; } diff --git a/src/database_export.c b/src/database_export.c index 7edc314d..1c604c69 100644 --- a/src/database_export.c +++ b/src/database_export.c @@ -195,6 +195,190 @@ typedef struct int skipped; // counter } export_ctx_t; +// Export a single contact's messages from SQLite to flatfile. +// Returns the number of newly exported messages, or -1 on error/skip. +static int +_export_one_contact(const char* contact) +{ + // 1. Read existing flatfile lines for dedup + GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + + GSList* existing = _ff_read_all_lines(contact); + for (GSList* l = existing; l; l = l->next) { + ff_parsed_line_t* pl = l->data; + char* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message); + g_hash_table_add(seen_keys, key); + } + int existing_count = g_slist_length(existing); + + // 2. Query SQLite for this contact — get ALL messages via direct SQL + GSList* sqlite_lines = db_sqlite_get_all_chat(contact); + + if (!sqlite_lines) { + log_debug("export: no SQLite messages for %s", contact); + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + return -1; + } + + // 3. Merge: write existing flatfile lines + new SQLite lines to temp file + auto_gchar gchar* log_path = ff_get_log_path(contact); + if (!log_path) { + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + return -1; + } + + // Ensure directory + auto_gchar gchar* dir = g_path_get_dirname(log_path); + ff_ensure_dir(dir); + + auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path); + + // Use open() with O_NOFOLLOW|O_EXCL to prevent symlink attacks, + // and explicit 0600 permissions (chat history is private). + int tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR); + if (tmp_fd < 0) { + if (errno == EEXIST) { + // Stale temp file from previous failed export — remove and retry + unlink(tmp_path); + tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR); + } + } + if (tmp_fd < 0) { + log_error("export: cannot create %s: %s", tmp_path, strerror(errno)); + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + return -1; + } + FILE* fp = fdopen(tmp_fd, "w"); + if (!fp) { + log_error("export: fdopen failed for %s: %s", tmp_path, strerror(errno)); + close(tmp_fd); + unlink(tmp_path); + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + return -1; + } + + // Lock the temp file + int fd = fileno(fp); + if (flock(fd, LOCK_EX) != 0) { + log_warning("export: flock(%s) failed: %s", tmp_path, strerror(errno)); + } + + fprintf(fp, "%s", FLATFILE_HEADER); + + int contact_exported = 0; + int contact_skipped = 0; + + // Collect ALL lines into a merged list for sorted output + GSList* merged = NULL; + + // Add existing flatfile lines (mark as already seen in dedup) + for (GSList* l = existing; l; l = l->next) { + ff_parsed_line_t* pl = l->data; + // Create a shallow copy for the merged list (originals freed separately) + ff_parsed_line_t* copy = g_malloc0(sizeof(ff_parsed_line_t)); + copy->timestamp_str = g_strdup(pl->timestamp_str); + copy->timestamp = pl->timestamp ? g_date_time_ref(pl->timestamp) : NULL; + copy->type = g_strdup(pl->type); + copy->enc = g_strdup(pl->enc); + copy->stanza_id = g_strdup(pl->stanza_id); + copy->archive_id = g_strdup(pl->archive_id); + copy->replace_id = g_strdup(pl->replace_id); + copy->from_jid = g_strdup(pl->from_jid); + copy->from_resource = g_strdup(pl->from_resource); + copy->to_jid = g_strdup(pl->to_jid); + copy->to_resource = g_strdup(pl->to_resource); + copy->marked_read = pl->marked_read; + copy->message = g_strdup(pl->message); + merged = g_slist_prepend(merged, copy); + } + + // Add SQLite messages that aren't already in the flatfile + for (GSList* l = sqlite_lines; l; l = l->next) { + ProfMessage* msg = l->data; + if (!msg || !msg->timestamp) + continue; + + auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp); + const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown"; + const char* body = msg->plain ? msg->plain : ""; + const char* sid = msg->id ? msg->id : ""; + + char* key = _make_dedup_key(sid, ts, from_jid, body); + if (g_hash_table_contains(seen_keys, key)) { + g_free(key); + contact_skipped++; + continue; + } + g_hash_table_add(seen_keys, key); + + ff_parsed_line_t* pl = g_malloc0(sizeof(ff_parsed_line_t)); + pl->timestamp_str = g_strdup(ts); + pl->timestamp = g_date_time_ref(msg->timestamp); + pl->type = g_strdup(ff_get_message_type_str(msg->type)); + pl->enc = g_strdup(ff_get_message_enc_str(msg->enc)); + pl->stanza_id = g_strdup(sid); + pl->archive_id = msg->stanzaid ? g_strdup(msg->stanzaid) : NULL; + pl->replace_id = NULL; + pl->from_jid = g_strdup(from_jid); + pl->from_resource = msg->from_jid ? g_strdup(msg->from_jid->resourcepart) : NULL; + pl->to_jid = msg->to_jid ? g_strdup(msg->to_jid->barejid) : NULL; + pl->to_resource = msg->to_jid ? g_strdup(msg->to_jid->resourcepart) : NULL; + pl->marked_read = msg->marked_read; + pl->message = g_strdup(body); + merged = g_slist_prepend(merged, pl); + contact_exported++; + } + + // Sort merged list by timestamp (ISO8601 is lexicographically sortable) + merged = g_slist_sort(merged, _compare_parsed_lines_by_timestamp); + + // Write all lines sorted + int written = 0; + for (GSList* l = merged; l; l = l->next) { + ff_parsed_line_t* pl = l->data; + ff_write_line(fp, pl->timestamp_str, + pl->type, pl->enc, + pl->stanza_id, pl->archive_id, pl->replace_id, + pl->from_jid, pl->from_resource, + pl->to_jid, pl->to_resource, pl->marked_read, + pl->message); + written++; + if (written % EXPORT_PROGRESS_INTERVAL == 0) { + cons_show(" ... %s: %d/%d written so far", contact, written, (int)g_slist_length(merged)); + } + } + + g_slist_free_full(merged, (GDestroyNotify)ff_parsed_line_free); + + fflush(fp); + fsync(fd); // ensure data is on disk before atomic rename + fclose(fp); // also releases flock + + // Atomic rename + int result = -1; + if (rename(tmp_path, log_path) != 0) { + log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno)); + cons_show_error("Export failed for %s: could not rename temp file (%s)", contact, strerror(errno)); + unlink(tmp_path); + } else { + cons_show("Exported %s: %d new, %d skipped (already existed: %d)", + contact, contact_exported, contact_skipped, existing_count); + result = contact_exported; + } + + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + return result; +} + int log_database_export_to_flatfile(const gchar* const contact_jid) { @@ -234,182 +418,9 @@ log_database_export_to_flatfile(const gchar* const contact_jid) for (GSList* c = contacts; c; c = c->next) { const char* contact = c->data; - - // 1. Read existing flatfile lines for dedup - GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); - - GSList* existing = _ff_read_all_lines(contact); - for (GSList* l = existing; l; l = l->next) { - ff_parsed_line_t* pl = l->data; - char* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message); - g_hash_table_add(seen_keys, key); - } - int existing_count = g_slist_length(existing); - - // 2. Query SQLite for this contact — get ALL messages via direct SQL - GSList* sqlite_lines = db_sqlite_get_all_chat(contact); - - if (!sqlite_lines) { - log_debug("export: no SQLite messages for %s", contact); - g_hash_table_destroy(seen_keys); - g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); - continue; - } - - // 3. Merge: write existing flatfile lines + new SQLite lines to temp file - auto_gchar gchar* log_path = ff_get_log_path(contact); - if (!log_path) { - g_hash_table_destroy(seen_keys); - g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); - g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); - continue; - } - - // Ensure directory - auto_gchar gchar* dir = g_path_get_dirname(log_path); - ff_ensure_dir(dir); - - auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path); - - // Use open() with O_NOFOLLOW|O_EXCL to prevent symlink attacks, - // and explicit 0600 permissions (chat history is private). - int tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR); - if (tmp_fd < 0) { - if (errno == EEXIST) { - // Stale temp file from previous failed export — remove and retry - unlink(tmp_path); - tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR); - } - } - if (tmp_fd < 0) { - log_error("export: cannot create %s: %s", tmp_path, strerror(errno)); - g_hash_table_destroy(seen_keys); - g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); - g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); - continue; - } - FILE* fp = fdopen(tmp_fd, "w"); - if (!fp) { - log_error("export: fdopen failed for %s: %s", tmp_path, strerror(errno)); - close(tmp_fd); - unlink(tmp_path); - g_hash_table_destroy(seen_keys); - g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); - g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); - continue; - } - - // Lock the temp file - int fd = fileno(fp); - if (flock(fd, LOCK_EX) != 0) { - log_warning("export: flock(%s) failed: %s", tmp_path, strerror(errno)); - } - - fprintf(fp, "%s", FLATFILE_HEADER); - - int contact_exported = 0; - int contact_skipped = 0; - - // Collect ALL lines into a merged list for sorted output - GSList* merged = NULL; - - // Add existing flatfile lines (mark as already seen in dedup) - for (GSList* l = existing; l; l = l->next) { - ff_parsed_line_t* pl = l->data; - // Create a shallow copy for the merged list (originals freed separately) - ff_parsed_line_t* copy = g_malloc0(sizeof(ff_parsed_line_t)); - copy->timestamp_str = g_strdup(pl->timestamp_str); - copy->timestamp = pl->timestamp ? g_date_time_ref(pl->timestamp) : NULL; - copy->type = g_strdup(pl->type); - copy->enc = g_strdup(pl->enc); - copy->stanza_id = g_strdup(pl->stanza_id); - copy->archive_id = g_strdup(pl->archive_id); - copy->replace_id = g_strdup(pl->replace_id); - copy->from_jid = g_strdup(pl->from_jid); - copy->from_resource = g_strdup(pl->from_resource); - copy->to_jid = g_strdup(pl->to_jid); - copy->to_resource = g_strdup(pl->to_resource); - copy->marked_read = pl->marked_read; - copy->message = g_strdup(pl->message); - merged = g_slist_prepend(merged, copy); - } - - // Add SQLite messages that aren't already in the flatfile - for (GSList* l = sqlite_lines; l; l = l->next) { - ProfMessage* msg = l->data; - if (!msg || !msg->timestamp) - continue; - - auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp); - const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown"; - const char* body = msg->plain ? msg->plain : ""; - const char* sid = msg->id ? msg->id : ""; - - char* key = _make_dedup_key(sid, ts, from_jid, body); - if (g_hash_table_contains(seen_keys, key)) { - g_free(key); - contact_skipped++; - continue; - } - g_hash_table_add(seen_keys, key); - - ff_parsed_line_t* pl = g_malloc0(sizeof(ff_parsed_line_t)); - pl->timestamp_str = g_strdup(ts); - pl->timestamp = g_date_time_ref(msg->timestamp); - pl->type = g_strdup(ff_get_message_type_str(msg->type)); - pl->enc = g_strdup(ff_get_message_enc_str(msg->enc)); - pl->stanza_id = g_strdup(sid); - pl->archive_id = msg->stanzaid ? g_strdup(msg->stanzaid) : NULL; - pl->replace_id = NULL; - pl->from_jid = g_strdup(from_jid); - pl->from_resource = msg->from_jid ? g_strdup(msg->from_jid->resourcepart) : NULL; - pl->to_jid = msg->to_jid ? g_strdup(msg->to_jid->barejid) : NULL; - pl->to_resource = msg->to_jid ? g_strdup(msg->to_jid->resourcepart) : NULL; - pl->marked_read = msg->marked_read; - pl->message = g_strdup(body); - merged = g_slist_prepend(merged, pl); - contact_exported++; - } - - // Sort merged list by timestamp (ISO8601 is lexicographically sortable) - merged = g_slist_sort(merged, _compare_parsed_lines_by_timestamp); - - // Write all lines sorted - int written = 0; - for (GSList* l = merged; l; l = l->next) { - ff_parsed_line_t* pl = l->data; - ff_write_line(fp, pl->timestamp_str, - pl->type, pl->enc, - pl->stanza_id, pl->archive_id, pl->replace_id, - pl->from_jid, pl->from_resource, - pl->to_jid, pl->to_resource, pl->marked_read, - pl->message); - written++; - if (written % EXPORT_PROGRESS_INTERVAL == 0) { - cons_show(" ... %s: %d/%d written so far", contact, written, (int)g_slist_length(merged)); - } - } - - g_slist_free_full(merged, (GDestroyNotify)ff_parsed_line_free); - - fflush(fp); - fsync(fd); // ensure data is on disk before atomic rename - fclose(fp); // also releases flock - - // Atomic rename - if (rename(tmp_path, log_path) != 0) { - log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno)); - cons_show_error("Export failed for %s: could not rename temp file (%s)", contact, strerror(errno)); - unlink(tmp_path); - } else { - cons_show("Exported %s: %d new, %d skipped (already existed: %d)", - contact, contact_exported, contact_skipped, existing_count); - total_exported += contact_exported; - } - - g_hash_table_destroy(seen_keys); - g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); - g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + int exported = _export_one_contact(contact); + if (exported > 0) + total_exported += exported; } g_slist_free_full(contacts, g_free); diff --git a/src/database_flatfile.c b/src/database_flatfile.c index fbaece96..29f700bf 100644 --- a/src/database_flatfile.c +++ b/src/database_flatfile.c @@ -163,23 +163,12 @@ _ff_maybe_index_line(ff_contact_state_t* state, const char* buf, off_t pos) g_date_time_unref(dt); } -static gboolean -_ff_state_build(ff_contact_state_t* state) +// Scan lines from an already-positioned file pointer, caching IDs and +// building index entries. Returns the number of messages scanned. +static size_t +_ff_scan_lines(ff_contact_state_t* state, FILE* fp) { - FILE* fp = fopen(state->filepath, "r"); - if (!fp) - return FALSE; - - state->bom_len = ff_skip_bom(fp); - - state->n_entries = 0; - state->total_lines = 0; - size_t msg_count = 0; - - // Clear ID caches for full rebuild - g_hash_table_remove_all(state->archive_ids); - g_hash_table_remove_all(state->stanza_senders); - + size_t count = 0; while (1) { off_t pos = ftell(fp); gboolean truncated = FALSE; @@ -197,16 +186,34 @@ _ff_state_build(ff_contact_state_t* state) } state->total_lines++; - - // Extract IDs for O(1) dedup/LMC cache _ff_cache_line_ids(state, buf); - if (msg_count % FF_INDEX_STEP == 0) { + if (count % FF_INDEX_STEP == 0) { _ff_maybe_index_line(state, buf, pos); } - msg_count++; + count++; free(buf); } + return count; +} + +static gboolean +_ff_state_build(ff_contact_state_t* state) +{ + FILE* fp = fopen(state->filepath, "r"); + if (!fp) + return FALSE; + + state->bom_len = ff_skip_bom(fp); + + state->n_entries = 0; + state->total_lines = 0; + + // Clear ID caches for full rebuild + g_hash_table_remove_all(state->archive_ids); + g_hash_table_remove_all(state->stanza_senders); + + _ff_scan_lines(state, fp); fclose(fp); @@ -247,37 +254,8 @@ _ff_state_extend(ff_contact_state_t* state) } } - // Use a local counter for new messages so index step alignment - // doesn't depend on the total from the initial build. - size_t new_count = 0; - - while (1) { - off_t pos = ftell(fp); - gboolean truncated = FALSE; - char* buf = ff_readline(fp, &truncated); - if (!buf) - break; - if (truncated) { - free(buf); - break; - } - - if (buf[0] == '\0' || buf[0] == '#') { - free(buf); - continue; - } - - state->total_lines++; - - // Extract IDs for O(1) dedup/LMC cache - _ff_cache_line_ids(state, buf); - - if (new_count % FF_INDEX_STEP == 0) { - _ff_maybe_index_line(state, buf, pos); - } - new_count++; - free(buf); - } + // Use _ff_scan_lines for the shared read/index/cache loop. + _ff_scan_lines(state, fp); fclose(fp);