/* * database_export.c * vim: expandtab:ts=4:sts=4:sw=4 * * Copyright (C) 2026 Profanity Contributors * * This file is part of Profanity. * * Profanity is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Profanity is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Profanity. If not, see . * * Cross-backend export/import for message history. * Reads from one backend, writes to the other, with merge + dedup. */ #include "config.h" #include #include #include #include #include #include #include #include #ifdef HAVE_SQLITE #include #endif #include "log.h" #include "common.h" #include "config/files.h" #include "database.h" #include "database_flatfile.h" #include "config/preferences.h" #include "ui/ui.h" #include "xmpp/xmpp.h" #include "xmpp/message.h" // ========================================================================= // Everything below is only used when HAVE_SQLITE is defined // ========================================================================= #ifdef HAVE_SQLITE // ========================================================================= // Dedup key: stanza_id if present, else hash of timestamp+from+body // ========================================================================= static char* _make_dedup_key(const char* stanza_id, const char* timestamp, const char* from_jid, const char* body) { if (stanza_id && strlen(stanza_id) > 0) return g_strdup(stanza_id); // Fallback: simple composite key auto_gchar gchar* raw = g_strdup_printf("%s|%s|%.64s", timestamp ? timestamp : "", from_jid ? from_jid : "", body ? body : ""); return g_compute_checksum_for_string(G_CHECKSUM_SHA256, raw, -1); } // ========================================================================= // Helper: reverse ff_jid_to_dir (best-effort) // '_at_' -> '@' // ========================================================================= static char* _dir_to_jid(const char* dirname) { if (!dirname) return NULL; return str_replace(dirname, "_at_", "@"); } // ========================================================================= // Helper: list all contact JIDs from flatfile directory // Returns a GSList of g_strdup'd JID strings. Caller frees with g_slist_free_full(list, g_free). // ========================================================================= static GSList* _ff_list_contacts(void) { GSList* contacts = NULL; if (!g_flatfile_account_jid) return NULL; auto_gchar gchar* data_path = files_get_data_path(DIR_FLATLOG); auto_gchar gchar* my_dir = ff_jid_to_dir(g_flatfile_account_jid); auto_gchar gchar* base_dir = g_strdup_printf("%s/%s", data_path, my_dir); GError* err = NULL; GDir* dir = g_dir_open(base_dir, 0, &err); if (!dir) { if (err) { log_debug("export: cannot open flatlog dir %s: %s", base_dir, err->message); g_error_free(err); } return NULL; } const gchar* entry; while ((entry = g_dir_read_name(dir)) != NULL) { auto_gchar gchar* full_path = g_strdup_printf("%s/%s/history.log", base_dir, entry); if (g_file_test(full_path, G_FILE_TEST_IS_REGULAR)) { char* jid = _dir_to_jid(entry); if (jid) { contacts = g_slist_append(contacts, jid); } } } g_dir_close(dir); return contacts; } // ========================================================================= // Helper: read ALL parsed lines from a flatfile for a contact // Returns GSList. Caller frees with g_slist_free_full(list, (GDestroyNotify)ff_parsed_line_free). // ========================================================================= static GSList* _ff_read_all_lines(const char* contact_barejid) { GSList* lines = NULL; auto_gchar gchar* log_path = ff_get_log_path(contact_barejid); if (!log_path) return NULL; FILE* fp = fopen(log_path, "r"); if (!fp) return NULL; while (1) { gboolean truncated = FALSE; char* buf = ff_readline(fp, &truncated); if (!buf) break; if (truncated) { free(buf); break; } if (buf[0] == '\0' || buf[0] == '#') { free(buf); continue; } ff_parsed_line_t* pl = ff_parse_line(buf); free(buf); if (pl) { lines = g_slist_append(lines, pl); } } fclose(fp); return lines; } // ========================================================================= // Export: SQLite -> flatfile (merge) // ========================================================================= // Callback data for the SQLite query typedef struct { GHashTable* seen_keys; // dedup set FILE* fp; // output file int exported; // counter int skipped; // counter } export_ctx_t; int log_database_export_to_flatfile(const gchar* const contact_jid) { const Jid* myjid = connection_get_jid(); if (!myjid || !myjid->barejid) { cons_show_error("Export failed: not connected."); return -1; } if (!g_flatfile_account_jid) { // Flatfile backend not initialized — init it temporarily // We need g_flatfile_account_jid for path construction g_free(g_flatfile_account_jid); g_flatfile_account_jid = g_strdup(myjid->barejid); } // Get the SQLite backend db_backend_t* sqlite_be = db_backend_sqlite(); if (!sqlite_be) { cons_show_error("Export failed: SQLite backend not available."); return -1; } // Determine contact list GSList* contacts = NULL; if (contact_jid) { contacts = g_slist_append(contacts, g_strdup(contact_jid)); } else { contacts = db_sqlite_list_contacts(); if (!contacts) { cons_show("No contacts found in SQLite database."); return 0; } cons_show("Exporting %d contact(s) to flat-file format...", g_slist_length(contacts)); } int total_exported = 0; int total_skipped = 0; for (GSList* c = contacts; c; c = c->next) { const char* contact = c->data; // 1. Read existing flatfile lines for dedup GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); GSList* existing = _ff_read_all_lines(contact); for (GSList* l = existing; l; l = l->next) { ff_parsed_line_t* pl = l->data; char* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message); g_hash_table_add(seen_keys, key); } int existing_count = g_slist_length(existing); // 2. Query SQLite for this contact — get ALL messages ordered by timestamp // We read all messages by paginating through the existing get_previous_chat // But it limits to MESSAGES_TO_RETRIEVE... We need a direct SQL query. // Use the same approach as verify — iterate with raw SQL. // For now, collect all messages into a list via repeated calls // Actually, let's use a simpler approach: read all from flatfile existing data, // then query SQLite for everything, merge, write back. // Read all SQLite messages for this contact GSList* sqlite_lines = NULL; // Use the backend's get_previous_chat with a wide time range // This is limited to MESSAGES_TO_RETRIEVE, so we need to paginate gboolean has_more = TRUE; gchar* start_time = NULL; while (has_more) { GSList* batch = NULL; db_history_result_t res = sqlite_be->get_previous_chat( contact, start_time, NULL, TRUE, FALSE, &batch); if (res != DB_RESPONSE_SUCCESS || !batch) { has_more = FALSE; break; } int batch_size = g_slist_length(batch); // Get the last timestamp for the next query ProfMessage* last_msg = g_slist_last(batch)->data; if (last_msg && last_msg->timestamp) { g_free(start_time); start_time = g_date_time_format_iso8601(last_msg->timestamp); } sqlite_lines = g_slist_concat(sqlite_lines, batch); if (batch_size < MESSAGES_TO_RETRIEVE) { has_more = FALSE; } } g_free(start_time); if (!sqlite_lines) { log_debug("export: no SQLite messages for %s", contact); g_hash_table_destroy(seen_keys); g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); continue; } // 3. Merge: write existing flatfile lines + new SQLite lines to temp file auto_gchar gchar* log_path = ff_get_log_path(contact); if (!log_path) { g_hash_table_destroy(seen_keys); g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); continue; } // Ensure directory auto_gchar gchar* dir = g_path_get_dirname(log_path); ff_ensure_dir(dir); auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path); FILE* fp = fopen(tmp_path, "w"); if (!fp) { log_error("export: cannot create %s: %s", tmp_path, strerror(errno)); g_hash_table_destroy(seen_keys); g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); continue; } // Lock the temp file int fd = fileno(fp); flock(fd, LOCK_EX); fprintf(fp, "%s", FLATFILE_HEADER); int contact_exported = 0; int contact_skipped = 0; // Write existing flatfile lines first (preserve originals) for (GSList* l = existing; l; l = l->next) { ff_parsed_line_t* pl = l->data; ff_write_line(fp, pl->timestamp_str, pl->type, pl->enc, pl->stanza_id, pl->archive_id, pl->replace_id, pl->from_jid, pl->from_resource, pl->message); } // Write SQLite messages that aren't already in the flatfile for (GSList* l = sqlite_lines; l; l = l->next) { ProfMessage* msg = l->data; if (!msg || !msg->timestamp) continue; auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp); const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown"; const char* body = msg->plain ? msg->plain : ""; const char* sid = msg->id ? msg->id : ""; char* key = _make_dedup_key(sid, ts, from_jid, body); if (g_hash_table_contains(seen_keys, key)) { g_free(key); contact_skipped++; continue; } g_hash_table_add(seen_keys, key); const char* type_str = ff_get_message_type_str(msg->type); const char* enc_str = ff_get_message_enc_str(msg->enc); const char* from_res = msg->from_jid ? msg->from_jid->resourcepart : NULL; ff_write_line(fp, ts, type_str, enc_str, sid, msg->stanzaid, NULL, from_jid, from_res, body); contact_exported++; } fflush(fp); fclose(fp); // also releases flock // Atomic rename if (rename(tmp_path, log_path) != 0) { log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno)); unlink(tmp_path); } cons_show("Exported %s: %d new, %d skipped (already existed: %d)", contact, contact_exported, contact_skipped, existing_count); total_exported += contact_exported; total_skipped += contact_skipped; g_hash_table_destroy(seen_keys); g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); } g_slist_free_full(contacts, g_free); return total_exported; } // ========================================================================= // Import: flatfile -> SQLite (merge) // ========================================================================= int log_database_import_from_flatfile(const gchar* const contact_jid) { const Jid* myjid = connection_get_jid(); if (!myjid || !myjid->barejid) { cons_show_error("Import failed: not connected."); return -1; } if (!g_flatfile_account_jid) { g_free(g_flatfile_account_jid); g_flatfile_account_jid = g_strdup(myjid->barejid); } db_backend_t* sqlite_be = db_backend_sqlite(); if (!sqlite_be) { cons_show_error("Import failed: SQLite backend not available."); return -1; } // Determine contacts GSList* contacts = NULL; if (contact_jid) { contacts = g_slist_append(contacts, g_strdup(contact_jid)); } else { contacts = _ff_list_contacts(); if (!contacts) { cons_show("No flat-file history found to import."); return 0; } } int total_imported = 0; int total_skipped = 0; for (GSList* c = contacts; c; c = c->next) { const char* contact = c->data; // 1. Read all flatfile lines GSList* ff_lines = _ff_read_all_lines(contact); if (!ff_lines) { log_debug("import: no flatfile messages for %s", contact); continue; } // 2. Build dedup set from SQLite (existing stanza_ids + fallback keys) GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); // Read all SQLite messages for this contact gboolean has_more = TRUE; gchar* start_time = NULL; while (has_more) { GSList* batch = NULL; db_history_result_t res = sqlite_be->get_previous_chat( contact, start_time, NULL, TRUE, FALSE, &batch); if (res != DB_RESPONSE_SUCCESS || !batch) { has_more = FALSE; break; } int batch_size = g_slist_length(batch); ProfMessage* last_msg = g_slist_last(batch)->data; if (last_msg && last_msg->timestamp) { g_free(start_time); start_time = g_date_time_format_iso8601(last_msg->timestamp); } for (GSList* l = batch; l; l = l->next) { ProfMessage* msg = l->data; if (!msg) continue; auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup(""); char* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain); g_hash_table_add(seen_keys, key); } g_slist_free_full(batch, (GDestroyNotify)message_free); if (batch_size < MESSAGES_TO_RETRIEVE) { has_more = FALSE; } } g_free(start_time); // 3. For each flatfile line not in seen_keys, insert via add_incoming/add_outgoing int contact_imported = 0; int contact_skipped = 0; for (GSList* l = ff_lines; l; l = l->next) { ff_parsed_line_t* pl = l->data; if (!pl || !pl->timestamp) continue; auto_gchar gchar* ts = g_date_time_format_iso8601(pl->timestamp); char* key = _make_dedup_key(pl->stanza_id, ts, pl->from_jid, pl->message); if (g_hash_table_contains(seen_keys, key)) { g_free(key); contact_skipped++; continue; } g_hash_table_add(seen_keys, key); // Build a ProfMessage and insert via add_incoming // (add_incoming uses _add_to_db which preserves the original timestamp) gboolean is_outgoing = (g_strcmp0(pl->from_jid, myjid->barejid) == 0); ProfMessage* msg = message_init(); msg->id = pl->stanza_id ? g_strdup(pl->stanza_id) : NULL; msg->stanzaid = pl->archive_id ? g_strdup(pl->archive_id) : NULL; msg->plain = g_strdup(pl->message ? pl->message : ""); msg->timestamp = g_date_time_ref(pl->timestamp); msg->type = ff_get_message_type_type(pl->type); msg->enc = ff_get_message_enc_type(pl->enc); msg->replace_id = pl->replace_id ? g_strdup(pl->replace_id) : NULL; msg->is_mam = TRUE; // Suppress dedup check in the backend if (is_outgoing) { msg->from_jid = jid_create(myjid->barejid); msg->to_jid = jid_create(contact); } else { msg->from_jid = jid_create_from_bare_and_resource(pl->from_jid, pl->from_resource); msg->to_jid = jid_create(myjid->barejid); } sqlite_be->add_incoming(msg); message_free(msg); contact_imported++; } cons_show("Imported %s: %d new, %d skipped (already in SQLite)", contact, contact_imported, contact_skipped); total_imported += contact_imported; total_skipped += contact_skipped; g_hash_table_destroy(seen_keys); g_slist_free_full(ff_lines, (GDestroyNotify)ff_parsed_line_free); } g_slist_free_full(contacts, g_free); return total_imported; } #endif /* HAVE_SQLITE */