diff --git a/Makefile.am b/Makefile.am index 9e884d2c..021491d0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,6 +6,7 @@ core_sources = \ src/database_flatfile.c src/database_flatfile.h \ src/database_flatfile_parser.c \ src/database_flatfile_verify.c \ + src/database_export.c \ src/log.h src/profanity.c src/common.h \ src/profanity.h src/xmpp/chat_session.c \ src/xmpp/chat_session.h src/xmpp/muc.c src/xmpp/muc.h src/xmpp/jid.h src/xmpp/jid.c \ diff --git a/docs/profanity.1 b/docs/profanity.1 index 934e4d64..b00e30c7 100755 --- a/docs/profanity.1 +++ b/docs/profanity.1 @@ -226,6 +226,14 @@ Each line has the format: Use .B /history verify to check integrity of stored history (both SQLite and flat-file). +.PP +Use +.B /history export [] +to copy messages from SQLite to flat-file format (merge with existing data), or +.B /history import [] +to copy from flat-file to SQLite. +Both operations skip duplicates. +Omit the JID to process all contacts. .SH BUGS Bugs can either be reported by raising an issue at the Github issue tracker: .br diff --git a/src/command/cmd_ac.c b/src/command/cmd_ac.c index bebac5de..7c133fc6 100644 --- a/src/command/cmd_ac.c +++ b/src/command/cmd_ac.c @@ -1145,6 +1145,8 @@ cmd_ac_init(void) autocomplete_add(history_ac, "on"); autocomplete_add(history_ac, "off"); autocomplete_add(history_ac, "verify"); + autocomplete_add(history_ac, "export"); + autocomplete_add(history_ac, "import"); autocomplete_add(logging_group_ac, "on"); autocomplete_add(logging_group_ac, "off"); diff --git a/src/command/cmd_defs.c b/src/command/cmd_defs.c index 5c5125ef..3f80fa39 100644 --- a/src/command/cmd_defs.c +++ b/src/command/cmd_defs.c @@ -1882,14 +1882,20 @@ static const struct cmd_t command_defs[] = { CMD_TAG_CHAT) CMD_SYN( "/history on|off", - "/history verify []") + "/history verify []", + "/history export []", + "/history import []") CMD_DESC( "Switch chat history on or off, /logging chat will automatically be enabled when this setting is on. " "When history is enabled, previous messages are shown in chat windows. " - "Use 'verify' to check integrity of stored message history.") + "Use 'verify' to check integrity of stored message history. " + "Use 'export' to copy messages from SQLite to flat-file format, or 'import' to copy from flat-file to SQLite. " + "Both export and import merge with existing data (duplicates are skipped).") CMD_ARGS( { "on|off", "Enable or disable showing chat history." }, - { "verify []", "Verify integrity of message history. Optionally specify a JID to check only one contact." }) + { "verify []", "Verify integrity of message history. Optionally specify a JID to check only one contact." }, + { "export []", "Export SQLite history to flat-file format. Optionally specify a JID to export only one contact." }, + { "import []", "Import flat-file history into SQLite. Optionally specify a JID to import only one contact." }) }, { CMD_PREAMBLE("/log", diff --git a/src/command/cmd_funcs.c b/src/command/cmd_funcs.c index a08608e7..5258afe5 100644 --- a/src/command/cmd_funcs.c +++ b/src/command/cmd_funcs.c @@ -6787,6 +6787,42 @@ cmd_history(ProfWin* window, const char* const command, gchar** args) return TRUE; } + if (g_strcmp0(args[0], "export") == 0) { +#ifdef HAVE_SQLITE + const gchar* contact_jid = args[1]; // may be NULL (export all) + if (contact_jid) { + cons_show("Exporting SQLite history for %s to flat-file...", contact_jid); + } else { + cons_show("Exporting all SQLite history to flat-file..."); + } + int n = log_database_export_to_flatfile(contact_jid); + if (n >= 0) { + cons_show("Export complete: %d message(s) exported.", n); + } +#else + cons_show_error("Export requires SQLite support. Rebuild with --with-sqlite."); +#endif + return TRUE; + } + + if (g_strcmp0(args[0], "import") == 0) { +#ifdef HAVE_SQLITE + const gchar* contact_jid = args[1]; // may be NULL (import all) + if (contact_jid) { + cons_show("Importing flat-file history for %s into SQLite...", contact_jid); + } else { + cons_show("Importing all flat-file history into SQLite..."); + } + int n = log_database_import_from_flatfile(contact_jid); + if (n >= 0) { + cons_show("Import complete: %d message(s) imported.", n); + } +#else + cons_show_error("Import requires SQLite support. Rebuild with --with-sqlite."); +#endif + return TRUE; + } + _cmd_set_boolean_preference(args[0], "Chat history", PREF_HISTORY); // if set to on, set chlog (/logging chat on) diff --git a/src/database.h b/src/database.h index 1a9e0bf9..9b29f014 100644 --- a/src/database.h +++ b/src/database.h @@ -87,6 +87,7 @@ extern db_backend_t* active_db_backend; // Backend registry #ifdef HAVE_SQLITE db_backend_t* db_backend_sqlite(void); +GSList* db_sqlite_list_contacts(void); #endif db_backend_t* db_backend_flatfile(void); @@ -101,4 +102,10 @@ ProfMessage* log_database_get_limits_info(const gchar* const contact_barejid, gb void log_database_close(void); GSList* log_database_verify_integrity(const gchar* const contact_barejid); +// Cross-backend export/import (requires HAVE_SQLITE) +#ifdef HAVE_SQLITE +int log_database_export_to_flatfile(const gchar* const contact_jid); +int log_database_import_from_flatfile(const gchar* const contact_jid); +#endif + #endif // DATABASE_H diff --git a/src/database_export.c b/src/database_export.c new file mode 100644 index 00000000..03b561cb --- /dev/null +++ b/src/database_export.c @@ -0,0 +1,531 @@ +/* + * database_export.c + * vim: expandtab:ts=4:sts=4:sw=4 + * + * Copyright (C) 2026 Profanity Contributors + * + * This file is part of Profanity. + * + * Profanity is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Profanity is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Profanity. If not, see . + * + * Cross-backend export/import for message history. + * Reads from one backend, writes to the other, with merge + dedup. + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_SQLITE +#include +#endif + +#include "log.h" +#include "common.h" +#include "config/files.h" +#include "database.h" +#include "database_flatfile.h" +#include "config/preferences.h" +#include "ui/ui.h" +#include "xmpp/xmpp.h" +#include "xmpp/message.h" + +// ========================================================================= +// Everything below is only used when HAVE_SQLITE is defined +// ========================================================================= + +#ifdef HAVE_SQLITE + +// ========================================================================= +// Dedup key: stanza_id if present, else hash of timestamp+from+body +// ========================================================================= + +static char* +_make_dedup_key(const char* stanza_id, const char* timestamp, const char* from_jid, const char* body) +{ + if (stanza_id && strlen(stanza_id) > 0) + return g_strdup(stanza_id); + + // Fallback: simple composite key + auto_gchar gchar* raw = g_strdup_printf("%s|%s|%.64s", + timestamp ? timestamp : "", + from_jid ? from_jid : "", + body ? body : ""); + return g_compute_checksum_for_string(G_CHECKSUM_SHA256, raw, -1); +} + +// ========================================================================= +// Helper: reverse ff_jid_to_dir (best-effort) +// '_at_' -> '@' +// ========================================================================= + +static char* +_dir_to_jid(const char* dirname) +{ + if (!dirname) + return NULL; + return str_replace(dirname, "_at_", "@"); +} + +// ========================================================================= +// Helper: list all contact JIDs from flatfile directory +// Returns a GSList of g_strdup'd JID strings. Caller frees with g_slist_free_full(list, g_free). +// ========================================================================= + +static GSList* +_ff_list_contacts(void) +{ + GSList* contacts = NULL; + + if (!g_flatfile_account_jid) + return NULL; + + auto_gchar gchar* data_path = files_get_data_path(DIR_FLATLOG); + auto_gchar gchar* my_dir = ff_jid_to_dir(g_flatfile_account_jid); + auto_gchar gchar* base_dir = g_strdup_printf("%s/%s", data_path, my_dir); + + GError* err = NULL; + GDir* dir = g_dir_open(base_dir, 0, &err); + if (!dir) { + if (err) { + log_debug("export: cannot open flatlog dir %s: %s", base_dir, err->message); + g_error_free(err); + } + return NULL; + } + + const gchar* entry; + while ((entry = g_dir_read_name(dir)) != NULL) { + auto_gchar gchar* full_path = g_strdup_printf("%s/%s/history.log", base_dir, entry); + if (g_file_test(full_path, G_FILE_TEST_IS_REGULAR)) { + char* jid = _dir_to_jid(entry); + if (jid) { + contacts = g_slist_append(contacts, jid); + } + } + } + g_dir_close(dir); + + return contacts; +} + +// ========================================================================= +// Helper: read ALL parsed lines from a flatfile for a contact +// Returns GSList. Caller frees with g_slist_free_full(list, (GDestroyNotify)ff_parsed_line_free). +// ========================================================================= + +static GSList* +_ff_read_all_lines(const char* contact_barejid) +{ + GSList* lines = NULL; + auto_gchar gchar* log_path = ff_get_log_path(contact_barejid); + if (!log_path) + return NULL; + + FILE* fp = fopen(log_path, "r"); + if (!fp) + return NULL; + + while (1) { + gboolean truncated = FALSE; + char* buf = ff_readline(fp, &truncated); + if (!buf) + break; + if (truncated) { + free(buf); + break; + } + if (buf[0] == '\0' || buf[0] == '#') { + free(buf); + continue; + } + + ff_parsed_line_t* pl = ff_parse_line(buf); + free(buf); + if (pl) { + lines = g_slist_append(lines, pl); + } + } + fclose(fp); + + return lines; +} + +// ========================================================================= +// Export: SQLite -> flatfile (merge) +// ========================================================================= + +// Callback data for the SQLite query +typedef struct +{ + GHashTable* seen_keys; // dedup set + FILE* fp; // output file + int exported; // counter + int skipped; // counter +} export_ctx_t; + +int +log_database_export_to_flatfile(const gchar* const contact_jid) +{ + const Jid* myjid = connection_get_jid(); + if (!myjid || !myjid->barejid) { + cons_show_error("Export failed: not connected."); + return -1; + } + + if (!g_flatfile_account_jid) { + // Flatfile backend not initialized — init it temporarily + // We need g_flatfile_account_jid for path construction + g_free(g_flatfile_account_jid); + g_flatfile_account_jid = g_strdup(myjid->barejid); + } + + // Get the SQLite backend + db_backend_t* sqlite_be = db_backend_sqlite(); + if (!sqlite_be) { + cons_show_error("Export failed: SQLite backend not available."); + return -1; + } + + // Determine contact list + GSList* contacts = NULL; + if (contact_jid) { + contacts = g_slist_append(contacts, g_strdup(contact_jid)); + } else { + contacts = db_sqlite_list_contacts(); + if (!contacts) { + cons_show("No contacts found in SQLite database."); + return 0; + } + cons_show("Exporting %d contact(s) to flat-file format...", g_slist_length(contacts)); + } + + int total_exported = 0; + int total_skipped = 0; + + for (GSList* c = contacts; c; c = c->next) { + const char* contact = c->data; + + // 1. Read existing flatfile lines for dedup + GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + + GSList* existing = _ff_read_all_lines(contact); + for (GSList* l = existing; l; l = l->next) { + ff_parsed_line_t* pl = l->data; + char* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message); + g_hash_table_add(seen_keys, key); + } + int existing_count = g_slist_length(existing); + + // 2. Query SQLite for this contact — get ALL messages ordered by timestamp + // We read all messages by paginating through the existing get_previous_chat + // But it limits to MESSAGES_TO_RETRIEVE... We need a direct SQL query. + // Use the same approach as verify — iterate with raw SQL. + + // For now, collect all messages into a list via repeated calls + // Actually, let's use a simpler approach: read all from flatfile existing data, + // then query SQLite for everything, merge, write back. + + // Read all SQLite messages for this contact + GSList* sqlite_lines = NULL; + // Use the backend's get_previous_chat with a wide time range + // This is limited to MESSAGES_TO_RETRIEVE, so we need to paginate + gboolean has_more = TRUE; + gchar* start_time = NULL; + + while (has_more) { + GSList* batch = NULL; + db_history_result_t res = sqlite_be->get_previous_chat( + contact, start_time, NULL, TRUE, FALSE, &batch); + + if (res != DB_RESPONSE_SUCCESS || !batch) { + has_more = FALSE; + break; + } + + int batch_size = g_slist_length(batch); + + // Get the last timestamp for the next query + ProfMessage* last_msg = g_slist_last(batch)->data; + if (last_msg && last_msg->timestamp) { + g_free(start_time); + start_time = g_date_time_format_iso8601(last_msg->timestamp); + } + + sqlite_lines = g_slist_concat(sqlite_lines, batch); + + if (batch_size < MESSAGES_TO_RETRIEVE) { + has_more = FALSE; + } + } + g_free(start_time); + + if (!sqlite_lines) { + log_debug("export: no SQLite messages for %s", contact); + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + continue; + } + + // 3. Merge: write existing flatfile lines + new SQLite lines to temp file + auto_gchar gchar* log_path = ff_get_log_path(contact); + if (!log_path) { + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + continue; + } + + // Ensure directory + auto_gchar gchar* dir = g_path_get_dirname(log_path); + ff_ensure_dir(dir); + + auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path); + FILE* fp = fopen(tmp_path, "w"); + if (!fp) { + log_error("export: cannot create %s: %s", tmp_path, strerror(errno)); + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + continue; + } + + // Lock the temp file + int fd = fileno(fp); + flock(fd, LOCK_EX); + + fprintf(fp, "%s", FLATFILE_HEADER); + + int contact_exported = 0; + int contact_skipped = 0; + + // Write existing flatfile lines first (preserve originals) + for (GSList* l = existing; l; l = l->next) { + ff_parsed_line_t* pl = l->data; + ff_write_line(fp, pl->timestamp_str, + pl->type, pl->enc, + pl->stanza_id, pl->archive_id, pl->replace_id, + pl->from_jid, pl->from_resource, pl->message); + } + + // Write SQLite messages that aren't already in the flatfile + for (GSList* l = sqlite_lines; l; l = l->next) { + ProfMessage* msg = l->data; + if (!msg || !msg->timestamp) + continue; + + auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp); + const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown"; + const char* body = msg->plain ? msg->plain : ""; + const char* sid = msg->id ? msg->id : ""; + + char* key = _make_dedup_key(sid, ts, from_jid, body); + if (g_hash_table_contains(seen_keys, key)) { + g_free(key); + contact_skipped++; + continue; + } + g_hash_table_add(seen_keys, key); + + const char* type_str = ff_get_message_type_str(msg->type); + const char* enc_str = ff_get_message_enc_str(msg->enc); + const char* from_res = msg->from_jid ? msg->from_jid->resourcepart : NULL; + + ff_write_line(fp, ts, type_str, enc_str, + sid, msg->stanzaid, NULL, + from_jid, from_res, body); + contact_exported++; + } + + fflush(fp); + fclose(fp); // also releases flock + + // Atomic rename + if (rename(tmp_path, log_path) != 0) { + log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno)); + unlink(tmp_path); + } + + cons_show("Exported %s: %d new, %d skipped (already existed: %d)", + contact, contact_exported, contact_skipped, existing_count); + total_exported += contact_exported; + total_skipped += contact_skipped; + + g_hash_table_destroy(seen_keys); + g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free); + g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free); + } + + g_slist_free_full(contacts, g_free); + return total_exported; +} + +// ========================================================================= +// Import: flatfile -> SQLite (merge) +// ========================================================================= + +int +log_database_import_from_flatfile(const gchar* const contact_jid) +{ + const Jid* myjid = connection_get_jid(); + if (!myjid || !myjid->barejid) { + cons_show_error("Import failed: not connected."); + return -1; + } + + if (!g_flatfile_account_jid) { + g_free(g_flatfile_account_jid); + g_flatfile_account_jid = g_strdup(myjid->barejid); + } + + db_backend_t* sqlite_be = db_backend_sqlite(); + if (!sqlite_be) { + cons_show_error("Import failed: SQLite backend not available."); + return -1; + } + + // Determine contacts + GSList* contacts = NULL; + if (contact_jid) { + contacts = g_slist_append(contacts, g_strdup(contact_jid)); + } else { + contacts = _ff_list_contacts(); + if (!contacts) { + cons_show("No flat-file history found to import."); + return 0; + } + } + + int total_imported = 0; + int total_skipped = 0; + + for (GSList* c = contacts; c; c = c->next) { + const char* contact = c->data; + + // 1. Read all flatfile lines + GSList* ff_lines = _ff_read_all_lines(contact); + if (!ff_lines) { + log_debug("import: no flatfile messages for %s", contact); + continue; + } + + // 2. Build dedup set from SQLite (existing stanza_ids + fallback keys) + GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + + // Read all SQLite messages for this contact + gboolean has_more = TRUE; + gchar* start_time = NULL; + + while (has_more) { + GSList* batch = NULL; + db_history_result_t res = sqlite_be->get_previous_chat( + contact, start_time, NULL, TRUE, FALSE, &batch); + + if (res != DB_RESPONSE_SUCCESS || !batch) { + has_more = FALSE; + break; + } + + int batch_size = g_slist_length(batch); + + ProfMessage* last_msg = g_slist_last(batch)->data; + if (last_msg && last_msg->timestamp) { + g_free(start_time); + start_time = g_date_time_format_iso8601(last_msg->timestamp); + } + + for (GSList* l = batch; l; l = l->next) { + ProfMessage* msg = l->data; + if (!msg) + continue; + auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup(""); + char* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain); + g_hash_table_add(seen_keys, key); + } + + g_slist_free_full(batch, (GDestroyNotify)message_free); + + if (batch_size < MESSAGES_TO_RETRIEVE) { + has_more = FALSE; + } + } + g_free(start_time); + + // 3. For each flatfile line not in seen_keys, insert via add_incoming/add_outgoing + int contact_imported = 0; + int contact_skipped = 0; + + for (GSList* l = ff_lines; l; l = l->next) { + ff_parsed_line_t* pl = l->data; + if (!pl || !pl->timestamp) + continue; + + auto_gchar gchar* ts = g_date_time_format_iso8601(pl->timestamp); + char* key = _make_dedup_key(pl->stanza_id, ts, pl->from_jid, pl->message); + + if (g_hash_table_contains(seen_keys, key)) { + g_free(key); + contact_skipped++; + continue; + } + g_hash_table_add(seen_keys, key); + + // Build a ProfMessage and insert via add_incoming + // (add_incoming uses _add_to_db which preserves the original timestamp) + gboolean is_outgoing = (g_strcmp0(pl->from_jid, myjid->barejid) == 0); + + ProfMessage* msg = message_init(); + msg->id = pl->stanza_id ? g_strdup(pl->stanza_id) : NULL; + msg->stanzaid = pl->archive_id ? g_strdup(pl->archive_id) : NULL; + msg->plain = g_strdup(pl->message ? pl->message : ""); + msg->timestamp = g_date_time_ref(pl->timestamp); + msg->type = ff_get_message_type_type(pl->type); + msg->enc = ff_get_message_enc_type(pl->enc); + msg->replace_id = pl->replace_id ? g_strdup(pl->replace_id) : NULL; + msg->is_mam = TRUE; // Suppress dedup check in the backend + + if (is_outgoing) { + msg->from_jid = jid_create(myjid->barejid); + msg->to_jid = jid_create(contact); + } else { + msg->from_jid = jid_create_from_bare_and_resource(pl->from_jid, pl->from_resource); + msg->to_jid = jid_create(myjid->barejid); + } + + sqlite_be->add_incoming(msg); + message_free(msg); + contact_imported++; + } + + cons_show("Imported %s: %d new, %d skipped (already in SQLite)", + contact, contact_imported, contact_skipped); + total_imported += contact_imported; + total_skipped += contact_skipped; + + g_hash_table_destroy(seen_keys); + g_slist_free_full(ff_lines, (GDestroyNotify)ff_parsed_line_free); + } + + g_slist_free_full(contacts, g_free); + return total_imported; +} + +#endif /* HAVE_SQLITE */ diff --git a/src/database_sqlite.c b/src/database_sqlite.c index 9ba2d63b..761f8515 100644 --- a/src/database_sqlite.c +++ b/src/database_sqlite.c @@ -817,3 +817,37 @@ db_backend_sqlite(void) { return &sqlite_backend; } + +GSList* +db_sqlite_list_contacts(void) +{ + GSList* contacts = NULL; + if (!g_chatlog_database) + return NULL; + + const Jid* myjid = connection_get_jid(); + if (!myjid || !myjid->barejid) + return NULL; + + const char* q = "SELECT DISTINCT `from_jid` FROM `ChatLogs` " + "WHERE `from_jid` != ? " + "UNION " + "SELECT DISTINCT `to_jid` FROM `ChatLogs` " + "WHERE `to_jid` != ?"; + sqlite3_stmt* stmt = NULL; + if (!_db_prepare_ctx(q, &stmt, "db_sqlite_list_contacts")) + return NULL; + + sqlite3_bind_text(stmt, 1, myjid->barejid, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, myjid->barejid, -1, SQLITE_STATIC); + + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char* jid = (const char*)sqlite3_column_text(stmt, 0); + if (jid && strlen(jid) > 0) { + contacts = g_slist_append(contacts, g_strdup(jid)); + } + } + sqlite3_finalize(stmt); + + return contacts; +} \ No newline at end of file diff --git a/tests/unittests/database/stub_database.c b/tests/unittests/database/stub_database.c index 9d6f72df..282fb805 100644 --- a/tests/unittests/database/stub_database.c +++ b/tests/unittests/database/stub_database.c @@ -73,6 +73,21 @@ db_backend_sqlite(void) { return NULL; } +GSList* +db_sqlite_list_contacts(void) +{ + return NULL; +} +int +log_database_export_to_flatfile(const gchar* const contact_jid) +{ + return 0; +} +int +log_database_import_from_flatfile(const gchar* const contact_jid) +{ + return 0; +} #endif db_backend_t* db_backend_flatfile(void)