feat(draft): add /history export|import for SQLite<->flatfile migration
All checks were successful
CI Code / Check spelling (pull_request) Successful in 18s
CI Code / Check coding style (pull_request) Successful in 31s
CI Code / Code Coverage (pull_request) Successful in 4m48s
CI Code / Linux (debian) (pull_request) Successful in 6m11s
CI Code / Linux (ubuntu) (pull_request) Successful in 6m16s
CI Code / Linux (arch) (pull_request) Successful in 6m28s
All checks were successful
CI Code / Check spelling (pull_request) Successful in 18s
CI Code / Check coding style (pull_request) Successful in 31s
CI Code / Code Coverage (pull_request) Successful in 4m48s
CI Code / Linux (debian) (pull_request) Successful in 6m11s
CI Code / Linux (ubuntu) (pull_request) Successful in 6m16s
CI Code / Linux (arch) (pull_request) Successful in 6m28s
DRAFT — not yet tested end-to-end with a live XMPP session. New commands: /history export [<jid>] — copy messages from SQLite to flat-file /history import [<jid>] — copy messages from flat-file to SQLite Both merge with existing data; duplicates are skipped using stanza-id or a SHA-256 fallback key (timestamp + sender + body prefix). Implementation (src/database_export.c): - Export: paginate SQLite via get_previous_chat, read existing flatfile for dedup, write merged result via ff_write_line + atomic rename - Import: parse flatfile lines via ff_parse_line, build dedup set from SQLite, insert new messages via add_incoming (preserves original timestamps for both directions) - List all contacts: db_sqlite_list_contacts() queries UNION of DISTINCT from_jid/to_jid; flatfile enumerates flatlog directories Wiring: - database.h: declare export/import functions + db_sqlite_list_contacts - database_sqlite.c: add db_sqlite_list_contacts() - cmd_defs.c: add export/import to /history synopsis and args - cmd_funcs.c: add export/import handlers in cmd_history() - cmd_ac.c: add 'export' and 'import' to history_ac - Makefile.am: add database_export.c to core_sources - stub_database.c: add stubs for test linking - profanity.1: document export/import in man page All code guarded with #ifdef HAVE_SQLITE — builds cleanly without it.
This commit is contained in:
@@ -6,6 +6,7 @@ core_sources = \
|
|||||||
src/database_flatfile.c src/database_flatfile.h \
|
src/database_flatfile.c src/database_flatfile.h \
|
||||||
src/database_flatfile_parser.c \
|
src/database_flatfile_parser.c \
|
||||||
src/database_flatfile_verify.c \
|
src/database_flatfile_verify.c \
|
||||||
|
src/database_export.c \
|
||||||
src/log.h src/profanity.c src/common.h \
|
src/log.h src/profanity.c src/common.h \
|
||||||
src/profanity.h src/xmpp/chat_session.c \
|
src/profanity.h src/xmpp/chat_session.c \
|
||||||
src/xmpp/chat_session.h src/xmpp/muc.c src/xmpp/muc.h src/xmpp/jid.h src/xmpp/jid.c \
|
src/xmpp/chat_session.h src/xmpp/muc.c src/xmpp/muc.h src/xmpp/jid.h src/xmpp/jid.c \
|
||||||
|
|||||||
@@ -226,6 +226,14 @@ Each line has the format:
|
|||||||
Use
|
Use
|
||||||
.B /history verify
|
.B /history verify
|
||||||
to check integrity of stored history (both SQLite and flat-file).
|
to check integrity of stored history (both SQLite and flat-file).
|
||||||
|
.PP
|
||||||
|
Use
|
||||||
|
.B /history export [<jid>]
|
||||||
|
to copy messages from SQLite to flat-file format (merge with existing data), or
|
||||||
|
.B /history import [<jid>]
|
||||||
|
to copy from flat-file to SQLite.
|
||||||
|
Both operations skip duplicates.
|
||||||
|
Omit the JID to process all contacts.
|
||||||
.SH BUGS
|
.SH BUGS
|
||||||
Bugs can either be reported by raising an issue at the Github issue tracker:
|
Bugs can either be reported by raising an issue at the Github issue tracker:
|
||||||
.br
|
.br
|
||||||
|
|||||||
@@ -1145,6 +1145,8 @@ cmd_ac_init(void)
|
|||||||
autocomplete_add(history_ac, "on");
|
autocomplete_add(history_ac, "on");
|
||||||
autocomplete_add(history_ac, "off");
|
autocomplete_add(history_ac, "off");
|
||||||
autocomplete_add(history_ac, "verify");
|
autocomplete_add(history_ac, "verify");
|
||||||
|
autocomplete_add(history_ac, "export");
|
||||||
|
autocomplete_add(history_ac, "import");
|
||||||
|
|
||||||
autocomplete_add(logging_group_ac, "on");
|
autocomplete_add(logging_group_ac, "on");
|
||||||
autocomplete_add(logging_group_ac, "off");
|
autocomplete_add(logging_group_ac, "off");
|
||||||
|
|||||||
@@ -1882,14 +1882,20 @@ static const struct cmd_t command_defs[] = {
|
|||||||
CMD_TAG_CHAT)
|
CMD_TAG_CHAT)
|
||||||
CMD_SYN(
|
CMD_SYN(
|
||||||
"/history on|off",
|
"/history on|off",
|
||||||
"/history verify [<jid>]")
|
"/history verify [<jid>]",
|
||||||
|
"/history export [<jid>]",
|
||||||
|
"/history import [<jid>]")
|
||||||
CMD_DESC(
|
CMD_DESC(
|
||||||
"Switch chat history on or off, /logging chat will automatically be enabled when this setting is on. "
|
"Switch chat history on or off, /logging chat will automatically be enabled when this setting is on. "
|
||||||
"When history is enabled, previous messages are shown in chat windows. "
|
"When history is enabled, previous messages are shown in chat windows. "
|
||||||
"Use 'verify' to check integrity of stored message history.")
|
"Use 'verify' to check integrity of stored message history. "
|
||||||
|
"Use 'export' to copy messages from SQLite to flat-file format, or 'import' to copy from flat-file to SQLite. "
|
||||||
|
"Both export and import merge with existing data (duplicates are skipped).")
|
||||||
CMD_ARGS(
|
CMD_ARGS(
|
||||||
{ "on|off", "Enable or disable showing chat history." },
|
{ "on|off", "Enable or disable showing chat history." },
|
||||||
{ "verify [<jid>]", "Verify integrity of message history. Optionally specify a JID to check only one contact." })
|
{ "verify [<jid>]", "Verify integrity of message history. Optionally specify a JID to check only one contact." },
|
||||||
|
{ "export [<jid>]", "Export SQLite history to flat-file format. Optionally specify a JID to export only one contact." },
|
||||||
|
{ "import [<jid>]", "Import flat-file history into SQLite. Optionally specify a JID to import only one contact." })
|
||||||
},
|
},
|
||||||
|
|
||||||
{ CMD_PREAMBLE("/log",
|
{ CMD_PREAMBLE("/log",
|
||||||
|
|||||||
@@ -6787,6 +6787,42 @@ cmd_history(ProfWin* window, const char* const command, gchar** args)
|
|||||||
return TRUE;
|
return TRUE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (g_strcmp0(args[0], "export") == 0) {
|
||||||
|
#ifdef HAVE_SQLITE
|
||||||
|
const gchar* contact_jid = args[1]; // may be NULL (export all)
|
||||||
|
if (contact_jid) {
|
||||||
|
cons_show("Exporting SQLite history for %s to flat-file...", contact_jid);
|
||||||
|
} else {
|
||||||
|
cons_show("Exporting all SQLite history to flat-file...");
|
||||||
|
}
|
||||||
|
int n = log_database_export_to_flatfile(contact_jid);
|
||||||
|
if (n >= 0) {
|
||||||
|
cons_show("Export complete: %d message(s) exported.", n);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
cons_show_error("Export requires SQLite support. Rebuild with --with-sqlite.");
|
||||||
|
#endif
|
||||||
|
return TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (g_strcmp0(args[0], "import") == 0) {
|
||||||
|
#ifdef HAVE_SQLITE
|
||||||
|
const gchar* contact_jid = args[1]; // may be NULL (import all)
|
||||||
|
if (contact_jid) {
|
||||||
|
cons_show("Importing flat-file history for %s into SQLite...", contact_jid);
|
||||||
|
} else {
|
||||||
|
cons_show("Importing all flat-file history into SQLite...");
|
||||||
|
}
|
||||||
|
int n = log_database_import_from_flatfile(contact_jid);
|
||||||
|
if (n >= 0) {
|
||||||
|
cons_show("Import complete: %d message(s) imported.", n);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
cons_show_error("Import requires SQLite support. Rebuild with --with-sqlite.");
|
||||||
|
#endif
|
||||||
|
return TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
_cmd_set_boolean_preference(args[0], "Chat history", PREF_HISTORY);
|
_cmd_set_boolean_preference(args[0], "Chat history", PREF_HISTORY);
|
||||||
|
|
||||||
// if set to on, set chlog (/logging chat on)
|
// if set to on, set chlog (/logging chat on)
|
||||||
|
|||||||
@@ -87,6 +87,7 @@ extern db_backend_t* active_db_backend;
|
|||||||
// Backend registry
|
// Backend registry
|
||||||
#ifdef HAVE_SQLITE
|
#ifdef HAVE_SQLITE
|
||||||
db_backend_t* db_backend_sqlite(void);
|
db_backend_t* db_backend_sqlite(void);
|
||||||
|
GSList* db_sqlite_list_contacts(void);
|
||||||
#endif
|
#endif
|
||||||
db_backend_t* db_backend_flatfile(void);
|
db_backend_t* db_backend_flatfile(void);
|
||||||
|
|
||||||
@@ -101,4 +102,10 @@ ProfMessage* log_database_get_limits_info(const gchar* const contact_barejid, gb
|
|||||||
void log_database_close(void);
|
void log_database_close(void);
|
||||||
GSList* log_database_verify_integrity(const gchar* const contact_barejid);
|
GSList* log_database_verify_integrity(const gchar* const contact_barejid);
|
||||||
|
|
||||||
|
// Cross-backend export/import (requires HAVE_SQLITE)
|
||||||
|
#ifdef HAVE_SQLITE
|
||||||
|
int log_database_export_to_flatfile(const gchar* const contact_jid);
|
||||||
|
int log_database_import_from_flatfile(const gchar* const contact_jid);
|
||||||
|
#endif
|
||||||
|
|
||||||
#endif // DATABASE_H
|
#endif // DATABASE_H
|
||||||
|
|||||||
531
src/database_export.c
Normal file
531
src/database_export.c
Normal file
@@ -0,0 +1,531 @@
|
|||||||
|
/*
|
||||||
|
* database_export.c
|
||||||
|
* vim: expandtab:ts=4:sts=4:sw=4
|
||||||
|
*
|
||||||
|
* Copyright (C) 2026 Profanity Contributors
|
||||||
|
*
|
||||||
|
* This file is part of Profanity.
|
||||||
|
*
|
||||||
|
* Profanity is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Profanity is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*
|
||||||
|
* Cross-backend export/import for message history.
|
||||||
|
* Reads from one backend, writes to the other, with merge + dedup.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
|
||||||
|
#include <glib.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <errno.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/file.h>
|
||||||
|
|
||||||
|
#ifdef HAVE_SQLITE
|
||||||
|
#include <sqlite3.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include "log.h"
|
||||||
|
#include "common.h"
|
||||||
|
#include "config/files.h"
|
||||||
|
#include "database.h"
|
||||||
|
#include "database_flatfile.h"
|
||||||
|
#include "config/preferences.h"
|
||||||
|
#include "ui/ui.h"
|
||||||
|
#include "xmpp/xmpp.h"
|
||||||
|
#include "xmpp/message.h"
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Everything below is only used when HAVE_SQLITE is defined
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
#ifdef HAVE_SQLITE
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Dedup key: stanza_id if present, else hash of timestamp+from+body
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
static char*
|
||||||
|
_make_dedup_key(const char* stanza_id, const char* timestamp, const char* from_jid, const char* body)
|
||||||
|
{
|
||||||
|
if (stanza_id && strlen(stanza_id) > 0)
|
||||||
|
return g_strdup(stanza_id);
|
||||||
|
|
||||||
|
// Fallback: simple composite key
|
||||||
|
auto_gchar gchar* raw = g_strdup_printf("%s|%s|%.64s",
|
||||||
|
timestamp ? timestamp : "",
|
||||||
|
from_jid ? from_jid : "",
|
||||||
|
body ? body : "");
|
||||||
|
return g_compute_checksum_for_string(G_CHECKSUM_SHA256, raw, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Helper: reverse ff_jid_to_dir (best-effort)
|
||||||
|
// '_at_' -> '@'
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
static char*
|
||||||
|
_dir_to_jid(const char* dirname)
|
||||||
|
{
|
||||||
|
if (!dirname)
|
||||||
|
return NULL;
|
||||||
|
return str_replace(dirname, "_at_", "@");
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Helper: list all contact JIDs from flatfile directory
|
||||||
|
// Returns a GSList of g_strdup'd JID strings. Caller frees with g_slist_free_full(list, g_free).
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
static GSList*
|
||||||
|
_ff_list_contacts(void)
|
||||||
|
{
|
||||||
|
GSList* contacts = NULL;
|
||||||
|
|
||||||
|
if (!g_flatfile_account_jid)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
auto_gchar gchar* data_path = files_get_data_path(DIR_FLATLOG);
|
||||||
|
auto_gchar gchar* my_dir = ff_jid_to_dir(g_flatfile_account_jid);
|
||||||
|
auto_gchar gchar* base_dir = g_strdup_printf("%s/%s", data_path, my_dir);
|
||||||
|
|
||||||
|
GError* err = NULL;
|
||||||
|
GDir* dir = g_dir_open(base_dir, 0, &err);
|
||||||
|
if (!dir) {
|
||||||
|
if (err) {
|
||||||
|
log_debug("export: cannot open flatlog dir %s: %s", base_dir, err->message);
|
||||||
|
g_error_free(err);
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
const gchar* entry;
|
||||||
|
while ((entry = g_dir_read_name(dir)) != NULL) {
|
||||||
|
auto_gchar gchar* full_path = g_strdup_printf("%s/%s/history.log", base_dir, entry);
|
||||||
|
if (g_file_test(full_path, G_FILE_TEST_IS_REGULAR)) {
|
||||||
|
char* jid = _dir_to_jid(entry);
|
||||||
|
if (jid) {
|
||||||
|
contacts = g_slist_append(contacts, jid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g_dir_close(dir);
|
||||||
|
|
||||||
|
return contacts;
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Helper: read ALL parsed lines from a flatfile for a contact
|
||||||
|
// Returns GSList<ff_parsed_line_t*>. Caller frees with g_slist_free_full(list, (GDestroyNotify)ff_parsed_line_free).
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
static GSList*
|
||||||
|
_ff_read_all_lines(const char* contact_barejid)
|
||||||
|
{
|
||||||
|
GSList* lines = NULL;
|
||||||
|
auto_gchar gchar* log_path = ff_get_log_path(contact_barejid);
|
||||||
|
if (!log_path)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
FILE* fp = fopen(log_path, "r");
|
||||||
|
if (!fp)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
while (1) {
|
||||||
|
gboolean truncated = FALSE;
|
||||||
|
char* buf = ff_readline(fp, &truncated);
|
||||||
|
if (!buf)
|
||||||
|
break;
|
||||||
|
if (truncated) {
|
||||||
|
free(buf);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (buf[0] == '\0' || buf[0] == '#') {
|
||||||
|
free(buf);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
ff_parsed_line_t* pl = ff_parse_line(buf);
|
||||||
|
free(buf);
|
||||||
|
if (pl) {
|
||||||
|
lines = g_slist_append(lines, pl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fclose(fp);
|
||||||
|
|
||||||
|
return lines;
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Export: SQLite -> flatfile (merge)
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
// Callback data for the SQLite query
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
GHashTable* seen_keys; // dedup set
|
||||||
|
FILE* fp; // output file
|
||||||
|
int exported; // counter
|
||||||
|
int skipped; // counter
|
||||||
|
} export_ctx_t;
|
||||||
|
|
||||||
|
int
|
||||||
|
log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||||
|
{
|
||||||
|
const Jid* myjid = connection_get_jid();
|
||||||
|
if (!myjid || !myjid->barejid) {
|
||||||
|
cons_show_error("Export failed: not connected.");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!g_flatfile_account_jid) {
|
||||||
|
// Flatfile backend not initialized — init it temporarily
|
||||||
|
// We need g_flatfile_account_jid for path construction
|
||||||
|
g_free(g_flatfile_account_jid);
|
||||||
|
g_flatfile_account_jid = g_strdup(myjid->barejid);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the SQLite backend
|
||||||
|
db_backend_t* sqlite_be = db_backend_sqlite();
|
||||||
|
if (!sqlite_be) {
|
||||||
|
cons_show_error("Export failed: SQLite backend not available.");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine contact list
|
||||||
|
GSList* contacts = NULL;
|
||||||
|
if (contact_jid) {
|
||||||
|
contacts = g_slist_append(contacts, g_strdup(contact_jid));
|
||||||
|
} else {
|
||||||
|
contacts = db_sqlite_list_contacts();
|
||||||
|
if (!contacts) {
|
||||||
|
cons_show("No contacts found in SQLite database.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
cons_show("Exporting %d contact(s) to flat-file format...", g_slist_length(contacts));
|
||||||
|
}
|
||||||
|
|
||||||
|
int total_exported = 0;
|
||||||
|
int total_skipped = 0;
|
||||||
|
|
||||||
|
for (GSList* c = contacts; c; c = c->next) {
|
||||||
|
const char* contact = c->data;
|
||||||
|
|
||||||
|
// 1. Read existing flatfile lines for dedup
|
||||||
|
GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
|
||||||
|
|
||||||
|
GSList* existing = _ff_read_all_lines(contact);
|
||||||
|
for (GSList* l = existing; l; l = l->next) {
|
||||||
|
ff_parsed_line_t* pl = l->data;
|
||||||
|
char* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message);
|
||||||
|
g_hash_table_add(seen_keys, key);
|
||||||
|
}
|
||||||
|
int existing_count = g_slist_length(existing);
|
||||||
|
|
||||||
|
// 2. Query SQLite for this contact — get ALL messages ordered by timestamp
|
||||||
|
// We read all messages by paginating through the existing get_previous_chat
|
||||||
|
// But it limits to MESSAGES_TO_RETRIEVE... We need a direct SQL query.
|
||||||
|
// Use the same approach as verify — iterate with raw SQL.
|
||||||
|
|
||||||
|
// For now, collect all messages into a list via repeated calls
|
||||||
|
// Actually, let's use a simpler approach: read all from flatfile existing data,
|
||||||
|
// then query SQLite for everything, merge, write back.
|
||||||
|
|
||||||
|
// Read all SQLite messages for this contact
|
||||||
|
GSList* sqlite_lines = NULL;
|
||||||
|
// Use the backend's get_previous_chat with a wide time range
|
||||||
|
// This is limited to MESSAGES_TO_RETRIEVE, so we need to paginate
|
||||||
|
gboolean has_more = TRUE;
|
||||||
|
gchar* start_time = NULL;
|
||||||
|
|
||||||
|
while (has_more) {
|
||||||
|
GSList* batch = NULL;
|
||||||
|
db_history_result_t res = sqlite_be->get_previous_chat(
|
||||||
|
contact, start_time, NULL, TRUE, FALSE, &batch);
|
||||||
|
|
||||||
|
if (res != DB_RESPONSE_SUCCESS || !batch) {
|
||||||
|
has_more = FALSE;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
int batch_size = g_slist_length(batch);
|
||||||
|
|
||||||
|
// Get the last timestamp for the next query
|
||||||
|
ProfMessage* last_msg = g_slist_last(batch)->data;
|
||||||
|
if (last_msg && last_msg->timestamp) {
|
||||||
|
g_free(start_time);
|
||||||
|
start_time = g_date_time_format_iso8601(last_msg->timestamp);
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlite_lines = g_slist_concat(sqlite_lines, batch);
|
||||||
|
|
||||||
|
if (batch_size < MESSAGES_TO_RETRIEVE) {
|
||||||
|
has_more = FALSE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g_free(start_time);
|
||||||
|
|
||||||
|
if (!sqlite_lines) {
|
||||||
|
log_debug("export: no SQLite messages for %s", contact);
|
||||||
|
g_hash_table_destroy(seen_keys);
|
||||||
|
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Merge: write existing flatfile lines + new SQLite lines to temp file
|
||||||
|
auto_gchar gchar* log_path = ff_get_log_path(contact);
|
||||||
|
if (!log_path) {
|
||||||
|
g_hash_table_destroy(seen_keys);
|
||||||
|
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||||
|
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure directory
|
||||||
|
auto_gchar gchar* dir = g_path_get_dirname(log_path);
|
||||||
|
ff_ensure_dir(dir);
|
||||||
|
|
||||||
|
auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path);
|
||||||
|
FILE* fp = fopen(tmp_path, "w");
|
||||||
|
if (!fp) {
|
||||||
|
log_error("export: cannot create %s: %s", tmp_path, strerror(errno));
|
||||||
|
g_hash_table_destroy(seen_keys);
|
||||||
|
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||||
|
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lock the temp file
|
||||||
|
int fd = fileno(fp);
|
||||||
|
flock(fd, LOCK_EX);
|
||||||
|
|
||||||
|
fprintf(fp, "%s", FLATFILE_HEADER);
|
||||||
|
|
||||||
|
int contact_exported = 0;
|
||||||
|
int contact_skipped = 0;
|
||||||
|
|
||||||
|
// Write existing flatfile lines first (preserve originals)
|
||||||
|
for (GSList* l = existing; l; l = l->next) {
|
||||||
|
ff_parsed_line_t* pl = l->data;
|
||||||
|
ff_write_line(fp, pl->timestamp_str,
|
||||||
|
pl->type, pl->enc,
|
||||||
|
pl->stanza_id, pl->archive_id, pl->replace_id,
|
||||||
|
pl->from_jid, pl->from_resource, pl->message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write SQLite messages that aren't already in the flatfile
|
||||||
|
for (GSList* l = sqlite_lines; l; l = l->next) {
|
||||||
|
ProfMessage* msg = l->data;
|
||||||
|
if (!msg || !msg->timestamp)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp);
|
||||||
|
const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown";
|
||||||
|
const char* body = msg->plain ? msg->plain : "";
|
||||||
|
const char* sid = msg->id ? msg->id : "";
|
||||||
|
|
||||||
|
char* key = _make_dedup_key(sid, ts, from_jid, body);
|
||||||
|
if (g_hash_table_contains(seen_keys, key)) {
|
||||||
|
g_free(key);
|
||||||
|
contact_skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
g_hash_table_add(seen_keys, key);
|
||||||
|
|
||||||
|
const char* type_str = ff_get_message_type_str(msg->type);
|
||||||
|
const char* enc_str = ff_get_message_enc_str(msg->enc);
|
||||||
|
const char* from_res = msg->from_jid ? msg->from_jid->resourcepart : NULL;
|
||||||
|
|
||||||
|
ff_write_line(fp, ts, type_str, enc_str,
|
||||||
|
sid, msg->stanzaid, NULL,
|
||||||
|
from_jid, from_res, body);
|
||||||
|
contact_exported++;
|
||||||
|
}
|
||||||
|
|
||||||
|
fflush(fp);
|
||||||
|
fclose(fp); // also releases flock
|
||||||
|
|
||||||
|
// Atomic rename
|
||||||
|
if (rename(tmp_path, log_path) != 0) {
|
||||||
|
log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno));
|
||||||
|
unlink(tmp_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
cons_show("Exported %s: %d new, %d skipped (already existed: %d)",
|
||||||
|
contact, contact_exported, contact_skipped, existing_count);
|
||||||
|
total_exported += contact_exported;
|
||||||
|
total_skipped += contact_skipped;
|
||||||
|
|
||||||
|
g_hash_table_destroy(seen_keys);
|
||||||
|
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||||
|
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||||
|
}
|
||||||
|
|
||||||
|
g_slist_free_full(contacts, g_free);
|
||||||
|
return total_exported;
|
||||||
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Import: flatfile -> SQLite (merge)
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
int
|
||||||
|
log_database_import_from_flatfile(const gchar* const contact_jid)
|
||||||
|
{
|
||||||
|
const Jid* myjid = connection_get_jid();
|
||||||
|
if (!myjid || !myjid->barejid) {
|
||||||
|
cons_show_error("Import failed: not connected.");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!g_flatfile_account_jid) {
|
||||||
|
g_free(g_flatfile_account_jid);
|
||||||
|
g_flatfile_account_jid = g_strdup(myjid->barejid);
|
||||||
|
}
|
||||||
|
|
||||||
|
db_backend_t* sqlite_be = db_backend_sqlite();
|
||||||
|
if (!sqlite_be) {
|
||||||
|
cons_show_error("Import failed: SQLite backend not available.");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine contacts
|
||||||
|
GSList* contacts = NULL;
|
||||||
|
if (contact_jid) {
|
||||||
|
contacts = g_slist_append(contacts, g_strdup(contact_jid));
|
||||||
|
} else {
|
||||||
|
contacts = _ff_list_contacts();
|
||||||
|
if (!contacts) {
|
||||||
|
cons_show("No flat-file history found to import.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int total_imported = 0;
|
||||||
|
int total_skipped = 0;
|
||||||
|
|
||||||
|
for (GSList* c = contacts; c; c = c->next) {
|
||||||
|
const char* contact = c->data;
|
||||||
|
|
||||||
|
// 1. Read all flatfile lines
|
||||||
|
GSList* ff_lines = _ff_read_all_lines(contact);
|
||||||
|
if (!ff_lines) {
|
||||||
|
log_debug("import: no flatfile messages for %s", contact);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Build dedup set from SQLite (existing stanza_ids + fallback keys)
|
||||||
|
GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
|
||||||
|
|
||||||
|
// Read all SQLite messages for this contact
|
||||||
|
gboolean has_more = TRUE;
|
||||||
|
gchar* start_time = NULL;
|
||||||
|
|
||||||
|
while (has_more) {
|
||||||
|
GSList* batch = NULL;
|
||||||
|
db_history_result_t res = sqlite_be->get_previous_chat(
|
||||||
|
contact, start_time, NULL, TRUE, FALSE, &batch);
|
||||||
|
|
||||||
|
if (res != DB_RESPONSE_SUCCESS || !batch) {
|
||||||
|
has_more = FALSE;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
int batch_size = g_slist_length(batch);
|
||||||
|
|
||||||
|
ProfMessage* last_msg = g_slist_last(batch)->data;
|
||||||
|
if (last_msg && last_msg->timestamp) {
|
||||||
|
g_free(start_time);
|
||||||
|
start_time = g_date_time_format_iso8601(last_msg->timestamp);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (GSList* l = batch; l; l = l->next) {
|
||||||
|
ProfMessage* msg = l->data;
|
||||||
|
if (!msg)
|
||||||
|
continue;
|
||||||
|
auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");
|
||||||
|
char* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain);
|
||||||
|
g_hash_table_add(seen_keys, key);
|
||||||
|
}
|
||||||
|
|
||||||
|
g_slist_free_full(batch, (GDestroyNotify)message_free);
|
||||||
|
|
||||||
|
if (batch_size < MESSAGES_TO_RETRIEVE) {
|
||||||
|
has_more = FALSE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g_free(start_time);
|
||||||
|
|
||||||
|
// 3. For each flatfile line not in seen_keys, insert via add_incoming/add_outgoing
|
||||||
|
int contact_imported = 0;
|
||||||
|
int contact_skipped = 0;
|
||||||
|
|
||||||
|
for (GSList* l = ff_lines; l; l = l->next) {
|
||||||
|
ff_parsed_line_t* pl = l->data;
|
||||||
|
if (!pl || !pl->timestamp)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
auto_gchar gchar* ts = g_date_time_format_iso8601(pl->timestamp);
|
||||||
|
char* key = _make_dedup_key(pl->stanza_id, ts, pl->from_jid, pl->message);
|
||||||
|
|
||||||
|
if (g_hash_table_contains(seen_keys, key)) {
|
||||||
|
g_free(key);
|
||||||
|
contact_skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
g_hash_table_add(seen_keys, key);
|
||||||
|
|
||||||
|
// Build a ProfMessage and insert via add_incoming
|
||||||
|
// (add_incoming uses _add_to_db which preserves the original timestamp)
|
||||||
|
gboolean is_outgoing = (g_strcmp0(pl->from_jid, myjid->barejid) == 0);
|
||||||
|
|
||||||
|
ProfMessage* msg = message_init();
|
||||||
|
msg->id = pl->stanza_id ? g_strdup(pl->stanza_id) : NULL;
|
||||||
|
msg->stanzaid = pl->archive_id ? g_strdup(pl->archive_id) : NULL;
|
||||||
|
msg->plain = g_strdup(pl->message ? pl->message : "");
|
||||||
|
msg->timestamp = g_date_time_ref(pl->timestamp);
|
||||||
|
msg->type = ff_get_message_type_type(pl->type);
|
||||||
|
msg->enc = ff_get_message_enc_type(pl->enc);
|
||||||
|
msg->replace_id = pl->replace_id ? g_strdup(pl->replace_id) : NULL;
|
||||||
|
msg->is_mam = TRUE; // Suppress dedup check in the backend
|
||||||
|
|
||||||
|
if (is_outgoing) {
|
||||||
|
msg->from_jid = jid_create(myjid->barejid);
|
||||||
|
msg->to_jid = jid_create(contact);
|
||||||
|
} else {
|
||||||
|
msg->from_jid = jid_create_from_bare_and_resource(pl->from_jid, pl->from_resource);
|
||||||
|
msg->to_jid = jid_create(myjid->barejid);
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlite_be->add_incoming(msg);
|
||||||
|
message_free(msg);
|
||||||
|
contact_imported++;
|
||||||
|
}
|
||||||
|
|
||||||
|
cons_show("Imported %s: %d new, %d skipped (already in SQLite)",
|
||||||
|
contact, contact_imported, contact_skipped);
|
||||||
|
total_imported += contact_imported;
|
||||||
|
total_skipped += contact_skipped;
|
||||||
|
|
||||||
|
g_hash_table_destroy(seen_keys);
|
||||||
|
g_slist_free_full(ff_lines, (GDestroyNotify)ff_parsed_line_free);
|
||||||
|
}
|
||||||
|
|
||||||
|
g_slist_free_full(contacts, g_free);
|
||||||
|
return total_imported;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* HAVE_SQLITE */
|
||||||
@@ -817,3 +817,37 @@ db_backend_sqlite(void)
|
|||||||
{
|
{
|
||||||
return &sqlite_backend;
|
return &sqlite_backend;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
GSList*
|
||||||
|
db_sqlite_list_contacts(void)
|
||||||
|
{
|
||||||
|
GSList* contacts = NULL;
|
||||||
|
if (!g_chatlog_database)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
const Jid* myjid = connection_get_jid();
|
||||||
|
if (!myjid || !myjid->barejid)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
const char* q = "SELECT DISTINCT `from_jid` FROM `ChatLogs` "
|
||||||
|
"WHERE `from_jid` != ? "
|
||||||
|
"UNION "
|
||||||
|
"SELECT DISTINCT `to_jid` FROM `ChatLogs` "
|
||||||
|
"WHERE `to_jid` != ?";
|
||||||
|
sqlite3_stmt* stmt = NULL;
|
||||||
|
if (!_db_prepare_ctx(q, &stmt, "db_sqlite_list_contacts"))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
sqlite3_bind_text(stmt, 1, myjid->barejid, -1, SQLITE_STATIC);
|
||||||
|
sqlite3_bind_text(stmt, 2, myjid->barejid, -1, SQLITE_STATIC);
|
||||||
|
|
||||||
|
while (sqlite3_step(stmt) == SQLITE_ROW) {
|
||||||
|
const char* jid = (const char*)sqlite3_column_text(stmt, 0);
|
||||||
|
if (jid && strlen(jid) > 0) {
|
||||||
|
contacts = g_slist_append(contacts, g_strdup(jid));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sqlite3_finalize(stmt);
|
||||||
|
|
||||||
|
return contacts;
|
||||||
|
}
|
||||||
@@ -73,6 +73,21 @@ db_backend_sqlite(void)
|
|||||||
{
|
{
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
GSList*
|
||||||
|
db_sqlite_list_contacts(void)
|
||||||
|
{
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
int
|
||||||
|
log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
int
|
||||||
|
log_database_import_from_flatfile(const gchar* const contact_jid)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
db_backend_t*
|
db_backend_t*
|
||||||
db_backend_flatfile(void)
|
db_backend_flatfile(void)
|
||||||
|
|||||||
Reference in New Issue
Block a user