mirror of
https://git.jabber.space/devs/cproof.git
synced 2026-07-18 21:36:21 +00:00
feat(draft): add /history export|import for SQLite<->flatfile migration
DRAFT — not yet tested end-to-end with a live XMPP session. New commands: /history export [<jid>] — copy messages from SQLite to flat-file /history import [<jid>] — copy messages from flat-file to SQLite Both merge with existing data; duplicates are skipped using stanza-id or a SHA-256 fallback key (timestamp + sender + body prefix). Implementation (src/database_export.c): - Export: paginate SQLite via get_previous_chat, read existing flatfile for dedup, write merged result via ff_write_line + atomic rename - Import: parse flatfile lines via ff_parse_line, build dedup set from SQLite, insert new messages via add_incoming (preserves original timestamps for both directions) - List all contacts: db_sqlite_list_contacts() queries UNION of DISTINCT from_jid/to_jid; flatfile enumerates flatlog directories Wiring: - database.h: declare export/import functions + db_sqlite_list_contacts - database_sqlite.c: add db_sqlite_list_contacts() - cmd_defs.c: add export/import to /history synopsis and args - cmd_funcs.c: add export/import handlers in cmd_history() - cmd_ac.c: add 'export' and 'import' to history_ac - Makefile.am: add database_export.c to core_sources - stub_database.c: add stubs for test linking - profanity.1: document export/import in man page All code guarded with #ifdef HAVE_SQLITE — builds cleanly without it.
This commit is contained in:
@@ -6,6 +6,7 @@ core_sources = \
|
||||
src/database_flatfile.c src/database_flatfile.h \
|
||||
src/database_flatfile_parser.c \
|
||||
src/database_flatfile_verify.c \
|
||||
src/database_export.c \
|
||||
src/log.h src/profanity.c src/common.h \
|
||||
src/profanity.h src/xmpp/chat_session.c \
|
||||
src/xmpp/chat_session.h src/xmpp/muc.c src/xmpp/muc.h src/xmpp/jid.h src/xmpp/jid.c \
|
||||
|
||||
@@ -226,6 +226,14 @@ Each line has the format:
|
||||
Use
|
||||
.B /history verify
|
||||
to check integrity of stored history (both SQLite and flat-file).
|
||||
.PP
|
||||
Use
|
||||
.B /history export [<jid>]
|
||||
to copy messages from SQLite to flat-file format (merge with existing data), or
|
||||
.B /history import [<jid>]
|
||||
to copy from flat-file to SQLite.
|
||||
Both operations skip duplicates.
|
||||
Omit the JID to process all contacts.
|
||||
.SH BUGS
|
||||
Bugs can either be reported by raising an issue at the Github issue tracker:
|
||||
.br
|
||||
|
||||
@@ -1145,6 +1145,8 @@ cmd_ac_init(void)
|
||||
autocomplete_add(history_ac, "on");
|
||||
autocomplete_add(history_ac, "off");
|
||||
autocomplete_add(history_ac, "verify");
|
||||
autocomplete_add(history_ac, "export");
|
||||
autocomplete_add(history_ac, "import");
|
||||
|
||||
autocomplete_add(logging_group_ac, "on");
|
||||
autocomplete_add(logging_group_ac, "off");
|
||||
|
||||
@@ -1882,14 +1882,20 @@ static const struct cmd_t command_defs[] = {
|
||||
CMD_TAG_CHAT)
|
||||
CMD_SYN(
|
||||
"/history on|off",
|
||||
"/history verify [<jid>]")
|
||||
"/history verify [<jid>]",
|
||||
"/history export [<jid>]",
|
||||
"/history import [<jid>]")
|
||||
CMD_DESC(
|
||||
"Switch chat history on or off, /logging chat will automatically be enabled when this setting is on. "
|
||||
"When history is enabled, previous messages are shown in chat windows. "
|
||||
"Use 'verify' to check integrity of stored message history.")
|
||||
"Use 'verify' to check integrity of stored message history. "
|
||||
"Use 'export' to copy messages from SQLite to flat-file format, or 'import' to copy from flat-file to SQLite. "
|
||||
"Both export and import merge with existing data (duplicates are skipped).")
|
||||
CMD_ARGS(
|
||||
{ "on|off", "Enable or disable showing chat history." },
|
||||
{ "verify [<jid>]", "Verify integrity of message history. Optionally specify a JID to check only one contact." })
|
||||
{ "verify [<jid>]", "Verify integrity of message history. Optionally specify a JID to check only one contact." },
|
||||
{ "export [<jid>]", "Export SQLite history to flat-file format. Optionally specify a JID to export only one contact." },
|
||||
{ "import [<jid>]", "Import flat-file history into SQLite. Optionally specify a JID to import only one contact." })
|
||||
},
|
||||
|
||||
{ CMD_PREAMBLE("/log",
|
||||
|
||||
@@ -6787,6 +6787,42 @@ cmd_history(ProfWin* window, const char* const command, gchar** args)
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
if (g_strcmp0(args[0], "export") == 0) {
|
||||
#ifdef HAVE_SQLITE
|
||||
const gchar* contact_jid = args[1]; // may be NULL (export all)
|
||||
if (contact_jid) {
|
||||
cons_show("Exporting SQLite history for %s to flat-file...", contact_jid);
|
||||
} else {
|
||||
cons_show("Exporting all SQLite history to flat-file...");
|
||||
}
|
||||
int n = log_database_export_to_flatfile(contact_jid);
|
||||
if (n >= 0) {
|
||||
cons_show("Export complete: %d message(s) exported.", n);
|
||||
}
|
||||
#else
|
||||
cons_show_error("Export requires SQLite support. Rebuild with --with-sqlite.");
|
||||
#endif
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
if (g_strcmp0(args[0], "import") == 0) {
|
||||
#ifdef HAVE_SQLITE
|
||||
const gchar* contact_jid = args[1]; // may be NULL (import all)
|
||||
if (contact_jid) {
|
||||
cons_show("Importing flat-file history for %s into SQLite...", contact_jid);
|
||||
} else {
|
||||
cons_show("Importing all flat-file history into SQLite...");
|
||||
}
|
||||
int n = log_database_import_from_flatfile(contact_jid);
|
||||
if (n >= 0) {
|
||||
cons_show("Import complete: %d message(s) imported.", n);
|
||||
}
|
||||
#else
|
||||
cons_show_error("Import requires SQLite support. Rebuild with --with-sqlite.");
|
||||
#endif
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
_cmd_set_boolean_preference(args[0], "Chat history", PREF_HISTORY);
|
||||
|
||||
// if set to on, set chlog (/logging chat on)
|
||||
|
||||
@@ -87,6 +87,7 @@ extern db_backend_t* active_db_backend;
|
||||
// Backend registry
|
||||
#ifdef HAVE_SQLITE
|
||||
db_backend_t* db_backend_sqlite(void);
|
||||
GSList* db_sqlite_list_contacts(void);
|
||||
#endif
|
||||
db_backend_t* db_backend_flatfile(void);
|
||||
|
||||
@@ -101,4 +102,10 @@ ProfMessage* log_database_get_limits_info(const gchar* const contact_barejid, gb
|
||||
void log_database_close(void);
|
||||
GSList* log_database_verify_integrity(const gchar* const contact_barejid);
|
||||
|
||||
// Cross-backend export/import (requires HAVE_SQLITE)
|
||||
#ifdef HAVE_SQLITE
|
||||
int log_database_export_to_flatfile(const gchar* const contact_jid);
|
||||
int log_database_import_from_flatfile(const gchar* const contact_jid);
|
||||
#endif
|
||||
|
||||
#endif // DATABASE_H
|
||||
|
||||
531
src/database_export.c
Normal file
531
src/database_export.c
Normal file
@@ -0,0 +1,531 @@
|
||||
/*
|
||||
* database_export.c
|
||||
* vim: expandtab:ts=4:sts=4:sw=4
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
*
|
||||
* This file is part of Profanity.
|
||||
*
|
||||
* Profanity is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Profanity is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* Cross-backend export/import for message history.
|
||||
* Reads from one backend, writes to the other, with merge + dedup.
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include <glib.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/file.h>
|
||||
|
||||
#ifdef HAVE_SQLITE
|
||||
#include <sqlite3.h>
|
||||
#endif
|
||||
|
||||
#include "log.h"
|
||||
#include "common.h"
|
||||
#include "config/files.h"
|
||||
#include "database.h"
|
||||
#include "database_flatfile.h"
|
||||
#include "config/preferences.h"
|
||||
#include "ui/ui.h"
|
||||
#include "xmpp/xmpp.h"
|
||||
#include "xmpp/message.h"
|
||||
|
||||
// =========================================================================
|
||||
// Everything below is only used when HAVE_SQLITE is defined
|
||||
// =========================================================================
|
||||
|
||||
#ifdef HAVE_SQLITE
|
||||
|
||||
// =========================================================================
|
||||
// Dedup key: stanza_id if present, else hash of timestamp+from+body
|
||||
// =========================================================================
|
||||
|
||||
static char*
|
||||
_make_dedup_key(const char* stanza_id, const char* timestamp, const char* from_jid, const char* body)
|
||||
{
|
||||
if (stanza_id && strlen(stanza_id) > 0)
|
||||
return g_strdup(stanza_id);
|
||||
|
||||
// Fallback: simple composite key
|
||||
auto_gchar gchar* raw = g_strdup_printf("%s|%s|%.64s",
|
||||
timestamp ? timestamp : "",
|
||||
from_jid ? from_jid : "",
|
||||
body ? body : "");
|
||||
return g_compute_checksum_for_string(G_CHECKSUM_SHA256, raw, -1);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Helper: reverse ff_jid_to_dir (best-effort)
|
||||
// '_at_' -> '@'
|
||||
// =========================================================================
|
||||
|
||||
static char*
|
||||
_dir_to_jid(const char* dirname)
|
||||
{
|
||||
if (!dirname)
|
||||
return NULL;
|
||||
return str_replace(dirname, "_at_", "@");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Helper: list all contact JIDs from flatfile directory
|
||||
// Returns a GSList of g_strdup'd JID strings. Caller frees with g_slist_free_full(list, g_free).
|
||||
// =========================================================================
|
||||
|
||||
static GSList*
|
||||
_ff_list_contacts(void)
|
||||
{
|
||||
GSList* contacts = NULL;
|
||||
|
||||
if (!g_flatfile_account_jid)
|
||||
return NULL;
|
||||
|
||||
auto_gchar gchar* data_path = files_get_data_path(DIR_FLATLOG);
|
||||
auto_gchar gchar* my_dir = ff_jid_to_dir(g_flatfile_account_jid);
|
||||
auto_gchar gchar* base_dir = g_strdup_printf("%s/%s", data_path, my_dir);
|
||||
|
||||
GError* err = NULL;
|
||||
GDir* dir = g_dir_open(base_dir, 0, &err);
|
||||
if (!dir) {
|
||||
if (err) {
|
||||
log_debug("export: cannot open flatlog dir %s: %s", base_dir, err->message);
|
||||
g_error_free(err);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const gchar* entry;
|
||||
while ((entry = g_dir_read_name(dir)) != NULL) {
|
||||
auto_gchar gchar* full_path = g_strdup_printf("%s/%s/history.log", base_dir, entry);
|
||||
if (g_file_test(full_path, G_FILE_TEST_IS_REGULAR)) {
|
||||
char* jid = _dir_to_jid(entry);
|
||||
if (jid) {
|
||||
contacts = g_slist_append(contacts, jid);
|
||||
}
|
||||
}
|
||||
}
|
||||
g_dir_close(dir);
|
||||
|
||||
return contacts;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Helper: read ALL parsed lines from a flatfile for a contact
|
||||
// Returns GSList<ff_parsed_line_t*>. Caller frees with g_slist_free_full(list, (GDestroyNotify)ff_parsed_line_free).
|
||||
// =========================================================================
|
||||
|
||||
static GSList*
|
||||
_ff_read_all_lines(const char* contact_barejid)
|
||||
{
|
||||
GSList* lines = NULL;
|
||||
auto_gchar gchar* log_path = ff_get_log_path(contact_barejid);
|
||||
if (!log_path)
|
||||
return NULL;
|
||||
|
||||
FILE* fp = fopen(log_path, "r");
|
||||
if (!fp)
|
||||
return NULL;
|
||||
|
||||
while (1) {
|
||||
gboolean truncated = FALSE;
|
||||
char* buf = ff_readline(fp, &truncated);
|
||||
if (!buf)
|
||||
break;
|
||||
if (truncated) {
|
||||
free(buf);
|
||||
break;
|
||||
}
|
||||
if (buf[0] == '\0' || buf[0] == '#') {
|
||||
free(buf);
|
||||
continue;
|
||||
}
|
||||
|
||||
ff_parsed_line_t* pl = ff_parse_line(buf);
|
||||
free(buf);
|
||||
if (pl) {
|
||||
lines = g_slist_append(lines, pl);
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Export: SQLite -> flatfile (merge)
|
||||
// =========================================================================
|
||||
|
||||
// Callback data for the SQLite query
|
||||
typedef struct
|
||||
{
|
||||
GHashTable* seen_keys; // dedup set
|
||||
FILE* fp; // output file
|
||||
int exported; // counter
|
||||
int skipped; // counter
|
||||
} export_ctx_t;
|
||||
|
||||
int
|
||||
log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||
{
|
||||
const Jid* myjid = connection_get_jid();
|
||||
if (!myjid || !myjid->barejid) {
|
||||
cons_show_error("Export failed: not connected.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!g_flatfile_account_jid) {
|
||||
// Flatfile backend not initialized — init it temporarily
|
||||
// We need g_flatfile_account_jid for path construction
|
||||
g_free(g_flatfile_account_jid);
|
||||
g_flatfile_account_jid = g_strdup(myjid->barejid);
|
||||
}
|
||||
|
||||
// Get the SQLite backend
|
||||
db_backend_t* sqlite_be = db_backend_sqlite();
|
||||
if (!sqlite_be) {
|
||||
cons_show_error("Export failed: SQLite backend not available.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Determine contact list
|
||||
GSList* contacts = NULL;
|
||||
if (contact_jid) {
|
||||
contacts = g_slist_append(contacts, g_strdup(contact_jid));
|
||||
} else {
|
||||
contacts = db_sqlite_list_contacts();
|
||||
if (!contacts) {
|
||||
cons_show("No contacts found in SQLite database.");
|
||||
return 0;
|
||||
}
|
||||
cons_show("Exporting %d contact(s) to flat-file format...", g_slist_length(contacts));
|
||||
}
|
||||
|
||||
int total_exported = 0;
|
||||
int total_skipped = 0;
|
||||
|
||||
for (GSList* c = contacts; c; c = c->next) {
|
||||
const char* contact = c->data;
|
||||
|
||||
// 1. Read existing flatfile lines for dedup
|
||||
GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
|
||||
|
||||
GSList* existing = _ff_read_all_lines(contact);
|
||||
for (GSList* l = existing; l; l = l->next) {
|
||||
ff_parsed_line_t* pl = l->data;
|
||||
char* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message);
|
||||
g_hash_table_add(seen_keys, key);
|
||||
}
|
||||
int existing_count = g_slist_length(existing);
|
||||
|
||||
// 2. Query SQLite for this contact — get ALL messages ordered by timestamp
|
||||
// We read all messages by paginating through the existing get_previous_chat
|
||||
// But it limits to MESSAGES_TO_RETRIEVE... We need a direct SQL query.
|
||||
// Use the same approach as verify — iterate with raw SQL.
|
||||
|
||||
// For now, collect all messages into a list via repeated calls
|
||||
// Actually, let's use a simpler approach: read all from flatfile existing data,
|
||||
// then query SQLite for everything, merge, write back.
|
||||
|
||||
// Read all SQLite messages for this contact
|
||||
GSList* sqlite_lines = NULL;
|
||||
// Use the backend's get_previous_chat with a wide time range
|
||||
// This is limited to MESSAGES_TO_RETRIEVE, so we need to paginate
|
||||
gboolean has_more = TRUE;
|
||||
gchar* start_time = NULL;
|
||||
|
||||
while (has_more) {
|
||||
GSList* batch = NULL;
|
||||
db_history_result_t res = sqlite_be->get_previous_chat(
|
||||
contact, start_time, NULL, TRUE, FALSE, &batch);
|
||||
|
||||
if (res != DB_RESPONSE_SUCCESS || !batch) {
|
||||
has_more = FALSE;
|
||||
break;
|
||||
}
|
||||
|
||||
int batch_size = g_slist_length(batch);
|
||||
|
||||
// Get the last timestamp for the next query
|
||||
ProfMessage* last_msg = g_slist_last(batch)->data;
|
||||
if (last_msg && last_msg->timestamp) {
|
||||
g_free(start_time);
|
||||
start_time = g_date_time_format_iso8601(last_msg->timestamp);
|
||||
}
|
||||
|
||||
sqlite_lines = g_slist_concat(sqlite_lines, batch);
|
||||
|
||||
if (batch_size < MESSAGES_TO_RETRIEVE) {
|
||||
has_more = FALSE;
|
||||
}
|
||||
}
|
||||
g_free(start_time);
|
||||
|
||||
if (!sqlite_lines) {
|
||||
log_debug("export: no SQLite messages for %s", contact);
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3. Merge: write existing flatfile lines + new SQLite lines to temp file
|
||||
auto_gchar gchar* log_path = ff_get_log_path(contact);
|
||||
if (!log_path) {
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ensure directory
|
||||
auto_gchar gchar* dir = g_path_get_dirname(log_path);
|
||||
ff_ensure_dir(dir);
|
||||
|
||||
auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path);
|
||||
FILE* fp = fopen(tmp_path, "w");
|
||||
if (!fp) {
|
||||
log_error("export: cannot create %s: %s", tmp_path, strerror(errno));
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Lock the temp file
|
||||
int fd = fileno(fp);
|
||||
flock(fd, LOCK_EX);
|
||||
|
||||
fprintf(fp, "%s", FLATFILE_HEADER);
|
||||
|
||||
int contact_exported = 0;
|
||||
int contact_skipped = 0;
|
||||
|
||||
// Write existing flatfile lines first (preserve originals)
|
||||
for (GSList* l = existing; l; l = l->next) {
|
||||
ff_parsed_line_t* pl = l->data;
|
||||
ff_write_line(fp, pl->timestamp_str,
|
||||
pl->type, pl->enc,
|
||||
pl->stanza_id, pl->archive_id, pl->replace_id,
|
||||
pl->from_jid, pl->from_resource, pl->message);
|
||||
}
|
||||
|
||||
// Write SQLite messages that aren't already in the flatfile
|
||||
for (GSList* l = sqlite_lines; l; l = l->next) {
|
||||
ProfMessage* msg = l->data;
|
||||
if (!msg || !msg->timestamp)
|
||||
continue;
|
||||
|
||||
auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp);
|
||||
const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown";
|
||||
const char* body = msg->plain ? msg->plain : "";
|
||||
const char* sid = msg->id ? msg->id : "";
|
||||
|
||||
char* key = _make_dedup_key(sid, ts, from_jid, body);
|
||||
if (g_hash_table_contains(seen_keys, key)) {
|
||||
g_free(key);
|
||||
contact_skipped++;
|
||||
continue;
|
||||
}
|
||||
g_hash_table_add(seen_keys, key);
|
||||
|
||||
const char* type_str = ff_get_message_type_str(msg->type);
|
||||
const char* enc_str = ff_get_message_enc_str(msg->enc);
|
||||
const char* from_res = msg->from_jid ? msg->from_jid->resourcepart : NULL;
|
||||
|
||||
ff_write_line(fp, ts, type_str, enc_str,
|
||||
sid, msg->stanzaid, NULL,
|
||||
from_jid, from_res, body);
|
||||
contact_exported++;
|
||||
}
|
||||
|
||||
fflush(fp);
|
||||
fclose(fp); // also releases flock
|
||||
|
||||
// Atomic rename
|
||||
if (rename(tmp_path, log_path) != 0) {
|
||||
log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno));
|
||||
unlink(tmp_path);
|
||||
}
|
||||
|
||||
cons_show("Exported %s: %d new, %d skipped (already existed: %d)",
|
||||
contact, contact_exported, contact_skipped, existing_count);
|
||||
total_exported += contact_exported;
|
||||
total_skipped += contact_skipped;
|
||||
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||
}
|
||||
|
||||
g_slist_free_full(contacts, g_free);
|
||||
return total_exported;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Import: flatfile -> SQLite (merge)
|
||||
// =========================================================================
|
||||
|
||||
int
|
||||
log_database_import_from_flatfile(const gchar* const contact_jid)
|
||||
{
|
||||
const Jid* myjid = connection_get_jid();
|
||||
if (!myjid || !myjid->barejid) {
|
||||
cons_show_error("Import failed: not connected.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!g_flatfile_account_jid) {
|
||||
g_free(g_flatfile_account_jid);
|
||||
g_flatfile_account_jid = g_strdup(myjid->barejid);
|
||||
}
|
||||
|
||||
db_backend_t* sqlite_be = db_backend_sqlite();
|
||||
if (!sqlite_be) {
|
||||
cons_show_error("Import failed: SQLite backend not available.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Determine contacts
|
||||
GSList* contacts = NULL;
|
||||
if (contact_jid) {
|
||||
contacts = g_slist_append(contacts, g_strdup(contact_jid));
|
||||
} else {
|
||||
contacts = _ff_list_contacts();
|
||||
if (!contacts) {
|
||||
cons_show("No flat-file history found to import.");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int total_imported = 0;
|
||||
int total_skipped = 0;
|
||||
|
||||
for (GSList* c = contacts; c; c = c->next) {
|
||||
const char* contact = c->data;
|
||||
|
||||
// 1. Read all flatfile lines
|
||||
GSList* ff_lines = _ff_read_all_lines(contact);
|
||||
if (!ff_lines) {
|
||||
log_debug("import: no flatfile messages for %s", contact);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Build dedup set from SQLite (existing stanza_ids + fallback keys)
|
||||
GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
|
||||
|
||||
// Read all SQLite messages for this contact
|
||||
gboolean has_more = TRUE;
|
||||
gchar* start_time = NULL;
|
||||
|
||||
while (has_more) {
|
||||
GSList* batch = NULL;
|
||||
db_history_result_t res = sqlite_be->get_previous_chat(
|
||||
contact, start_time, NULL, TRUE, FALSE, &batch);
|
||||
|
||||
if (res != DB_RESPONSE_SUCCESS || !batch) {
|
||||
has_more = FALSE;
|
||||
break;
|
||||
}
|
||||
|
||||
int batch_size = g_slist_length(batch);
|
||||
|
||||
ProfMessage* last_msg = g_slist_last(batch)->data;
|
||||
if (last_msg && last_msg->timestamp) {
|
||||
g_free(start_time);
|
||||
start_time = g_date_time_format_iso8601(last_msg->timestamp);
|
||||
}
|
||||
|
||||
for (GSList* l = batch; l; l = l->next) {
|
||||
ProfMessage* msg = l->data;
|
||||
if (!msg)
|
||||
continue;
|
||||
auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");
|
||||
char* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain);
|
||||
g_hash_table_add(seen_keys, key);
|
||||
}
|
||||
|
||||
g_slist_free_full(batch, (GDestroyNotify)message_free);
|
||||
|
||||
if (batch_size < MESSAGES_TO_RETRIEVE) {
|
||||
has_more = FALSE;
|
||||
}
|
||||
}
|
||||
g_free(start_time);
|
||||
|
||||
// 3. For each flatfile line not in seen_keys, insert via add_incoming/add_outgoing
|
||||
int contact_imported = 0;
|
||||
int contact_skipped = 0;
|
||||
|
||||
for (GSList* l = ff_lines; l; l = l->next) {
|
||||
ff_parsed_line_t* pl = l->data;
|
||||
if (!pl || !pl->timestamp)
|
||||
continue;
|
||||
|
||||
auto_gchar gchar* ts = g_date_time_format_iso8601(pl->timestamp);
|
||||
char* key = _make_dedup_key(pl->stanza_id, ts, pl->from_jid, pl->message);
|
||||
|
||||
if (g_hash_table_contains(seen_keys, key)) {
|
||||
g_free(key);
|
||||
contact_skipped++;
|
||||
continue;
|
||||
}
|
||||
g_hash_table_add(seen_keys, key);
|
||||
|
||||
// Build a ProfMessage and insert via add_incoming
|
||||
// (add_incoming uses _add_to_db which preserves the original timestamp)
|
||||
gboolean is_outgoing = (g_strcmp0(pl->from_jid, myjid->barejid) == 0);
|
||||
|
||||
ProfMessage* msg = message_init();
|
||||
msg->id = pl->stanza_id ? g_strdup(pl->stanza_id) : NULL;
|
||||
msg->stanzaid = pl->archive_id ? g_strdup(pl->archive_id) : NULL;
|
||||
msg->plain = g_strdup(pl->message ? pl->message : "");
|
||||
msg->timestamp = g_date_time_ref(pl->timestamp);
|
||||
msg->type = ff_get_message_type_type(pl->type);
|
||||
msg->enc = ff_get_message_enc_type(pl->enc);
|
||||
msg->replace_id = pl->replace_id ? g_strdup(pl->replace_id) : NULL;
|
||||
msg->is_mam = TRUE; // Suppress dedup check in the backend
|
||||
|
||||
if (is_outgoing) {
|
||||
msg->from_jid = jid_create(myjid->barejid);
|
||||
msg->to_jid = jid_create(contact);
|
||||
} else {
|
||||
msg->from_jid = jid_create_from_bare_and_resource(pl->from_jid, pl->from_resource);
|
||||
msg->to_jid = jid_create(myjid->barejid);
|
||||
}
|
||||
|
||||
sqlite_be->add_incoming(msg);
|
||||
message_free(msg);
|
||||
contact_imported++;
|
||||
}
|
||||
|
||||
cons_show("Imported %s: %d new, %d skipped (already in SQLite)",
|
||||
contact, contact_imported, contact_skipped);
|
||||
total_imported += contact_imported;
|
||||
total_skipped += contact_skipped;
|
||||
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(ff_lines, (GDestroyNotify)ff_parsed_line_free);
|
||||
}
|
||||
|
||||
g_slist_free_full(contacts, g_free);
|
||||
return total_imported;
|
||||
}
|
||||
|
||||
#endif /* HAVE_SQLITE */
|
||||
@@ -817,3 +817,37 @@ db_backend_sqlite(void)
|
||||
{
|
||||
return &sqlite_backend;
|
||||
}
|
||||
|
||||
GSList*
|
||||
db_sqlite_list_contacts(void)
|
||||
{
|
||||
GSList* contacts = NULL;
|
||||
if (!g_chatlog_database)
|
||||
return NULL;
|
||||
|
||||
const Jid* myjid = connection_get_jid();
|
||||
if (!myjid || !myjid->barejid)
|
||||
return NULL;
|
||||
|
||||
const char* q = "SELECT DISTINCT `from_jid` FROM `ChatLogs` "
|
||||
"WHERE `from_jid` != ? "
|
||||
"UNION "
|
||||
"SELECT DISTINCT `to_jid` FROM `ChatLogs` "
|
||||
"WHERE `to_jid` != ?";
|
||||
sqlite3_stmt* stmt = NULL;
|
||||
if (!_db_prepare_ctx(q, &stmt, "db_sqlite_list_contacts"))
|
||||
return NULL;
|
||||
|
||||
sqlite3_bind_text(stmt, 1, myjid->barejid, -1, SQLITE_STATIC);
|
||||
sqlite3_bind_text(stmt, 2, myjid->barejid, -1, SQLITE_STATIC);
|
||||
|
||||
while (sqlite3_step(stmt) == SQLITE_ROW) {
|
||||
const char* jid = (const char*)sqlite3_column_text(stmt, 0);
|
||||
if (jid && strlen(jid) > 0) {
|
||||
contacts = g_slist_append(contacts, g_strdup(jid));
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(stmt);
|
||||
|
||||
return contacts;
|
||||
}
|
||||
@@ -73,6 +73,21 @@ db_backend_sqlite(void)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
GSList*
|
||||
db_sqlite_list_contacts(void)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
int
|
||||
log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
int
|
||||
log_database_import_from_flatfile(const gchar* const contact_jid)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
db_backend_t*
|
||||
db_backend_flatfile(void)
|
||||
|
||||
Reference in New Issue
Block a user