no-DB mode implementation #94

Manually merged
jabber.developer merged 34 commits from feat/no-db-mode into master 2026-05-05 19:32:42 +00:00
20 changed files with 3998 additions and 647 deletions
Showing only changes of commit 1f3d3117bf - Show all commits

View File

@@ -6,6 +6,7 @@ core_sources = \
src/database_flatfile.c src/database_flatfile.h \ src/database_flatfile.c src/database_flatfile.h \
src/database_flatfile_parser.c \ src/database_flatfile_parser.c \
src/database_flatfile_verify.c \ src/database_flatfile_verify.c \
src/database_export.c \
src/log.h src/profanity.c src/common.h \ src/log.h src/profanity.c src/common.h \
src/profanity.h src/xmpp/chat_session.c \ src/profanity.h src/xmpp/chat_session.c \
src/xmpp/chat_session.h src/xmpp/muc.c src/xmpp/muc.h src/xmpp/jid.h src/xmpp/jid.c \ src/xmpp/chat_session.h src/xmpp/muc.c src/xmpp/muc.h src/xmpp/jid.h src/xmpp/jid.c \

View File

@@ -226,6 +226,14 @@ Each line has the format:
Use Use
.B /history verify .B /history verify
to check integrity of stored history (both SQLite and flat-file). to check integrity of stored history (both SQLite and flat-file).
.PP
Use
.B /history export [<jid>]
to copy messages from SQLite to flat-file format (merge with existing data), or
.B /history import [<jid>]
to copy from flat-file to SQLite.
Both operations skip duplicates.
Omit the JID to process all contacts.
.SH BUGS .SH BUGS
Bugs can either be reported by raising an issue at the Github issue tracker: Bugs can either be reported by raising an issue at the Github issue tracker:
.br .br

View File

@@ -1145,6 +1145,8 @@ cmd_ac_init(void)
autocomplete_add(history_ac, "on"); autocomplete_add(history_ac, "on");
autocomplete_add(history_ac, "off"); autocomplete_add(history_ac, "off");
autocomplete_add(history_ac, "verify"); autocomplete_add(history_ac, "verify");
autocomplete_add(history_ac, "export");
autocomplete_add(history_ac, "import");
autocomplete_add(logging_group_ac, "on"); autocomplete_add(logging_group_ac, "on");
autocomplete_add(logging_group_ac, "off"); autocomplete_add(logging_group_ac, "off");

View File

@@ -1882,14 +1882,20 @@ static const struct cmd_t command_defs[] = {
CMD_TAG_CHAT) CMD_TAG_CHAT)
CMD_SYN( CMD_SYN(
"/history on|off", "/history on|off",
"/history verify [<jid>]") "/history verify [<jid>]",
"/history export [<jid>]",
"/history import [<jid>]")
CMD_DESC( CMD_DESC(
"Switch chat history on or off, /logging chat will automatically be enabled when this setting is on. " "Switch chat history on or off, /logging chat will automatically be enabled when this setting is on. "
"When history is enabled, previous messages are shown in chat windows. " "When history is enabled, previous messages are shown in chat windows. "
"Use 'verify' to check integrity of stored message history.") "Use 'verify' to check integrity of stored message history. "
"Use 'export' to copy messages from SQLite to flat-file format, or 'import' to copy from flat-file to SQLite. "
"Both export and import merge with existing data (duplicates are skipped).")
CMD_ARGS( CMD_ARGS(
{ "on|off", "Enable or disable showing chat history." }, { "on|off", "Enable or disable showing chat history." },
{ "verify [<jid>]", "Verify integrity of message history. Optionally specify a JID to check only one contact." }) { "verify [<jid>]", "Verify integrity of message history. Optionally specify a JID to check only one contact." },
{ "export [<jid>]", "Export SQLite history to flat-file format. Optionally specify a JID to export only one contact." },
{ "import [<jid>]", "Import flat-file history into SQLite. Optionally specify a JID to import only one contact." })
jabber.developer marked this conversation as resolved Outdated

Does the switch automatically export/import? It is unclear from documentation.

Does the switch automatically export/import? It is unclear from documentation.

documentation-only clarification, queued. /history switch only swaps the active backend; data is not migrated unless the user runs /history export and /history import explicitly.

documentation-only clarification, queued. /history switch only swaps the active backend; data is not migrated unless the user runs /history export and /history import explicitly.
}, },
jabber.developer marked this conversation as resolved Outdated

How does it align with flatfile manual editability?

How does it align with flatfile manual editability?

needs a doc paragraph explaining that ff_verify_integrity flags timestamp drift, dup IDs, broken LMC refs etc. so manual edits remain auditable. Doc-only follow-up

needs a doc paragraph explaining that ff_verify_integrity flags timestamp drift, dup IDs, broken LMC refs etc. so manual edits remain auditable. Doc-only follow-up
{ CMD_PREAMBLE("/log", { CMD_PREAMBLE("/log",

View File

@@ -6787,6 +6787,42 @@ cmd_history(ProfWin* window, const char* const command, gchar** args)
return TRUE; return TRUE;
} }
if (g_strcmp0(args[0], "export") == 0) {
#ifdef HAVE_SQLITE
const gchar* contact_jid = args[1]; // may be NULL (export all)
if (contact_jid) {
cons_show("Exporting SQLite history for %s to flat-file...", contact_jid);
} else {
cons_show("Exporting all SQLite history to flat-file...");
}
int n = log_database_export_to_flatfile(contact_jid);
jabber.developer marked this conversation as resolved Outdated

nit: It might be more readable if it's in the separate method. I suggest extraction of the error printing in a separate method.

nit: It might be more readable if it's in the separate method. I suggest extraction of the error printing in a separate method.

Corrected

Corrected
if (n >= 0) {
cons_show("Export complete: %d message(s) exported.", n);
}
#else
cons_show_error("Export requires SQLite support. Rebuild with --with-sqlite.");
#endif
return TRUE;
}
if (g_strcmp0(args[0], "import") == 0) {
#ifdef HAVE_SQLITE
const gchar* contact_jid = args[1]; // may be NULL (import all)
if (contact_jid) {
cons_show("Importing flat-file history for %s into SQLite...", contact_jid);
} else {
cons_show("Importing all flat-file history into SQLite...");
}
int n = log_database_import_from_flatfile(contact_jid);
if (n >= 0) {
cons_show("Import complete: %d message(s) imported.", n);
}
#else
cons_show_error("Import requires SQLite support. Rebuild with --with-sqlite.");
#endif
return TRUE;
}
_cmd_set_boolean_preference(args[0], "Chat history", PREF_HISTORY); _cmd_set_boolean_preference(args[0], "Chat history", PREF_HISTORY);
// if set to on, set chlog (/logging chat on) // if set to on, set chlog (/logging chat on)

View File

@@ -87,6 +87,7 @@ extern db_backend_t* active_db_backend;
// Backend registry // Backend registry
#ifdef HAVE_SQLITE #ifdef HAVE_SQLITE
db_backend_t* db_backend_sqlite(void); db_backend_t* db_backend_sqlite(void);
GSList* db_sqlite_list_contacts(void);
#endif #endif
db_backend_t* db_backend_flatfile(void); db_backend_t* db_backend_flatfile(void);
@@ -101,4 +102,10 @@ ProfMessage* log_database_get_limits_info(const gchar* const contact_barejid, gb
void log_database_close(void); void log_database_close(void);
GSList* log_database_verify_integrity(const gchar* const contact_barejid); GSList* log_database_verify_integrity(const gchar* const contact_barejid);
// Cross-backend export/import (requires HAVE_SQLITE)
#ifdef HAVE_SQLITE
int log_database_export_to_flatfile(const gchar* const contact_jid);
int log_database_import_from_flatfile(const gchar* const contact_jid);
#endif
#endif // DATABASE_H #endif // DATABASE_H

531
src/database_export.c Normal file
View File

@@ -0,0 +1,531 @@
/*
* database_export.c
* vim: expandtab:ts=4:sts=4:sw=4
*
* Copyright (C) 2026 Profanity Contributors
jabber.developer marked this conversation as resolved Outdated

We are not affiliated with Profanity.

Use the following lines for copyright notice:

// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of CProof.
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
We are not affiliated with Profanity. Use the following lines for copyright notice: ```c // SPDX-License-Identifier: GPL-3.0-or-later // // This file is part of CProof. // See LICENSE for the full GPLv3 text and the special OpenSSL linking exception. ```

Corrected

Corrected
*
* This file is part of Profanity.
*
* Profanity is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Profanity is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
*
* Cross-backend export/import for message history.
* Reads from one backend, writes to the other, with merge + dedup.
*/
#include "config.h"
#include <glib.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/file.h>
#ifdef HAVE_SQLITE
#include <sqlite3.h>
#endif
#include "log.h"
#include "common.h"
#include "config/files.h"
#include "database.h"
#include "database_flatfile.h"
#include "config/preferences.h"
#include "ui/ui.h"
#include "xmpp/xmpp.h"
#include "xmpp/message.h"
// =========================================================================
// Everything below is only used when HAVE_SQLITE is defined
// =========================================================================
#ifdef HAVE_SQLITE
// =========================================================================
// Dedup key: stanza_id if present, else hash of timestamp+from+body
// =========================================================================
static char*
_make_dedup_key(const char* stanza_id, const char* timestamp, const char* from_jid, const char* body)
{
if (stanza_id && strlen(stanza_id) > 0)
return g_strdup(stanza_id);

is it efficient? Might g_strdup_printf first be more efficient than updating checksum each time?

is it efficient? Might g_strdup_printf first be more efficient than updating checksum each time?
// Fallback: simple composite key
auto_gchar gchar* raw = g_strdup_printf("%s|%s|%.64s",
timestamp ? timestamp : "",
from_jid ? from_jid : "",
body ? body : "");
return g_compute_checksum_for_string(G_CHECKSUM_SHA256, raw, -1);
}
// =========================================================================
// Helper: reverse ff_jid_to_dir (best-effort)
// '_at_' -> '@'
// =========================================================================
static char*
_dir_to_jid(const char* dirname)
{
if (!dirname)
return NULL;
return str_replace(dirname, "_at_", "@");
}
// =========================================================================
// Helper: list all contact JIDs from flatfile directory
// Returns a GSList of g_strdup'd JID strings. Caller frees with g_slist_free_full(list, g_free).
// =========================================================================
static GSList*
_ff_list_contacts(void)
{
GSList* contacts = NULL;
if (!g_flatfile_account_jid)
return NULL;
auto_gchar gchar* data_path = files_get_data_path(DIR_FLATLOG);
auto_gchar gchar* my_dir = ff_jid_to_dir(g_flatfile_account_jid);
auto_gchar gchar* base_dir = g_strdup_printf("%s/%s", data_path, my_dir);
GError* err = NULL;
GDir* dir = g_dir_open(base_dir, 0, &err);
if (!dir) {
if (err) {
log_debug("export: cannot open flatlog dir %s: %s", base_dir, err->message);
jabber.developer marked this conversation as resolved Outdated

the issue doesn't appear to be pure debug level, but rather warn at the very least

the issue doesn't appear to be pure `debug` level, but rather `warn` at the very least

Corrected

Corrected
g_error_free(err);
}
return NULL;
jabber.developer marked this conversation as resolved Outdated

will fail silently if no err is present.

will fail silently if no `err` is present.

Corrected

Corrected
}
const gchar* entry;
while ((entry = g_dir_read_name(dir)) != NULL) {
auto_gchar gchar* full_path = g_strdup_printf("%s/%s/history.log", base_dir, entry);
if (g_file_test(full_path, G_FILE_TEST_IS_REGULAR)) {
char* jid = _dir_to_jid(entry);
if (jid) {
contacts = g_slist_append(contacts, jid);
}
}
}
g_dir_close(dir);
return contacts;
}
// =========================================================================
// Helper: read ALL parsed lines from a flatfile for a contact
// Returns GSList<ff_parsed_line_t*>. Caller frees with g_slist_free_full(list, (GDestroyNotify)ff_parsed_line_free).
// =========================================================================
static GSList*
_ff_read_all_lines(const char* contact_barejid)
{
GSList* lines = NULL;
auto_gchar gchar* log_path = ff_get_log_path(contact_barejid);
if (!log_path)
return NULL;
FILE* fp = fopen(log_path, "r");
if (!fp)
return NULL;
while (1) {
gboolean truncated = FALSE;
char* buf = ff_readline(fp, &truncated);
jabber.developer marked this conversation as resolved Outdated

why not auto_char instead of free'ing?

why not `auto_char` instead of `free`'ing?

Corrected

Corrected
if (!buf)
break;
if (truncated) {
free(buf);
break;
}
if (buf[0] == '\0' || buf[0] == '#') {
free(buf);
continue;
}
ff_parsed_line_t* pl = ff_parse_line(buf);
free(buf);
if (pl) {
lines = g_slist_append(lines, pl);
}
}
fclose(fp);
return lines;
}
// =========================================================================
// Export: SQLite -> flatfile (merge)
// =========================================================================
// Callback data for the SQLite query
typedef struct
{
GHashTable* seen_keys; // dedup set
FILE* fp; // output file
int exported; // counter
int skipped; // counter
} export_ctx_t;
int
log_database_export_to_flatfile(const gchar* const contact_jid)
{
const Jid* myjid = connection_get_jid();
if (!myjid || !myjid->barejid) {
cons_show_error("Export failed: not connected.");
return -1;
}
if (!g_flatfile_account_jid) {
// Flatfile backend not initialized — init it temporarily
// We need g_flatfile_account_jid for path construction
g_free(g_flatfile_account_jid);
g_flatfile_account_jid = g_strdup(myjid->barejid);
}
// Get the SQLite backend
db_backend_t* sqlite_be = db_backend_sqlite();
if (!sqlite_be) {
cons_show_error("Export failed: SQLite backend not available.");
return -1;
}
// Determine contact list
GSList* contacts = NULL;
if (contact_jid) {
contacts = g_slist_append(contacts, g_strdup(contact_jid));
} else {
contacts = db_sqlite_list_contacts();
if (!contacts) {
cons_show("No contacts found in SQLite database.");
return 0;
}
cons_show("Exporting %d contact(s) to flat-file format...", g_slist_length(contacts));
}
int total_exported = 0;
int total_skipped = 0;
for (GSList* c = contacts; c; c = c->next) {
const char* contact = c->data;
// 1. Read existing flatfile lines for dedup
GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
GSList* existing = _ff_read_all_lines(contact);
for (GSList* l = existing; l; l = l->next) {
ff_parsed_line_t* pl = l->data;
char* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message);
g_hash_table_add(seen_keys, key);
}
int existing_count = g_slist_length(existing);
// 2. Query SQLite for this contact — get ALL messages ordered by timestamp
// We read all messages by paginating through the existing get_previous_chat
// But it limits to MESSAGES_TO_RETRIEVE... We need a direct SQL query.
// Use the same approach as verify — iterate with raw SQL.
// For now, collect all messages into a list via repeated calls
// Actually, let's use a simpler approach: read all from flatfile existing data,
// then query SQLite for everything, merge, write back.
// Read all SQLite messages for this contact
GSList* sqlite_lines = NULL;
// Use the backend's get_previous_chat with a wide time range
// This is limited to MESSAGES_TO_RETRIEVE, so we need to paginate
gboolean has_more = TRUE;
gchar* start_time = NULL;
while (has_more) {
GSList* batch = NULL;
db_history_result_t res = sqlite_be->get_previous_chat(
contact, start_time, NULL, TRUE, FALSE, &batch);
if (res != DB_RESPONSE_SUCCESS || !batch) {
has_more = FALSE;
break;
}
int batch_size = g_slist_length(batch);
// Get the last timestamp for the next query
ProfMessage* last_msg = g_slist_last(batch)->data;
if (last_msg && last_msg->timestamp) {
g_free(start_time);
start_time = g_date_time_format_iso8601(last_msg->timestamp);
}
sqlite_lines = g_slist_concat(sqlite_lines, batch);
if (batch_size < MESSAGES_TO_RETRIEVE) {
has_more = FALSE;
}
}
g_free(start_time);
if (!sqlite_lines) {
log_debug("export: no SQLite messages for %s", contact);
g_hash_table_destroy(seen_keys);
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
continue;
}
// 3. Merge: write existing flatfile lines + new SQLite lines to temp file
auto_gchar gchar* log_path = ff_get_log_path(contact);
if (!log_path) {
g_hash_table_destroy(seen_keys);
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
continue;
}
// Ensure directory
auto_gchar gchar* dir = g_path_get_dirname(log_path);
ff_ensure_dir(dir);
auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path);
FILE* fp = fopen(tmp_path, "w");
if (!fp) {
log_error("export: cannot create %s: %s", tmp_path, strerror(errno));
g_hash_table_destroy(seen_keys);
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
continue;
}
// Lock the temp file
int fd = fileno(fp);
flock(fd, LOCK_EX);
fprintf(fp, "%s", FLATFILE_HEADER);
int contact_exported = 0;
int contact_skipped = 0;
// Write existing flatfile lines first (preserve originals)
for (GSList* l = existing; l; l = l->next) {
ff_parsed_line_t* pl = l->data;
ff_write_line(fp, pl->timestamp_str,
pl->type, pl->enc,
pl->stanza_id, pl->archive_id, pl->replace_id,
pl->from_jid, pl->from_resource, pl->message);
}
// Write SQLite messages that aren't already in the flatfile
for (GSList* l = sqlite_lines; l; l = l->next) {
ProfMessage* msg = l->data;
if (!msg || !msg->timestamp)
continue;
auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp);
const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown";
const char* body = msg->plain ? msg->plain : "";
jabber.developer marked this conversation as resolved Outdated

why g_free if key is char? Normally, we do this:
g_free for gchar
free for char

why `g_free` if `key` is `char`? Normally, we do this: `g_free` for `gchar` `free` for `char`

Corrected

Corrected
const char* sid = msg->id ? msg->id : "";
char* key = _make_dedup_key(sid, ts, from_jid, body);
if (g_hash_table_contains(seen_keys, key)) {
g_free(key);
contact_skipped++;
continue;
}
g_hash_table_add(seen_keys, key);
const char* type_str = ff_get_message_type_str(msg->type);
const char* enc_str = ff_get_message_enc_str(msg->enc);
const char* from_res = msg->from_jid ? msg->from_jid->resourcepart : NULL;
ff_write_line(fp, ts, type_str, enc_str,
sid, msg->stanzaid, NULL,
from_jid, from_res, body);
contact_exported++;
}
fflush(fp);
fclose(fp); // also releases flock
// Atomic rename
if (rename(tmp_path, log_path) != 0) {
log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno));
unlink(tmp_path);
}
cons_show("Exported %s: %d new, %d skipped (already existed: %d)",
contact, contact_exported, contact_skipped, existing_count);
total_exported += contact_exported;
total_skipped += contact_skipped;
g_hash_table_destroy(seen_keys);
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
}
g_slist_free_full(contacts, g_free);
return total_exported;
}
// =========================================================================
// Import: flatfile -> SQLite (merge)
// =========================================================================
int
log_database_import_from_flatfile(const gchar* const contact_jid)
{
const Jid* myjid = connection_get_jid();
if (!myjid || !myjid->barejid) {
cons_show_error("Import failed: not connected.");
jabber.developer marked this conversation as resolved Outdated

We can't just print Exported %s... to user after this case. It's a error and should be handled as such.

We can't just print `Exported %s`... to user after this case. It's a error and should be handled as such.

Corrected

Corrected
return -1;
}
if (!g_flatfile_account_jid) {
g_free(g_flatfile_account_jid);
g_flatfile_account_jid = g_strdup(myjid->barejid);
}
db_backend_t* sqlite_be = db_backend_sqlite();
if (!sqlite_be) {
cons_show_error("Import failed: SQLite backend not available.");
return -1;
}
jabber.developer marked this conversation as resolved Outdated

nit: This method is quite fat. In some sense, it's needed for coherency, but I might've overlooked some potential issues with early returns. If possible, I suggest extracting at least body of the cycle.

nit: This method is quite fat. In some sense, it's needed for coherency, but I might've overlooked some potential issues with early returns. If possible, I suggest extracting at least body of the cycle.

Corrected

Corrected
// Determine contacts
GSList* contacts = NULL;
if (contact_jid) {
contacts = g_slist_append(contacts, g_strdup(contact_jid));
} else {
contacts = _ff_list_contacts();
if (!contacts) {
cons_show("No flat-file history found to import.");
return 0;
}
}
int total_imported = 0;
int total_skipped = 0;
for (GSList* c = contacts; c; c = c->next) {
const char* contact = c->data;
// 1. Read all flatfile lines
GSList* ff_lines = _ff_read_all_lines(contact);
if (!ff_lines) {
log_debug("import: no flatfile messages for %s", contact);
continue;
}
// 2. Build dedup set from SQLite (existing stanza_ids + fallback keys)
GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
// Read all SQLite messages for this contact
gboolean has_more = TRUE;
gchar* start_time = NULL;
while (has_more) {
GSList* batch = NULL;
db_history_result_t res = sqlite_be->get_previous_chat(
contact, start_time, NULL, TRUE, FALSE, &batch);
if (res != DB_RESPONSE_SUCCESS || !batch) {
has_more = FALSE;
break;
}
int batch_size = g_slist_length(batch);
ProfMessage* last_msg = g_slist_last(batch)->data;
if (last_msg && last_msg->timestamp) {
g_free(start_time);
start_time = g_date_time_format_iso8601(last_msg->timestamp);
}
for (GSList* l = batch; l; l = l->next) {
ProfMessage* msg = l->data;
if (!msg)
continue;
auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");
char* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain);
g_hash_table_add(seen_keys, key);
}
g_slist_free_full(batch, (GDestroyNotify)message_free);
if (batch_size < MESSAGES_TO_RETRIEVE) {
has_more = FALSE;
}
}
g_free(start_time);
// 3. For each flatfile line not in seen_keys, insert via add_incoming/add_outgoing
int contact_imported = 0;
int contact_skipped = 0;
for (GSList* l = ff_lines; l; l = l->next) {
ff_parsed_line_t* pl = l->data;
if (!pl || !pl->timestamp)
continue;
auto_gchar gchar* ts = g_date_time_format_iso8601(pl->timestamp);
char* key = _make_dedup_key(pl->stanza_id, ts, pl->from_jid, pl->message);
jabber.developer marked this conversation as resolved
Review

nit: this conversion is done twice (see l463: auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");).

nit: this conversion is done twice (see l463: ` auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");`).

Postponed — investigated: the two ts conversions on the SAME message happen in the import path (once for dedup-key building, once inside _add_to_db when serializing for SQL). Removing the duplicate would require an _add_to_db API change to accept a pre-formatted ts. Microsecond per row × 1M = ~1 s saving — accepted as deferred until that signature changes for other reasons.

Postponed — investigated: the two ts conversions on the SAME message happen in the import path (once for dedup-key building, once inside _add_to_db when serializing for SQL). Removing the duplicate would require an _add_to_db API change to accept a pre-formatted ts. Microsecond per row × 1M = ~1 s saving — accepted as deferred until that signature changes for other reasons.
if (g_hash_table_contains(seen_keys, key)) {
g_free(key);
contact_skipped++;
continue;
}
g_hash_table_add(seen_keys, key);
// Build a ProfMessage and insert via add_incoming
// (add_incoming uses _add_to_db which preserves the original timestamp)
gboolean is_outgoing = (g_strcmp0(pl->from_jid, myjid->barejid) == 0);
ProfMessage* msg = message_init();
msg->id = pl->stanza_id ? g_strdup(pl->stanza_id) : NULL;
msg->stanzaid = pl->archive_id ? g_strdup(pl->archive_id) : NULL;
msg->plain = g_strdup(pl->message ? pl->message : "");
msg->timestamp = g_date_time_ref(pl->timestamp);
msg->type = ff_get_message_type_type(pl->type);
msg->enc = ff_get_message_enc_type(pl->enc);
msg->replace_id = pl->replace_id ? g_strdup(pl->replace_id) : NULL;
msg->is_mam = TRUE; // Suppress dedup check in the backend
if (is_outgoing) {
msg->from_jid = jid_create(myjid->barejid);
msg->to_jid = jid_create(contact);
} else {
msg->from_jid = jid_create_from_bare_and_resource(pl->from_jid, pl->from_resource);
msg->to_jid = jid_create(myjid->barejid);
}
sqlite_be->add_incoming(msg);
message_free(msg);
contact_imported++;
}
cons_show("Imported %s: %d new, %d skipped (already in SQLite)",
contact, contact_imported, contact_skipped);
total_imported += contact_imported;
total_skipped += contact_skipped;
g_hash_table_destroy(seen_keys);
jabber.developer marked this conversation as resolved Outdated

note to check: do we sort by timestamp when we fetch from the sqlite? Or is it default sorting by id which presumes that all messages are in order?

note to check: do we sort by timestamp when we fetch from the sqlite? Or is it default sorting by id which presumes that all messages are in order?

Verified — no change needed. db_sqlite_get_all_chat already has "ORDER BY A.timestamp ASC" (database_sqlite.c:916). Documented in 555faaa23 commit body.

Verified — no change needed. db_sqlite_get_all_chat already has "ORDER BY A.timestamp ASC" (database_sqlite.c:916). Documented in 555faaa23 commit body.
g_slist_free_full(ff_lines, (GDestroyNotify)ff_parsed_line_free);
}
g_slist_free_full(contacts, g_free);
return total_imported;
}
#endif /* HAVE_SQLITE */

View File

@@ -817,3 +817,37 @@ db_backend_sqlite(void)
{ {
return &sqlite_backend; return &sqlite_backend;
} }
GSList*
db_sqlite_list_contacts(void)
{
GSList* contacts = NULL;
if (!g_chatlog_database)
return NULL;
const Jid* myjid = connection_get_jid();
if (!myjid || !myjid->barejid)
return NULL;
const char* q = "SELECT DISTINCT `from_jid` FROM `ChatLogs` "
"WHERE `from_jid` != ? "
"UNION "
"SELECT DISTINCT `to_jid` FROM `ChatLogs` "
"WHERE `to_jid` != ?";
sqlite3_stmt* stmt = NULL;
if (!_db_prepare_ctx(q, &stmt, "db_sqlite_list_contacts"))
return NULL;
sqlite3_bind_text(stmt, 1, myjid->barejid, -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, myjid->barejid, -1, SQLITE_STATIC);
while (sqlite3_step(stmt) == SQLITE_ROW) {
const char* jid = (const char*)sqlite3_column_text(stmt, 0);
if (jid && strlen(jid) > 0) {
contacts = g_slist_append(contacts, g_strdup(jid));
}
}
sqlite3_finalize(stmt);
return contacts;
}

View File

@@ -73,6 +73,21 @@ db_backend_sqlite(void)
{ {
return NULL; return NULL;
} }
GSList*
db_sqlite_list_contacts(void)
{
return NULL;
}
int
log_database_export_to_flatfile(const gchar* const contact_jid)
{
return 0;
}
int
log_database_import_from_flatfile(const gchar* const contact_jid)
{
return 0;
}
#endif #endif
db_backend_t* db_backend_t*
db_backend_flatfile(void) db_backend_flatfile(void)