cproof/src/database_export.c

// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of CProof.
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.

/*
 * database_export.c
 * vim: expandtab:ts=4:sts=4:sw=4
 *
 * Cross-backend export/import for message history.
 * Reads from one backend, writes to the other, with merge + dedup.
 */

#include "config.h"

#include <glib.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/file.h>

#ifdef HAVE_SQLITE
#include <sqlite3.h>
#endif

#include "log.h"
#include "common.h"
#include "config/files.h"
#include "config/accounts.h"
#include "database.h"
#include "database_flatfile.h"
#include "config/preferences.h"
#include "ui/ui.h"
#include "xmpp/session.h"
#include "xmpp/xmpp.h"
#include "xmpp/message.h"

// =========================================================================
// Everything below is only used when HAVE_SQLITE is defined
// =========================================================================

#ifdef HAVE_SQLITE

// Print progress to console every N messages during export/import
#define EXPORT_PROGRESS_INTERVAL 500

// Open SQLite if it isn't already (e.g. when flatfile is the active backend
// and sqlite was never initialised this session, or was closed by /history
// switch). Returns 1 if we opened it (caller must close), 0 if it was already
// open, -1 on failure.
static int
_ensure_sqlite_open(void)
{
    if (db_sqlite_is_open())
        return 0;

    db_backend_t* sqlite_be = db_backend_sqlite();
    if (!sqlite_be || !sqlite_be->init)
        return -1;

    const char* account_name = session_get_account_name();
    if (!account_name)
        return -1;

    ProfAccount* account = accounts_get_account(account_name);
    if (!account)
        return -1;

    gboolean ok = sqlite_be->init(account);
    account_free(account);
    return ok ? 1 : -1;
}

static void
_close_temp_sqlite(int opened)
{
    if (opened != 1)
        return;
    db_backend_t* sqlite_be = db_backend_sqlite();
    if (sqlite_be && sqlite_be->close)
        sqlite_be->close();
}

// =========================================================================
// Dedup key: stanza_id if present, else hash of timestamp+from+body
// =========================================================================

// Returns a g_strdup'd key — callers free with g_free (gchar* convention).
//
// stanza_id is mixed into the hash but is NOT used as the sole key, since
// some clients (older Pidgin, Adium) reuse incremental ids per session and
// servers echo them back unchanged. Trusting stanza_id alone silently drops
// legitimate distinct messages on collision. By hashing id + timestamp +
// from_jid + body together we preserve id as a disambiguator without making
// it a single point of failure: true exact duplicates still dedup, but two
// messages that share an id with different content (or different timestamps)
// stay distinct.
static gchar*
_make_dedup_key(const char* stanza_id, const char* timestamp, const char* from_jid, const char* body)
{
    GChecksum* sum = g_checksum_new(G_CHECKSUM_SHA256);
    g_checksum_update(sum, (const guchar*)(stanza_id ? stanza_id : ""), -1);
    g_checksum_update(sum, (const guchar*)"|", 1);
    g_checksum_update(sum, (const guchar*)(timestamp ? timestamp : ""), -1);
    g_checksum_update(sum, (const guchar*)"|", 1);
    g_checksum_update(sum, (const guchar*)(from_jid ? from_jid : ""), -1);
    g_checksum_update(sum, (const guchar*)"|", 1);
    g_checksum_update(sum, (const guchar*)(body ? body : ""), -1);
    gchar* key = g_strdup(g_checksum_get_string(sum));
    g_checksum_free(sum);
    return key;
}

// =========================================================================
// Helper: reverse ff_jid_to_dir (best-effort)
// '_at_' -> '@'
// Returns a g_strdup'd jid — callers free with g_free (gchar* convention).
// str_replace allocates via libc malloc, so we hand over to g_-allocated.
// =========================================================================

static gchar*
_dir_to_jid(const char* dirname)
{
    if (!dirname)
        return NULL;
    char* raw = str_replace(dirname, "_at_", "@");
    if (!raw)
        return NULL;
    gchar* gres = g_strdup(raw);
    free(raw);
    return gres;
}

// =========================================================================
// Helper: list all contact JIDs from flatfile directory
// Returns a GSList of g_strdup'd JID strings. Caller frees with g_slist_free_full(list, g_free).
// =========================================================================

static GSList*
_ff_list_contacts(void)
{
    GSList* contacts = NULL;

    if (!g_flatfile_account_jid)
        return NULL;

    auto_gchar gchar* data_path = files_get_data_path(DIR_FLATLOG);
    auto_gchar gchar* my_dir = ff_jid_to_dir(g_flatfile_account_jid);
    auto_gchar gchar* base_dir = g_strdup_printf("%s/%s", data_path, my_dir);

    GError* err = NULL;
    GDir* dir = g_dir_open(base_dir, 0, &err);
    if (!dir) {
        if (err) {
            log_warning("export: cannot open flatlog dir %s: %s", base_dir, err->message);
            g_error_free(err);
        } else {
            log_warning("export: cannot open flatlog dir %s (no error details)", base_dir);
        }
        return NULL;
    }

    const gchar* entry;
    while ((entry = g_dir_read_name(dir)) != NULL) {
        auto_gchar gchar* full_path = g_strdup_printf("%s/%s/history.log", base_dir, entry);
        if (g_file_test(full_path, G_FILE_TEST_IS_REGULAR)) {
            gchar* jid = _dir_to_jid(entry);
            if (jid) {
                contacts = g_slist_prepend(contacts, jid);
            }
        }
    }
    g_dir_close(dir);

    return g_slist_reverse(contacts);
}

// =========================================================================
// Helper: read ALL parsed lines from a flatfile for a contact
// Returns GSList<ff_parsed_line_t*>. Caller frees with g_slist_free_full(list, (GDestroyNotify)ff_parsed_line_free).
// =========================================================================

// Hard ceiling on the total size of a single contact's history.log that
// _ff_read_all_lines is willing to slurp into memory. Each line expands
// ~10x on parse (12 g_strdup'd fields + GDateTime), so 2 GB on disk
// becomes ~20 GB peak heap — well past any realistic working set.
#define FF_EXPORT_MAX_FILE_BYTES (2LL * 1024 * 1024 * 1024)

static GSList*
_ff_read_all_lines(const char* contact_barejid)
{
    GSList* lines = NULL;
    auto_gchar gchar* log_path = ff_get_log_path(contact_barejid);
    if (!log_path)
        return NULL;

    // Refuse upfront if the file is so large that parsing will OOM the
    // process. We have no streaming-parse path; the whole file becomes
    // a GSList of ff_parsed_line_t in memory. Surface a clear error so
    // the user knows what hit them, rather than the kernel killing prof.
    struct stat st;
    if (stat(log_path, &st) == 0 && st.st_size > FF_EXPORT_MAX_FILE_BYTES) {
        log_error("export: %s is %lld bytes (> %lld limit); refusing to load.",
                  log_path, (long long)st.st_size,
                  (long long)FF_EXPORT_MAX_FILE_BYTES);
        cons_show_error("Flat-file for %s exceeds %lld MB load limit; cannot export.",
                        contact_barejid, (long long)(FF_EXPORT_MAX_FILE_BYTES / (1024 * 1024)));
        return NULL;
    }

    FILE* fp = fopen(log_path, "r");
    if (!fp)
        return NULL;

    while (1) {
        gboolean truncated = FALSE;
        auto_char char* buf = ff_readline(fp, &truncated);
        if (!buf)
            break;
        if (truncated)
            break;
        if (buf[0] == '\0' || buf[0] == '#')
            continue;

        ff_parsed_line_t* pl = ff_parse_line(buf);
        if (pl) {
            lines = g_slist_prepend(lines, pl);
        }
    }
    fclose(fp);

    return g_slist_reverse(lines);
}

// =========================================================================
// Comparator for sorting ff_parsed_line_t by timestamp (ISO8601 is lexicographic)
// =========================================================================

static gint
_compare_parsed_lines_by_timestamp(gconstpointer a, gconstpointer b)
{
    const ff_parsed_line_t* la = a;
    const ff_parsed_line_t* lb = b;
    int cmp = g_strcmp0(la->timestamp_str, lb->timestamp_str);
    if (cmp != 0)
        return cmp;
    // Secondary key: stanza_id (stabilises sort for same-second messages)
    cmp = g_strcmp0(la->stanza_id, lb->stanza_id);
    if (cmp != 0)
        return cmp;
    return g_strcmp0(la->from_jid, lb->from_jid);
}

// =========================================================================
// Export: SQLite -> flatfile (merge)
// =========================================================================

// Callback data for the SQLite query
typedef struct
{
    GHashTable* seen_keys; // dedup set
    FILE* fp;              // output file
    int exported;          // counter
    int skipped;           // counter
} export_ctx_t;

// Export a single contact's messages from SQLite to flatfile.
// Returns the number of newly exported messages, or -1 on error/skip.
static int
_export_one_contact(const char* contact)
{
    // 1. Read existing flatfile lines for dedup
    GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);

    GSList* existing = _ff_read_all_lines(contact);
    for (GSList* l = existing; l; l = l->next) {
        ff_parsed_line_t* pl = l->data;
        gchar* key = _make_dedup_key(pl->stanza_id, pl->timestamp_str, pl->from_jid, pl->message);
        g_hash_table_add(seen_keys, key);
    }
    int existing_count = g_slist_length(existing);

    // 2. Query SQLite for this contact — get ALL messages via direct SQL
    GSList* sqlite_lines = db_sqlite_get_all_chat(contact);

    if (!sqlite_lines) {
        log_debug("export: no SQLite messages for %s", contact);
        g_hash_table_destroy(seen_keys);
        g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
        return -1;
    }

    // 3. Merge: write existing flatfile lines + new SQLite lines to temp file
    auto_gchar gchar* log_path = ff_get_log_path(contact);
    if (!log_path) {
        g_hash_table_destroy(seen_keys);
        g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
        g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
        return -1;
    }

    // Ensure directory
    auto_gchar gchar* dir = g_path_get_dirname(log_path);
    ff_ensure_dir(dir);

    // Use mkstemp() to create a unique-named temp file. The previous fixed
    // "{log_path}.export.tmp" name + unlink-retry was vulnerable to two
    // concurrent exports clobbering each other's in-flight tmp files
    // (process B unlinks A's mid-write tmp, both race to write, second
    // rename wins). Random suffix eliminates that.
    //
    // mkstemp on glibc creates the file with mode 0600. We fchmod to be
    // sure across libc implementations. mkstemp uses O_RDWR|O_CREAT|O_EXCL
    // internally; symlink attacks aren't possible because the name was
    // just generated and no other process knows it.
    auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.XXXXXX", log_path);
    int tmp_fd = mkstemp(tmp_path);
    if (tmp_fd < 0) {
        log_error("export: mkstemp(%s) failed: %s", tmp_path, strerror(errno));
        g_hash_table_destroy(seen_keys);
        g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
        g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
        return -1;
    }
    if (fchmod(tmp_fd, S_IRUSR | S_IWUSR) != 0) {
        log_warning("export: fchmod(%s, 0600) failed: %s", tmp_path, strerror(errno));
    }
    FILE* fp = fdopen(tmp_fd, "w");
    if (!fp) {
        log_error("export: fdopen failed for %s: %s", tmp_path, strerror(errno));
        close(tmp_fd);
        unlink(tmp_path);
        g_hash_table_destroy(seen_keys);
        g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
        g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
        return -1;
    }

    // Lock the temp file
    int fd = fileno(fp);
    if (flock(fd, LOCK_EX) != 0) {
        log_warning("export: flock(%s) failed: %s", tmp_path, strerror(errno));
    }

    fprintf(fp, "%s", FLATFILE_HEADER);

    int contact_exported = 0;
    int contact_skipped = 0;

    // Collect ALL lines into a merged list for sorted output.
    // Transfer ownership of every parsed line from `existing` directly into
    // `merged` instead of deep-copying — saves ~12 g_strdup calls per row,
    // which dominate wall time at 100k+ rows. After splicing we free the
    // list nodes (not the data) and NULL out existing so the final cleanup
    // doesn't double-free.
    GSList* merged = NULL;
    for (GSList* l = existing; l; l = l->next) {
        merged = g_slist_prepend(merged, l->data);
    }
    g_slist_free(existing);
    existing = NULL;

    // Add SQLite messages that aren't already in the flatfile
    for (GSList* l = sqlite_lines; l; l = l->next) {
        ProfMessage* msg = l->data;
        if (!msg || !msg->timestamp)
            continue;

        auto_gchar gchar* ts = g_date_time_format_iso8601(msg->timestamp);
        const char* from_jid = msg->from_jid ? msg->from_jid->barejid : "unknown";
        const char* body = msg->plain ? msg->plain : "";
        const char* sid = msg->id ? msg->id : "";

        gchar* key = _make_dedup_key(sid, ts, from_jid, body);
        if (g_hash_table_contains(seen_keys, key)) {
            g_free(key);
            contact_skipped++;
            continue;
        }
        g_hash_table_add(seen_keys, key);

        ff_parsed_line_t* pl = g_malloc0(sizeof(ff_parsed_line_t));
        pl->timestamp_str = g_strdup(ts);
        pl->timestamp = g_date_time_ref(msg->timestamp);
        pl->type = g_strdup(ff_get_message_type_str(msg->type));
        pl->enc = g_strdup(ff_get_message_enc_str(msg->enc));
        pl->stanza_id = g_strdup(sid);
        pl->archive_id = msg->stanzaid ? g_strdup(msg->stanzaid) : NULL;
        pl->replace_id = NULL;
        pl->from_jid = g_strdup(from_jid);
        pl->from_resource = msg->from_jid ? g_strdup(msg->from_jid->resourcepart) : NULL;
        pl->to_jid = msg->to_jid ? g_strdup(msg->to_jid->barejid) : NULL;
        pl->to_resource = msg->to_jid ? g_strdup(msg->to_jid->resourcepart) : NULL;
        pl->marked_read = msg->marked_read;
        pl->message = g_strdup(body);
        merged = g_slist_prepend(merged, pl);
        contact_exported++;
    }

    // Sort merged list by timestamp (ISO8601 is lexicographically sortable)
    merged = g_slist_sort(merged, _compare_parsed_lines_by_timestamp);

    // Cache the total length once — calling g_slist_length() inside the
    // write loop is O(n) per call, turning the loop into O(n²) wall time
    // (catastrophic at >100k rows).
    int merged_total = (int)g_slist_length(merged);

    // Write all lines sorted
    int written = 0;
    for (GSList* l = merged; l; l = l->next) {
        ff_parsed_line_t* pl = l->data;
        ff_write_line(fp, pl->timestamp_str,
                      pl->type, pl->enc,
                      pl->stanza_id, pl->archive_id, pl->replace_id,
                      pl->from_jid, pl->from_resource,
                      pl->to_jid, pl->to_resource, pl->marked_read,
                      pl->message);
        written++;
        if (written % EXPORT_PROGRESS_INTERVAL == 0) {
            cons_show("  ... %s: %d/%d written so far", contact, written, merged_total);
        }
    }

    g_slist_free_full(merged, (GDestroyNotify)ff_parsed_line_free);

    fflush(fp);
    fsync(fd);  // ensure data is on disk before atomic rename
    fclose(fp); // also releases flock

    // Atomic rename
    int result = -1;
    if (rename(tmp_path, log_path) != 0) {
        log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno));
        cons_show_error("Export failed for %s: could not rename temp file (%s)", contact, strerror(errno));
        unlink(tmp_path);
    } else {
        cons_show("Exported %s: %d new, %d skipped (already existed: %d)",
                  contact, contact_exported, contact_skipped, existing_count);
        result = contact_exported;
    }

    g_hash_table_destroy(seen_keys);
    g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
    g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
    return result;
}

int
log_database_export_to_flatfile(const gchar* const contact_jid)
{
    const Jid* myjid = connection_get_jid();
    if (!myjid || !myjid->barejid) {
        cons_show_error("Export failed: not connected.");
        return -1;
    }

    if (!g_flatfile_account_jid) {
        // Flatfile backend not initialized — init it temporarily
        // We need g_flatfile_account_jid for path construction
        g_flatfile_account_jid = g_strdup(myjid->barejid);
    }

    // Get the SQLite backend
    db_backend_t* sqlite_be = db_backend_sqlite();
    if (!sqlite_be) {
        cons_show_error("Export failed: SQLite backend not available.");
        return -1;
    }

    // SQLite must be open for db_sqlite_list_contacts/db_sqlite_get_all_chat
    // to return rows. When the flatfile backend is active, sqlite is closed —
    // queries silently return NULL and the export reports zero changes.
    int sqlite_opened = _ensure_sqlite_open();
    if (sqlite_opened < 0) {
        cons_show_error("Export failed: could not open SQLite database.");
        return -1;
    }

    // Determine contact list
    GSList* contacts = NULL;
    if (contact_jid) {
        contacts = g_slist_append(contacts, g_strdup(contact_jid));
    } else {
        contacts = db_sqlite_list_contacts();
        if (!contacts) {
            cons_show("No contacts found in SQLite database.");
            _close_temp_sqlite(sqlite_opened);
            return 0;
        }
        cons_show("Exporting %d contact(s) to flat-file format...", g_slist_length(contacts));
    }

    int total_exported = 0;

    for (GSList* c = contacts; c; c = c->next) {
        const char* contact = c->data;
        int exported = _export_one_contact(contact);
        if (exported > 0)
            total_exported += exported;
    }

    g_slist_free_full(contacts, g_free);
    _close_temp_sqlite(sqlite_opened);
    return total_exported;
}

// =========================================================================
// Import: flatfile -> SQLite (merge)
// =========================================================================

int
log_database_import_from_flatfile(const gchar* const contact_jid)
{
    const Jid* myjid = connection_get_jid();
    if (!myjid || !myjid->barejid) {
        cons_show_error("Import failed: not connected.");
        return -1;
    }

    if (!g_flatfile_account_jid) {
        g_flatfile_account_jid = g_strdup(myjid->barejid);
    }

    db_backend_t* sqlite_be = db_backend_sqlite();
    if (!sqlite_be) {
        cons_show_error("Import failed: SQLite backend not available.");
        return -1;
    }

    int sqlite_opened = _ensure_sqlite_open();
    if (sqlite_opened < 0) {
        cons_show_error("Import failed: could not open SQLite database.");
        return -1;
    }

    // Determine contacts
    GSList* contacts = NULL;
    if (contact_jid) {
        contacts = g_slist_append(contacts, g_strdup(contact_jid));
    } else {
        contacts = _ff_list_contacts();
        if (!contacts) {
            cons_show("No flat-file history found to import.");
            _close_temp_sqlite(sqlite_opened);
            return 0;
        }
    }

    int total_imported = 0;

    for (GSList* c = contacts; c; c = c->next) {
        const char* contact = c->data;

        // 1. Read all flatfile lines
        GSList* ff_lines = _ff_read_all_lines(contact);
        if (!ff_lines) {
            log_debug("import: no flatfile messages for %s", contact);
            continue;
        }

        // 2. Build dedup set from SQLite (existing stanza_ids + fallback keys)
        GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);

        // Read all SQLite messages for this contact via direct SQL query
        GSList* existing_msgs = db_sqlite_get_all_chat(contact);
        for (GSList* l = existing_msgs; l; l = l->next) {
            ProfMessage* msg = l->data;
            if (!msg)
                continue;
            auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");
            gchar* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain);
            g_hash_table_add(seen_keys, key);
        }
        g_slist_free_full(existing_msgs, (GDestroyNotify)message_free);

        // 3. For each flatfile line not in seen_keys, insert via add_incoming
        //    Wrap in a transaction for atomicity and performance
        int contact_imported = 0;
        int contact_skipped = 0;
        int total_lines = g_slist_length(ff_lines);

        db_sqlite_begin_transaction();
        gboolean import_ok = TRUE;

        for (GSList* l = ff_lines; l; l = l->next) {
            ff_parsed_line_t* pl = l->data;
            if (!pl || !pl->timestamp)
                continue;

            auto_gchar gchar* ts = g_date_time_format_iso8601(pl->timestamp);
            gchar* key = _make_dedup_key(pl->stanza_id, ts, pl->from_jid, pl->message);

            if (g_hash_table_contains(seen_keys, key)) {
                g_free(key);
                contact_skipped++;
                continue;
            }
            g_hash_table_add(seen_keys, key);

            // Build a ProfMessage and insert via add_incoming
            // (add_incoming uses _add_to_db which preserves the original timestamp)
            gboolean is_outgoing = (g_strcmp0(pl->from_jid, myjid->barejid) == 0);

            ProfMessage* msg = message_init();
            msg->id = pl->stanza_id ? g_strdup(pl->stanza_id) : NULL;
            msg->stanzaid = pl->archive_id ? g_strdup(pl->archive_id) : NULL;
            msg->plain = g_strdup(pl->message ? pl->message : "");
            msg->timestamp = g_date_time_ref(pl->timestamp);
            msg->type = ff_get_message_type_type(pl->type);
            msg->enc = ff_get_message_enc_type(pl->enc);
            msg->replace_id = pl->replace_id ? g_strdup(pl->replace_id) : NULL;
            msg->is_mam = TRUE; // Suppress dedup check in the backend

            if (is_outgoing) {
                msg->from_jid = jid_create(myjid->barejid);
                if (pl->to_jid) {
                    msg->to_jid = jid_create_from_bare_and_resource(pl->to_jid, pl->to_resource);
                } else {
                    msg->to_jid = jid_create(contact);
                }
            } else {
                msg->from_jid = jid_create_from_bare_and_resource(pl->from_jid, pl->from_resource);
                if (pl->to_jid) {
                    msg->to_jid = jid_create_from_bare_and_resource(pl->to_jid, pl->to_resource);
                } else {
                    msg->to_jid = jid_create(myjid->barejid);
                }
            }

            sqlite_be->add_incoming(msg);
            message_free(msg);

            // Verify the row was actually inserted
            if (db_sqlite_last_changes() < 1) {
                log_error("import: insert failed for %s at %s", contact, ts);
                import_ok = FALSE;
                break;
            }
            contact_imported++;

            if (contact_imported % EXPORT_PROGRESS_INTERVAL == 0) {
                cons_show("  ... %s: %d/%d imported so far", contact, contact_imported, total_lines);
            }
        }

        if (import_ok) {
            db_sqlite_end_transaction();
        } else {
            db_sqlite_rollback_transaction();
            cons_show_error("Import of %s failed — transaction rolled back.", contact);
        }

        cons_show("Imported %s: %d new, %d skipped (already in SQLite)",
                  contact, contact_imported, contact_skipped);
        total_imported += contact_imported;

        g_hash_table_destroy(seen_keys);
        g_slist_free_full(ff_lines, (GDestroyNotify)ff_parsed_line_free);

        // If the per-contact import bailed mid-stream (likely a system-level
        // problem: disk full, sqlite locked, etc.) — don't blindly try the
        // remaining contacts; they'll almost certainly hit the same error
        // and produce a confusing wall of "Import of X failed" messages.
        // Stop here so the user can investigate the root cause.
        if (!import_ok) {
            cons_show_error("Aborting import — system-level failure suspected. Fix the underlying issue and retry.");
            break;
        }
    }

    g_slist_free_full(contacts, g_free);
    _close_temp_sqlite(sqlite_opened);
    return total_imported;
}

#endif /* HAVE_SQLITE */