devs/cproof
1
2
Fork 1
Code Issues 43 Pull Requests 6 Actions Packages Projects 1 Releases Wiki Activity

feat: complete field parity, harden export/import, add tests

Browse Source 
Export/Import improvements:
- Replace pagination with direct SQL query (db_sqlite_get_all_chat)
- Wrap import in SQL transaction with rollback on error
- Add fsync before fclose in export for data safety
- Sort merged output by timestamp with secondary key (stanza_id, from_jid)
- Export archive_id and marked_read from SQLite (lossless migration)
- Add progress indication every 500 messages during write/import
- Expand dedup key body prefix from 64 to 256 chars
- Fix g_slist_append O(n²) → g_slist_prepend + g_slist_reverse O(n)

Field parity (to_jid, to_resource, marked_read):
- Add fields to ff_parsed_line_t struct
- Write/parse to:|to_res:|read: metadata tags in flatfile format
- Pass to_resource through _ff_add_message and all callers
- Add marked_read to ProfMessage struct with -1 default (unset)
- Preserve fields across export/import round-trips

Tests (19 new: 11 unit + 8 functional):
- Unit: to_jid_and_marked_read, bracket_in_stanza_id, backslash_in_resource,
  mucpm_type, all_enc_types, crlf_handling, to_jid_special_chars,
  multiple_lines, parsed_line_free_null_safe, no_space_rejected,
  unclosed_bracket
- Functional: export_idempotent_no_duplicates, export_lmc_correction_survives,
  switch_preserves_old_backend_data, export_all_contacts,
  import_double_dedup, verify_after_export,
  switch_backends_independent_messages, export_empty_contact
- Rebalance test groups: move Chat Session from Group 3 to Group 4
  (25/33/30/27 instead of 25/33/36/21)
- Remove hardcoded test counts from group comments

Man page:
- Document /history switch sqlite|flatfile
This commit is contained in:
jabber.developer2
2026-03-09 12:14:41 +03:00
parent d5d67592ab
commit e91e8403b0
16 changed files with 1037 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
src/database_export.c
View File

@@ -64,8 +64,8 @@ _make_dedup_key(const char* stanza_id, const char* timestamp, const char* from_j
if (stanza_id && strlen(stanza_id) > 0)
return g_strdup(stanza_id);
// Fallback: simple composite key
auto_gchar gchar* raw = g_strdup_printf("%s|%s|%.64s",
// Fallback: composite key from timestamp + sender + body prefix
auto_gchar gchar* raw = g_strdup_printf("%s|%s|%.256s",
timestamp ? timestamp : "",
from_jid ? from_jid : "",
body ? body : "");
@@ -118,13 +118,13 @@ _ff_list_contacts(void)
if (g_file_test(full_path, G_FILE_TEST_IS_REGULAR)) {
char* jid = _dir_to_jid(entry);
if (jid) {
contacts = g_slist_append(contacts, jid);
contacts = g_slist_prepend(contacts, jid);
}
}
}
g_dir_close(dir);
return contacts;
return g_slist_reverse(contacts);
}
// =========================================================================
@@ -161,12 +161,31 @@ _ff_read_all_lines(const char* contact_barejid)
ff_parsed_line_t* pl = ff_parse_line(buf);
free(buf);
if (pl) {
lines = g_slist_append(lines, pl);
lines = g_slist_prepend(lines, pl);
}
}
fclose(fp);
return lines;
return g_slist_reverse(lines);
}
// =========================================================================
// Comparator for sorting ff_parsed_line_t by timestamp (ISO8601 is lexicographic)
// =========================================================================
static gint
_compare_parsed_lines_by_timestamp(gconstpointer a, gconstpointer b)
{
const ff_parsed_line_t* la = a;
const ff_parsed_line_t* lb = b;
int cmp = g_strcmp0(la->timestamp_str, lb->timestamp_str);
if (cmp != 0)
return cmp;
// Secondary key: stanza_id (stabilises sort for same-second messages)
cmp = g_strcmp0(la->stanza_id, lb->stanza_id);
if (cmp != 0)
return cmp;
return g_strcmp0(la->from_jid, lb->from_jid);
}
// =========================================================================
@@ -235,48 +254,8 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
}
int existing_count = g_slist_length(existing);
// 2. Query SQLite for this contact — get ALL messages ordered by timestamp
// We read all messages by paginating through the existing get_previous_chat
// But it limits to MESSAGES_TO_RETRIEVE... We need a direct SQL query.
// Use the same approach as verify — iterate with raw SQL.
// For now, collect all messages into a list via repeated calls
// Actually, let's use a simpler approach: read all from flatfile existing data,
// then query SQLite for everything, merge, write back.
// Read all SQLite messages for this contact
GSList* sqlite_lines = NULL;
// Use the backend's get_previous_chat with a wide time range
// This is limited to MESSAGES_TO_RETRIEVE, so we need to paginate
gboolean has_more = TRUE;
gchar* start_time = NULL;
while (has_more) {
GSList* batch = NULL;
db_history_result_t res = sqlite_be->get_previous_chat(
contact, start_time, NULL, TRUE, FALSE, &batch);
if (res != DB_RESPONSE_SUCCESS || !batch) {
has_more = FALSE;
break;
}
int batch_size = g_slist_length(batch);
// Get the last timestamp for the next query
ProfMessage* last_msg = g_slist_last(batch)->data;
if (last_msg && last_msg->timestamp) {
g_free(start_time);
start_time = g_date_time_format_iso8601(last_msg->timestamp);
}
sqlite_lines = g_slist_concat(sqlite_lines, batch);
if (batch_size < MESSAGES_TO_RETRIEVE) {
has_more = FALSE;
}
}
g_free(start_time);
// 2. Query SQLite for this contact — get ALL messages via direct SQL
GSList* sqlite_lines = db_sqlite_get_all_chat(contact);
if (!sqlite_lines) {
log_debug("export: no SQLite messages for %s", contact);
@@ -317,16 +296,31 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
int contact_exported = 0;
int contact_skipped = 0;
// Write existing flatfile lines first (preserve originals)
// Collect ALL lines into a merged list for sorted output
GSList* merged = NULL;
// Add existing flatfile lines (mark as already seen in dedup)
for (GSList* l = existing; l; l = l->next) {
ff_parsed_line_t* pl = l->data;
ff_write_line(fp, pl->timestamp_str,
pl->type, pl->enc,
pl->stanza_id, pl->archive_id, pl->replace_id,
pl->from_jid, pl->from_resource, pl->message);
// Create a shallow copy for the merged list (originals freed separately)
ff_parsed_line_t* copy = g_malloc0(sizeof(ff_parsed_line_t));
copy->timestamp_str = g_strdup(pl->timestamp_str);
copy->timestamp = pl->timestamp ? g_date_time_ref(pl->timestamp) : NULL;
copy->type = g_strdup(pl->type);
copy->enc = g_strdup(pl->enc);
copy->stanza_id = g_strdup(pl->stanza_id);
copy->archive_id = g_strdup(pl->archive_id);
copy->replace_id = g_strdup(pl->replace_id);
copy->from_jid = g_strdup(pl->from_jid);
copy->from_resource = g_strdup(pl->from_resource);
copy->to_jid = g_strdup(pl->to_jid);
copy->to_resource = g_strdup(pl->to_resource);
copy->marked_read = pl->marked_read;
copy->message = g_strdup(pl->message);
merged = g_slist_prepend(merged, copy);
}
// Write SQLite messages that aren't already in the flatfile
// Add SQLite messages that aren't already in the flatfile
for (GSList* l = sqlite_lines; l; l = l->next) {
ProfMessage* msg = l->data;
if (!msg || !msg->timestamp)
@@ -345,17 +339,47 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
}
g_hash_table_add(seen_keys, key);
const char* type_str = ff_get_message_type_str(msg->type);
const char* enc_str = ff_get_message_enc_str(msg->enc);
const char* from_res = msg->from_jid ? msg->from_jid->resourcepart : NULL;
ff_write_line(fp, ts, type_str, enc_str,
sid, msg->stanzaid, NULL,
from_jid, from_res, body);
ff_parsed_line_t* pl = g_malloc0(sizeof(ff_parsed_line_t));
pl->timestamp_str = g_strdup(ts);
pl->timestamp = g_date_time_ref(msg->timestamp);
pl->type = g_strdup(ff_get_message_type_str(msg->type));
pl->enc = g_strdup(ff_get_message_enc_str(msg->enc));
pl->stanza_id = g_strdup(sid);
pl->archive_id = msg->stanzaid ? g_strdup(msg->stanzaid) : NULL;
pl->replace_id = NULL;
pl->from_jid = g_strdup(from_jid);
pl->from_resource = msg->from_jid ? g_strdup(msg->from_jid->resourcepart) : NULL;
pl->to_jid = msg->to_jid ? g_strdup(msg->to_jid->barejid) : NULL;
pl->to_resource = msg->to_jid ? g_strdup(msg->to_jid->resourcepart) : NULL;
pl->marked_read = msg->marked_read;
pl->message = g_strdup(body);
merged = g_slist_prepend(merged, pl);
contact_exported++;
}
// Sort merged list by timestamp (ISO8601 is lexicographically sortable)
merged = g_slist_sort(merged, _compare_parsed_lines_by_timestamp);
// Write all lines sorted
int written = 0;
for (GSList* l = merged; l; l = l->next) {
ff_parsed_line_t* pl = l->data;
ff_write_line(fp, pl->timestamp_str,
pl->type, pl->enc,
pl->stanza_id, pl->archive_id, pl->replace_id,
pl->from_jid, pl->from_resource,
pl->to_jid, pl->to_resource, pl->marked_read,
pl->message);
written++;
if (written % 500 == 0) {
cons_show(" ... %s: %d/%d written so far", contact, written, (int)g_slist_length(merged));
}
}
g_slist_free_full(merged, (GDestroyNotify)ff_parsed_line_free);
fflush(fp);
fsync(fd); // ensure data is on disk before atomic rename
fclose(fp); // also releases flock
// Atomic rename
@@ -430,48 +454,26 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
// 2. Build dedup set from SQLite (existing stanza_ids + fallback keys)
GHashTable* seen_keys = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
// Read all SQLite messages for this contact
gboolean has_more = TRUE;
gchar* start_time = NULL;
while (has_more) {
GSList* batch = NULL;
db_history_result_t res = sqlite_be->get_previous_chat(
contact, start_time, NULL, TRUE, FALSE, &batch);
if (res != DB_RESPONSE_SUCCESS || !batch) {
has_more = FALSE;
break;
}
int batch_size = g_slist_length(batch);
ProfMessage* last_msg = g_slist_last(batch)->data;
if (last_msg && last_msg->timestamp) {
g_free(start_time);
start_time = g_date_time_format_iso8601(last_msg->timestamp);
}
for (GSList* l = batch; l; l = l->next) {
ProfMessage* msg = l->data;
if (!msg)
continue;
auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");
char* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain);
g_hash_table_add(seen_keys, key);
}
g_slist_free_full(batch, (GDestroyNotify)message_free);
if (batch_size < MESSAGES_TO_RETRIEVE) {
has_more = FALSE;
}
// Read all SQLite messages for this contact via direct SQL query
GSList* existing_msgs = db_sqlite_get_all_chat(contact);
for (GSList* l = existing_msgs; l; l = l->next) {
ProfMessage* msg = l->data;
if (!msg)
continue;
auto_gchar gchar* ts = msg->timestamp ? g_date_time_format_iso8601(msg->timestamp) : g_strdup("");
char* key = _make_dedup_key(msg->id, ts, msg->from_jid ? msg->from_jid->barejid : "", msg->plain);
g_hash_table_add(seen_keys, key);
}
g_free(start_time);
g_slist_free_full(existing_msgs, (GDestroyNotify)message_free);
// 3. For each flatfile line not in seen_keys, insert via add_incoming/add_outgoing
// 3. For each flatfile line not in seen_keys, insert via add_incoming
// Wrap in a transaction for atomicity and performance
int contact_imported = 0;
int contact_skipped = 0;
int total_lines = g_slist_length(ff_lines);
db_sqlite_begin_transaction();
gboolean import_ok = TRUE;
for (GSList* l = ff_lines; l; l = l->next) {
ff_parsed_line_t* pl = l->data;
@@ -504,15 +506,34 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
if (is_outgoing) {
msg->from_jid = jid_create(myjid->barejid);
msg->to_jid = jid_create(contact);
if (pl->to_jid) {
msg->to_jid = jid_create_from_bare_and_resource(pl->to_jid, pl->to_resource);
} else {
msg->to_jid = jid_create(contact);
}
} else {
msg->from_jid = jid_create_from_bare_and_resource(pl->from_jid, pl->from_resource);
msg->to_jid = jid_create(myjid->barejid);
if (pl->to_jid) {
msg->to_jid = jid_create_from_bare_and_resource(pl->to_jid, pl->to_resource);
} else {
msg->to_jid = jid_create(myjid->barejid);
}
}
sqlite_be->add_incoming(msg);
message_free(msg);
contact_imported++;
if (contact_imported % 500 == 0) {
cons_show(" ... %s: %d/%d imported so far", contact, contact_imported, total_lines);
}
}
if (import_ok) {
db_sqlite_end_transaction();
} else {
db_sqlite_rollback_transaction();
cons_show_error("Import of %s failed — transaction rolled back.", contact);
}
cons_show("Imported %s: %d new, %d skipped (already in SQLite)",