From f8ef504a09d4aa938ff84df3e90bf66ed87b6d31 Mon Sep 17 00:00:00 2001 From: "jabber.developer2" Date: Thu, 19 Feb 2026 16:09:23 +0300 Subject: [PATCH] database_flatfile: single-file storage with sparse index Replace per-contact directory structure with a single flat file per contact. Each file uses pipe-delimited records with a sparse byte-offset index that is rebuilt on demand and cached in memory. - Rewrite _ff_write_record / _ff_read_records for single-file format - Add sparse index (every Nth record) for O(log N) seek on history read - Cursor-based pagination: _ff_get_previous_chat uses saved byte offset - LMC corrections applied in-place during read (_ff_apply_lmc) - Newline escaping (\n) in message bodies for line-based storage - Simplify verify: parse + timestamp order + LMC reference check - Update parser helpers for new field layout --- src/database_flatfile.c | 578 ++++++++++++++++++++++++--------- src/database_flatfile.h | 39 ++- src/database_flatfile_parser.c | 9 +- src/database_flatfile_verify.c | 359 +++++++++----------- 4 files changed, 608 insertions(+), 377 deletions(-) diff --git a/src/database_flatfile.c b/src/database_flatfile.c index b78869c9..e8b36316 100644 --- a/src/database_flatfile.c +++ b/src/database_flatfile.c @@ -31,6 +31,8 @@ #include #include +#include +#include #include #include #include @@ -52,6 +54,261 @@ // ========================================================================= char* g_flatfile_account_jid = NULL; +static GHashTable* g_contact_states = NULL; + +// ========================================================================= +// Per-contact state / sparse index +// ========================================================================= + +ff_contact_state_t* +ff_state_new(const char* filepath) +{ + ff_contact_state_t* state = g_malloc0(sizeof(ff_contact_state_t)); + state->filepath = g_strdup(filepath); + state->cursor_offset = -1; + return state; +} + +void +ff_state_free(ff_contact_state_t* state) +{ + if (!state) + return; + g_free(state->filepath); + g_free(state->entries); + g_free(state); +} + +static gboolean +_ff_state_build(ff_contact_state_t* state) +{ + FILE* fp = fopen(state->filepath, "r"); + if (!fp) + return FALSE; + + int c1 = fgetc(fp); + int c2 = fgetc(fp); + int c3 = fgetc(fp); + if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) { + state->bom_len = 3; + } else { + state->bom_len = 0; + fseek(fp, 0, SEEK_SET); + } + + state->n_entries = 0; + state->total_lines = 0; + size_t msg_count = 0; + + while (1) { + off_t pos = ftell(fp); + gboolean truncated = FALSE; + char* buf = ff_readline(fp, &truncated); + if (!buf) + break; + if (truncated) { + free(buf); + break; + } + + if (buf[0] == '\0' || buf[0] == '#') { + free(buf); + continue; + } + + state->total_lines++; + + if (msg_count % FF_INDEX_STEP == 0) { + char* space = strchr(buf, ' '); + if (space) { + char* ts_str = g_strndup(buf, space - buf); + GDateTime* dt = g_date_time_new_from_iso8601(ts_str, NULL); + if (dt) { + if (state->n_entries >= state->cap_entries) { + state->cap_entries = state->cap_entries ? state->cap_entries * 2 : 64; + state->entries = g_realloc(state->entries, + state->cap_entries * sizeof(ff_index_entry_t)); + } + state->entries[state->n_entries].byte_offset = pos; + state->entries[state->n_entries].timestamp_epoch = g_date_time_to_unix(dt); + state->n_entries++; + g_date_time_unref(dt); + } + g_free(ts_str); + } + } + msg_count++; + free(buf); + } + + fclose(fp); + + struct stat st; + if (stat(state->filepath, &st) == 0) { + state->stamp.mtime = st.st_mtime; + state->stamp.size = st.st_size; + state->stamp.inode = st.st_ino; + } + + log_debug("flatfile: index built for %s (%zu entries, %zu lines)", + state->filepath, state->n_entries, state->total_lines); + return TRUE; +} + +static gboolean +_ff_state_extend(ff_contact_state_t* state) +{ + struct stat st; + if (stat(state->filepath, &st) != 0) + return FALSE; + + if (st.st_size <= state->stamp.size) + return _ff_state_build(state); + + FILE* fp = fopen(state->filepath, "r"); + if (!fp) + return FALSE; + + fseek(fp, state->stamp.size, SEEK_SET); + if (state->stamp.size > 0) { + fseek(fp, state->stamp.size - 1, SEEK_SET); + int prev = fgetc(fp); + if (prev != '\n' && prev != EOF) { + int ch; + while ((ch = fgetc(fp)) != EOF && ch != '\n') { + } + } + } + + size_t msg_count = state->total_lines; + + while (1) { + off_t pos = ftell(fp); + gboolean truncated = FALSE; + char* buf = ff_readline(fp, &truncated); + if (!buf) + break; + if (truncated) { + free(buf); + break; + } + + if (buf[0] == '\0' || buf[0] == '#') { + free(buf); + continue; + } + + state->total_lines++; + + if (msg_count % FF_INDEX_STEP == 0) { + char* space = strchr(buf, ' '); + if (space) { + char* ts_str = g_strndup(buf, space - buf); + GDateTime* dt = g_date_time_new_from_iso8601(ts_str, NULL); + if (dt) { + if (state->n_entries >= state->cap_entries) { + state->cap_entries = state->cap_entries ? state->cap_entries * 2 : 64; + state->entries = g_realloc(state->entries, + state->cap_entries * sizeof(ff_index_entry_t)); + } + state->entries[state->n_entries].byte_offset = pos; + state->entries[state->n_entries].timestamp_epoch = g_date_time_to_unix(dt); + state->n_entries++; + g_date_time_unref(dt); + } + g_free(ts_str); + } + } + msg_count++; + free(buf); + } + + fclose(fp); + + state->stamp.mtime = st.st_mtime; + state->stamp.size = st.st_size; + state->stamp.inode = st.st_ino; + + log_debug("flatfile: index extended for %s (%zu entries, %zu lines)", + state->filepath, state->n_entries, state->total_lines); + return TRUE; +} + +gboolean +ff_state_ensure_fresh(ff_contact_state_t* state) +{ + if (!state || !state->filepath) + return FALSE; + + struct stat st; + if (stat(state->filepath, &st) != 0) { + state->total_lines = 0; + state->n_entries = 0; + state->stamp.size = 0; + return FALSE; + } + + if (state->stamp.size == 0 && state->n_entries == 0) + return _ff_state_build(state); + + if (st.st_mtime == state->stamp.mtime + && st.st_size == state->stamp.size + && st.st_ino == state->stamp.inode) { + return TRUE; + } + + if (st.st_ino == state->stamp.inode && st.st_size > state->stamp.size) + return _ff_state_extend(state); + + state->cursor_offset = -1; + return _ff_state_build(state); +} + +off_t +ff_state_offset_for_time(ff_contact_state_t* state, const char* iso_time) +{ + if (!state || !iso_time || state->n_entries == 0) + return state ? (off_t)state->bom_len : 0; + + GDateTime* dt = g_date_time_new_from_iso8601(iso_time, NULL); + if (!dt) + return state->bom_len; + + gint64 target = g_date_time_to_unix(dt); + g_date_time_unref(dt); + + size_t lo = 0, hi = state->n_entries; + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + if (state->entries[mid].timestamp_epoch <= target) + lo = mid + 1; + else + hi = mid; + } + + if (lo == 0) + return state->bom_len; + return state->entries[lo - 1].byte_offset; +} + +static ff_contact_state_t* +_ff_get_state(const char* contact_barejid) +{ + if (!g_contact_states || !contact_barejid) + return NULL; + + ff_contact_state_t* state = g_hash_table_lookup(g_contact_states, contact_barejid); + if (state) + return state; + + auto_gchar gchar* filepath = ff_get_log_path(contact_barejid); + if (!filepath) + return NULL; + + state = ff_state_new(filepath); + g_hash_table_insert(g_contact_states, g_strdup(contact_barejid), state); + return state; +} // ========================================================================= // Core write logic @@ -79,6 +336,7 @@ _ff_add_message(const char* type, const char* stanza_id, const char* archive_id, GDateTime* dt = timestamp ? g_date_time_ref(timestamp) : g_date_time_new_now_local(); auto_gchar gchar* date_fmt = g_date_time_format_iso8601(dt); + g_date_time_unref(dt); // Determine which JID to use for the log file path (the "other" party) const char* contact = NULL; @@ -89,8 +347,7 @@ _ff_add_message(const char* type, const char* stanza_id, const char* archive_id, contact = from_barejid; } - auto_gchar gchar* log_path = ff_get_log_path(contact, dt); - g_date_time_unref(dt); + auto_gchar gchar* log_path = ff_get_log_path(contact); if (!log_path) { log_error("flatfile: could not determine log path for %s", contact); @@ -170,6 +427,12 @@ _flatfile_init(ProfAccount* account) return FALSE; } + if (g_contact_states) { + g_hash_table_destroy(g_contact_states); + } + g_contact_states = g_hash_table_new_full(g_str_hash, g_str_equal, + g_free, (GDestroyNotify)ff_state_free); + log_info("Initialized flat-file database backend: %s", base_dir); return TRUE; } @@ -177,6 +440,10 @@ _flatfile_init(ProfAccount* account) static void _flatfile_close(void) { + if (g_contact_states) { + g_hash_table_destroy(g_contact_states); + g_contact_states = NULL; + } g_free(g_flatfile_account_jid); g_flatfile_account_jid = NULL; log_debug("flatfile: closed"); @@ -235,113 +502,6 @@ _flatfile_add_outgoing_muc_pm(const char* const id, const char* const barejid, // Read logic // ========================================================================= -// List all log files for a contact, sorted by filename (date order) -static GSList* -_ff_list_log_files(const char* contact_barejid) -{ - auto_gchar gchar* contact_dir = ff_get_contact_dir(contact_barejid); - if (!contact_dir || !g_file_test(contact_dir, G_FILE_TEST_IS_DIR)) { - return NULL; - } - - GDir* dir = g_dir_open(contact_dir, 0, NULL); - if (!dir) - return NULL; - - GSList* files = NULL; - const gchar* fname; - while ((fname = g_dir_read_name(dir)) != NULL) { - if (g_str_has_suffix(fname, ".log")) { - files = g_slist_insert_sorted(files, - g_strdup_printf("%s/%s", contact_dir, fname), - (GCompareFunc)g_strcmp0); - } - } - g_dir_close(dir); - return files; -} - -// Read all parsed lines from a file, applying filters -static GSList* -_ff_read_file_messages(const char* filepath, const char* contact_barejid, - const char* start_time, const char* end_time, - const Jid* myjid) -{ - GSList* messages = NULL; - - FILE* fp = fopen(filepath, "r"); - if (!fp) - return NULL; - - // BOM detection - int c1 = fgetc(fp); - int c2 = fgetc(fp); - int c3 = fgetc(fp); - if (!(c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)) { - // Not a BOM, rewind - fseek(fp, 0, SEEK_SET); - } - - char* buf = NULL; - gboolean truncated = FALSE; - while ((buf = ff_readline(fp, &truncated)) != NULL) { - if (truncated) { - log_warning("flatfile: truncated line at EOF in %s, skipping", filepath); - free(buf); - break; - } - - ff_parsed_line_t* pl = ff_parse_line(buf); - free(buf); - if (!pl) - continue; - - // Check timestamp bounds - if (start_time) { - GDateTime* start_dt = g_date_time_new_from_iso8601(start_time, NULL); - if (start_dt) { - if (g_date_time_compare(pl->timestamp, start_dt) <= 0) { - g_date_time_unref(start_dt); - ff_parsed_line_free(pl); - continue; - } - g_date_time_unref(start_dt); - } - } - if (end_time) { - GDateTime* end_dt = g_date_time_new_from_iso8601(end_time, NULL); - if (end_dt) { - if (g_date_time_compare(pl->timestamp, end_dt) >= 0) { - g_date_time_unref(end_dt); - ff_parsed_line_free(pl); - continue; - } - g_date_time_unref(end_dt); - } - } - - // Check JID filter: message must involve contact and our JID - gboolean matches = FALSE; - if (myjid && myjid->barejid && contact_barejid) { - if ((g_strcmp0(pl->from_jid, contact_barejid) == 0) || (g_strcmp0(pl->from_jid, myjid->barejid) == 0)) { - matches = TRUE; - } - } else { - matches = TRUE; // no filter - } - - if (!matches) { - ff_parsed_line_free(pl); - continue; - } - - messages = g_slist_append(messages, pl); - } - - fclose(fp); - return messages; -} - // Apply LMC: for messages with corrects:X, find original and replace its text static void _ff_apply_lmc(GSList* parsed_lines, GSList** result) @@ -435,52 +595,160 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta return DB_RESPONSE_ERROR; } - // If no end_time, use now - auto_gchar gchar* effective_end = NULL; - if (end_time) { - effective_end = g_strdup(end_time); - } else { - GDateTime* now = g_date_time_new_now_local(); - effective_end = g_date_time_format_iso8601(now); - g_date_time_unref(now); - } + // Get or create per-contact state with sparse index + ff_contact_state_t* state = _ff_get_state(contact_barejid); + if (!state) + return DB_RESPONSE_ERROR; - // Get all log files for this contact - GSList* files = _ff_list_log_files(contact_barejid); - if (!files) { + // Ensure index is up-to-date (stat check -> build/extend if file changed) + if (!ff_state_ensure_fresh(state)) return DB_RESPONSE_EMPTY; + if (state->total_lines == 0) + return DB_RESPONSE_EMPTY; + + // Reset cursor for filtered (non-Page-Up) queries + if (start_time) + state->cursor_offset = -1; + + // Determine read byte-range using the sparse index + off_t read_from = state->bom_len; + off_t read_to = state->stamp.size; + + // Use stored cursor for sequential Page Up (skip bisect) + if (!start_time && end_time && state->cursor_offset >= 0) { + read_to = state->cursor_offset; + } else if (end_time) { + // Upper-bound: find first index entry AFTER end_time + GDateTime* edt = g_date_time_new_from_iso8601(end_time, NULL); + if (edt) { + gint64 end_epoch = g_date_time_to_unix(edt); + g_date_time_unref(edt); + size_t lo = 0, hi = state->n_entries; + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + if (state->entries[mid].timestamp_epoch <= end_epoch) + lo = mid + 1; + else + hi = mid; + } + // lo = first entry with timestamp > end_epoch + // Use next entry's offset as read_to (+ 1 entry margin) + if (lo + 1 < state->n_entries) + read_to = state->entries[lo + 1].byte_offset; + // else read_to stays at EOF + } } - // Collect all matching parsed lines across files + if (start_time) { + read_from = ff_state_offset_for_time(state, start_time); + } + + // For "last N messages": back up from read_to using index + if (!from_start) { + int margin_entries = (MESSAGES_TO_RETRIEVE * 3) / FF_INDEX_STEP + 2; + + // Find index entry closest to (but before) read_to + size_t entry_idx = 0; + for (size_t i = 0; i < state->n_entries; i++) { + if (state->entries[i].byte_offset >= read_to) + break; + entry_idx = i; + } + + size_t start_entry = entry_idx > (size_t)margin_entries + ? entry_idx - margin_entries + : 0; + off_t backed = state->entries[start_entry].byte_offset; + if (backed > read_from) + read_from = backed; + } + + // Open file and read lines in [read_from, read_to) + FILE* fp = fopen(state->filepath, "r"); + if (!fp) + return DB_RESPONSE_EMPTY; + + fseek(fp, read_from, SEEK_SET); + + // Pre-parse time filter boundaries once (avoid per-line allocation) + GDateTime* start_dt = start_time ? g_date_time_new_from_iso8601(start_time, NULL) : NULL; + GDateTime* end_dt = end_time ? g_date_time_new_from_iso8601(end_time, NULL) : NULL; + GSList* all_parsed = NULL; - for (GSList* f = files; f; f = f->next) { - GSList* file_msgs = _ff_read_file_messages(f->data, contact_barejid, - start_time, effective_end, myjid); - all_parsed = g_slist_concat(all_parsed, file_msgs); + while (1) { + off_t pos = ftell(fp); + if (pos < 0 || pos >= read_to) + break; + + gboolean truncated = FALSE; + char* buf = ff_readline(fp, &truncated); + if (!buf) + break; + if (truncated) { + free(buf); + break; + } + + ff_parsed_line_t* pl = ff_parse_line(buf); + free(buf); + if (!pl) + continue; + + pl->file_offset = pos; + + // Time filters + if (start_dt && g_date_time_compare(pl->timestamp, start_dt) <= 0) { + ff_parsed_line_free(pl); + continue; + } + if (end_dt && g_date_time_compare(pl->timestamp, end_dt) >= 0) { + ff_parsed_line_free(pl); + continue; + } + + // JID filter + gboolean matches = FALSE; + if (myjid->barejid && contact_barejid) { + if ((g_strcmp0(pl->from_jid, contact_barejid) == 0) + || (g_strcmp0(pl->from_jid, myjid->barejid) == 0)) { + matches = TRUE; + } + } else { + matches = TRUE; + } + if (!matches) { + ff_parsed_line_free(pl); + continue; + } + + all_parsed = g_slist_append(all_parsed, pl); } + fclose(fp); - g_slist_free_full(files, g_free); + if (start_dt) + g_date_time_unref(start_dt); + if (end_dt) + g_date_time_unref(end_dt); - if (!all_parsed) { + if (!all_parsed) return DB_RESPONSE_EMPTY; - } + + // Update cursor: byte offset of oldest parsed message in this batch + ff_parsed_line_t* oldest = all_parsed->data; + state->cursor_offset = oldest->file_offset; // Apply LMC corrections and build ProfMessage list GSList* pre_result = NULL; _ff_apply_lmc(all_parsed, &pre_result); - - // Free parsed lines g_slist_free_full(all_parsed, (GDestroyNotify)ff_parsed_line_free); - if (!pre_result) { + if (!pre_result) return DB_RESPONSE_EMPTY; - } - // Apply pagination: take first or last MESSAGES_TO_RETRIEVE + // Paginate: keep first or last MESSAGES_TO_RETRIEVE guint total = g_slist_length(pre_result); if (total > MESSAGES_TO_RETRIEVE) { if (from_start) { - // Keep first N, free rest GSList* nth = g_slist_nth(pre_result, MESSAGES_TO_RETRIEVE); if (nth) { GSList* prev = g_slist_nth(pre_result, MESSAGES_TO_RETRIEVE - 1); @@ -489,7 +757,6 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta g_slist_free_full(nth, (GDestroyNotify)message_free); } } else { - // Keep last N, free first (total - N) guint skip = total - MESSAGES_TO_RETRIEVE; GSList* keep_start = g_slist_nth(pre_result, skip); GSList* prev = g_slist_nth(pre_result, skip - 1); @@ -500,7 +767,6 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta } } - // Apply flip (reverse order) if (flip) { pre_result = g_slist_reverse(pre_result); } @@ -520,39 +786,32 @@ _flatfile_get_limits_info(const gchar* const contact_barejid, gboolean is_last) return msg; } - GSList* files = _ff_list_log_files(contact_barejid); - if (!files) { + ff_contact_state_t* state = _ff_get_state(contact_barejid); + if (!state) { if (is_last) msg->timestamp = g_date_time_new_now_utc(); return msg; } - // For first message: read first non-comment line of first file - // For last message: read last non-comment line of last file - const char* target_file = is_last ? (const char*)g_slist_last(files)->data - : (const char*)files->data; + if (!ff_state_ensure_fresh(state) || state->total_lines == 0) { + if (is_last) + msg->timestamp = g_date_time_new_now_utc(); + return msg; + } - FILE* fp = fopen(target_file, "r"); + FILE* fp = fopen(state->filepath, "r"); if (!fp) { - g_slist_free_full(files, g_free); if (is_last) msg->timestamp = g_date_time_new_now_utc(); return msg; } - // BOM skip - int c1 = fgetc(fp); - int c2 = fgetc(fp); - int c3 = fgetc(fp); - if (!(c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)) { - fseek(fp, 0, SEEK_SET); - } - - char* buf = NULL; ff_parsed_line_t* found = NULL; if (!is_last) { - // Find first valid line + // First message: start from beginning (skip BOM) + fseek(fp, state->bom_len, SEEK_SET); + char* buf; while ((buf = ff_readline(fp, NULL)) != NULL) { found = ff_parse_line(buf); free(buf); @@ -560,7 +819,13 @@ _flatfile_get_limits_info(const gchar* const contact_barejid, gboolean is_last) break; } } else { - // Find last valid line — read all and keep last + // Last message: seek near end using index for efficiency + off_t seek_pos = 0; + if (state->n_entries > 0) + seek_pos = state->entries[state->n_entries - 1].byte_offset; + fseek(fp, seek_pos, SEEK_SET); + + char* buf; while ((buf = ff_readline(fp, NULL)) != NULL) { ff_parsed_line_t* pl = ff_parse_line(buf); free(buf); @@ -581,7 +846,6 @@ _flatfile_get_limits_info(const gchar* const contact_barejid, gboolean is_last) msg->timestamp = g_date_time_new_now_utc(); } - g_slist_free_full(files, g_free); return msg; } diff --git a/src/database_flatfile.h b/src/database_flatfile.h index febdd311..49c0671e 100644 --- a/src/database_flatfile.h +++ b/src/database_flatfile.h @@ -28,6 +28,7 @@ #include #include +#include #include "database.h" #include "xmpp/xmpp.h" @@ -60,8 +61,44 @@ typedef struct char* from_jid; char* from_resource; char* message; + off_t file_offset; } ff_parsed_line_t; +// --- Sparse index for single-file lookup --- + +#define FF_INDEX_STEP 500 + +typedef struct +{ + off_t byte_offset; + gint64 timestamp_epoch; +} ff_index_entry_t; + +typedef struct +{ + time_t mtime; + off_t size; + ino_t inode; +} ff_file_stamp_t; + +typedef struct +{ + char* filepath; + ff_index_entry_t* entries; + size_t n_entries; + size_t cap_entries; + size_t total_lines; + ff_file_stamp_t stamp; + off_t cursor_offset; + int bom_len; +} ff_contact_state_t; + +// State management +ff_contact_state_t* ff_state_new(const char* filepath); +void ff_state_free(ff_contact_state_t* state); +gboolean ff_state_ensure_fresh(ff_contact_state_t* state); +off_t ff_state_offset_for_time(ff_contact_state_t* state, const char* iso_time); + // --- Type conversion helpers --- const char* ff_get_message_type_str(prof_msg_type_t type); @@ -73,7 +110,7 @@ prof_enc_t ff_get_message_enc_type(const char* const encstr); char* ff_jid_to_dir(const char* jid); char* ff_get_contact_dir(const char* contact_barejid); -char* ff_get_log_path(const char* contact_barejid, GDateTime* dt); +char* ff_get_log_path(const char* contact_barejid); gboolean ff_ensure_dir(const char* path); // --- Escape / unescape --- diff --git a/src/database_flatfile_parser.c b/src/database_flatfile_parser.c index a97927f2..12a553c5 100644 --- a/src/database_flatfile_parser.c +++ b/src/database_flatfile_parser.c @@ -167,17 +167,15 @@ ff_get_contact_dir(const char* contact_barejid) return result; } -// Get the log file path for a contact on a specific date: -// {contact_dir}/{YYYY_MM_DD}.log +// Get the single log file path for a contact: {contact_dir}/history.log char* -ff_get_log_path(const char* contact_barejid, GDateTime* dt) +ff_get_log_path(const char* contact_barejid) { auto_gchar gchar* contact_dir = ff_get_contact_dir(contact_barejid); if (!contact_dir) return NULL; - auto_gchar gchar* date_str = g_date_time_format(dt, "%Y_%m_%d"); - char* result = g_strdup_printf("%s/%s.log", contact_dir, date_str); + char* result = g_strdup_printf("%s/history.log", contact_dir); return result; } @@ -616,6 +614,7 @@ ff_parse_line(const char* line) } ff_parsed_line_t* result = g_malloc0(sizeof(ff_parsed_line_t)); + result->file_offset = -1; // Parse timestamp — everything up to first space followed by '[' char* bracket_start = strchr(work, '['); diff --git a/src/database_flatfile_verify.c b/src/database_flatfile_verify.c index 515b1886..bc304b4b 100644 --- a/src/database_flatfile_verify.c +++ b/src/database_flatfile_verify.c @@ -89,253 +89,185 @@ ff_verify_integrity(const gchar* const contact_barejid) } } - // Verify each contact directory + // Verify each contact directory (single history.log per contact) for (GSList* cd = contact_dirs; cd; cd = cd->next) { const char* cdir_path = cd->data; - GDir* dir = g_dir_open(cdir_path, 0, NULL); - if (!dir) + auto_gchar gchar* filepath = g_strdup_printf("%s/history.log", cdir_path); + if (!g_file_test(filepath, G_FILE_TEST_EXISTS)) continue; - GSList* log_files = NULL; - const gchar* fname; - while ((fname = g_dir_read_name(dir)) != NULL) { - if (g_str_has_suffix(fname, ".log")) { - log_files = g_slist_insert_sorted(log_files, - g_strdup_printf("%s/%s", cdir_path, fname), - (GCompareFunc)g_strcmp0); - } - } - g_dir_close(dir); + const char* basename = "history.log"; - GDateTime* prev_file_last_ts = NULL; - GHashTable* seen_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); - GHashTable* all_stanza_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); - - for (GSList* lf = log_files; lf; lf = lf->next) { - const char* filepath = lf->data; - const char* basename = strrchr(filepath, '/'); - basename = basename ? basename + 1 : filepath; - - // Check file permissions - struct stat st; - if (g_stat(filepath, &st) == 0) { - if ((st.st_mode & 0777) != (S_IRUSR | S_IWUSR)) { - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_WARNING; - issue->file = g_strdup(basename); - issue->line = 0; - issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777); - issues = g_slist_append(issues, issue); - } - } - - FILE* fp = fopen(filepath, "r"); - if (!fp) - continue; - - // BOM check - int c1 = fgetc(fp); - int c2 = fgetc(fp); - int c3 = fgetc(fp); - if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) { - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_INFO; - issue->file = g_strdup(basename); - issue->line = 0; - issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary"); - issues = g_slist_append(issues, issue); - } else { - fseek(fp, 0, SEEK_SET); - } - - char* buf = NULL; - int lineno = 0; - GDateTime* prev_ts = NULL; - GDateTime* first_ts = NULL; - GDateTime* last_ts = NULL; - gboolean has_crlf = FALSE; - gboolean is_empty = TRUE; - - while ((buf = ff_readline(fp, NULL)) != NULL) { - lineno++; - gsize len = strlen(buf); - - // CRLF check - if (len > 0 && buf[len - 1] == '\r') { - has_crlf = TRUE; - buf[--len] = '\0'; - } - - // Skip empty lines and comments - if (len == 0 || buf[0] == '#') { - free(buf); - continue; - } - is_empty = FALSE; - - // UTF-8 validation - const gchar* end; - if (!g_utf8_validate(buf, -1, &end)) { - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_ERROR; - issue->file = g_strdup(basename); - issue->line = lineno; - issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf)); - issues = g_slist_append(issues, issue); - free(buf); - continue; - } - - // Control character check - for (gsize i = 0; i < len; i++) { - unsigned char ch = (unsigned char)buf[i]; - if (ch < 0x20 && ch != '\t') { - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_WARNING; - issue->file = g_strdup(basename); - issue->line = lineno; - issue->message = g_strdup_printf("Contains control character 0x%02x", ch); - issues = g_slist_append(issues, issue); - break; - } - } - - // Parse line - ff_parsed_line_t* pl = ff_parse_line(buf); - if (!pl) { - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_ERROR; - issue->file = g_strdup(basename); - issue->line = lineno; - issue->message = g_strdup("Unparsable line"); - issues = g_slist_append(issues, issue); - free(buf); - continue; - } - - free(buf); // done with raw line - - if (!first_ts) - first_ts = g_date_time_ref(pl->timestamp); - if (last_ts) - g_date_time_unref(last_ts); - last_ts = g_date_time_ref(pl->timestamp); - - // Timestamp order within file - if (prev_ts && g_date_time_compare(pl->timestamp, prev_ts) < 0) { - auto_gchar gchar* ts_cur = g_date_time_format_iso8601(pl->timestamp); - auto_gchar gchar* ts_prev = g_date_time_format_iso8601(prev_ts); - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_WARNING; - issue->file = g_strdup(basename); - issue->line = lineno; - issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev); - issues = g_slist_append(issues, issue); - } - if (prev_ts) - g_date_time_unref(prev_ts); - prev_ts = g_date_time_ref(pl->timestamp); - - // Duplicate stanza-id / archive-id - if (pl->stanza_id && strlen(pl->stanza_id) > 0) { - if (g_hash_table_contains(seen_ids, pl->stanza_id)) { - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_WARNING; - issue->file = g_strdup(basename); - issue->line = lineno; - issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id); - issues = g_slist_append(issues, issue); - } else { - g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno)); - } - g_hash_table_insert(all_stanza_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno)); - } - if (pl->archive_id && strlen(pl->archive_id) > 0) { - if (g_hash_table_contains(seen_ids, pl->archive_id)) { - integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_WARNING; - issue->file = g_strdup(basename); - issue->line = lineno; - issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id); - issues = g_slist_append(issues, issue); - } else { - g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno)); - } - } - - ff_parsed_line_free(pl); - } - - fclose(fp); - - // CRLF warning for file - if (has_crlf) { + // Check file permissions + struct stat st; + if (g_stat(filepath, &st) == 0) { + if ((st.st_mode & 0777) != (S_IRUSR | S_IWUSR)) { integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); issue->level = INTEGRITY_WARNING; issue->file = g_strdup(basename); issue->line = 0; - issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF"); + issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777); issues = g_slist_append(issues, issue); } + } - // Empty file - if (is_empty) { + FILE* fp = fopen(filepath, "r"); + if (!fp) + continue; + + // BOM check + int c1 = fgetc(fp); + int c2 = fgetc(fp); + int c3 = fgetc(fp); + if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) { + integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); + issue->level = INTEGRITY_INFO; + issue->file = g_strdup(basename); + issue->line = 0; + issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary"); + issues = g_slist_append(issues, issue); + } else { + fseek(fp, 0, SEEK_SET); + } + + char* buf = NULL; + int lineno = 0; + GDateTime* prev_ts = NULL; + GHashTable* seen_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + GHashTable* all_stanza_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + gboolean has_crlf = FALSE; + gboolean is_empty = TRUE; + + while ((buf = ff_readline(fp, NULL)) != NULL) { + lineno++; + gsize len = strlen(buf); + + if (len > 0 && buf[len - 1] == '\r') { + has_crlf = TRUE; + buf[--len] = '\0'; + } + + if (len == 0 || buf[0] == '#') { + free(buf); + continue; + } + is_empty = FALSE; + + const gchar* end; + if (!g_utf8_validate(buf, -1, &end)) { integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); - issue->level = INTEGRITY_INFO; + issue->level = INTEGRITY_ERROR; issue->file = g_strdup(basename); - issue->line = 0; - issue->message = g_strdup("File is empty (no message lines)"); + issue->line = lineno; + issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf)); issues = g_slist_append(issues, issue); + free(buf); + continue; } - // Cross-file timestamp ordering - if (first_ts && prev_file_last_ts) { - if (g_date_time_compare(first_ts, prev_file_last_ts) < 0) { - auto_gchar gchar* ts_first = g_date_time_format_iso8601(first_ts); - auto_gchar gchar* ts_prev_last = g_date_time_format_iso8601(prev_file_last_ts); + for (gsize i = 0; i < len; i++) { + unsigned char ch = (unsigned char)buf[i]; + if (ch < 0x20 && ch != '\t') { integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); issue->level = INTEGRITY_WARNING; issue->file = g_strdup(basename); - issue->line = 0; - issue->message = g_strdup_printf("First timestamp (%s) is before previous file's last timestamp (%s)", - ts_first, ts_prev_last); + issue->line = lineno; + issue->message = g_strdup_printf("Contains control character 0x%02x", ch); issues = g_slist_append(issues, issue); + break; } } - if (prev_file_last_ts) - g_date_time_unref(prev_file_last_ts); - prev_file_last_ts = last_ts ? g_date_time_ref(last_ts) : NULL; + ff_parsed_line_t* pl = ff_parse_line(buf); + if (!pl) { + integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); + issue->level = INTEGRITY_ERROR; + issue->file = g_strdup(basename); + issue->line = lineno; + issue->message = g_strdup("Unparsable line"); + issues = g_slist_append(issues, issue); + free(buf); + continue; + } + free(buf); + + // Timestamp ordering + if (prev_ts && g_date_time_compare(pl->timestamp, prev_ts) < 0) { + auto_gchar gchar* ts_cur = g_date_time_format_iso8601(pl->timestamp); + auto_gchar gchar* ts_prev = g_date_time_format_iso8601(prev_ts); + integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); + issue->level = INTEGRITY_WARNING; + issue->file = g_strdup(basename); + issue->line = lineno; + issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev); + issues = g_slist_append(issues, issue); + } if (prev_ts) g_date_time_unref(prev_ts); - if (first_ts) - g_date_time_unref(first_ts); - if (last_ts) - g_date_time_unref(last_ts); + prev_ts = g_date_time_ref(pl->timestamp); + + // Duplicate stanza-id / archive-id + if (pl->stanza_id && strlen(pl->stanza_id) > 0) { + if (g_hash_table_contains(seen_ids, pl->stanza_id)) { + integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); + issue->level = INTEGRITY_WARNING; + issue->file = g_strdup(basename); + issue->line = lineno; + issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id); + issues = g_slist_append(issues, issue); + } else { + g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno)); + } + g_hash_table_insert(all_stanza_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno)); + } + if (pl->archive_id && strlen(pl->archive_id) > 0) { + if (g_hash_table_contains(seen_ids, pl->archive_id)) { + integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); + issue->level = INTEGRITY_WARNING; + issue->file = g_strdup(basename); + issue->line = lineno; + issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id); + issues = g_slist_append(issues, issue); + } else { + g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno)); + } + } + + ff_parsed_line_free(pl); } - // Second pass: check LMC references across all files - for (GSList* lf = log_files; lf; lf = lf->next) { - const char* filepath = lf->data; - const char* basename_lmc = strrchr(filepath, '/'); - basename_lmc = basename_lmc ? basename_lmc + 1 : filepath; + fclose(fp); - FILE* fp = fopen(filepath, "r"); - if (!fp) - continue; + if (has_crlf) { + integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); + issue->level = INTEGRITY_WARNING; + issue->file = g_strdup(basename); + issue->line = 0; + issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF"); + issues = g_slist_append(issues, issue); + } - // Skip BOM + if (is_empty) { + integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); + issue->level = INTEGRITY_INFO; + issue->file = g_strdup(basename); + issue->line = 0; + issue->message = g_strdup("File is empty (no message lines)"); + issues = g_slist_append(issues, issue); + } + + // Second pass: check LMC references + fp = fopen(filepath, "r"); + if (fp) { int b1 = fgetc(fp); int b2 = fgetc(fp); int b3 = fgetc(fp); - if (!(b1 == 0xEF && b2 == 0xBB && b3 == 0xBF)) { + if (!(b1 == 0xEF && b2 == 0xBB && b3 == 0xBF)) fseek(fp, 0, SEEK_SET); - } - char* buf = NULL; - int lineno = 0; + lineno = 0; while ((buf = ff_readline(fp, NULL)) != NULL) { lineno++; gsize len = strlen(buf); @@ -355,7 +287,7 @@ ff_verify_integrity(const gchar* const contact_barejid) if (!g_hash_table_contains(all_stanza_ids, pl->replace_id)) { integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t)); issue->level = INTEGRITY_ERROR; - issue->file = g_strdup(basename_lmc); + issue->file = g_strdup(basename); issue->line = lineno; issue->message = g_strdup_printf("Broken correction reference: corrects:%s not found", pl->replace_id); issues = g_slist_append(issues, issue); @@ -366,11 +298,10 @@ ff_verify_integrity(const gchar* const contact_barejid) fclose(fp); } - if (prev_file_last_ts) - g_date_time_unref(prev_file_last_ts); + if (prev_ts) + g_date_time_unref(prev_ts); g_hash_table_destroy(seen_ids); g_hash_table_destroy(all_stanza_ids); - g_slist_free_full(log_files, g_free); } g_slist_free_full(contact_dirs, g_free);