devs/cproof
1
2
Fork 1
Code Issues 43 Pull Requests 6 Actions Packages Projects 1 Releases Wiki Activity

database_flatfile: single-file storage with sparse index

Browse Source 
Replace per-contact directory structure with a single flat file per contact.
Each file uses pipe-delimited records with a sparse byte-offset index
that is rebuilt on demand and cached in memory.

- Rewrite _ff_write_record / _ff_read_records for single-file format
- Add sparse index (every Nth record) for O(log N) seek on history read
- Cursor-based pagination: _ff_get_previous_chat uses saved byte offset
- LMC corrections applied in-place during read (_ff_apply_lmc)
- Newline escaping (\n) in message bodies for line-based storage
- Simplify verify: parse + timestamp order + LMC reference check
- Update parser helpers for new field layout
This commit is contained in:
jabber.developer2
2026-02-19 16:09:23 +03:00
parent 786117cf62
commit 611fe0e50c
4 changed files with 608 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

578
src/database_flatfile.c
View File

@@ -31,6 +31,8 @@
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <glib.h>
#include <stdio.h>
#include <stdlib.h>
@@ -52,6 +54,261 @@
// =========================================================================
char* g_flatfile_account_jid = NULL;
static GHashTable* g_contact_states = NULL;
// =========================================================================
// Per-contact state / sparse index
// =========================================================================
ff_contact_state_t*
ff_state_new(const char* filepath)
{
ff_contact_state_t* state = g_malloc0(sizeof(ff_contact_state_t));
state->filepath = g_strdup(filepath);
state->cursor_offset = -1;
return state;
}
void
ff_state_free(ff_contact_state_t* state)
{
if (!state)
return;
g_free(state->filepath);
g_free(state->entries);
g_free(state);
}
static gboolean
_ff_state_build(ff_contact_state_t* state)
{
FILE* fp = fopen(state->filepath, "r");
if (!fp)
return FALSE;
int c1 = fgetc(fp);
int c2 = fgetc(fp);
int c3 = fgetc(fp);
if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
state->bom_len = 3;
} else {
state->bom_len = 0;
fseek(fp, 0, SEEK_SET);
}
state->n_entries = 0;
state->total_lines = 0;
size_t msg_count = 0;
while (1) {
off_t pos = ftell(fp);
gboolean truncated = FALSE;
char* buf = ff_readline(fp, &truncated);
if (!buf)
break;
if (truncated) {
free(buf);
break;
}
if (buf[0] == '\0' || buf[0] == '#') {
free(buf);
continue;
}
state->total_lines++;
if (msg_count % FF_INDEX_STEP == 0) {
char* space = strchr(buf, ' ');
if (space) {
char* ts_str = g_strndup(buf, space - buf);
GDateTime* dt = g_date_time_new_from_iso8601(ts_str, NULL);
if (dt) {
if (state->n_entries >= state->cap_entries) {
state->cap_entries = state->cap_entries ? state->cap_entries * 2 : 64;
state->entries = g_realloc(state->entries,
state->cap_entries * sizeof(ff_index_entry_t));
}
state->entries[state->n_entries].byte_offset = pos;
state->entries[state->n_entries].timestamp_epoch = g_date_time_to_unix(dt);
state->n_entries++;
g_date_time_unref(dt);
}
g_free(ts_str);
}
}
msg_count++;
free(buf);
}
fclose(fp);
struct stat st;
if (stat(state->filepath, &st) == 0) {
state->stamp.mtime = st.st_mtime;
state->stamp.size = st.st_size;
state->stamp.inode = st.st_ino;
}
log_debug("flatfile: index built for %s (%zu entries, %zu lines)",
state->filepath, state->n_entries, state->total_lines);
return TRUE;
}
static gboolean
_ff_state_extend(ff_contact_state_t* state)
{
struct stat st;
if (stat(state->filepath, &st) != 0)
return FALSE;
if (st.st_size <= state->stamp.size)
return _ff_state_build(state);
FILE* fp = fopen(state->filepath, "r");
if (!fp)
return FALSE;
fseek(fp, state->stamp.size, SEEK_SET);
if (state->stamp.size > 0) {
fseek(fp, state->stamp.size - 1, SEEK_SET);
int prev = fgetc(fp);
if (prev != '\n' && prev != EOF) {
int ch;
while ((ch = fgetc(fp)) != EOF && ch != '\n') {
}
}
}
size_t msg_count = state->total_lines;
while (1) {
off_t pos = ftell(fp);
gboolean truncated = FALSE;
char* buf = ff_readline(fp, &truncated);
if (!buf)
break;
if (truncated) {
free(buf);
break;
}
if (buf[0] == '\0' || buf[0] == '#') {
free(buf);
continue;
}
state->total_lines++;
if (msg_count % FF_INDEX_STEP == 0) {
char* space = strchr(buf, ' ');
if (space) {
char* ts_str = g_strndup(buf, space - buf);
GDateTime* dt = g_date_time_new_from_iso8601(ts_str, NULL);
if (dt) {
if (state->n_entries >= state->cap_entries) {
state->cap_entries = state->cap_entries ? state->cap_entries * 2 : 64;
state->entries = g_realloc(state->entries,
state->cap_entries * sizeof(ff_index_entry_t));
}
state->entries[state->n_entries].byte_offset = pos;
state->entries[state->n_entries].timestamp_epoch = g_date_time_to_unix(dt);
state->n_entries++;
g_date_time_unref(dt);
}
g_free(ts_str);
}
}
msg_count++;
free(buf);
}
fclose(fp);
state->stamp.mtime = st.st_mtime;
state->stamp.size = st.st_size;
state->stamp.inode = st.st_ino;
log_debug("flatfile: index extended for %s (%zu entries, %zu lines)",
state->filepath, state->n_entries, state->total_lines);
return TRUE;
}
gboolean
ff_state_ensure_fresh(ff_contact_state_t* state)
{
if (!state || !state->filepath)
return FALSE;
struct stat st;
if (stat(state->filepath, &st) != 0) {
state->total_lines = 0;
state->n_entries = 0;
state->stamp.size = 0;
return FALSE;
}
if (state->stamp.size == 0 && state->n_entries == 0)
return _ff_state_build(state);
if (st.st_mtime == state->stamp.mtime
&& st.st_size == state->stamp.size
&& st.st_ino == state->stamp.inode) {
return TRUE;
}
if (st.st_ino == state->stamp.inode && st.st_size > state->stamp.size)
return _ff_state_extend(state);
state->cursor_offset = -1;
return _ff_state_build(state);
}
off_t
ff_state_offset_for_time(ff_contact_state_t* state, const char* iso_time)
{
if (!state || !iso_time || state->n_entries == 0)
return state ? (off_t)state->bom_len : 0;
GDateTime* dt = g_date_time_new_from_iso8601(iso_time, NULL);
if (!dt)
return state->bom_len;
gint64 target = g_date_time_to_unix(dt);
g_date_time_unref(dt);
size_t lo = 0, hi = state->n_entries;
while (lo < hi) {
size_t mid = lo + (hi - lo) / 2;
if (state->entries[mid].timestamp_epoch <= target)
lo = mid + 1;
else
hi = mid;
}
if (lo == 0)
return state->bom_len;
return state->entries[lo - 1].byte_offset;
}
static ff_contact_state_t*
_ff_get_state(const char* contact_barejid)
{
if (!g_contact_states || !contact_barejid)
return NULL;
ff_contact_state_t* state = g_hash_table_lookup(g_contact_states, contact_barejid);
if (state)
return state;
auto_gchar gchar* filepath = ff_get_log_path(contact_barejid);
if (!filepath)
return NULL;
state = ff_state_new(filepath);
g_hash_table_insert(g_contact_states, g_strdup(contact_barejid), state);
return state;
}
// =========================================================================
// Core write logic
@@ -79,6 +336,7 @@ _ff_add_message(const char* type, const char* stanza_id, const char* archive_id,
GDateTime* dt = timestamp ? g_date_time_ref(timestamp) : g_date_time_new_now_local();
auto_gchar gchar* date_fmt = g_date_time_format_iso8601(dt);
g_date_time_unref(dt);
// Determine which JID to use for the log file path (the "other" party)
const char* contact = NULL;
@@ -89,8 +347,7 @@ _ff_add_message(const char* type, const char* stanza_id, const char* archive_id,
contact = from_barejid;
}
auto_gchar gchar* log_path = ff_get_log_path(contact, dt);
g_date_time_unref(dt);
auto_gchar gchar* log_path = ff_get_log_path(contact);
if (!log_path) {
log_error("flatfile: could not determine log path for %s", contact);
@@ -170,6 +427,12 @@ _flatfile_init(ProfAccount* account)
return FALSE;
}
if (g_contact_states) {
g_hash_table_destroy(g_contact_states);
}
g_contact_states = g_hash_table_new_full(g_str_hash, g_str_equal,
g_free, (GDestroyNotify)ff_state_free);
log_info("Initialized flat-file database backend: %s", base_dir);
return TRUE;
}
@@ -177,6 +440,10 @@ _flatfile_init(ProfAccount* account)
static void
_flatfile_close(void)
{
if (g_contact_states) {
g_hash_table_destroy(g_contact_states);
g_contact_states = NULL;
}
g_free(g_flatfile_account_jid);
g_flatfile_account_jid = NULL;
log_debug("flatfile: closed");
@@ -235,113 +502,6 @@ _flatfile_add_outgoing_muc_pm(const char* const id, const char* const barejid,
// Read logic
// =========================================================================
// List all log files for a contact, sorted by filename (date order)
static GSList*
_ff_list_log_files(const char* contact_barejid)
{
auto_gchar gchar* contact_dir = ff_get_contact_dir(contact_barejid);
if (!contact_dir || !g_file_test(contact_dir, G_FILE_TEST_IS_DIR)) {
return NULL;
}
GDir* dir = g_dir_open(contact_dir, 0, NULL);
if (!dir)
return NULL;
GSList* files = NULL;
const gchar* fname;
while ((fname = g_dir_read_name(dir)) != NULL) {
if (g_str_has_suffix(fname, ".log")) {
files = g_slist_insert_sorted(files,
g_strdup_printf("%s/%s", contact_dir, fname),
(GCompareFunc)g_strcmp0);
}
}
g_dir_close(dir);
return files;
}
// Read all parsed lines from a file, applying filters
static GSList*
_ff_read_file_messages(const char* filepath, const char* contact_barejid,
const char* start_time, const char* end_time,
const Jid* myjid)
{
GSList* messages = NULL;
FILE* fp = fopen(filepath, "r");
if (!fp)
return NULL;
// BOM detection
int c1 = fgetc(fp);
int c2 = fgetc(fp);
int c3 = fgetc(fp);
if (!(c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)) {
// Not a BOM, rewind
fseek(fp, 0, SEEK_SET);
}
char* buf = NULL;
gboolean truncated = FALSE;
while ((buf = ff_readline(fp, &truncated)) != NULL) {
if (truncated) {
log_warning("flatfile: truncated line at EOF in %s, skipping", filepath);
free(buf);
break;
}
ff_parsed_line_t* pl = ff_parse_line(buf);
free(buf);
if (!pl)
continue;
// Check timestamp bounds
if (start_time) {
GDateTime* start_dt = g_date_time_new_from_iso8601(start_time, NULL);
if (start_dt) {
if (g_date_time_compare(pl->timestamp, start_dt) <= 0) {
g_date_time_unref(start_dt);
ff_parsed_line_free(pl);
continue;
}
g_date_time_unref(start_dt);
}
}
if (end_time) {
GDateTime* end_dt = g_date_time_new_from_iso8601(end_time, NULL);
if (end_dt) {
if (g_date_time_compare(pl->timestamp, end_dt) >= 0) {
g_date_time_unref(end_dt);
ff_parsed_line_free(pl);
continue;
}
g_date_time_unref(end_dt);
}
}
// Check JID filter: message must involve contact and our JID
gboolean matches = FALSE;
if (myjid && myjid->barejid && contact_barejid) {
if ((g_strcmp0(pl->from_jid, contact_barejid) == 0) || (g_strcmp0(pl->from_jid, myjid->barejid) == 0)) {
matches = TRUE;
}
} else {
matches = TRUE; // no filter
}
if (!matches) {
ff_parsed_line_free(pl);
continue;
}
messages = g_slist_append(messages, pl);
}
fclose(fp);
return messages;
}
// Apply LMC: for messages with corrects:X, find original and replace its text
static void
_ff_apply_lmc(GSList* parsed_lines, GSList** result)
@@ -435,52 +595,160 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta
return DB_RESPONSE_ERROR;
}
// If no end_time, use now
auto_gchar gchar* effective_end = NULL;
if (end_time) {
effective_end = g_strdup(end_time);
} else {
GDateTime* now = g_date_time_new_now_local();
effective_end = g_date_time_format_iso8601(now);
g_date_time_unref(now);
}
// Get or create per-contact state with sparse index
ff_contact_state_t* state = _ff_get_state(contact_barejid);
if (!state)
return DB_RESPONSE_ERROR;
// Get all log files for this contact
GSList* files = _ff_list_log_files(contact_barejid);
if (!files) {
// Ensure index is up-to-date (stat check -> build/extend if file changed)
if (!ff_state_ensure_fresh(state))
return DB_RESPONSE_EMPTY;
if (state->total_lines == 0)
return DB_RESPONSE_EMPTY;
// Reset cursor for filtered (non-Page-Up) queries
if (start_time)
state->cursor_offset = -1;
// Determine read byte-range using the sparse index
off_t read_from = state->bom_len;
off_t read_to = state->stamp.size;
// Use stored cursor for sequential Page Up (skip bisect)
if (!start_time && end_time && state->cursor_offset >= 0) {
read_to = state->cursor_offset;
} else if (end_time) {
// Upper-bound: find first index entry AFTER end_time
GDateTime* edt = g_date_time_new_from_iso8601(end_time, NULL);
if (edt) {
gint64 end_epoch = g_date_time_to_unix(edt);
g_date_time_unref(edt);
size_t lo = 0, hi = state->n_entries;
while (lo < hi) {
size_t mid = lo + (hi - lo) / 2;
if (state->entries[mid].timestamp_epoch <= end_epoch)
lo = mid + 1;
else
hi = mid;
}
// lo = first entry with timestamp > end_epoch
// Use next entry's offset as read_to (+ 1 entry margin)
if (lo + 1 < state->n_entries)
read_to = state->entries[lo + 1].byte_offset;
// else read_to stays at EOF
}
}
// Collect all matching parsed lines across files
if (start_time) {
read_from = ff_state_offset_for_time(state, start_time);
}
// For "last N messages": back up from read_to using index
if (!from_start) {
int margin_entries = (MESSAGES_TO_RETRIEVE * 3) / FF_INDEX_STEP + 2;
// Find index entry closest to (but before) read_to
size_t entry_idx = 0;
for (size_t i = 0; i < state->n_entries; i++) {
if (state->entries[i].byte_offset >= read_to)
break;
entry_idx = i;
}
size_t start_entry = entry_idx > (size_t)margin_entries
? entry_idx - margin_entries
: 0;
off_t backed = state->entries[start_entry].byte_offset;
if (backed > read_from)
read_from = backed;
}
// Open file and read lines in [read_from, read_to)
FILE* fp = fopen(state->filepath, "r");
if (!fp)
return DB_RESPONSE_EMPTY;
fseek(fp, read_from, SEEK_SET);
// Pre-parse time filter boundaries once (avoid per-line allocation)
GDateTime* start_dt = start_time ? g_date_time_new_from_iso8601(start_time, NULL) : NULL;
GDateTime* end_dt = end_time ? g_date_time_new_from_iso8601(end_time, NULL) : NULL;
GSList* all_parsed = NULL;
for (GSList* f = files; f; f = f->next) {
GSList* file_msgs = _ff_read_file_messages(f->data, contact_barejid,
start_time, effective_end, myjid);
all_parsed = g_slist_concat(all_parsed, file_msgs);
while (1) {
off_t pos = ftell(fp);
if (pos < 0 || pos >= read_to)
break;
gboolean truncated = FALSE;
char* buf = ff_readline(fp, &truncated);
if (!buf)
break;
if (truncated) {
free(buf);
break;
}
ff_parsed_line_t* pl = ff_parse_line(buf);
free(buf);
if (!pl)
continue;
pl->file_offset = pos;
// Time filters
if (start_dt && g_date_time_compare(pl->timestamp, start_dt) <= 0) {
ff_parsed_line_free(pl);
continue;
}
if (end_dt && g_date_time_compare(pl->timestamp, end_dt) >= 0) {
ff_parsed_line_free(pl);
continue;
}
// JID filter
gboolean matches = FALSE;
if (myjid->barejid && contact_barejid) {
if ((g_strcmp0(pl->from_jid, contact_barejid) == 0)
|| (g_strcmp0(pl->from_jid, myjid->barejid) == 0)) {
matches = TRUE;
}
} else {
matches = TRUE;
}
if (!matches) {
ff_parsed_line_free(pl);
continue;
}
all_parsed = g_slist_append(all_parsed, pl);
}
fclose(fp);
g_slist_free_full(files, g_free);
if (start_dt)
g_date_time_unref(start_dt);
if (end_dt)
g_date_time_unref(end_dt);
if (!all_parsed) {
if (!all_parsed)
return DB_RESPONSE_EMPTY;
}
// Update cursor: byte offset of oldest parsed message in this batch
ff_parsed_line_t* oldest = all_parsed->data;
state->cursor_offset = oldest->file_offset;
// Apply LMC corrections and build ProfMessage list
GSList* pre_result = NULL;
_ff_apply_lmc(all_parsed, &pre_result);
// Free parsed lines
g_slist_free_full(all_parsed, (GDestroyNotify)ff_parsed_line_free);
if (!pre_result) {
if (!pre_result)
return DB_RESPONSE_EMPTY;
}
// Apply pagination: take first or last MESSAGES_TO_RETRIEVE
// Paginate: keep first or last MESSAGES_TO_RETRIEVE
guint total = g_slist_length(pre_result);
if (total > MESSAGES_TO_RETRIEVE) {
if (from_start) {
// Keep first N, free rest
GSList* nth = g_slist_nth(pre_result, MESSAGES_TO_RETRIEVE);
if (nth) {
GSList* prev = g_slist_nth(pre_result, MESSAGES_TO_RETRIEVE - 1);
@@ -489,7 +757,6 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta
g_slist_free_full(nth, (GDestroyNotify)message_free);
}
} else {
// Keep last N, free first (total - N)
guint skip = total - MESSAGES_TO_RETRIEVE;
GSList* keep_start = g_slist_nth(pre_result, skip);
GSList* prev = g_slist_nth(pre_result, skip - 1);
@@ -500,7 +767,6 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta
}
}
// Apply flip (reverse order)
if (flip) {
pre_result = g_slist_reverse(pre_result);
}
@@ -520,39 +786,32 @@ _flatfile_get_limits_info(const gchar* const contact_barejid, gboolean is_last)
return msg;
}
GSList* files = _ff_list_log_files(contact_barejid);
if (!files) {
ff_contact_state_t* state = _ff_get_state(contact_barejid);
if (!state) {
if (is_last)
msg->timestamp = g_date_time_new_now_utc();
return msg;
}
// For first message: read first non-comment line of first file
// For last message: read last non-comment line of last file
const char* target_file = is_last ? (const char*)g_slist_last(files)->data
: (const char*)files->data;
if (!ff_state_ensure_fresh(state) || state->total_lines == 0) {
if (is_last)
msg->timestamp = g_date_time_new_now_utc();
return msg;
}
FILE* fp = fopen(target_file, "r");
FILE* fp = fopen(state->filepath, "r");
if (!fp) {
g_slist_free_full(files, g_free);
if (is_last)
msg->timestamp = g_date_time_new_now_utc();
return msg;
}
// BOM skip
int c1 = fgetc(fp);
int c2 = fgetc(fp);
int c3 = fgetc(fp);
if (!(c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)) {
fseek(fp, 0, SEEK_SET);
}
char* buf = NULL;
ff_parsed_line_t* found = NULL;
if (!is_last) {
// Find first valid line
// First message: start from beginning (skip BOM)
fseek(fp, state->bom_len, SEEK_SET);
char* buf;
while ((buf = ff_readline(fp, NULL)) != NULL) {
found = ff_parse_line(buf);
free(buf);
@@ -560,7 +819,13 @@ _flatfile_get_limits_info(const gchar* const contact_barejid, gboolean is_last)
break;
}
} else {
// Find last valid line — read all and keep last
// Last message: seek near end using index for efficiency
off_t seek_pos = 0;
if (state->n_entries > 0)
seek_pos = state->entries[state->n_entries - 1].byte_offset;
fseek(fp, seek_pos, SEEK_SET);
char* buf;
while ((buf = ff_readline(fp, NULL)) != NULL) {
ff_parsed_line_t* pl = ff_parse_line(buf);
free(buf);
@@ -581,7 +846,6 @@ _flatfile_get_limits_info(const gchar* const contact_barejid, gboolean is_last)
msg->timestamp = g_date_time_new_now_utc();
}
g_slist_free_full(files, g_free);
return msg;
}

39
src/database_flatfile.h
View File

@@ -28,6 +28,7 @@
#include <glib.h>
#include <stdio.h>
#include <sys/types.h>
#include "database.h"
#include "xmpp/xmpp.h"
@@ -60,8 +61,44 @@ typedef struct
char* from_jid;
char* from_resource;
char* message;
off_t file_offset;
} ff_parsed_line_t;
// --- Sparse index for single-file lookup ---
#define FF_INDEX_STEP 500
typedef struct
{
off_t byte_offset;
gint64 timestamp_epoch;
} ff_index_entry_t;
typedef struct
{
time_t mtime;
off_t size;
ino_t inode;
} ff_file_stamp_t;
typedef struct
{
char* filepath;
ff_index_entry_t* entries;
size_t n_entries;
size_t cap_entries;
size_t total_lines;
ff_file_stamp_t stamp;
off_t cursor_offset;
int bom_len;
} ff_contact_state_t;
// State management
ff_contact_state_t* ff_state_new(const char* filepath);
void ff_state_free(ff_contact_state_t* state);
gboolean ff_state_ensure_fresh(ff_contact_state_t* state);
off_t ff_state_offset_for_time(ff_contact_state_t* state, const char* iso_time);
// --- Type conversion helpers ---
const char* ff_get_message_type_str(prof_msg_type_t type);
@@ -73,7 +110,7 @@ prof_enc_t ff_get_message_enc_type(const char* const encstr);
char* ff_jid_to_dir(const char* jid);
char* ff_get_contact_dir(const char* contact_barejid);
char* ff_get_log_path(const char* contact_barejid, GDateTime* dt);
char* ff_get_log_path(const char* contact_barejid);
gboolean ff_ensure_dir(const char* path);
// --- Escape / unescape ---

9
src/database_flatfile_parser.c
View File

@@ -167,17 +167,15 @@ ff_get_contact_dir(const char* contact_barejid)
return result;
}
// Get the log file path for a contact on a specific date:
// {contact_dir}/{YYYY_MM_DD}.log
// Get the single log file path for a contact: {contact_dir}/history.log
char*
ff_get_log_path(const char* contact_barejid, GDateTime* dt)
ff_get_log_path(const char* contact_barejid)
{
auto_gchar gchar* contact_dir = ff_get_contact_dir(contact_barejid);
if (!contact_dir)
return NULL;
auto_gchar gchar* date_str = g_date_time_format(dt, "%Y_%m_%d");
char* result = g_strdup_printf("%s/%s.log", contact_dir, date_str);
char* result = g_strdup_printf("%s/history.log", contact_dir);
return result;
}
@@ -616,6 +614,7 @@ ff_parse_line(const char* line)
}
ff_parsed_line_t* result = g_malloc0(sizeof(ff_parsed_line_t));
result->file_offset = -1;
// Parse timestamp — everything up to first space followed by '['
char* bracket_start = strchr(work, '[');

359
src/database_flatfile_verify.c
View File

@@ -89,253 +89,185 @@ ff_verify_integrity(const gchar* const contact_barejid)
}
}
// Verify each contact directory
// Verify each contact directory (single history.log per contact)
for (GSList* cd = contact_dirs; cd; cd = cd->next) {
const char* cdir_path = cd->data;
GDir* dir = g_dir_open(cdir_path, 0, NULL);
if (!dir)
auto_gchar gchar* filepath = g_strdup_printf("%s/history.log", cdir_path);
if (!g_file_test(filepath, G_FILE_TEST_EXISTS))
continue;
GSList* log_files = NULL;
const gchar* fname;
while ((fname = g_dir_read_name(dir)) != NULL) {
if (g_str_has_suffix(fname, ".log")) {
log_files = g_slist_insert_sorted(log_files,
g_strdup_printf("%s/%s", cdir_path, fname),
(GCompareFunc)g_strcmp0);
}
}
g_dir_close(dir);
const char* basename = "history.log";
GDateTime* prev_file_last_ts = NULL;
GHashTable* seen_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
GHashTable* all_stanza_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
for (GSList* lf = log_files; lf; lf = lf->next) {
const char* filepath = lf->data;
const char* basename = strrchr(filepath, '/');
basename = basename ? basename + 1 : filepath;
// Check file permissions
struct stat st;
if (g_stat(filepath, &st) == 0) {
if ((st.st_mode & 0777) != (S_IRUSR | S_IWUSR)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777);
issues = g_slist_append(issues, issue);
}
}
FILE* fp = fopen(filepath, "r");
if (!fp)
continue;
// BOM check
int c1 = fgetc(fp);
int c2 = fgetc(fp);
int c3 = fgetc(fp);
if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_INFO;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary");
issues = g_slist_append(issues, issue);
} else {
fseek(fp, 0, SEEK_SET);
}
char* buf = NULL;
int lineno = 0;
GDateTime* prev_ts = NULL;
GDateTime* first_ts = NULL;
GDateTime* last_ts = NULL;
gboolean has_crlf = FALSE;
gboolean is_empty = TRUE;
while ((buf = ff_readline(fp, NULL)) != NULL) {
lineno++;
gsize len = strlen(buf);
// CRLF check
if (len > 0 && buf[len - 1] == '\r') {
has_crlf = TRUE;
buf[--len] = '\0';
}
// Skip empty lines and comments
if (len == 0 || buf[0] == '#') {
free(buf);
continue;
}
is_empty = FALSE;
// UTF-8 validation
const gchar* end;
if (!g_utf8_validate(buf, -1, &end)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_ERROR;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf));
issues = g_slist_append(issues, issue);
free(buf);
continue;
}
// Control character check
for (gsize i = 0; i < len; i++) {
unsigned char ch = (unsigned char)buf[i];
if (ch < 0x20 && ch != '\t') {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Contains control character 0x%02x", ch);
issues = g_slist_append(issues, issue);
break;
}
}
// Parse line
ff_parsed_line_t* pl = ff_parse_line(buf);
if (!pl) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_ERROR;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup("Unparsable line");
issues = g_slist_append(issues, issue);
free(buf);
continue;
}
free(buf); // done with raw line
if (!first_ts)
first_ts = g_date_time_ref(pl->timestamp);
if (last_ts)
g_date_time_unref(last_ts);
last_ts = g_date_time_ref(pl->timestamp);
// Timestamp order within file
if (prev_ts && g_date_time_compare(pl->timestamp, prev_ts) < 0) {
auto_gchar gchar* ts_cur = g_date_time_format_iso8601(pl->timestamp);
auto_gchar gchar* ts_prev = g_date_time_format_iso8601(prev_ts);
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev);
issues = g_slist_append(issues, issue);
}
if (prev_ts)
g_date_time_unref(prev_ts);
prev_ts = g_date_time_ref(pl->timestamp);
// Duplicate stanza-id / archive-id
if (pl->stanza_id && strlen(pl->stanza_id) > 0) {
if (g_hash_table_contains(seen_ids, pl->stanza_id)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id);
issues = g_slist_append(issues, issue);
} else {
g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
}
g_hash_table_insert(all_stanza_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
}
if (pl->archive_id && strlen(pl->archive_id) > 0) {
if (g_hash_table_contains(seen_ids, pl->archive_id)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id);
issues = g_slist_append(issues, issue);
} else {
g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno));
}
}
ff_parsed_line_free(pl);
}
fclose(fp);
// CRLF warning for file
if (has_crlf) {
// Check file permissions
struct stat st;
if (g_stat(filepath, &st) == 0) {
if ((st.st_mode & 0777) != (S_IRUSR | S_IWUSR)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF");
issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777);
issues = g_slist_append(issues, issue);
}
}
// Empty file
if (is_empty) {
FILE* fp = fopen(filepath, "r");
if (!fp)
continue;
// BOM check
int c1 = fgetc(fp);
int c2 = fgetc(fp);
int c3 = fgetc(fp);
if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_INFO;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary");
issues = g_slist_append(issues, issue);
} else {
fseek(fp, 0, SEEK_SET);
}
char* buf = NULL;
int lineno = 0;
GDateTime* prev_ts = NULL;
GHashTable* seen_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
GHashTable* all_stanza_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
gboolean has_crlf = FALSE;
gboolean is_empty = TRUE;
while ((buf = ff_readline(fp, NULL)) != NULL) {
lineno++;
gsize len = strlen(buf);
if (len > 0 && buf[len - 1] == '\r') {
has_crlf = TRUE;
buf[--len] = '\0';
}
if (len == 0 || buf[0] == '#') {
free(buf);
continue;
}
is_empty = FALSE;
const gchar* end;
if (!g_utf8_validate(buf, -1, &end)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_INFO;
issue->level = INTEGRITY_ERROR;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup("File is empty (no message lines)");
issue->line = lineno;
issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf));
issues = g_slist_append(issues, issue);
free(buf);
continue;
}
// Cross-file timestamp ordering
if (first_ts && prev_file_last_ts) {
if (g_date_time_compare(first_ts, prev_file_last_ts) < 0) {
auto_gchar gchar* ts_first = g_date_time_format_iso8601(first_ts);
auto_gchar gchar* ts_prev_last = g_date_time_format_iso8601(prev_file_last_ts);
for (gsize i = 0; i < len; i++) {
unsigned char ch = (unsigned char)buf[i];
if (ch < 0x20 && ch != '\t') {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup_printf("First timestamp (%s) is before previous file's last timestamp (%s)",
ts_first, ts_prev_last);
issue->line = lineno;
issue->message = g_strdup_printf("Contains control character 0x%02x", ch);
issues = g_slist_append(issues, issue);
break;
}
}
if (prev_file_last_ts)
g_date_time_unref(prev_file_last_ts);
prev_file_last_ts = last_ts ? g_date_time_ref(last_ts) : NULL;
ff_parsed_line_t* pl = ff_parse_line(buf);
if (!pl) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_ERROR;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup("Unparsable line");
issues = g_slist_append(issues, issue);
free(buf);
continue;
}
free(buf);
// Timestamp ordering
if (prev_ts && g_date_time_compare(pl->timestamp, prev_ts) < 0) {
auto_gchar gchar* ts_cur = g_date_time_format_iso8601(pl->timestamp);
auto_gchar gchar* ts_prev = g_date_time_format_iso8601(prev_ts);
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev);
issues = g_slist_append(issues, issue);
}
if (prev_ts)
g_date_time_unref(prev_ts);
if (first_ts)
g_date_time_unref(first_ts);
if (last_ts)
g_date_time_unref(last_ts);
prev_ts = g_date_time_ref(pl->timestamp);
// Duplicate stanza-id / archive-id
if (pl->stanza_id && strlen(pl->stanza_id) > 0) {
if (g_hash_table_contains(seen_ids, pl->stanza_id)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id);
issues = g_slist_append(issues, issue);
} else {
g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
}
g_hash_table_insert(all_stanza_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
}
if (pl->archive_id && strlen(pl->archive_id) > 0) {
if (g_hash_table_contains(seen_ids, pl->archive_id)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id);
issues = g_slist_append(issues, issue);
} else {
g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno));
}
}
ff_parsed_line_free(pl);
}
// Second pass: check LMC references across all files
for (GSList* lf = log_files; lf; lf = lf->next) {
const char* filepath = lf->data;
const char* basename_lmc = strrchr(filepath, '/');
basename_lmc = basename_lmc ? basename_lmc + 1 : filepath;
fclose(fp);
FILE* fp = fopen(filepath, "r");
if (!fp)
continue;
if (has_crlf) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_WARNING;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF");
issues = g_slist_append(issues, issue);
}
// Skip BOM
if (is_empty) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_INFO;
issue->file = g_strdup(basename);
issue->line = 0;
issue->message = g_strdup("File is empty (no message lines)");
issues = g_slist_append(issues, issue);
}
// Second pass: check LMC references
fp = fopen(filepath, "r");
if (fp) {
int b1 = fgetc(fp);
int b2 = fgetc(fp);
int b3 = fgetc(fp);
if (!(b1 == 0xEF && b2 == 0xBB && b3 == 0xBF)) {
if (!(b1 == 0xEF && b2 == 0xBB && b3 == 0xBF))
fseek(fp, 0, SEEK_SET);
}
char* buf = NULL;
int lineno = 0;
lineno = 0;
while ((buf = ff_readline(fp, NULL)) != NULL) {
lineno++;
gsize len = strlen(buf);
@@ -355,7 +287,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
if (!g_hash_table_contains(all_stanza_ids, pl->replace_id)) {
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
issue->level = INTEGRITY_ERROR;
issue->file = g_strdup(basename_lmc);
issue->file = g_strdup(basename);
issue->line = lineno;
issue->message = g_strdup_printf("Broken correction reference: corrects:%s not found", pl->replace_id);
issues = g_slist_append(issues, issue);
@@ -366,11 +298,10 @@ ff_verify_integrity(const gchar* const contact_barejid)
fclose(fp);
}
if (prev_file_last_ts)
g_date_time_unref(prev_file_last_ts);
if (prev_ts)
g_date_time_unref(prev_ts);
g_hash_table_destroy(seen_ids);
g_hash_table_destroy(all_stanza_ids);
g_slist_free_full(log_files, g_free);
}
g_slist_free_full(contact_dirs, g_free);