database_flatfile: single-file storage with sparse index

Replace per-contact directory structure with a single flat file per contact.
Each file uses pipe-delimited records with a sparse byte-offset index
that is rebuilt on demand and cached in memory.

- Rewrite _ff_write_record / _ff_read_records for single-file format
- Add sparse index (every Nth record) for O(log N) seek on history read
- Cursor-based pagination: _ff_get_previous_chat uses saved byte offset
- LMC corrections applied in-place during read (_ff_apply_lmc)
- Newline escaping (\n) in message bodies for line-based storage
- Simplify verify: parse + timestamp order + LMC reference check
- Update parser helpers for new field layout

src/database_flatfile_verify.c

@@ -89,253 +89,185 @@ ff_verify_integrity(const gchar* const contact_barejid)
         }
     }
 
-    // Verify each contact directory
+    // Verify each contact directory (single history.log per contact)
     for (GSList* cd = contact_dirs; cd; cd = cd->next) {
         const char* cdir_path = cd->data;
 
-        GDir* dir = g_dir_open(cdir_path, 0, NULL);
-        if (!dir)
+        auto_gchar gchar* filepath = g_strdup_printf("%s/history.log", cdir_path);
+        if (!g_file_test(filepath, G_FILE_TEST_EXISTS))
             continue;
 
-        GSList* log_files = NULL;
-        const gchar* fname;
-        while ((fname = g_dir_read_name(dir)) != NULL) {
-            if (g_str_has_suffix(fname, ".log")) {
-                log_files = g_slist_insert_sorted(log_files,
-                                                  g_strdup_printf("%s/%s", cdir_path, fname),
-                                                  (GCompareFunc)g_strcmp0);
-            }
-        }
-        g_dir_close(dir);
+        const char* basename = "history.log";
 
-        GDateTime* prev_file_last_ts = NULL;
-        GHashTable* seen_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
-        GHashTable* all_stanza_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
-
-        for (GSList* lf = log_files; lf; lf = lf->next) {
-            const char* filepath = lf->data;
-            const char* basename = strrchr(filepath, '/');
-            basename = basename ? basename + 1 : filepath;
-
-            // Check file permissions
-            struct stat st;
-            if (g_stat(filepath, &st) == 0) {
-                if ((st.st_mode & 0777) != (S_IRUSR | S_IWUSR)) {
-                    integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                    issue->level = INTEGRITY_WARNING;
-                    issue->file = g_strdup(basename);
-                    issue->line = 0;
-                    issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777);
-                    issues = g_slist_append(issues, issue);
-                }
-            }
-
-            FILE* fp = fopen(filepath, "r");
-            if (!fp)
-                continue;
-
-            // BOM check
-            int c1 = fgetc(fp);
-            int c2 = fgetc(fp);
-            int c3 = fgetc(fp);
-            if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
-                integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                issue->level = INTEGRITY_INFO;
-                issue->file = g_strdup(basename);
-                issue->line = 0;
-                issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary");
-                issues = g_slist_append(issues, issue);
-            } else {
-                fseek(fp, 0, SEEK_SET);
-            }
-
-            char* buf = NULL;
-            int lineno = 0;
-            GDateTime* prev_ts = NULL;
-            GDateTime* first_ts = NULL;
-            GDateTime* last_ts = NULL;
-            gboolean has_crlf = FALSE;
-            gboolean is_empty = TRUE;
-
-            while ((buf = ff_readline(fp, NULL)) != NULL) {
-                lineno++;
-                gsize len = strlen(buf);
-
-                // CRLF check
-                if (len > 0 && buf[len - 1] == '\r') {
-                    has_crlf = TRUE;
-                    buf[--len] = '\0';
-                }
-
-                // Skip empty lines and comments
-                if (len == 0 || buf[0] == '#') {
-                    free(buf);
-                    continue;
-                }
-                is_empty = FALSE;
-
-                // UTF-8 validation
-                const gchar* end;
-                if (!g_utf8_validate(buf, -1, &end)) {
-                    integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                    issue->level = INTEGRITY_ERROR;
-                    issue->file = g_strdup(basename);
-                    issue->line = lineno;
-                    issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf));
-                    issues = g_slist_append(issues, issue);
-                    free(buf);
-                    continue;
-                }
-
-                // Control character check
-                for (gsize i = 0; i < len; i++) {
-                    unsigned char ch = (unsigned char)buf[i];
-                    if (ch < 0x20 && ch != '\t') {
-                        integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                        issue->level = INTEGRITY_WARNING;
-                        issue->file = g_strdup(basename);
-                        issue->line = lineno;
-                        issue->message = g_strdup_printf("Contains control character 0x%02x", ch);
-                        issues = g_slist_append(issues, issue);
-                        break;
-                    }
-                }
-
-                // Parse line
-                ff_parsed_line_t* pl = ff_parse_line(buf);
-                if (!pl) {
-                    integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                    issue->level = INTEGRITY_ERROR;
-                    issue->file = g_strdup(basename);
-                    issue->line = lineno;
-                    issue->message = g_strdup("Unparsable line");
-                    issues = g_slist_append(issues, issue);
-                    free(buf);
-                    continue;
-                }
-
-                free(buf); // done with raw line
-
-                if (!first_ts)
-                    first_ts = g_date_time_ref(pl->timestamp);
-                if (last_ts)
-                    g_date_time_unref(last_ts);
-                last_ts = g_date_time_ref(pl->timestamp);
-
-                // Timestamp order within file
-                if (prev_ts && g_date_time_compare(pl->timestamp, prev_ts) < 0) {
-                    auto_gchar gchar* ts_cur = g_date_time_format_iso8601(pl->timestamp);
-                    auto_gchar gchar* ts_prev = g_date_time_format_iso8601(prev_ts);
-                    integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                    issue->level = INTEGRITY_WARNING;
-                    issue->file = g_strdup(basename);
-                    issue->line = lineno;
-                    issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev);
-                    issues = g_slist_append(issues, issue);
-                }
-                if (prev_ts)
-                    g_date_time_unref(prev_ts);
-                prev_ts = g_date_time_ref(pl->timestamp);
-
-                // Duplicate stanza-id / archive-id
-                if (pl->stanza_id && strlen(pl->stanza_id) > 0) {
-                    if (g_hash_table_contains(seen_ids, pl->stanza_id)) {
-                        integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                        issue->level = INTEGRITY_WARNING;
-                        issue->file = g_strdup(basename);
-                        issue->line = lineno;
-                        issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id);
-                        issues = g_slist_append(issues, issue);
-                    } else {
-                        g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
-                    }
-                    g_hash_table_insert(all_stanza_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
-                }
-                if (pl->archive_id && strlen(pl->archive_id) > 0) {
-                    if (g_hash_table_contains(seen_ids, pl->archive_id)) {
-                        integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                        issue->level = INTEGRITY_WARNING;
-                        issue->file = g_strdup(basename);
-                        issue->line = lineno;
-                        issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id);
-                        issues = g_slist_append(issues, issue);
-                    } else {
-                        g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno));
-                    }
-                }
-
-                ff_parsed_line_free(pl);
-            }
-
-            fclose(fp);
-
-            // CRLF warning for file
-            if (has_crlf) {
+        // Check file permissions
+        struct stat st;
+        if (g_stat(filepath, &st) == 0) {
+            if ((st.st_mode & 0777) != (S_IRUSR | S_IWUSR)) {
                 integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
                 issue->level = INTEGRITY_WARNING;
                 issue->file = g_strdup(basename);
                 issue->line = 0;
-                issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF");
+                issue->message = g_strdup_printf("File permissions are %o, expected 600 (sensitive data)", st.st_mode & 0777);
                 issues = g_slist_append(issues, issue);
             }
+        }
 
-            // Empty file
-            if (is_empty) {
+        FILE* fp = fopen(filepath, "r");
+        if (!fp)
+            continue;
+
+        // BOM check
+        int c1 = fgetc(fp);
+        int c2 = fgetc(fp);
+        int c3 = fgetc(fp);
+        if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
+            integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
+            issue->level = INTEGRITY_INFO;
+            issue->file = g_strdup(basename);
+            issue->line = 0;
+            issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary");
+            issues = g_slist_append(issues, issue);
+        } else {
+            fseek(fp, 0, SEEK_SET);
+        }
+
+        char* buf = NULL;
+        int lineno = 0;
+        GDateTime* prev_ts = NULL;
+        GHashTable* seen_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+        GHashTable* all_stanza_ids = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+        gboolean has_crlf = FALSE;
+        gboolean is_empty = TRUE;
+
+        while ((buf = ff_readline(fp, NULL)) != NULL) {
+            lineno++;
+            gsize len = strlen(buf);
+
+            if (len > 0 && buf[len - 1] == '\r') {
+                has_crlf = TRUE;
+                buf[--len] = '\0';
+            }
+
+            if (len == 0 || buf[0] == '#') {
+                free(buf);
+                continue;
+            }
+            is_empty = FALSE;
+
+            const gchar* end;
+            if (!g_utf8_validate(buf, -1, &end)) {
                 integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
-                issue->level = INTEGRITY_INFO;
+                issue->level = INTEGRITY_ERROR;
                 issue->file = g_strdup(basename);
-                issue->line = 0;
-                issue->message = g_strdup("File is empty (no message lines)");
+                issue->line = lineno;
+                issue->message = g_strdup_printf("Invalid UTF-8 at byte offset %ld", (long)(end - buf));
                 issues = g_slist_append(issues, issue);
+                free(buf);
+                continue;
             }
 
-            // Cross-file timestamp ordering
-            if (first_ts && prev_file_last_ts) {
-                if (g_date_time_compare(first_ts, prev_file_last_ts) < 0) {
-                    auto_gchar gchar* ts_first = g_date_time_format_iso8601(first_ts);
-                    auto_gchar gchar* ts_prev_last = g_date_time_format_iso8601(prev_file_last_ts);
+            for (gsize i = 0; i < len; i++) {
+                unsigned char ch = (unsigned char)buf[i];
+                if (ch < 0x20 && ch != '\t') {
                     integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
                     issue->level = INTEGRITY_WARNING;
                     issue->file = g_strdup(basename);
-                    issue->line = 0;
-                    issue->message = g_strdup_printf("First timestamp (%s) is before previous file's last timestamp (%s)",
-                                                     ts_first, ts_prev_last);
+                    issue->line = lineno;
+                    issue->message = g_strdup_printf("Contains control character 0x%02x", ch);
                     issues = g_slist_append(issues, issue);
+                    break;
                 }
             }
 
-            if (prev_file_last_ts)
-                g_date_time_unref(prev_file_last_ts);
-            prev_file_last_ts = last_ts ? g_date_time_ref(last_ts) : NULL;
+            ff_parsed_line_t* pl = ff_parse_line(buf);
+            if (!pl) {
+                integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
+                issue->level = INTEGRITY_ERROR;
+                issue->file = g_strdup(basename);
+                issue->line = lineno;
+                issue->message = g_strdup("Unparsable line");
+                issues = g_slist_append(issues, issue);
+                free(buf);
+                continue;
+            }
 
+            free(buf);
+
+            // Timestamp ordering
+            if (prev_ts && g_date_time_compare(pl->timestamp, prev_ts) < 0) {
+                auto_gchar gchar* ts_cur = g_date_time_format_iso8601(pl->timestamp);
+                auto_gchar gchar* ts_prev = g_date_time_format_iso8601(prev_ts);
+                integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
+                issue->level = INTEGRITY_WARNING;
+                issue->file = g_strdup(basename);
+                issue->line = lineno;
+                issue->message = g_strdup_printf("Timestamp out of order (%s after %s)", ts_cur, ts_prev);
+                issues = g_slist_append(issues, issue);
+            }
             if (prev_ts)
                 g_date_time_unref(prev_ts);
-            if (first_ts)
-                g_date_time_unref(first_ts);
-            if (last_ts)
-                g_date_time_unref(last_ts);
+            prev_ts = g_date_time_ref(pl->timestamp);
+
+            // Duplicate stanza-id / archive-id
+            if (pl->stanza_id && strlen(pl->stanza_id) > 0) {
+                if (g_hash_table_contains(seen_ids, pl->stanza_id)) {
+                    integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
+                    issue->level = INTEGRITY_WARNING;
+                    issue->file = g_strdup(basename);
+                    issue->line = lineno;
+                    issue->message = g_strdup_printf("Duplicate stanza-id \"%s\"", pl->stanza_id);
+                    issues = g_slist_append(issues, issue);
+                } else {
+                    g_hash_table_insert(seen_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
+                }
+                g_hash_table_insert(all_stanza_ids, g_strdup(pl->stanza_id), GINT_TO_POINTER(lineno));
+            }
+            if (pl->archive_id && strlen(pl->archive_id) > 0) {
+                if (g_hash_table_contains(seen_ids, pl->archive_id)) {
+                    integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
+                    issue->level = INTEGRITY_WARNING;
+                    issue->file = g_strdup(basename);
+                    issue->line = lineno;
+                    issue->message = g_strdup_printf("Duplicate archive-id \"%s\"", pl->archive_id);
+                    issues = g_slist_append(issues, issue);
+                } else {
+                    g_hash_table_insert(seen_ids, g_strdup(pl->archive_id), GINT_TO_POINTER(lineno));
+                }
+            }
+
+            ff_parsed_line_free(pl);
         }
 
-        // Second pass: check LMC references across all files
-        for (GSList* lf = log_files; lf; lf = lf->next) {
-            const char* filepath = lf->data;
-            const char* basename_lmc = strrchr(filepath, '/');
-            basename_lmc = basename_lmc ? basename_lmc + 1 : filepath;
+        fclose(fp);
 
-            FILE* fp = fopen(filepath, "r");
-            if (!fp)
-                continue;
+        if (has_crlf) {
+            integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
+            issue->level = INTEGRITY_WARNING;
+            issue->file = g_strdup(basename);
+            issue->line = 0;
+            issue->message = g_strdup("File uses Windows line endings (CRLF) — consider converting to LF");
+            issues = g_slist_append(issues, issue);
+        }
 
-            // Skip BOM
+        if (is_empty) {
+            integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
+            issue->level = INTEGRITY_INFO;
+            issue->file = g_strdup(basename);
+            issue->line = 0;
+            issue->message = g_strdup("File is empty (no message lines)");
+            issues = g_slist_append(issues, issue);
+        }
+
+        // Second pass: check LMC references
+        fp = fopen(filepath, "r");
+        if (fp) {
             int b1 = fgetc(fp);
             int b2 = fgetc(fp);
             int b3 = fgetc(fp);
-            if (!(b1 == 0xEF && b2 == 0xBB && b3 == 0xBF)) {
+            if (!(b1 == 0xEF && b2 == 0xBB && b3 == 0xBF))
                 fseek(fp, 0, SEEK_SET);
-            }
 
-            char* buf = NULL;
-            int lineno = 0;
+            lineno = 0;
             while ((buf = ff_readline(fp, NULL)) != NULL) {
                 lineno++;
                 gsize len = strlen(buf);
@@ -355,7 +287,7 @@ ff_verify_integrity(const gchar* const contact_barejid)
                     if (!g_hash_table_contains(all_stanza_ids, pl->replace_id)) {
                         integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
                         issue->level = INTEGRITY_ERROR;
-                        issue->file = g_strdup(basename_lmc);
+                        issue->file = g_strdup(basename);
                         issue->line = lineno;
                         issue->message = g_strdup_printf("Broken correction reference: corrects:%s not found", pl->replace_id);
                         issues = g_slist_append(issues, issue);
@@ -366,11 +298,10 @@ ff_verify_integrity(const gchar* const contact_barejid)
             fclose(fp);
         }
 
-        if (prev_file_last_ts)
-            g_date_time_unref(prev_file_last_ts);
+        if (prev_ts)
+            g_date_time_unref(prev_ts);
         g_hash_table_destroy(seen_ids);
         g_hash_table_destroy(all_stanza_ids);
-        g_slist_free_full(log_files, g_free);
     }
 
     g_slist_free_full(contact_dirs, g_free);