// SPDX-License-Identifier: GPL-3.0-or-later // // This file is part of CProof. // See LICENSE for the full GPLv3 text and the special OpenSSL linking exception. /* * gen_history.c * vim: expandtab:ts=4:sts=4:sw=4 * * Synthetic history generator for the flat-file backend bench harness. * Writes deterministic, ff_write_line-compatible logs covering: * - configurable line counts, contact counts, year span; * - mixed message-length profiles (tiny/medium/long/paste-bomb/extreme); * - stanza-id strategies (uuid / libpurple-incremental / conversations / mixed); * - configurable LMC rate and MAM out-of-order rate; * - resource diversity per contact. * * Output layout (canonical, matches files_get_data_path/jid_to_dir): * /flatlog///history.log * /manifest.txt # one line per generated file with size + line count * * CLI: * gen_history [options] * * Options: * --account=JID account-side JID for path layout (default bench@bench.example) * --lines=N total lines to generate (default 10000) * --contacts=K distinct contact JIDs (default 1) * --years=Y span (default 1, ts spread over Y years ending now) * --seed=S RNG seed (default 42) * --stanza-id={uuid|libpurple|conversations|mixed} default uuid * --lmc-rate=PCT percent of lines that are LMC corrections (0..50) * --mam-ooo-rate=PCT percent of lines with intentionally non-monotonic ts (0..50) * --resources-per-contact=R default 3 * --msg-len-profile={short|mixed|long|extreme} default mixed * --output=DIR output directory (default /tmp/cproof-bench-corpus) * --quiet suppress progress prints * * Exit code: 0 on success, non-zero on error. */ #include "config.h" #include #include #include #include #include #include #include #include #include #include "database_flatfile.h" // --------------------------------------------------------------------------- // CLI options typedef enum { SID_UUID, SID_LIBPURPLE, SID_CONVERSATIONS, SID_MIXED, } sid_mode_t; typedef enum { LEN_SHORT, LEN_MIXED, LEN_LONG, LEN_EXTREME, } len_profile_t; typedef struct { int64_t lines; int contacts; int years; uint64_t seed; sid_mode_t sid_mode; int lmc_rate; int ooo_rate; int resources_per_contact; len_profile_t len_profile; const char* output_dir; const char* account_jid; int quiet; } opts_t; static void opts_default(opts_t* o) { o->lines = 10000; o->contacts = 1; o->years = 1; o->seed = 42; o->sid_mode = SID_UUID; o->lmc_rate = 3; o->ooo_rate = 0; o->resources_per_contact = 3; o->len_profile = LEN_MIXED; o->output_dir = "/tmp/cproof-bench-corpus"; o->account_jid = "bench@bench.example"; o->quiet = 0; } static int parse_sid_mode(const char* s, sid_mode_t* out) { if (g_strcmp0(s, "uuid") == 0) { *out = SID_UUID; return 0; } if (g_strcmp0(s, "libpurple") == 0) { *out = SID_LIBPURPLE; return 0; } if (g_strcmp0(s, "conversations") == 0) { *out = SID_CONVERSATIONS; return 0; } if (g_strcmp0(s, "mixed") == 0) { *out = SID_MIXED; return 0; } return -1; } static int parse_len_profile(const char* s, len_profile_t* out) { if (g_strcmp0(s, "short") == 0) { *out = LEN_SHORT; return 0; } if (g_strcmp0(s, "mixed") == 0) { *out = LEN_MIXED; return 0; } if (g_strcmp0(s, "long") == 0) { *out = LEN_LONG; return 0; } if (g_strcmp0(s, "extreme") == 0) { *out = LEN_EXTREME; return 0; } return -1; } static int parse_args(int argc, char** argv, opts_t* o) { opts_default(o); for (int i = 1; i < argc; i++) { const char* a = argv[i]; if (strncmp(a, "--lines=", 8) == 0) { o->lines = strtoll(a + 8, NULL, 10); } else if (strncmp(a, "--contacts=", 11) == 0) { o->contacts = atoi(a + 11); } else if (strncmp(a, "--years=", 8) == 0) { o->years = atoi(a + 8); } else if (strncmp(a, "--seed=", 7) == 0) { o->seed = strtoull(a + 7, NULL, 10); } else if (strncmp(a, "--stanza-id=", 12) == 0) { if (parse_sid_mode(a + 12, &o->sid_mode) != 0) { fprintf(stderr, "bad --stanza-id\n"); return -1; } } else if (strncmp(a, "--lmc-rate=", 11) == 0) { o->lmc_rate = atoi(a + 11); } else if (strncmp(a, "--mam-ooo-rate=", 15) == 0) { o->ooo_rate = atoi(a + 15); } else if (strncmp(a, "--resources-per-contact=", 24) == 0) { o->resources_per_contact = atoi(a + 24); } else if (strncmp(a, "--msg-len-profile=", 18) == 0) { if (parse_len_profile(a + 18, &o->len_profile) != 0) { fprintf(stderr, "bad --msg-len-profile\n"); return -1; } } else if (strncmp(a, "--output=", 9) == 0) { o->output_dir = a + 9; } else if (strncmp(a, "--account=", 10) == 0) { o->account_jid = a + 10; } else if (strcmp(a, "--quiet") == 0) { o->quiet = 1; } else if (strcmp(a, "--help") == 0 || strcmp(a, "-h") == 0) { return 1; } else { fprintf(stderr, "unknown option: %s\n", a); return -1; } } if (o->lines <= 0 || o->contacts <= 0 || o->years <= 0) return -1; if (o->lmc_rate < 0 || o->lmc_rate > 50) return -1; if (o->ooo_rate < 0 || o->ooo_rate > 50) return -1; if (o->resources_per_contact <= 0) return -1; return 0; } static void print_help(void) { fputs( "Usage: gen_history [options]\n" " --lines=N total lines (default 10000)\n" " --contacts=K contact count (default 1)\n" " --years=Y year span (default 1)\n" " --seed=S RNG seed (default 42)\n" " --stanza-id={uuid|libpurple|conversations|mixed}\n" " --lmc-rate=PCT 0..50, default 3\n" " --mam-ooo-rate=PCT 0..50, default 0\n" " --resources-per-contact=R default 3\n" " --msg-len-profile={short|mixed|long|extreme}\n" " --output=DIR default /tmp/cproof-bench-corpus\n" " --account=JID account JID (default bench@bench.example)\n" " --quiet suppress progress prints\n", stderr); } // --------------------------------------------------------------------------- // Body / stanza-id / resource generators (deterministic) static const char* MSG_BANK_TINY[] = { "ok", "yes", "no", "ack", "thx", "ttyl", "k", "lol", "+1", "yep", "nope", "sure", "got it", "great", "afk", "brb", "bye", "hi", "hello", "hey", }; static const int MSG_BANK_TINY_N = sizeof(MSG_BANK_TINY) / sizeof(MSG_BANK_TINY[0]); static const char* MSG_BANK_MEDIUM[] = { "Quick reminder, the meeting starts at 14:00 in conf room B.", "Pushed the patch to the staging branch, please review when you get a chance.", "Ran into a weird issue with the parser — investigating.", "Backups completed for the night, all green on the dashboard.", "Yeah I think the second approach is cleaner — let's go with that.", "Не могу подключиться к серверу с обеда, кто-то ещё видит проблему?", "私もそう思います。ところで、明日の予定はどうしますか?", "Just FYI: the migration will run during the maintenance window tonight.", }; static const int MSG_BANK_MEDIUM_N = sizeof(MSG_BANK_MEDIUM) / sizeof(MSG_BANK_MEDIUM[0]); // Linear-congruential — we don't need cryptographic strength, we need fast and // reproducible. xorshift64 hits both. static uint64_t xorshift64(uint64_t* s) { uint64_t x = *s; x ^= x << 13; x ^= x >> 7; x ^= x << 17; *s = x; return x; } static int rng_int(uint64_t* s, int min_inc, int max_exc) { if (max_exc <= min_inc) return min_inc; return min_inc + (int)(xorshift64(s) % (uint64_t)(max_exc - min_inc)); } static void fill_filler_text(char* buf, size_t n, uint64_t* rng) { static const char alpha[] = "abcdefghijklmnopqrstuvwxyz0123456789 "; static const int alpha_n = sizeof(alpha) - 1; for (size_t i = 0; i < n; i++) buf[i] = alpha[xorshift64(rng) % alpha_n]; if (n > 0) buf[n - 1] = '\0'; } // Pick a body length per the profile distribution (matches table in REVIEW.txt // Phase 9 plan section 2): // tiny 50% (5-100 B) long 8% (500-5000 B) extreme 0.5% (100KB-1MB) // medium 40% (100-500 B) paste 1.5%(5KB-100KB) static size_t pick_body_len(uint64_t* rng, len_profile_t profile) { int r = (int)(xorshift64(rng) % 1000); if (profile == LEN_SHORT) { return 5 + (xorshift64(rng) % 96); } if (profile == LEN_LONG) { if (r < 200) return 5 + (xorshift64(rng) % 96); if (r < 600) return 100 + (xorshift64(rng) % 400); if (r < 850) return 1000 + (xorshift64(rng) % 9000); // 1-10 KB if (r < 980) return 10000 + (xorshift64(rng) % 90000); // 10-100 KB return 100000 + (xorshift64(rng) % 900000); // 100KB-1MB } if (profile == LEN_EXTREME) { if (r < 100) return 5 + (xorshift64(rng) % 96); if (r < 400) return 100000 + (xorshift64(rng) % 900000); // 100KB-1MB if (r < 800) return 1000000 + (xorshift64(rng) % 4000000); // 1-5 MB return 5000000 + (xorshift64(rng) % 4000000); // 5-9 MB } // LEN_MIXED — realistic distribution if (r < 500) return 5 + (xorshift64(rng) % 96); if (r < 900) return 100 + (xorshift64(rng) % 400); if (r < 980) return 500 + (xorshift64(rng) % 4500); // 500B-5KB if (r < 995) return 5000 + (xorshift64(rng) % 95000); // 5-100 KB return 100000 + (xorshift64(rng) % 900000); // 100KB-1MB } static char* make_body(uint64_t* rng, size_t target_len) { if (target_len < 100) { // Pick from tiny bank, optionally pad with filler const char* base = MSG_BANK_TINY[xorshift64(rng) % MSG_BANK_TINY_N]; size_t blen = strlen(base); if (blen >= target_len) return g_strdup(base); char* buf = g_malloc(target_len + 1); memcpy(buf, base, blen); fill_filler_text(buf + blen, target_len - blen + 1, rng); buf[target_len] = '\0'; return buf; } if (target_len < 500) { const char* base = MSG_BANK_MEDIUM[xorshift64(rng) % MSG_BANK_MEDIUM_N]; size_t blen = strlen(base); if (blen >= target_len) { // Truncate at UTF-8 codepoint boundary so we don't emit invalid UTF-8. const char* p = base; const char* end = base + target_len; const char* last = base; while (*p && p < end) { const char* next = g_utf8_next_char(p); if (next > end) break; last = next; p = next; } return g_strndup(base, last - base); } char* buf = g_malloc(target_len + 1); memcpy(buf, base, blen); fill_filler_text(buf + blen, target_len - blen + 1, rng); buf[target_len] = '\0'; return buf; } // Large body — repeat a deterministic pseudo-paragraph. char* buf = g_malloc(target_len + 1); fill_filler_text(buf, target_len + 1, rng); // Sprinkle newlines so the escape path gets exercised on long bodies. for (size_t i = 80; i < target_len; i += 80) { if ((xorshift64(rng) & 0x7) == 0) buf[i] = '\n'; } buf[target_len] = '\0'; return buf; } static char* make_stanza_id(uint64_t* rng, sid_mode_t mode, int64_t global_seq, int contact_idx) { sid_mode_t effective = mode; if (mode == SID_MIXED) { int r = (int)(xorshift64(rng) % 3); effective = (r == 0) ? SID_UUID : (r == 1) ? SID_LIBPURPLE : SID_CONVERSATIONS; } switch (effective) { case SID_LIBPURPLE: // Per-contact incremental — collides across contacts (intentional, mirrors libpurple). return g_strdup_printf("%" PRId64, global_seq); case SID_CONVERSATIONS: { // Conversations-style: 22 base32-ish chars static const char alpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; char buf[24]; for (int i = 0; i < 22; i++) buf[i] = alpha[xorshift64(rng) % (sizeof(alpha) - 1)]; buf[22] = '\0'; return g_strdup(buf); } case SID_UUID: case SID_MIXED: default: { // RFC4122-ish UUID v4 (deterministic from RNG, not cryptographic) uint64_t a = xorshift64(rng); uint64_t b = xorshift64(rng); return g_strdup_printf("%08x-%04x-4%03x-%04x-%012lx", (unsigned)(a >> 32), (unsigned)((a >> 16) & 0xffff), (unsigned)(a & 0xfff), (unsigned)((b >> 48) & 0xffff) | 0x8000, (unsigned long)(b & 0xffffffffffffUL)); } } (void)contact_idx; } // --------------------------------------------------------------------------- // Output typedef struct { char* jid; char** resources; // n = opts.resources_per_contact int n_resources; char* dir; char* path; FILE* fp; int64_t seq; // libpurple-style counter int64_t lines_written; int64_t bytes_written; char* last_stanza_id; // for LMC corrections } contact_t; static contact_t* contact_init(int idx, const opts_t* o, const char* account_dir) { contact_t* c = g_new0(contact_t, 1); c->jid = g_strdup_printf("buddy%03d@bench.example", idx); c->n_resources = o->resources_per_contact; c->resources = g_new0(char*, c->n_resources); for (int r = 0; r < c->n_resources; r++) { c->resources[r] = g_strdup_printf("res%d-c%d", r, idx); } auto_gchar gchar* contact_dir = ff_jid_to_dir(c->jid); c->dir = g_strdup_printf("%s/flatlog/%s/%s", o->output_dir, account_dir, contact_dir); c->path = g_strdup_printf("%s/history.log", c->dir); if (g_mkdir_with_parents(c->dir, 0755) != 0) { fprintf(stderr, "mkdir %s failed\n", c->dir); exit(2); } c->fp = fopen(c->path, "w"); if (!c->fp) { fprintf(stderr, "fopen %s failed\n", c->path); exit(2); } fprintf(c->fp, "%s", FLATFILE_HEADER); return c; } static void contact_close(contact_t* c) { if (c->fp) { fclose(c->fp); struct stat st; if (stat(c->path, &st) == 0) c->bytes_written = st.st_size; } g_free(c->jid); for (int r = 0; r < c->n_resources; r++) g_free(c->resources[r]); g_free(c->resources); g_free(c->dir); g_free(c->path); g_free(c->last_stanza_id); g_free(c); } // --------------------------------------------------------------------------- // Main loop static char* iso_for_seq(int64_t seq, int64_t total, int years, int ooo_jitter_min) { // Spread seq evenly across [now - years, now], add small jitter for ooo. time_t now = time(NULL); time_t span = (time_t)years * 365 * 24 * 3600; time_t base = now - span; double frac = (double)seq / (double)(total > 0 ? total : 1); time_t t = base + (time_t)(frac * (double)span); // ooo_jitter_min in minutes — subtract for effect. t -= (time_t)ooo_jitter_min * 60; GDateTime* dt = g_date_time_new_from_unix_utc((gint64)t); char* iso = g_date_time_format_iso8601(dt); g_date_time_unref(dt); return iso; } int main(int argc, char** argv) { opts_t o; int r = parse_args(argc, argv, &o); if (r == 1) { print_help(); return 0; } if (r != 0) { print_help(); return 2; } if (g_mkdir_with_parents(o.output_dir, 0755) != 0) { fprintf(stderr, "cannot create %s\n", o.output_dir); return 2; } auto_gchar gchar* account_dir = ff_jid_to_dir(o.account_jid); if (!o.quiet) { fprintf(stderr, "gen_history: account=%s (dir=%s) lines=%" PRId64 " contacts=%d years=%d " "seed=%" PRIu64 " sid=%d lmc=%d%% ooo=%d%% res=%d profile=%d output=%s\n", o.account_jid, account_dir, o.lines, o.contacts, o.years, o.seed, (int)o.sid_mode, o.lmc_rate, o.ooo_rate, o.resources_per_contact, (int)o.len_profile, o.output_dir); } contact_t** contacts = g_new0(contact_t*, o.contacts); for (int i = 0; i < o.contacts; i++) contacts[i] = contact_init(i, &o, account_dir); uint64_t rng = o.seed ? o.seed : 1; double t0 = (double)g_get_monotonic_time() / 1000.0; int64_t per_contact_target = o.lines / o.contacts; int64_t leftover = o.lines % o.contacts; int64_t total_written = 0; int64_t total_lmc = 0; int64_t total_ooo = 0; for (int ci = 0; ci < o.contacts; ci++) { contact_t* c = contacts[ci]; int64_t target = per_contact_target + (ci < leftover ? 1 : 0); for (int64_t li = 0; li < target; li++) { c->seq++; int is_lmc = c->last_stanza_id && o.lmc_rate > 0 && ((int)(xorshift64(&rng) % 100) < o.lmc_rate); int is_ooo = o.ooo_rate > 0 && ((int)(xorshift64(&rng) % 100) < o.ooo_rate); int ooo_jitter = is_ooo ? rng_int(&rng, 1, 600) : 0; // up to 10h backwards auto_gchar gchar* iso = iso_for_seq(total_written, o.lines, o.years, ooo_jitter); const char* res = c->resources[xorshift64(&rng) % c->n_resources]; auto_gchar gchar* sid = make_stanza_id(&rng, o.sid_mode, c->seq, ci); auto_gchar gchar* aid = (xorshift64(&rng) & 1) ? make_stanza_id(&rng, SID_CONVERSATIONS, c->seq, ci) : NULL; const char* replace_id = is_lmc ? c->last_stanza_id : NULL; size_t blen = pick_body_len(&rng, o.len_profile); auto_gchar gchar* body = make_body(&rng, blen); ff_write_line(c->fp, iso, "chat", "none", sid, aid, replace_id, c->jid, res, NULL, NULL, -1, body); if (!is_lmc) { g_free(c->last_stanza_id); c->last_stanza_id = g_strdup(sid); } else { total_lmc++; } if (is_ooo) total_ooo++; c->lines_written++; total_written++; if (!o.quiet && (total_written % 100000) == 0) { double dt = (double)g_get_monotonic_time() / 1000.0 - t0; fprintf(stderr, " written %" PRId64 " / %" PRId64 " lines (%.1fs, %.0f lines/s)\n", total_written, o.lines, dt / 1000.0, (double)total_written / (dt / 1000.0 + 1e-9)); } } } // Manifest auto_gchar gchar* manifest_path = g_strdup_printf("%s/manifest.txt", o.output_dir); FILE* mf = fopen(manifest_path, "w"); int64_t total_bytes = 0; for (int i = 0; i < o.contacts; i++) { contact_close(contacts[i]); } for (int i = 0; i < o.contacts; i++) { // contact_close already closed the file; use stat for sizes. // The contact_t was freed though, so this is a noop. We re-stat via path: } // Manifest after close — re-stat per contact via known path layout. if (mf) { for (int i = 0; i < o.contacts; i++) { auto_gchar gchar* jid = g_strdup_printf("buddy%03d@bench.example", i); auto_gchar gchar* contact_dir = ff_jid_to_dir(jid); auto_gchar gchar* path = g_strdup_printf("%s/flatlog/%s/%s/history.log", o.output_dir, account_dir, contact_dir); struct stat st; int64_t sz = (stat(path, &st) == 0) ? (int64_t)st.st_size : -1; if (sz > 0) total_bytes += sz; fprintf(mf, "%s %" PRId64 " bytes\n", path, sz); } fprintf(mf, "TOTAL %" PRId64 " bytes\n", total_bytes); fclose(mf); } g_free(contacts); if (!o.quiet) { double dt = (double)g_get_monotonic_time() / 1000.0 - t0; fprintf(stderr, "gen_history: done in %.1fs — wrote %" PRId64 " lines (%" PRId64 " LMC, %" PRId64 " OOO), " "%" PRId64 " bytes total. Manifest: %s\n", dt / 1000.0, total_written, total_lmc, total_ooo, total_bytes, manifest_path); } return 0; }