cproof/tests/bench/gen_history.c

// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of CProof.
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.

/*
 * gen_history.c
 * vim: expandtab:ts=4:sts=4:sw=4
 *
 * Synthetic history generator for the flat-file backend bench harness.
 * Writes deterministic, ff_write_line-compatible logs covering:
 *   - configurable line counts, contact counts, year span;
 *   - mixed message-length profiles (tiny/medium/long/paste-bomb/extreme);
 *   - stanza-id strategies (uuid / libpurple-incremental / conversations / mixed);
 *   - configurable LMC rate and MAM out-of-order rate;
 *   - resource diversity per contact.
 *
 * Output layout (canonical, matches files_get_data_path/jid_to_dir):
 *   <output>/flatlog/<account_jid_to_dir>/<contact_jid_to_dir>/history.log
 *   <output>/manifest.txt              # one line per generated file with size + line count
 *
 * CLI:
 *   gen_history [options]
 *
 * Options:
 *   --account=JID              account-side JID for path layout (default bench@bench.example)
 *   --lines=N                  total lines to generate (default 10000)
 *   --contacts=K               distinct contact JIDs (default 1)
 *   --years=Y                  span (default 1, ts spread over Y years ending now)
 *   --seed=S                   RNG seed (default 42)
 *   --stanza-id={uuid|libpurple|conversations|mixed}  default uuid
 *   --lmc-rate=PCT             percent of lines that are LMC corrections (0..50)
 *   --mam-ooo-rate=PCT         percent of lines with intentionally non-monotonic ts (0..50)
 *   --resources-per-contact=R  default 3
 *   --msg-len-profile={short|mixed|long|extreme}      default mixed
 *   --output=DIR               output directory (default /tmp/cproof-bench-corpus)
 *   --quiet                    suppress progress prints
 *
 * Exit code: 0 on success, non-zero on error.
 */

#include "config.h"

#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>

#include <glib.h>
#include <glib/gstdio.h>

#include "database_flatfile.h"

// ---------------------------------------------------------------------------
// CLI options

typedef enum {
    SID_UUID,
    SID_LIBPURPLE,
    SID_CONVERSATIONS,
    SID_MIXED,
} sid_mode_t;

typedef enum {
    LEN_SHORT,
    LEN_MIXED,
    LEN_LONG,
    LEN_EXTREME,
} len_profile_t;

typedef struct
{
    int64_t lines;
    int contacts;
    int years;
    uint64_t seed;
    sid_mode_t sid_mode;
    int lmc_rate;
    int ooo_rate;
    int resources_per_contact;
    len_profile_t len_profile;
    const char* output_dir;
    const char* account_jid;
    int quiet;
} opts_t;

static void
opts_default(opts_t* o)
{
    o->lines = 10000;
    o->contacts = 1;
    o->years = 1;
    o->seed = 42;
    o->sid_mode = SID_UUID;
    o->lmc_rate = 3;
    o->ooo_rate = 0;
    o->resources_per_contact = 3;
    o->len_profile = LEN_MIXED;
    o->output_dir = "/tmp/cproof-bench-corpus";
    o->account_jid = "bench@bench.example";
    o->quiet = 0;
}

static int
parse_sid_mode(const char* s, sid_mode_t* out)
{
    if (g_strcmp0(s, "uuid") == 0) {
        *out = SID_UUID;
        return 0;
    }
    if (g_strcmp0(s, "libpurple") == 0) {
        *out = SID_LIBPURPLE;
        return 0;
    }
    if (g_strcmp0(s, "conversations") == 0) {
        *out = SID_CONVERSATIONS;
        return 0;
    }
    if (g_strcmp0(s, "mixed") == 0) {
        *out = SID_MIXED;
        return 0;
    }
    return -1;
}

static int
parse_len_profile(const char* s, len_profile_t* out)
{
    if (g_strcmp0(s, "short") == 0) {
        *out = LEN_SHORT;
        return 0;
    }
    if (g_strcmp0(s, "mixed") == 0) {
        *out = LEN_MIXED;
        return 0;
    }
    if (g_strcmp0(s, "long") == 0) {
        *out = LEN_LONG;
        return 0;
    }
    if (g_strcmp0(s, "extreme") == 0) {
        *out = LEN_EXTREME;
        return 0;
    }
    return -1;
}

static int
parse_args(int argc, char** argv, opts_t* o)
{
    opts_default(o);
    for (int i = 1; i < argc; i++) {
        const char* a = argv[i];
        if (strncmp(a, "--lines=", 8) == 0) {
            o->lines = strtoll(a + 8, NULL, 10);
        } else if (strncmp(a, "--contacts=", 11) == 0) {
            o->contacts = atoi(a + 11);
        } else if (strncmp(a, "--years=", 8) == 0) {
            o->years = atoi(a + 8);
        } else if (strncmp(a, "--seed=", 7) == 0) {
            o->seed = strtoull(a + 7, NULL, 10);
        } else if (strncmp(a, "--stanza-id=", 12) == 0) {
            if (parse_sid_mode(a + 12, &o->sid_mode) != 0) {
                fprintf(stderr, "bad --stanza-id\n");
                return -1;
            }
        } else if (strncmp(a, "--lmc-rate=", 11) == 0) {
            o->lmc_rate = atoi(a + 11);
        } else if (strncmp(a, "--mam-ooo-rate=", 15) == 0) {
            o->ooo_rate = atoi(a + 15);
        } else if (strncmp(a, "--resources-per-contact=", 24) == 0) {
            o->resources_per_contact = atoi(a + 24);
        } else if (strncmp(a, "--msg-len-profile=", 18) == 0) {
            if (parse_len_profile(a + 18, &o->len_profile) != 0) {
                fprintf(stderr, "bad --msg-len-profile\n");
                return -1;
            }
        } else if (strncmp(a, "--output=", 9) == 0) {
            o->output_dir = a + 9;
        } else if (strncmp(a, "--account=", 10) == 0) {
            o->account_jid = a + 10;
        } else if (strcmp(a, "--quiet") == 0) {
            o->quiet = 1;
        } else if (strcmp(a, "--help") == 0 || strcmp(a, "-h") == 0) {
            return 1;
        } else {
            fprintf(stderr, "unknown option: %s\n", a);
            return -1;
        }
    }
    if (o->lines <= 0 || o->contacts <= 0 || o->years <= 0)
        return -1;
    if (o->lmc_rate < 0 || o->lmc_rate > 50)
        return -1;
    if (o->ooo_rate < 0 || o->ooo_rate > 50)
        return -1;
    if (o->resources_per_contact <= 0)
        return -1;
    return 0;
}

static void
print_help(void)
{
    fputs(
        "Usage: gen_history [options]\n"
        "  --lines=N                       total lines (default 10000)\n"
        "  --contacts=K                    contact count (default 1)\n"
        "  --years=Y                       year span (default 1)\n"
        "  --seed=S                        RNG seed (default 42)\n"
        "  --stanza-id={uuid|libpurple|conversations|mixed}\n"
        "  --lmc-rate=PCT                  0..50, default 3\n"
        "  --mam-ooo-rate=PCT              0..50, default 0\n"
        "  --resources-per-contact=R       default 3\n"
        "  --msg-len-profile={short|mixed|long|extreme}\n"
        "  --output=DIR                    default /tmp/cproof-bench-corpus\n"
        "  --account=JID                   account JID (default bench@bench.example)\n"
        "  --quiet                         suppress progress prints\n",
        stderr);
}

// ---------------------------------------------------------------------------
// Body / stanza-id / resource generators (deterministic)

static const char* MSG_BANK_TINY[] = {
    "ok",
    "yes",
    "no",
    "ack",
    "thx",
    "ttyl",
    "k",
    "lol",
    "+1",
    "yep",
    "nope",
    "sure",
    "got it",
    "great",
    "afk",
    "brb",
    "bye",
    "hi",
    "hello",
    "hey",
};
static const int MSG_BANK_TINY_N = sizeof(MSG_BANK_TINY) / sizeof(MSG_BANK_TINY[0]);

static const char* MSG_BANK_MEDIUM[] = {
    "Quick reminder, the meeting starts at 14:00 in conf room B.",
    "Pushed the patch to the staging branch, please review when you get a chance.",
    "Ran into a weird issue with the parser — investigating.",
    "Backups completed for the night, all green on the dashboard.",
    "Yeah I think the second approach is cleaner — let's go with that.",
    "Не могу подключиться к серверу с обеда, кто-то ещё видит проблему?",
    "私もそう思います。ところで、明日の予定はどうしますか？",
    "Just FYI: the migration will run during the maintenance window tonight.",
};
static const int MSG_BANK_MEDIUM_N = sizeof(MSG_BANK_MEDIUM) / sizeof(MSG_BANK_MEDIUM[0]);

// Linear-congruential — we don't need cryptographic strength, we need fast and
// reproducible. xorshift64 hits both.
static uint64_t
xorshift64(uint64_t* s)
{
    uint64_t x = *s;
    x ^= x << 13;
    x ^= x >> 7;
    x ^= x << 17;
    *s = x;
    return x;
}

static int
rng_int(uint64_t* s, int min_inc, int max_exc)
{
    if (max_exc <= min_inc)
        return min_inc;
    return min_inc + (int)(xorshift64(s) % (uint64_t)(max_exc - min_inc));
}

static void
fill_filler_text(char* buf, size_t n, uint64_t* rng)
{
    static const char alpha[] = "abcdefghijklmnopqrstuvwxyz0123456789 ";
    static const int alpha_n = sizeof(alpha) - 1;
    for (size_t i = 0; i < n; i++)
        buf[i] = alpha[xorshift64(rng) % alpha_n];
    if (n > 0)
        buf[n - 1] = '\0';
}

// Pick a body length per the profile distribution (matches table in REVIEW.txt
// Phase 9 plan section 2):
//   tiny       50% (5-100 B)        long      8%  (500-5000 B)   extreme 0.5% (100KB-1MB)
//   medium     40% (100-500 B)      paste     1.5%(5KB-100KB)
static size_t
pick_body_len(uint64_t* rng, len_profile_t profile)
{
    int r = (int)(xorshift64(rng) % 1000);
    if (profile == LEN_SHORT) {
        return 5 + (xorshift64(rng) % 96);
    }
    if (profile == LEN_LONG) {
        if (r < 200)
            return 5 + (xorshift64(rng) % 96);
        if (r < 600)
            return 100 + (xorshift64(rng) % 400);
        if (r < 850)
            return 1000 + (xorshift64(rng) % 9000); // 1-10 KB
        if (r < 980)
            return 10000 + (xorshift64(rng) % 90000); // 10-100 KB
        return 100000 + (xorshift64(rng) % 900000);   // 100KB-1MB
    }
    if (profile == LEN_EXTREME) {
        if (r < 100)
            return 5 + (xorshift64(rng) % 96);
        if (r < 400)
            return 100000 + (xorshift64(rng) % 900000); // 100KB-1MB
        if (r < 800)
            return 1000000 + (xorshift64(rng) % 4000000); // 1-5 MB
        return 5000000 + (xorshift64(rng) % 4000000);     // 5-9 MB
    }
    // LEN_MIXED — realistic distribution
    if (r < 500)
        return 5 + (xorshift64(rng) % 96);
    if (r < 900)
        return 100 + (xorshift64(rng) % 400);
    if (r < 980)
        return 500 + (xorshift64(rng) % 4500); // 500B-5KB
    if (r < 995)
        return 5000 + (xorshift64(rng) % 95000); // 5-100 KB
    return 100000 + (xorshift64(rng) % 900000);  // 100KB-1MB
}

static char*
make_body(uint64_t* rng, size_t target_len)
{
    if (target_len < 100) {
        // Pick from tiny bank, optionally pad with filler
        const char* base = MSG_BANK_TINY[xorshift64(rng) % MSG_BANK_TINY_N];
        size_t blen = strlen(base);
        if (blen >= target_len)
            return g_strdup(base);
        char* buf = g_malloc(target_len + 1);
        memcpy(buf, base, blen);
        fill_filler_text(buf + blen, target_len - blen + 1, rng);
        buf[target_len] = '\0';
        return buf;
    }
    if (target_len < 500) {
        const char* base = MSG_BANK_MEDIUM[xorshift64(rng) % MSG_BANK_MEDIUM_N];
        size_t blen = strlen(base);
        if (blen >= target_len) {
            // Truncate at UTF-8 codepoint boundary so we don't emit invalid UTF-8.
            const char* p = base;
            const char* end = base + target_len;
            const char* last = base;
            while (*p && p < end) {
                const char* next = g_utf8_next_char(p);
                if (next > end)
                    break;
                last = next;
                p = next;
            }
            return g_strndup(base, last - base);
        }
        char* buf = g_malloc(target_len + 1);
        memcpy(buf, base, blen);
        fill_filler_text(buf + blen, target_len - blen + 1, rng);
        buf[target_len] = '\0';
        return buf;
    }
    // Large body — repeat a deterministic pseudo-paragraph.
    char* buf = g_malloc(target_len + 1);
    fill_filler_text(buf, target_len + 1, rng);
    // Sprinkle newlines so the escape path gets exercised on long bodies.
    for (size_t i = 80; i < target_len; i += 80) {
        if ((xorshift64(rng) & 0x7) == 0)
            buf[i] = '\n';
    }
    buf[target_len] = '\0';
    return buf;
}

static char*
make_stanza_id(uint64_t* rng, sid_mode_t mode, int64_t global_seq, int contact_idx)
{
    sid_mode_t effective = mode;
    if (mode == SID_MIXED) {
        int r = (int)(xorshift64(rng) % 3);
        effective = (r == 0) ? SID_UUID : (r == 1) ? SID_LIBPURPLE
                                                   : SID_CONVERSATIONS;
    }
    switch (effective) {
    case SID_LIBPURPLE:
        // Per-contact incremental — collides across contacts (intentional, mirrors libpurple).
        return g_strdup_printf("%" PRId64, global_seq);
    case SID_CONVERSATIONS:
    {
        // Conversations-style: 22 base32-ish chars
        static const char alpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
        char buf[24];
        for (int i = 0; i < 22; i++)
            buf[i] = alpha[xorshift64(rng) % (sizeof(alpha) - 1)];
        buf[22] = '\0';
        return g_strdup(buf);
    }
    case SID_UUID:
    case SID_MIXED:
    default:
    {
        // RFC4122-ish UUID v4 (deterministic from RNG, not cryptographic)
        uint64_t a = xorshift64(rng);
        uint64_t b = xorshift64(rng);
        return g_strdup_printf("%08x-%04x-4%03x-%04x-%012lx",
                               (unsigned)(a >> 32),
                               (unsigned)((a >> 16) & 0xffff),
                               (unsigned)(a & 0xfff),
                               (unsigned)((b >> 48) & 0xffff) | 0x8000,
                               (unsigned long)(b & 0xffffffffffffUL));
    }
    }
    (void)contact_idx;
}

// ---------------------------------------------------------------------------
// Output

typedef struct
{
    char* jid;
    char** resources; // n = opts.resources_per_contact
    int n_resources;
    char* dir;
    char* path;
    FILE* fp;
    int64_t seq; // libpurple-style counter
    int64_t lines_written;
    int64_t bytes_written;
    char* last_stanza_id; // for LMC corrections
} contact_t;

static contact_t*
contact_init(int idx, const opts_t* o, const char* account_dir)
{
    contact_t* c = g_new0(contact_t, 1);
    c->jid = g_strdup_printf("buddy%03d@bench.example", idx);
    c->n_resources = o->resources_per_contact;
    c->resources = g_new0(char*, c->n_resources);
    for (int r = 0; r < c->n_resources; r++) {
        c->resources[r] = g_strdup_printf("res%d-c%d", r, idx);
    }
    auto_gchar gchar* contact_dir = ff_jid_to_dir(c->jid);
    c->dir = g_strdup_printf("%s/flatlog/%s/%s",
                             o->output_dir, account_dir, contact_dir);
    c->path = g_strdup_printf("%s/history.log", c->dir);
    if (g_mkdir_with_parents(c->dir, 0755) != 0) {
        fprintf(stderr, "mkdir %s failed\n", c->dir);
        exit(2);
    }
    c->fp = fopen(c->path, "w");
    if (!c->fp) {
        fprintf(stderr, "fopen %s failed\n", c->path);
        exit(2);
    }
    fprintf(c->fp, "%s", FLATFILE_HEADER);
    return c;
}

static void
contact_close(contact_t* c)
{
    if (c->fp) {
        fclose(c->fp);
        struct stat st;
        if (stat(c->path, &st) == 0)
            c->bytes_written = st.st_size;
    }
    g_free(c->jid);
    for (int r = 0; r < c->n_resources; r++)
        g_free(c->resources[r]);
    g_free(c->resources);
    g_free(c->dir);
    g_free(c->path);
    g_free(c->last_stanza_id);
    g_free(c);
}

// ---------------------------------------------------------------------------
// Main loop

static char*
iso_for_seq(int64_t seq, int64_t total, int years, int ooo_jitter_min)
{
    // Spread seq evenly across [now - years, now], add small jitter for ooo.
    time_t now = time(NULL);
    time_t span = (time_t)years * 365 * 24 * 3600;
    time_t base = now - span;
    double frac = (double)seq / (double)(total > 0 ? total : 1);
    time_t t = base + (time_t)(frac * (double)span);
    // ooo_jitter_min in minutes — subtract for effect.
    t -= (time_t)ooo_jitter_min * 60;
    GDateTime* dt = g_date_time_new_from_unix_utc((gint64)t);
    char* iso = g_date_time_format_iso8601(dt);
    g_date_time_unref(dt);
    return iso;
}

int
main(int argc, char** argv)
{
    opts_t o;
    int r = parse_args(argc, argv, &o);
    if (r == 1) {
        print_help();
        return 0;
    }
    if (r != 0) {
        print_help();
        return 2;
    }

    if (g_mkdir_with_parents(o.output_dir, 0755) != 0) {
        fprintf(stderr, "cannot create %s\n", o.output_dir);
        return 2;
    }

    auto_gchar gchar* account_dir = ff_jid_to_dir(o.account_jid);

    if (!o.quiet) {
        fprintf(stderr,
                "gen_history: account=%s (dir=%s) lines=%" PRId64 " contacts=%d years=%d "
                "seed=%" PRIu64 " sid=%d lmc=%d%% ooo=%d%% res=%d profile=%d output=%s\n",
                o.account_jid, account_dir, o.lines, o.contacts, o.years, o.seed,
                (int)o.sid_mode, o.lmc_rate, o.ooo_rate,
                o.resources_per_contact, (int)o.len_profile, o.output_dir);
    }

    contact_t** contacts = g_new0(contact_t*, o.contacts);
    for (int i = 0; i < o.contacts; i++)
        contacts[i] = contact_init(i, &o, account_dir);

    uint64_t rng = o.seed ? o.seed : 1;
    double t0 = (double)g_get_monotonic_time() / 1000.0;

    int64_t per_contact_target = o.lines / o.contacts;
    int64_t leftover = o.lines % o.contacts;

    int64_t total_written = 0;
    int64_t total_lmc = 0;
    int64_t total_ooo = 0;

    for (int ci = 0; ci < o.contacts; ci++) {
        contact_t* c = contacts[ci];
        int64_t target = per_contact_target + (ci < leftover ? 1 : 0);
        for (int64_t li = 0; li < target; li++) {
            c->seq++;

            int is_lmc = c->last_stanza_id && o.lmc_rate > 0
                         && ((int)(xorshift64(&rng) % 100) < o.lmc_rate);
            int is_ooo = o.ooo_rate > 0
                         && ((int)(xorshift64(&rng) % 100) < o.ooo_rate);

            int ooo_jitter = is_ooo ? rng_int(&rng, 1, 600) : 0; // up to 10h backwards
            auto_gchar gchar* iso = iso_for_seq(total_written, o.lines, o.years, ooo_jitter);

            const char* res = c->resources[xorshift64(&rng) % c->n_resources];

            auto_gchar gchar* sid = make_stanza_id(&rng, o.sid_mode, c->seq, ci);
            auto_gchar gchar* aid = (xorshift64(&rng) & 1)
                                        ? make_stanza_id(&rng, SID_CONVERSATIONS, c->seq, ci)
                                        : NULL;
            const char* replace_id = is_lmc ? c->last_stanza_id : NULL;

            size_t blen = pick_body_len(&rng, o.len_profile);
            auto_gchar gchar* body = make_body(&rng, blen);

            ff_write_line(c->fp, iso, "chat", "none",
                          sid, aid, replace_id,
                          c->jid, res,
                          NULL, NULL, -1,
                          body);

            if (!is_lmc) {
                g_free(c->last_stanza_id);
                c->last_stanza_id = g_strdup(sid);
            } else {
                total_lmc++;
            }
            if (is_ooo)
                total_ooo++;

            c->lines_written++;
            total_written++;

            if (!o.quiet && (total_written % 100000) == 0) {
                double dt = (double)g_get_monotonic_time() / 1000.0 - t0;
                fprintf(stderr, "  written %" PRId64 " / %" PRId64 " lines  (%.1fs, %.0f lines/s)\n",
                        total_written, o.lines, dt / 1000.0,
                        (double)total_written / (dt / 1000.0 + 1e-9));
            }
        }
    }

    // Manifest
    auto_gchar gchar* manifest_path = g_strdup_printf("%s/manifest.txt", o.output_dir);
    FILE* mf = fopen(manifest_path, "w");
    int64_t total_bytes = 0;
    for (int i = 0; i < o.contacts; i++) {
        contact_close(contacts[i]);
    }
    for (int i = 0; i < o.contacts; i++) {
        // contact_close already closed the file; use stat for sizes.
        // The contact_t was freed though, so this is a noop. We re-stat via path:
    }
    // Manifest after close — re-stat per contact via known path layout.
    if (mf) {
        for (int i = 0; i < o.contacts; i++) {
            auto_gchar gchar* jid = g_strdup_printf("buddy%03d@bench.example", i);
            auto_gchar gchar* contact_dir = ff_jid_to_dir(jid);
            auto_gchar gchar* path = g_strdup_printf("%s/flatlog/%s/%s/history.log",
                                                     o.output_dir, account_dir, contact_dir);
            struct stat st;
            int64_t sz = (stat(path, &st) == 0) ? (int64_t)st.st_size : -1;
            if (sz > 0)
                total_bytes += sz;
            fprintf(mf, "%s %" PRId64 " bytes\n", path, sz);
        }
        fprintf(mf, "TOTAL %" PRId64 " bytes\n", total_bytes);
        fclose(mf);
    }
    g_free(contacts);

    if (!o.quiet) {
        double dt = (double)g_get_monotonic_time() / 1000.0 - t0;
        fprintf(stderr,
                "gen_history: done in %.1fs — wrote %" PRId64 " lines (%" PRId64 " LMC, %" PRId64 " OOO), "
                "%" PRId64 " bytes total. Manifest: %s\n",
                dt / 1000.0, total_written, total_lmc, total_ooo, total_bytes, manifest_path);
    }
    return 0;
}