no-DB mode implementation #94
@@ -195,9 +195,9 @@ functionaltest_sources = \
|
||||
tests/functionaltests/test_disconnect.c tests/functionaltests/test_disconnect.h \
|
||||
tests/functionaltests/test_history.c tests/functionaltests/test_history.h \
|
||||
tests/functionaltests/test_lastactivity.c tests/functionaltests/test_lastactivity.h \
|
||||
tests/functionaltests/test_autoping.c tests/functionaltests/test_autoping.h \
|
||||
tests/functionaltests/test_disco.c tests/functionaltests/test_disco.h \
|
||||
tests/functionaltests/test_export_import.c tests/functionaltests/test_export_import.h \
|
||||
tests/functionaltests/test_autoping.c tests/functionaltests/test_autoping.h \
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
|
||||
tests/functionaltests/test_disco.c tests/functionaltests/test_disco.h \
|
||||
tests/functionaltests/test_export_import.c tests/functionaltests/test_export_import.h \
|
||||
tests/functionaltests/functionaltests.c
|
||||
|
||||
main_source = src/main.c
|
||||
|
||||
@@ -197,7 +197,8 @@ Configuration for
|
||||
.B Profanity
|
||||
is stored in
|
||||
.I $XDG_CONFIG_HOME/profanity/profrc
|
||||
, details on commands for configuring Profanity can be found at <https://profanity-im.github.io/reference.html> or the respective built\-in help or man pages..SS Message History Storage
|
||||
, details on commands for configuring Profanity can be found at <https://jabber.space/command-reference/> or the respective built\-in help or man pages.
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Since the line is changed, it can also lead to the docs on our website, https://jabber.space/command-reference/ Also, why ".SS" is on the same page? Since the line is changed, it can also lead to the docs on our website, https://jabber.space/command-reference/
Also, why ".SS" is on the same page?
jabber.developer2
commented
.SS fixed and link to jabber.space/command-reference/ added .SS fixed and link to jabber.space/command-reference/ added
|
||||
.SS Message History Storage
|
||||
By default, message history is stored in an SQLite database. An alternative flat-file backend
|
||||
stores messages as human-readable plain text files that can be edited with any text editor.
|
||||
.PP
|
||||
|
||||
@@ -49,12 +49,7 @@ grlog=true
|
||||
maxsize=1048580
|
||||
rotate=true
|
||||
shared=true
|
||||
# Database backend for message history:
|
||||
# on - SQLite database (default)
|
||||
# off - no message logging
|
||||
# redact - store with redacted message content
|
||||
# flatfile - plain text files, editable with any text editor
|
||||
# stored in ~/.local/share/profanity/flatlog/
|
||||
# Database backend: on (SQLite), off, redact, flatfile
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
While the comment is useful, it violates general style of this example. I suggest moving documentation to more appropriate places. While the comment is useful, it violates general style of this example. I suggest moving documentation to more appropriate places.
jabber.developer2
commented
example simplified, the long comment removed; explanation lives in the man page now example simplified, the long comment removed; explanation lives in the man page now
|
||||
dblog=on
|
||||
|
||||
[otr]
|
||||
|
||||
@@ -6687,7 +6687,9 @@ cmd_privacy(ProfWin* window, const char* const command, gchar** args)
|
||||
cons_show("Messages are going to be redacted.");
|
||||
prefs_set_string(PREF_DBLOG, arg);
|
||||
} else if (g_strcmp0(arg, "flatfile") == 0) {
|
||||
auto_gchar gchar* ff_path = files_get_data_path(DIR_FLATLOG);
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
It might be beneficial to also output flatfile location to the user. It might be beneficial to also output flatfile location to the user.
jabber.developer2
commented
Corrected Corrected
|
||||
cons_show("Using flat-file backend for message logging. Takes effect on next connection.");
|
||||
cons_show("Flatfile directory: %s", ff_path);
|
||||
prefs_set_string(PREF_DBLOG, arg);
|
||||
prefs_set_boolean(PREF_CHLOG, TRUE);
|
||||
prefs_set_boolean(PREF_HISTORY, TRUE);
|
||||
|
||||
@@ -75,7 +75,7 @@ log_database_init(ProfAccount* account)
|
||||
active_db_backend = db_backend_sqlite();
|
||||
log_info("Using SQLite database backend");
|
||||
#else
|
||||
log_info("SQLite not compiled in, falling back to flat-file backend");
|
||||
log_warning("SQLite not compiled in, falling back to flat-file backend");
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Shouldn't it be a warning, since it's not an expected state? Shouldn't it be a warning, since it's not an expected state?
jabber.developer2
commented
Corrected Corrected
|
||||
active_db_backend = db_backend_flatfile();
|
||||
#endif
|
||||
}
|
||||
@@ -103,9 +103,8 @@ log_database_switch_backend(const char* new_backend)
|
||||
// Close current backend
|
||||
log_database_close();
|
||||
|
||||
// Update preference
|
||||
// Update preference (user must /save to persist across restarts)
|
||||
prefs_set_string(PREF_DBLOG, new_backend);
|
||||
prefs_save();
|
||||
|
||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
Typically, prefs are not autosaved and need manual saving via Typically, prefs are not autosaved and need manual saving via `/save` command. This place creates an exception, which needs to be discussed (e.g., later we might add `/autosave` command).
jabber.developer2
commented
Corrected Corrected
|
||||
// Get current account to reinitialize
|
||||
const char* account_name = session_get_account_name();
|
||||
@@ -130,6 +129,8 @@ log_database_add_incoming(ProfMessage* message)
|
||||
{
|
||||
if (active_db_backend && active_db_backend->add_incoming) {
|
||||
active_db_backend->add_incoming(message);
|
||||
} else {
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
here and in other places: if the method does not exists, then it just passes. I think that we need to log this invalid state. here and in other places: if the method does not exists, then it just passes. I think that we need to log this invalid state.
jabber.developer2
commented
Corrected Corrected
|
||||
log_warning("log_database_add_incoming: no backend or method available");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,6 +139,8 @@ log_database_add_outgoing_chat(const char* const id, const char* const barejid,
|
||||
{
|
||||
if (active_db_backend && active_db_backend->add_outgoing_chat) {
|
||||
active_db_backend->add_outgoing_chat(id, barejid, message, replace_id, enc);
|
||||
} else {
|
||||
log_warning("log_database_add_outgoing_chat: no backend or method available");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,6 +149,8 @@ log_database_add_outgoing_muc(const char* const id, const char* const barejid, c
|
||||
{
|
||||
if (active_db_backend && active_db_backend->add_outgoing_muc) {
|
||||
active_db_backend->add_outgoing_muc(id, barejid, message, replace_id, enc);
|
||||
} else {
|
||||
log_warning("log_database_add_outgoing_muc: no backend or method available");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,6 +159,8 @@ log_database_add_outgoing_muc_pm(const char* const id, const char* const barejid
|
||||
{
|
||||
if (active_db_backend && active_db_backend->add_outgoing_muc_pm) {
|
||||
active_db_backend->add_outgoing_muc_pm(id, barejid, message, replace_id, enc);
|
||||
} else {
|
||||
log_warning("log_database_add_outgoing_muc_pm: no backend or method available");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,24 +1,12 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//
|
||||
// This file is part of CProof.
|
||||
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
|
||||
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
We are not affiliated with Profanity. Use the following lines for copyright notice: We are not affiliated with Profanity.
Use the following lines for copyright notice:
```c
// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of CProof.
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
```
jabber.developer2
commented
Corrected Corrected
|
||||
/*
|
||||
* database_export.c
|
||||
* vim: expandtab:ts=4:sts=4:sw=4
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
*
|
||||
* This file is part of Profanity.
|
||||
*
|
||||
* Profanity is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Profanity is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* Cross-backend export/import for message history.
|
||||
* Reads from one backend, writes to the other, with merge + dedup.
|
||||
*/
|
||||
@@ -30,6 +18,7 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/file.h>
|
||||
@@ -54,6 +43,9 @@
|
||||
|
||||
#ifdef HAVE_SQLITE
|
||||
|
||||
// Print progress to console every N messages during export/import
|
||||
#define EXPORT_PROGRESS_INTERVAL 500
|
||||
|
||||
// =========================================================================
|
||||
// Dedup key: stanza_id if present, else hash of timestamp+from+body
|
||||
// =========================================================================
|
||||
@@ -106,8 +98,10 @@ _ff_list_contacts(void)
|
||||
GDir* dir = g_dir_open(base_dir, 0, &err);
|
||||
if (!dir) {
|
||||
if (err) {
|
||||
log_debug("export: cannot open flatlog dir %s: %s", base_dir, err->message);
|
||||
log_warning("export: cannot open flatlog dir %s: %s", base_dir, err->message);
|
||||
g_error_free(err);
|
||||
} else {
|
||||
log_warning("export: cannot open flatlog dir %s (no error details)", base_dir);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
@@ -237,7 +231,6 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||
}
|
||||
|
||||
int total_exported = 0;
|
||||
int total_skipped = 0;
|
||||
|
||||
for (GSList* c = contacts; c; c = c->next) {
|
||||
const char* contact = c->data;
|
||||
@@ -277,14 +270,34 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||
ff_ensure_dir(dir);
|
||||
|
||||
auto_gchar gchar* tmp_path = g_strdup_printf("%s.export.tmp", log_path);
|
||||
FILE* fp = fopen(tmp_path, "w");
|
||||
if (!fp) {
|
||||
|
||||
// Use open() with O_NOFOLLOW|O_EXCL to prevent symlink attacks,
|
||||
// and explicit 0600 permissions (chat history is private).
|
||||
int tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR);
|
||||
if (tmp_fd < 0) {
|
||||
if (errno == EEXIST) {
|
||||
// Stale temp file from previous failed export — remove and retry
|
||||
unlink(tmp_path);
|
||||
tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR);
|
||||
}
|
||||
}
|
||||
if (tmp_fd < 0) {
|
||||
log_error("export: cannot create %s: %s", tmp_path, strerror(errno));
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||
continue;
|
||||
}
|
||||
FILE* fp = fdopen(tmp_fd, "w");
|
||||
if (!fp) {
|
||||
log_error("export: fdopen failed for %s: %s", tmp_path, strerror(errno));
|
||||
close(tmp_fd);
|
||||
unlink(tmp_path);
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Lock the temp file
|
||||
int fd = fileno(fp);
|
||||
@@ -372,7 +385,7 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||
pl->to_jid, pl->to_resource, pl->marked_read,
|
||||
pl->message);
|
||||
written++;
|
||||
if (written % 500 == 0) {
|
||||
if (written % EXPORT_PROGRESS_INTERVAL == 0) {
|
||||
cons_show(" ... %s: %d/%d written so far", contact, written, (int)g_slist_length(merged));
|
||||
}
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
We can't just print We can't just print `Exported %s`... to user after this case. It's a error and should be handled as such.
jabber.developer2
commented
Corrected Corrected
|
||||
}
|
||||
@@ -386,14 +399,14 @@ log_database_export_to_flatfile(const gchar* const contact_jid)
|
||||
// Atomic rename
|
||||
if (rename(tmp_path, log_path) != 0) {
|
||||
log_error("export: rename %s -> %s failed: %s", tmp_path, log_path, strerror(errno));
|
||||
cons_show_error("Export failed for %s: could not rename temp file (%s)", contact, strerror(errno));
|
||||
unlink(tmp_path);
|
||||
} else {
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
nit: This method is quite fat. In some sense, it's needed for coherency, but I might've overlooked some potential issues with early returns. If possible, I suggest extracting at least body of the cycle. nit: This method is quite fat. In some sense, it's needed for coherency, but I might've overlooked some potential issues with early returns. If possible, I suggest extracting at least body of the cycle.
jabber.developer2
commented
Corrected Corrected
|
||||
cons_show("Exported %s: %d new, %d skipped (already existed: %d)",
|
||||
contact, contact_exported, contact_skipped, existing_count);
|
||||
total_exported += contact_exported;
|
||||
}
|
||||
|
||||
cons_show("Exported %s: %d new, %d skipped (already existed: %d)",
|
||||
contact, contact_exported, contact_skipped, existing_count);
|
||||
total_exported += contact_exported;
|
||||
total_skipped += contact_skipped;
|
||||
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(existing, (GDestroyNotify)ff_parsed_line_free);
|
||||
g_slist_free_full(sqlite_lines, (GDestroyNotify)message_free);
|
||||
@@ -439,7 +452,6 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
|
||||
}
|
||||
|
||||
int total_imported = 0;
|
||||
int total_skipped = 0;
|
||||
|
||||
for (GSList* c = contacts; c; c = c->next) {
|
||||
const char* contact = c->data;
|
||||
@@ -531,7 +543,7 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
|
||||
}
|
||||
contact_imported++;
|
||||
|
||||
if (contact_imported % 500 == 0) {
|
||||
if (contact_imported % EXPORT_PROGRESS_INTERVAL == 0) {
|
||||
cons_show(" ... %s: %d/%d imported so far", contact, contact_imported, total_lines);
|
||||
}
|
||||
}
|
||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
`total_skipped` is unused variable
jabber.developer2
commented
Corrected Corrected
|
||||
@@ -546,7 +558,6 @@ log_database_import_from_flatfile(const gchar* const contact_jid)
|
||||
cons_show("Imported %s: %d new, %d skipped (already in SQLite)",
|
||||
contact, contact_imported, contact_skipped);
|
||||
total_imported += contact_imported;
|
||||
total_skipped += contact_skipped;
|
||||
|
||||
g_hash_table_destroy(seen_keys);
|
||||
g_slist_free_full(ff_lines, (GDestroyNotify)ff_parsed_line_free);
|
||||
|
||||
@@ -1,24 +1,12 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//
|
||||
// This file is part of CProof.
|
||||
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
|
||||
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Same issue as earlier with copyright. We are not affiliated with Profanity. Use the following lines for copyright notice: Same issue with other files. Same issue as earlier with copyright. We are not affiliated with Profanity.
Use the following lines for copyright notice:
```c
// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of CProof.
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
```
Same issue with other files.
jabber.developer2
commented
Corrected Corrected
|
||||
/*
|
||||
* database_flatfile.c
|
||||
* vim: expandtab:ts=4:sts=4:sw=4
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
*
|
||||
* This file is part of Profanity.
|
||||
*
|
||||
* Profanity is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Profanity is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* Flat-file database backend: init/close, message write, message read,
|
||||
* LMC correction logic, and the backend vtable.
|
||||
*
|
||||
@@ -110,10 +98,10 @@ _ff_cache_line_ids(ff_contact_state_t* state, const char* line)
|
||||
|
||||
if (parts) {
|
||||
for (int i = 0; parts[i]; i++) {
|
||||
if (g_str_has_prefix(parts[i], "id:") && !stanza_id) {
|
||||
stanza_id = ff_unescape_meta_value(parts[i] + 3);
|
||||
} else if (g_str_has_prefix(parts[i], "aid:") && !archive_id) {
|
||||
archive_id = ff_unescape_meta_value(parts[i] + 4);
|
||||
if (g_str_has_prefix(parts[i], FF_META_PREFIX_ID) && !stanza_id) {
|
||||
stanza_id = ff_unescape_meta_value(parts[i] + strlen(FF_META_PREFIX_ID));
|
||||
} else if (g_str_has_prefix(parts[i], FF_META_PREFIX_AID) && !archive_id) {
|
||||
archive_id = ff_unescape_meta_value(parts[i] + strlen(FF_META_PREFIX_AID));
|
||||
}
|
||||
}
|
||||
g_strfreev(parts);
|
||||
@@ -121,17 +109,12 @@ _ff_cache_line_ids(ff_contact_state_t* state, const char* line)
|
||||
|
||||
// Extract from_jid (needed for stanza_senders map): after "] " before "/" or ": "
|
||||
char* from_jid = NULL;
|
||||
if (stanza_id && *(close + 1) == ' ') {
|
||||
if (stanza_id && close[1] != '\0' && close[1] == ' ' && close[2] != '\0') {
|
||||
const char* sender_start = close + 2;
|
||||
const char* colonspace = ff_find_unescaped_colonspace(sender_start);
|
||||
if (colonspace) {
|
||||
const char* jid_end = colonspace;
|
||||
for (const char* p = sender_start; p < colonspace; p++) {
|
||||
if (*p == '/') {
|
||||
jid_end = p;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const char* slash = memchr(sender_start, '/', colonspace - sender_start);
|
||||
const char* jid_end = slash ? slash : colonspace;
|
||||
from_jid = g_strndup(sender_start, jid_end - sender_start);
|
||||
|
jabber.developer marked this conversation as resolved
jabber.developer
commented
nit: I suggest this version for readability (avoid magic number of 3 and 4). Ideally, we can also move "id:" and "aid:" into separate constants. It would also increase robustness in case of prefix changes. nit: I suggest this version for readability (avoid magic number of 3 and 4).
```c
for (int i = 0; parts[i]; i++) {
if (g_str_has_prefix(parts[i], "id:") && !stanza_id) {
stanza_id = ff_unescape_meta_value(parts[i] + strlen("id:"));
} else if (g_str_has_prefix(parts[i], "aid:") && !archive_id) {
archive_id = ff_unescape_meta_value(parts[i] + strlen("aid:"));
}
}
```
Ideally, we can also move "id:" and "aid:" into separate constants. It would also increase robustness in case of prefix changes.
jabber.developer2
commented
Corrected Corrected
|
||||
}
|
||||
}
|
||||
@@ -150,6 +133,36 @@ _ff_cache_line_ids(ff_contact_state_t* state, const char* line)
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to add an index entry for the current line.
|
||||
// Called every FF_INDEX_STEP messages during build/extend.
|
||||
static void
|
||||
_ff_maybe_index_line(ff_contact_state_t* state, const char* buf, off_t pos)
|
||||
{
|
||||
char* space = strchr(buf, ' ');
|
||||
if (!space)
|
||||
return;
|
||||
|
||||
char* ts_str = g_strndup(buf, space - buf);
|
||||
GDateTime* dt = g_date_time_new_from_iso8601(ts_str, NULL);
|
||||
if (!dt) {
|
||||
log_warning("flatfile: unparseable timestamp in %s at offset %ld: %s",
|
||||
state->filepath, (long)pos, ts_str);
|
||||
g_free(ts_str);
|
||||
return;
|
||||
}
|
||||
g_free(ts_str);
|
||||
|
||||
if (state->n_entries >= state->cap_entries) {
|
||||
state->cap_entries = state->cap_entries ? state->cap_entries * 2 : 64;
|
||||
state->entries = g_realloc(state->entries,
|
||||
state->cap_entries * sizeof(ff_index_entry_t));
|
||||
}
|
||||
state->entries[state->n_entries].byte_offset = pos;
|
||||
state->entries[state->n_entries].timestamp_epoch = g_date_time_to_unix(dt);
|
||||
state->n_entries++;
|
||||
g_date_time_unref(dt);
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Please, extract bom-check to a different method. it is violates DRY and SRP. Please, extract bom-check to a different method. it is violates DRY and SRP. `database_flatfile_validate.c:123`
jabber.developer2
commented
Corrected Corrected
|
||||
}
|
||||
|
||||
static gboolean
|
||||
_ff_state_build(ff_contact_state_t* state)
|
||||
{
|
||||
@@ -157,15 +170,7 @@ _ff_state_build(ff_contact_state_t* state)
|
||||
if (!fp)
|
||||
return FALSE;
|
||||
|
||||
int c1 = fgetc(fp);
|
||||
int c2 = fgetc(fp);
|
||||
int c3 = fgetc(fp);
|
||||
if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
|
||||
state->bom_len = 3;
|
||||
} else {
|
||||
state->bom_len = 0;
|
||||
fseek(fp, 0, SEEK_SET);
|
||||
}
|
||||
state->bom_len = ff_skip_bom(fp);
|
||||
|
||||
state->n_entries = 0;
|
||||
state->total_lines = 0;
|
||||
@@ -197,23 +202,7 @@ _ff_state_build(ff_contact_state_t* state)
|
||||
_ff_cache_line_ids(state, buf);
|
||||
|
||||
if (msg_count % FF_INDEX_STEP == 0) {
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Are Are `!dt` and `!space` cases ok to be ignored?
jabber.developer2
commented
Postponed — discussion. Current behaviour is "skip silently" (lines outside index step are not indexed); this is by design. No specific change requested. Postponed — discussion. Current behaviour is "skip silently" (lines outside index step are not indexed); this is by design. No specific change requested.
|
||||
char* space = strchr(buf, ' ');
|
||||
if (space) {
|
||||
char* ts_str = g_strndup(buf, space - buf);
|
||||
GDateTime* dt = g_date_time_new_from_iso8601(ts_str, NULL);
|
||||
if (dt) {
|
||||
if (state->n_entries >= state->cap_entries) {
|
||||
state->cap_entries = state->cap_entries ? state->cap_entries * 2 : 64;
|
||||
state->entries = g_realloc(state->entries,
|
||||
state->cap_entries * sizeof(ff_index_entry_t));
|
||||
}
|
||||
state->entries[state->n_entries].byte_offset = pos;
|
||||
state->entries[state->n_entries].timestamp_epoch = g_date_time_to_unix(dt);
|
||||
state->n_entries++;
|
||||
g_date_time_unref(dt);
|
||||
}
|
||||
g_free(ts_str);
|
||||
}
|
||||
_ff_maybe_index_line(state, buf, pos);
|
||||
}
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
please, consider reduction of nesting please, consider reduction of nesting
jabber.developer2
commented
Corrected Corrected
|
||||
msg_count++;
|
||||
free(buf);
|
||||
@@ -284,23 +273,7 @@ _ff_state_extend(ff_contact_state_t* state)
|
||||
_ff_cache_line_ids(state, buf);
|
||||
|
||||
if (new_count % FF_INDEX_STEP == 0) {
|
||||
char* space = strchr(buf, ' ');
|
||||
if (space) {
|
||||
char* ts_str = g_strndup(buf, space - buf);
|
||||
GDateTime* dt = g_date_time_new_from_iso8601(ts_str, NULL);
|
||||
if (dt) {
|
||||
if (state->n_entries >= state->cap_entries) {
|
||||
state->cap_entries = state->cap_entries ? state->cap_entries * 2 : 64;
|
||||
state->entries = g_realloc(state->entries,
|
||||
state->cap_entries * sizeof(ff_index_entry_t));
|
||||
}
|
||||
state->entries[state->n_entries].byte_offset = pos;
|
||||
state->entries[state->n_entries].timestamp_epoch = g_date_time_to_unix(dt);
|
||||
state->n_entries++;
|
||||
g_date_time_unref(dt);
|
||||
}
|
||||
g_free(ts_str);
|
||||
}
|
||||
_ff_maybe_index_line(state, buf, pos);
|
||||
}
|
||||
new_count++;
|
||||
free(buf);
|
||||
@@ -445,9 +418,10 @@ _ff_add_message(const char* type, const char* stanza_id, const char* archive_id,
|
||||
return;
|
||||
}
|
||||
|
||||
// Open the file with O_NOFOLLOW to prevent symlink attacks,
|
||||
// and O_APPEND for safe concurrent appends.
|
||||
// Open the file with O_NOFOLLOW to prevent symlink attacks.
|
||||
// Try O_CREAT|O_EXCL first to detect new files atomically (no TOCTOU).
|
||||
// O_APPEND is included for consistency — it has no effect on a new file
|
||||
// but ensures append semantics if the fd is inherited after the fallback.
|
||||
int fd = open(log_path, O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_NOFOLLOW,
|
||||
S_IRUSR | S_IWUSR);
|
||||
gboolean is_new = (fd >= 0);
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
nit: for readability, maybe we can do something like that? nit: for readability, maybe we can do something like that?
```c
gboolean is_incoming = myjid && myjid->barejid && g_strcmp0(from_barejid, myjid->barejid) == 0;
contact = is_incoming ? to_barejid : from_barejid;
```
jabber.developer2
commented
Corrected Corrected
|
||||
@@ -601,14 +575,15 @@ _flatfile_add_incoming(ProfMessage* message)
|
||||
} else {
|
||||
contact = message->from_jid->barejid;
|
||||
}
|
||||
// MAM dedup: skip if this stanza-id already exists in the log (XEP-0313 replay)
|
||||
// Stanza-id duplicate warning (non-blocking).
|
||||
// XEP-0359 stanza-ids SHOULD be unique per server, but older clients
|
||||
// (Pidgin, Adium, old Profanity) may use incremental IDs that servers
|
||||
// echo back, causing false positives. Log but do NOT skip — matches
|
||||
// SQLite backend behavior. See https://github.com/profanity-im/profanity/issues/1899
|
||||
if (message->stanzaid && !message->is_mam) {
|
||||
if (_ff_has_archive_id(contact, message->stanzaid)) {
|
||||
log_error("flatfile: duplicate stanza-id '%s' from %s, skipping",
|
||||
message->stanzaid, message->from_jid->barejid);
|
||||
cons_show_error("Got a message with duplicate (server-generated) stanza-id from %s.",
|
||||
message->from_jid->fulljid);
|
||||
return;
|
||||
log_warning("flatfile: duplicate stanza-id '%s' from %s (may be non-unique client ID)",
|
||||
message->stanzaid, message->from_jid->barejid);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -903,9 +878,22 @@ _flatfile_get_previous_chat(const gchar* const contact_barejid, const gchar* sta
|
||||
ff_parsed_line_t* oldest = all_parsed->data;
|
||||
state->cursor_offset = oldest->file_offset;
|
||||
|
||||
// Apply LMC corrections and build ProfMessage list
|
||||
// Apply LMC corrections (if enabled) and build ProfMessage list
|
||||
GSList* pre_result = NULL;
|
||||
_ff_apply_lmc(all_parsed, &pre_result);
|
||||
if (prefs_get_boolean(PREF_CORRECTION_ALLOW)) {
|
||||
_ff_apply_lmc(all_parsed, &pre_result);
|
||||
} else {
|
||||
// LMC disabled — convert all lines to ProfMessage without correction
|
||||
for (GSList* cur = all_parsed; cur; cur = cur->next) {
|
||||
ff_parsed_line_t* pl = cur->data;
|
||||
if (pl) {
|
||||
ProfMessage* msg = ff_parsed_to_profmessage(pl);
|
||||
if (msg)
|
||||
pre_result = g_slist_prepend(pre_result, msg);
|
||||
}
|
||||
}
|
||||
pre_result = g_slist_reverse(pre_result);
|
||||
}
|
||||
g_slist_free_full(all_parsed, (GDestroyNotify)ff_parsed_line_free);
|
||||
|
||||
if (!pre_result)
|
||||
|
||||
@@ -1,24 +1,12 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//
|
||||
// This file is part of CProof.
|
||||
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
|
||||
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
Same issue as earlier with copyright. We are not affiliated with Profanity. Use the following lines for copyright notice: Same issue as earlier with copyright. We are not affiliated with Profanity.
Use the following lines for copyright notice:
```c
// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of CProof.
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
```
jabber.developer2
commented
Corrected Corrected
|
||||
/*
|
||||
* database_flatfile.h
|
||||
* vim: expandtab:ts=4:sts=4:sw=4
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
*
|
||||
* This file is part of Profanity.
|
||||
*
|
||||
* Profanity is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Profanity is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* Internal header shared between database_flatfile*.c modules.
|
||||
* Not part of the public API — do not include from outside the flatfile backend.
|
||||
*/
|
||||
@@ -69,8 +57,16 @@ typedef struct
|
||||
|
||||
// --- Sparse index for single-file lookup ---
|
||||
|
||||
// Sample every N-th log line for the sparse index.
|
||||
// 500 lines ≈ one index entry per ~50 KB of log data, so a 100K-message
|
||||
// contact requires only ~200 entries (~3 KB) for O(log n) time-range lookup.
|
||||
#define FF_INDEX_STEP 500
|
||||
|
||||
// --- Metadata field prefixes (used in _ff_cache_line_ids and ff_parse_line) ---
|
||||
|
||||
#define FF_META_PREFIX_ID "id:"
|
||||
#define FF_META_PREFIX_AID "aid:"
|
||||
|
||||
typedef struct
|
||||
{
|
||||
off_t byte_offset;
|
||||
@@ -127,6 +123,9 @@ char* ff_unescape_meta_value(const char* val);
|
||||
|
||||
// --- I/O ---
|
||||
|
||||
// Skip UTF-8 BOM if present; returns 3 if skipped, 0 otherwise.
|
||||
int ff_skip_bom(FILE* fp);
|
||||
|
||||
char* ff_readline(FILE* fp, gboolean* truncated);
|
||||
void ff_write_line(FILE* fp, const char* timestamp, const char* type, const char* enc,
|
||||
const char* stanza_id, const char* archive_id, const char* replace_id,
|
||||
|
||||
@@ -1,24 +1,12 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//
|
||||
// This file is part of CProof.
|
||||
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
|
||||
|
||||
/*
|
||||
* database_flatfile_parser.c
|
||||
* vim: expandtab:ts=4:sts=4:sw=4
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
*
|
||||
* This file is part of Profanity.
|
||||
*
|
||||
* Profanity is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Profanity is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* Flat-file backend: type helpers, path helpers, escape/unescape,
|
||||
* line I/O, and the tolerant log-line parser.
|
||||
*/
|
||||
@@ -339,6 +327,24 @@ ff_unescape_meta_value(const char* val)
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// BOM helper
|
||||
// =========================================================================
|
||||
//
|
||||
// Skip UTF-8 BOM (EF BB BF) at the beginning of a file.
|
||||
// Returns 3 if BOM was present, 0 otherwise. File position is set past
|
||||
// the BOM (or rewound to the start).
|
||||
int
|
||||
ff_skip_bom(FILE* fp)
|
||||
{
|
||||
int c1 = fgetc(fp);
|
||||
int c2 = fgetc(fp);
|
||||
int c3 = fgetc(fp);
|
||||
if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)
|
||||
return 3;
|
||||
fseek(fp, 0, SEEK_SET);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Readline helper
|
||||
// =========================================================================
|
||||
|
jabber.developer marked this conversation as resolved
Outdated
jabber.developer
commented
note for myself: come back to it and rereview note for myself: come back to it and rereview
|
||||
//
|
||||
@@ -658,18 +664,18 @@ ff_parse_line(const char* line)
|
||||
result->type = g_strdup(parts[i]);
|
||||
} else if (i == 1) {
|
||||
result->enc = g_strdup(parts[i]);
|
||||
} else if (g_str_has_prefix(parts[i], "id:")) {
|
||||
result->stanza_id = ff_unescape_meta_value(parts[i] + 3);
|
||||
} else if (g_str_has_prefix(parts[i], "aid:")) {
|
||||
result->archive_id = ff_unescape_meta_value(parts[i] + 4);
|
||||
} else if (g_str_has_prefix(parts[i], FF_META_PREFIX_ID)) {
|
||||
result->stanza_id = ff_unescape_meta_value(parts[i] + strlen(FF_META_PREFIX_ID));
|
||||
} else if (g_str_has_prefix(parts[i], FF_META_PREFIX_AID)) {
|
||||
result->archive_id = ff_unescape_meta_value(parts[i] + strlen(FF_META_PREFIX_AID));
|
||||
} else if (g_str_has_prefix(parts[i], "corrects:")) {
|
||||
result->replace_id = ff_unescape_meta_value(parts[i] + 9);
|
||||
result->replace_id = ff_unescape_meta_value(parts[i] + strlen("corrects:"));
|
||||
} else if (g_str_has_prefix(parts[i], "to:")) {
|
||||
result->to_jid = ff_unescape_meta_value(parts[i] + 3);
|
||||
result->to_jid = ff_unescape_meta_value(parts[i] + strlen("to:"));
|
||||
} else if (g_str_has_prefix(parts[i], "to_res:")) {
|
||||
result->to_resource = ff_unescape_meta_value(parts[i] + 7);
|
||||
result->to_resource = ff_unescape_meta_value(parts[i] + strlen("to_res:"));
|
||||
} else if (g_str_has_prefix(parts[i], "read:")) {
|
||||
result->marked_read = atoi(parts[i] + 5) ? 1 : 0;
|
||||
result->marked_read = atoi(parts[i] + strlen("read:")) ? 1 : 0;
|
||||
}
|
||||
}
|
||||
g_strfreev(parts);
|
||||
|
||||
@@ -1,24 +1,12 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//
|
||||
// This file is part of CProof.
|
||||
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
|
||||
|
||||
/*
|
||||
* database_flatfile_verify.c
|
||||
* vim: expandtab:ts=4:sts=4:sw=4
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
*
|
||||
* This file is part of Profanity.
|
||||
*
|
||||
* Profanity is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Profanity is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Profanity. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* Flat-file backend: integrity verification (/history verify).
|
||||
* Checks: parsability, timestamp ordering, duplicate IDs, broken LMC
|
||||
* references, file permissions, BOM, CRLF, UTF-8, control chars.
|
||||
@@ -117,18 +105,13 @@ ff_verify_integrity(const gchar* const contact_barejid)
|
||||
continue;
|
||||
|
||||
// BOM check
|
||||
int c1 = fgetc(fp);
|
||||
int c2 = fgetc(fp);
|
||||
int c3 = fgetc(fp);
|
||||
if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
|
||||
if (ff_skip_bom(fp)) {
|
||||
integrity_issue_t* issue = g_malloc0(sizeof(integrity_issue_t));
|
||||
issue->level = INTEGRITY_INFO;
|
||||
issue->file = g_strdup(basename);
|
||||
issue->line = 0;
|
||||
issue->message = g_strdup("File has UTF-8 BOM — harmless but unnecessary");
|
||||
issues = g_slist_prepend(issues, issue);
|
||||
} else {
|
||||
fseek(fp, 0, SEEK_SET);
|
||||
}
|
||||
|
||||
char* buf = NULL;
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
jabber.developer
commented
Add invalid date tests Add invalid date tests
jabber.developer2
commented
Corrected Corrected
|
||||
//
|
||||
// This file is part of CProof.
|
||||
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
|
||||
|
||||
/*
|
||||
* test_database_export.c
|
||||
*
|
||||
* Unit tests for the flatfile write→parse round-trip, escape/unescape helpers,
|
||||
* jid_to_dir path construction, and parser edge cases — these are the core
|
||||
* functions exercised by database_export.c during export and import.
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
* License: GPL-3.0-or-later (same as Profanity)
|
||||
*/
|
||||
|
||||
#include "prof_cmocka.h"
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//
|
||||
// This file is part of CProof.
|
||||
// See LICENSE for the full GPLv3 text and the special OpenSSL linking exception.
|
||||
|
||||
/*
|
||||
* test_database_stress.c
|
||||
*
|
||||
* Stress tests for flat-file database I/O: rapid-fire writes, MUC messages
|
||||
* with many participants, large message bodies, index/cache correctness,
|
||||
* export/import merge under load, and deep LMC correction chains.
|
||||
*
|
||||
* Copyright (C) 2026 Profanity Contributors
|
||||
* License: GPL-3.0-or-later (same as Profanity)
|
||||
*/
|
||||
|
||||
#include "prof_cmocka.h"
|
||||
|
||||
Is it a wrong offset or is it gitea's UI bug? It appears that the line is overly offset to the right, violating general formatting
Corrected