From e9c6b3b3d860a95d18932c990eef9222ebac8722 Mon Sep 17 00:00:00 2001 From: Michael Vetter Date: Thu, 19 Mar 2026 08:22:08 +0100 Subject: [PATCH 1/3] feat(tools): support UTF-8 characters in autocompletion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Autocompletion failed for nicknames using non Latin scripts. We used `g_str_to_ascii`, which replaces characters it cannot transliterate with '?', leading search failures and false matches between different scripts. Now we do: Use `g_utf8_casefold` for case-insensitive UTF-8 comparison. This ensures that like 'Σ' correctly match 'σ' across all Unicode scripts, providing correct results for non English nicknames. If we don't fine anything typing a base ASCII character matches an accented one (typing `e` matches `è`). This pass uses `g_str_to_ascii` followed by `g_ascii_strdown` for comparison. It is now restricted to run if the search string itself is valid ASCII, preventing the "everything matches '?'" bug in non Latin scripts. Autocomplete items are sorted using `strcmp`. `g_utf8_collate` provides linguistical ordering for a specific language. But its behavior is locale dependent and undefined when comparing strings from different scripts. `strcmp` does byte-wise ordering that correctly follows Unicode code order for UTF-8 strings. --- meson.build | 1 + src/tools/autocomplete.c | 28 +++++- tests/functionaltests/functionaltests.c | 4 + tests/functionaltests/test_i18n.c | 51 ++++++++++ tests/functionaltests/test_i18n.h | 9 ++ tests/unittests/tools/test_autocomplete.c | 112 ++++++++++++++++++++++ tests/unittests/tools/test_autocomplete.h | 7 ++ tests/unittests/unittests.c | 8 ++ tests/unittests/xmpp/test_roster_list.c | 22 +++++ tests/unittests/xmpp/test_roster_list.h | 1 + 10 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 tests/functionaltests/test_i18n.c create mode 100644 tests/functionaltests/test_i18n.h diff --git a/meson.build b/meson.build index aa4abc63..c58aa99a 100644 --- a/meson.build +++ b/meson.build @@ -698,6 +698,7 @@ if get_option('tests') 'tests/functionaltests/test_carbons.c', 'tests/functionaltests/test_receipts.c', 'tests/functionaltests/test_roster.c', + 'tests/functionaltests/test_i18n.c', 'tests/functionaltests/test_software.c', 'tests/functionaltests/test_muc.c', 'tests/functionaltests/test_disconnect.c', diff --git a/src/tools/autocomplete.c b/src/tools/autocomplete.c index 9b64ff10..cbbe856d 100644 --- a/src/tools/autocomplete.c +++ b/src/tools/autocomplete.c @@ -370,16 +370,34 @@ autocomplete_remove_older_than_max_reverse(Autocomplete ac, int maxsize) static gchar* _search(Autocomplete ac, GList* curr, gboolean quote, search_direction direction) { + auto_gchar gchar* search_str_fold = g_utf8_casefold(ac->search_str, -1); auto_gchar gchar* search_str_ascii = g_str_to_ascii(ac->search_str, NULL); - auto_gchar gchar* search_str_lower = g_ascii_strdown(search_str_ascii, -1); + auto_gchar gchar* search_str_ascii_lower = g_ascii_strdown(search_str_ascii, -1); while (curr) { - auto_gchar gchar* curr_ascii = g_str_to_ascii(curr->data, NULL); - auto_gchar gchar* curr_lower = g_ascii_strdown(curr_ascii, -1); + gboolean match = FALSE; - // match found - if (strncmp(curr_lower, search_str_lower, strlen(search_str_lower)) == 0) { + // Try exact UTF-8 case insensitive match + auto_gchar gchar* curr_fold = g_utf8_casefold(curr->data, -1); + if (g_str_has_prefix(curr_fold, search_str_fold)) { + match = TRUE; + } + // Try transliterated match (allow typing 'e' for 'è') + if (!match) { + auto_gchar gchar* curr_ascii = g_str_to_ascii(curr->data, NULL); + auto_gchar gchar* curr_ascii_lower = g_ascii_strdown(curr_ascii, -1); + + // We only use transliterated match if the search string conversion didn't result + // in unknown characters ('?'), to avoid false matches between different scripts. + if (strchr(search_str_ascii_lower, '?') == NULL) { + if (g_str_has_prefix(curr_ascii_lower, search_str_ascii_lower)) { + match = TRUE; + } + } + } + + if (match) { // set pointer to last found ac->last_found = curr; diff --git a/tests/functionaltests/functionaltests.c b/tests/functionaltests/functionaltests.c index 239937c0..e163b0f3 100644 --- a/tests/functionaltests/functionaltests.c +++ b/tests/functionaltests/functionaltests.c @@ -27,6 +27,7 @@ #include "test_software.h" #include "test_muc.h" #include "test_disconnect.h" +#include "test_i18n.h" #define PROF_FUNC_TEST(test) cmocka_unit_test_setup_teardown(test, init_prof_test, close_prof_test) @@ -116,6 +117,9 @@ main(int argc, char* argv[]) PROF_FUNC_TEST(shows_no_message_in_console_when_window_not_focussed), PROF_FUNC_TEST(disconnect_ends_session), + + PROF_FUNC_TEST(i18n_msg_nickname), + PROF_FUNC_TEST(i18n_win_nickname), }; return cmocka_run_group_tests(all_tests, NULL, NULL); diff --git a/tests/functionaltests/test_i18n.c b/tests/functionaltests/test_i18n.c new file mode 100644 index 00000000..572486dc --- /dev/null +++ b/tests/functionaltests/test_i18n.c @@ -0,0 +1,51 @@ +/* + * test_i18n.c + * + * Copyright (C) 2026 Michael Vetter + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ +#include +#include "prof_cmocka.h" +#include +#include +#include + +#include + +#include "proftest.h" + +void +i18n_msg_nickname(void** state) +{ + // Connect with a roster containing a UTF-8 nickname: Σωκράτης (Socrates) + prof_connect_with_roster( + "" + ""); + + // Test that we can /msg the UTF-8 nickname successfully. + // This verifies that the UI and command parsing handle UTF-8. + prof_input("/msg Σωκράτης Hello"); + + // Check if we switched to the socrates window + assert_true(prof_output_regex("socrates@localhost")); +} + +void +i18n_win_nickname(void** state) +{ + prof_connect_with_roster( + ""); + + // Open a chat with Socrates + prof_input("/msg Σωκράτης"); + assert_true(prof_output_regex("socrates@localhost")); + + // Switch back to console + prof_input("/win 1"); + assert_true(prof_output_regex("Console")); + + // Switch to Socrates window using name + prof_input("/win Σωκράτης"); + assert_true(prof_output_regex("socrates@localhost")); +} diff --git a/tests/functionaltests/test_i18n.h b/tests/functionaltests/test_i18n.h new file mode 100644 index 00000000..c37919cf --- /dev/null +++ b/tests/functionaltests/test_i18n.h @@ -0,0 +1,9 @@ +/* + * test_i18n.h + * + * Copyright (C) 2026 Michael Vetter + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ +void i18n_msg_nickname(void** state); +void i18n_win_nickname(void** state); diff --git a/tests/unittests/tools/test_autocomplete.c b/tests/unittests/tools/test_autocomplete.c index 77a6414d..45f03ebe 100644 --- a/tests/unittests/tools/test_autocomplete.c +++ b/tests/unittests/tools/test_autocomplete.c @@ -233,3 +233,115 @@ autocomplete_complete__returns__previous(void** state) free(result3); free(result4); } + +void +autocomplete_complete__returns__greek_false_match(void** state) +{ + Autocomplete ac = autocomplete_new(); + // Σωκράτης (Socrates) and Πλάτων (Plato) + autocomplete_add(ac, "Σωκράτης"); + autocomplete_add(ac, "Πλάτων"); + + char* result = autocomplete_complete(ac, "Π", TRUE, FALSE); + + assert_string_equal("Πλάτων", result); + + autocomplete_free(ac); + free(result); +} + +void +autocomplete_complete__returns__greek(void** state) +{ + Autocomplete ac = autocomplete_new(); + autocomplete_add(ac, "Αριστοτέλης"); + + char* result = autocomplete_complete(ac, "Αριστ", TRUE, FALSE); + + assert_non_null(result); + assert_string_equal("Αριστοτέλης", result); + + autocomplete_free(ac); + free(result); +} + +void +autocomplete_complete__returns__greek_case_insensitive(void** state) +{ + Autocomplete ac = autocomplete_new(); + autocomplete_add(ac, "Σωκράτης"); + + // Case insensitive search for Socrates + // σω is the lowercase of Σω + char* result = autocomplete_complete(ac, "σω", TRUE, FALSE); + + assert_non_null(result); + assert_string_equal("Σωκράτης", result); + + autocomplete_free(ac); + free(result); +} + +void +autocomplete_complete__returns__russian(void** state) +{ + Autocomplete ac = autocomplete_new(); + autocomplete_add(ac, "Достоевский"); + autocomplete_add(ac, "Толстой"); + autocomplete_add(ac, "Пушкин"); + + char* result = autocomplete_complete(ac, "дост", TRUE, FALSE); + + assert_non_null(result); + assert_string_equal("Достоевский", result); + + autocomplete_free(ac); + free(result); +} + +void +autocomplete_complete__returns__chinese(void** state) +{ + Autocomplete ac = autocomplete_new(); + autocomplete_add(ac, "孙子"); + autocomplete_add(ac, "诸葛亮"); + + char* result = autocomplete_complete(ac, "孙", TRUE, FALSE); + + assert_non_null(result); + assert_string_equal("孙子", result); + + autocomplete_free(ac); + free(result); +} + +void +autocomplete_complete__returns__transliterated(void** state) +{ + Autocomplete ac = autocomplete_new(); + autocomplete_add(ac, "München"); + + // Match 'ü' with 'u' + char* result = autocomplete_complete(ac, "mun", TRUE, FALSE); + + assert_non_null(result); + assert_string_equal("München", result); + + autocomplete_free(ac); + free(result); +} + +void +autocomplete_complete__returns__regular_ascii(void** state) +{ + Autocomplete ac = autocomplete_new(); + autocomplete_add(ac, "London"); + + char* result = autocomplete_complete(ac, "lon", TRUE, FALSE); + + assert_non_null(result); + assert_string_equal("London", result); + + autocomplete_free(ac); + free(result); +} diff --git a/tests/unittests/tools/test_autocomplete.h b/tests/unittests/tools/test_autocomplete.h index 047123da..09ec200d 100644 --- a/tests/unittests/tools/test_autocomplete.h +++ b/tests/unittests/tools/test_autocomplete.h @@ -17,5 +17,12 @@ void autocomplete_complete__returns__both_with_accented(void** state); void autocomplete_complete__returns__both_with_base(void** state); void autocomplete_complete__is__case_insensitive(void** state); void autocomplete_complete__returns__previous(void** state); +void autocomplete_complete__returns__greek(void** state); +void autocomplete_complete__returns__greek_false_match(void** state); +void autocomplete_complete__returns__greek_case_insensitive(void** state); +void autocomplete_complete__returns__russian(void** state); +void autocomplete_complete__returns__chinese(void** state); +void autocomplete_complete__returns__transliterated(void** state); +void autocomplete_complete__returns__regular_ascii(void** state); #endif diff --git a/tests/unittests/unittests.c b/tests/unittests/unittests.c index 1dede0e5..92aafd9e 100644 --- a/tests/unittests/unittests.c +++ b/tests/unittests/unittests.c @@ -132,6 +132,13 @@ main(int argc, char* argv[]) cmocka_unit_test(autocomplete_complete__returns__both_with_base), cmocka_unit_test(autocomplete_complete__is__case_insensitive), cmocka_unit_test(autocomplete_complete__returns__previous), + cmocka_unit_test(autocomplete_complete__returns__greek), + cmocka_unit_test(autocomplete_complete__returns__greek_false_match), + cmocka_unit_test(autocomplete_complete__returns__greek_case_insensitive), + cmocka_unit_test(autocomplete_complete__returns__russian), + cmocka_unit_test(autocomplete_complete__returns__chinese), + cmocka_unit_test(autocomplete_complete__returns__transliterated), + cmocka_unit_test(autocomplete_complete__returns__regular_ascii), cmocka_unit_test(jid_create__returns__null_from_null), cmocka_unit_test(jid_create__returns__null_from_empty_string), @@ -240,6 +247,7 @@ main(int argc, char* argv[]) cmocka_unit_test(roster_contact_autocomplete__returns__second_when_two_match), cmocka_unit_test(roster_contact_autocomplete__returns__fifth_when_multiple_match), cmocka_unit_test(roster_contact_autocomplete__returns__first_when_two_match_and_reset), + cmocka_unit_test(roster_contact_autocomplete__returns__utf8), cmocka_unit_test(roster_get_groups__returns__empty_for_no_group), cmocka_unit_test(roster_get_groups__returns__one_group), cmocka_unit_test(roster_get_groups__returns__two_groups), diff --git a/tests/unittests/xmpp/test_roster_list.c b/tests/unittests/xmpp/test_roster_list.c index 8eacfe78..4ade75ca 100644 --- a/tests/unittests/xmpp/test_roster_list.c +++ b/tests/unittests/xmpp/test_roster_list.c @@ -305,8 +305,30 @@ roster_contact_autocomplete__returns__first_when_two_match_and_reset(void** stat roster_destroy(); } +void +roster_contact_autocomplete__returns__utf8(void** state) +{ + roster_create(); + roster_add("Σωκράτης", NULL, NULL, NULL, FALSE); + roster_add("Πλάτων", NULL, NULL, NULL, FALSE); + + // Byte-wise (strcmp): Πλάτων (CE A0...) < Σωκράτης (CE A3...) + char* result = roster_contact_autocomplete("Π", FALSE, NULL); + assert_string_equal("Πλάτων", result); + g_free(result); + + roster_reset_search_attempts(); + + result = roster_contact_autocomplete("σω", FALSE, NULL); + assert_string_equal("Σωκράτης", result); + g_free(result); + + roster_destroy(); +} + void roster_get_groups__returns__empty_for_no_group(void** state) + { roster_create(); roster_add("person@server.org", NULL, NULL, NULL, FALSE); diff --git a/tests/unittests/xmpp/test_roster_list.h b/tests/unittests/xmpp/test_roster_list.h index 378ff275..c1787026 100644 --- a/tests/unittests/xmpp/test_roster_list.h +++ b/tests/unittests/xmpp/test_roster_list.h @@ -19,6 +19,7 @@ void roster_contact_autocomplete__returns__null_on_empty_roster(void** state); void roster_contact_autocomplete__returns__second_when_two_match(void** state); void roster_contact_autocomplete__returns__fifth_when_multiple_match(void** state); void roster_contact_autocomplete__returns__first_when_two_match_and_reset(void** state); +void roster_contact_autocomplete__returns__utf8(void** state); void roster_get_groups__returns__empty_for_no_group(void** state); void roster_get_groups__returns__one_group(void** state); void roster_get_groups__returns__two_groups(void** state); From ee0d325648929510e37d8edd918b4ca82704bf37 Mon Sep 17 00:00:00 2001 From: Michael Vetter Date: Thu, 19 Mar 2026 08:50:58 +0100 Subject: [PATCH 2/3] tests: add prof_send_raw Allow sending byte sequences to the Profanity PTY without a newline. Now we can simulate control characters, specifically the `TAB` key (`\t`), to trigger autocompletion. --- tests/functionaltests/proftest.c | 8 ++++++++ tests/functionaltests/proftest.h | 1 + 2 files changed, 9 insertions(+) diff --git a/tests/functionaltests/proftest.c b/tests/functionaltests/proftest.c index d91c2251..62384e4b 100644 --- a/tests/functionaltests/proftest.c +++ b/tests/functionaltests/proftest.c @@ -397,6 +397,14 @@ prof_input(const char* input) usleep(1000 * 100); // 100ms delay } +void +prof_send_raw(const char* bytes) +{ + write(fd, bytes, strlen(bytes)); + tcdrain(fd); + usleep(1000 * 100); +} + int prof_output_exact(const char* text) { diff --git a/tests/functionaltests/proftest.h b/tests/functionaltests/proftest.h index 85b62f7b..a5d2f679 100644 --- a/tests/functionaltests/proftest.h +++ b/tests/functionaltests/proftest.h @@ -28,6 +28,7 @@ void prof_start(void); void prof_connect(void); void prof_connect_with_roster(const char* roster); void prof_input(const char* input); +void prof_send_raw(const char* bytes); int prof_output_exact(const char* text); int prof_output_regex(const char* text); From f6b9903c313f0f1220daad47343c1408011557d4 Mon Sep 17 00:00:00 2001 From: Michael Vetter Date: Thu, 19 Mar 2026 08:55:20 +0100 Subject: [PATCH 3/3] tests: Add test for tab autocompletion of /msg Test both latin and UTF-8. --- tests/functionaltests/functionaltests.c | 2 ++ tests/functionaltests/test_i18n.c | 39 +++++++++++++++++++++++++ tests/functionaltests/test_i18n.h | 2 ++ 3 files changed, 43 insertions(+) diff --git a/tests/functionaltests/functionaltests.c b/tests/functionaltests/functionaltests.c index e163b0f3..14c5a8b6 100644 --- a/tests/functionaltests/functionaltests.c +++ b/tests/functionaltests/functionaltests.c @@ -120,6 +120,8 @@ main(int argc, char* argv[]) PROF_FUNC_TEST(i18n_msg_nickname), PROF_FUNC_TEST(i18n_win_nickname), + PROF_FUNC_TEST(i18n_autocomplete_tab_utf8), + PROF_FUNC_TEST(i18n_autocomplete_tab_latin), }; return cmocka_run_group_tests(all_tests, NULL, NULL); diff --git a/tests/functionaltests/test_i18n.c b/tests/functionaltests/test_i18n.c index 572486dc..78dccdf3 100644 --- a/tests/functionaltests/test_i18n.c +++ b/tests/functionaltests/test_i18n.c @@ -49,3 +49,42 @@ i18n_win_nickname(void** state) prof_input("/win Σωκράτης"); assert_true(prof_output_regex("socrates@localhost")); } + +void +i18n_autocomplete_tab_utf8(void** state) +{ + prof_connect_with_roster( + ""); + + prof_send_raw("/msg Σω"); + // TAB for completion + prof_send_raw("\t"); + prof_input(" Hello"); + + // If autocompletion worked, we should be in the Socrates window + assert_true(prof_output_regex("socrates@localhost")); + + // And stabber should have received the message + assert_true(stbbr_received( + "" + "Hello" + "")); +} + +void +i18n_autocomplete_tab_latin(void** state) +{ + prof_connect_with_roster( + ""); + + prof_send_raw("/msg Pl"); + prof_send_raw("\t"); + prof_input(" Hello"); + + assert_true(prof_output_regex("plato@localhost")); + + assert_true(stbbr_received( + "" + "Hello" + "")); +} diff --git a/tests/functionaltests/test_i18n.h b/tests/functionaltests/test_i18n.h index c37919cf..cb8ef612 100644 --- a/tests/functionaltests/test_i18n.h +++ b/tests/functionaltests/test_i18n.h @@ -7,3 +7,5 @@ */ void i18n_msg_nickname(void** state); void i18n_win_nickname(void** state); +void i18n_autocomplete_tab_utf8(void** state); +void i18n_autocomplete_tab_latin(void** state);