Merge pull request #2115 from profanity-im/fix-1986

Properly support UTF-8 characters in autocompletion

meson.build

@@ -698,6 +698,7 @@ if get_option('tests')
         'tests/functionaltests/test_carbons.c',
         'tests/functionaltests/test_receipts.c',
         'tests/functionaltests/test_roster.c',
+        'tests/functionaltests/test_i18n.c',
         'tests/functionaltests/test_software.c',
         'tests/functionaltests/test_muc.c',
         'tests/functionaltests/test_disconnect.c',

src/tools/autocomplete.c

@@ -370,16 +370,34 @@ autocomplete_remove_older_than_max_reverse(Autocomplete ac, int maxsize)
 static gchar*
 _search(Autocomplete ac, GList* curr, gboolean quote, search_direction direction)
 {
+    auto_gchar gchar* search_str_fold = g_utf8_casefold(ac->search_str, -1);
     auto_gchar gchar* search_str_ascii = g_str_to_ascii(ac->search_str, NULL);
-    auto_gchar gchar* search_str_lower = g_ascii_strdown(search_str_ascii, -1);
+    auto_gchar gchar* search_str_ascii_lower = g_ascii_strdown(search_str_ascii, -1);
 
     while (curr) {
-        auto_gchar gchar* curr_ascii = g_str_to_ascii(curr->data, NULL);
-        auto_gchar gchar* curr_lower = g_ascii_strdown(curr_ascii, -1);
+        gboolean match = FALSE;
 
-        // match found
-        if (strncmp(curr_lower, search_str_lower, strlen(search_str_lower)) == 0) {
+        // Try exact UTF-8 case insensitive match
+        auto_gchar gchar* curr_fold = g_utf8_casefold(curr->data, -1);
+        if (g_str_has_prefix(curr_fold, search_str_fold)) {
+            match = TRUE;
+        }
 
+        // Try transliterated match (allow typing 'e' for 'è')
+        if (!match) {
+            auto_gchar gchar* curr_ascii = g_str_to_ascii(curr->data, NULL);
+            auto_gchar gchar* curr_ascii_lower = g_ascii_strdown(curr_ascii, -1);
+
+            // We only use transliterated match if the search string conversion didn't result
+            // in unknown characters ('?'), to avoid false matches between different scripts.
+            if (strchr(search_str_ascii_lower, '?') == NULL) {
+                if (g_str_has_prefix(curr_ascii_lower, search_str_ascii_lower)) {
+                    match = TRUE;
+                }
+            }
+        }
+
+        if (match) {
             // set pointer to last found
             ac->last_found = curr;
 

tests/functionaltests/functionaltests.c

@@ -27,6 +27,7 @@
 #include "test_software.h"
 #include "test_muc.h"
 #include "test_disconnect.h"
+#include "test_i18n.h"
 
 #define PROF_FUNC_TEST(test) cmocka_unit_test_setup_teardown(test, init_prof_test, close_prof_test)
 
@@ -116,6 +117,11 @@ main(int argc, char* argv[])
         PROF_FUNC_TEST(shows_no_message_in_console_when_window_not_focussed),
 
         PROF_FUNC_TEST(disconnect_ends_session),
+
+        PROF_FUNC_TEST(i18n_msg_nickname),
+        PROF_FUNC_TEST(i18n_win_nickname),
+        PROF_FUNC_TEST(i18n_autocomplete_tab_utf8),
+        PROF_FUNC_TEST(i18n_autocomplete_tab_latin),
     };
 
     return cmocka_run_group_tests(all_tests, NULL, NULL);

tests/functionaltests/proftest.c

@@ -397,6 +397,14 @@ prof_input(const char* input)
     usleep(1000 * 100); // 100ms delay
 }
 
+void
+prof_send_raw(const char* bytes)
+{
+    write(fd, bytes, strlen(bytes));
+    tcdrain(fd);
+    usleep(1000 * 100);
+}
+
 int
 prof_output_exact(const char* text)
 {

tests/functionaltests/proftest.h

@@ -28,6 +28,7 @@ void prof_start(void);
 void prof_connect(void);
 void prof_connect_with_roster(const char* roster);
 void prof_input(const char* input);
+void prof_send_raw(const char* bytes);
 
 int prof_output_exact(const char* text);
 int prof_output_regex(const char* text);

tests/functionaltests/test_i18n.c

@@ -0,0 +1,90 @@
+/*
+ * test_i18n.c
+ *
+ * Copyright (C) 2026 Michael Vetter <jubalh@iodoru.org>
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+#include <glib.h>
+#include "prof_cmocka.h"
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <stabber.h>
+
+#include "proftest.h"
+
+void
+i18n_msg_nickname(void** state)
+{
+    // Connect with a roster containing a UTF-8 nickname: Σωκράτης (Socrates)
+    prof_connect_with_roster(
+        "<item jid='socrates@localhost' subscription='both' name='Σωκράτης'/>"
+        "<item jid='buddy2@localhost' subscription='both' name='Buddy2'/>");
+
+    // Test that we can /msg the UTF-8 nickname successfully.
+    // This verifies that the UI and command parsing handle UTF-8.
+    prof_input("/msg Σωκράτης Hello");
+
+    // Check if we switched to the socrates window
+    assert_true(prof_output_regex("socrates@localhost"));
+}
+
+void
+i18n_win_nickname(void** state)
+{
+    prof_connect_with_roster(
+        "<item jid='socrates@localhost' subscription='both' name='Σωκράτης'/>");
+
+    // Open a chat with Socrates
+    prof_input("/msg Σωκράτης");
+    assert_true(prof_output_regex("socrates@localhost"));
+
+    // Switch back to console
+    prof_input("/win 1");
+    assert_true(prof_output_regex("Console"));
+
+    // Switch to Socrates window using name
+    prof_input("/win Σωκράτης");
+    assert_true(prof_output_regex("socrates@localhost"));
+}
+
+void
+i18n_autocomplete_tab_utf8(void** state)
+{
+    prof_connect_with_roster(
+        "<item jid='socrates@localhost' subscription='both' name='Σωκράτης'/>");
+
+    prof_send_raw("/msg Σω");
+    // TAB for completion
+    prof_send_raw("\t");
+    prof_input(" Hello");
+
+    // If autocompletion worked, we should be in the Socrates window
+    assert_true(prof_output_regex("socrates@localhost"));
+
+    // And stabber should have received the message
+    assert_true(stbbr_received(
+        "<message to='socrates@localhost' id='*' type='chat'>"
+        "<body>Hello</body>"
+        "</message>"));
+}
+
+void
+i18n_autocomplete_tab_latin(void** state)
+{
+    prof_connect_with_roster(
+        "<item jid='plato@localhost' subscription='both' name='Plato'/>");
+
+    prof_send_raw("/msg Pl");
+    prof_send_raw("\t");
+    prof_input(" Hello");
+
+    assert_true(prof_output_regex("plato@localhost"));
+
+    assert_true(stbbr_received(
+        "<message to='plato@localhost' id='*' type='chat'>"
+        "<body>Hello</body>"
+        "</message>"));
+}

tests/functionaltests/test_i18n.h

@@ -0,0 +1,11 @@
+/*
+ * test_i18n.h
+ *
+ * Copyright (C) 2026 Michael Vetter <jubalh@iodoru.org>
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+void i18n_msg_nickname(void** state);
+void i18n_win_nickname(void** state);
+void i18n_autocomplete_tab_utf8(void** state);
+void i18n_autocomplete_tab_latin(void** state);

tests/unittests/tools/test_autocomplete.c

@@ -233,3 +233,115 @@ autocomplete_complete__returns__previous(void** state)
     free(result3);
     free(result4);
 }
+
+void
+autocomplete_complete__returns__greek_false_match(void** state)
+{
+    Autocomplete ac = autocomplete_new();
+    // Σωκράτης (Socrates) and Πλάτων (Plato)
+    autocomplete_add(ac, "Σωκράτης");
+    autocomplete_add(ac, "Πλάτων");
+
+    char* result = autocomplete_complete(ac, "Π", TRUE, FALSE);
+
+    assert_string_equal("Πλάτων", result);
+
+    autocomplete_free(ac);
+    free(result);
+}
+
+void
+autocomplete_complete__returns__greek(void** state)
+{
+    Autocomplete ac = autocomplete_new();
+    autocomplete_add(ac, "Αριστοτέλης");
+
+    char* result = autocomplete_complete(ac, "Αριστ", TRUE, FALSE);
+
+    assert_non_null(result);
+    assert_string_equal("Αριστοτέλης", result);
+
+    autocomplete_free(ac);
+    free(result);
+}
+
+void
+autocomplete_complete__returns__greek_case_insensitive(void** state)
+{
+    Autocomplete ac = autocomplete_new();
+    autocomplete_add(ac, "Σωκράτης");
+
+    // Case insensitive search for Socrates
+    // σω is the lowercase of Σω
+    char* result = autocomplete_complete(ac, "σω", TRUE, FALSE);
+
+    assert_non_null(result);
+    assert_string_equal("Σωκράτης", result);
+
+    autocomplete_free(ac);
+    free(result);
+}
+
+void
+autocomplete_complete__returns__russian(void** state)
+{
+    Autocomplete ac = autocomplete_new();
+    autocomplete_add(ac, "Достоевский");
+    autocomplete_add(ac, "Толстой");
+    autocomplete_add(ac, "Пушкин");
+
+    char* result = autocomplete_complete(ac, "дост", TRUE, FALSE);
+
+    assert_non_null(result);
+    assert_string_equal("Достоевский", result);
+
+    autocomplete_free(ac);
+    free(result);
+}
+
+void
+autocomplete_complete__returns__chinese(void** state)
+{
+    Autocomplete ac = autocomplete_new();
+    autocomplete_add(ac, "孙子");
+    autocomplete_add(ac, "诸葛亮");
+
+    char* result = autocomplete_complete(ac, "孙", TRUE, FALSE);
+
+    assert_non_null(result);
+    assert_string_equal("孙子", result);
+
+    autocomplete_free(ac);
+    free(result);
+}
+
+void
+autocomplete_complete__returns__transliterated(void** state)
+{
+    Autocomplete ac = autocomplete_new();
+    autocomplete_add(ac, "München");
+
+    // Match 'ü' with 'u'
+    char* result = autocomplete_complete(ac, "mun", TRUE, FALSE);
+
+    assert_non_null(result);
+    assert_string_equal("München", result);
+
+    autocomplete_free(ac);
+    free(result);
+}
+
+void
+autocomplete_complete__returns__regular_ascii(void** state)
+{
+    Autocomplete ac = autocomplete_new();
+    autocomplete_add(ac, "London");
+
+    char* result = autocomplete_complete(ac, "lon", TRUE, FALSE);
+
+    assert_non_null(result);
+    assert_string_equal("London", result);
+
+    autocomplete_free(ac);
+    free(result);
+}

tests/unittests/tools/test_autocomplete.h

@@ -17,5 +17,12 @@ void autocomplete_complete__returns__both_with_accented(void** state);
 void autocomplete_complete__returns__both_with_base(void** state);
 void autocomplete_complete__is__case_insensitive(void** state);
 void autocomplete_complete__returns__previous(void** state);
+void autocomplete_complete__returns__greek(void** state);
+void autocomplete_complete__returns__greek_false_match(void** state);
+void autocomplete_complete__returns__greek_case_insensitive(void** state);
+void autocomplete_complete__returns__russian(void** state);
+void autocomplete_complete__returns__chinese(void** state);
+void autocomplete_complete__returns__transliterated(void** state);
+void autocomplete_complete__returns__regular_ascii(void** state);
 
 #endif

tests/unittests/unittests.c

@@ -132,6 +132,13 @@ main(int argc, char* argv[])
         cmocka_unit_test(autocomplete_complete__returns__both_with_base),
         cmocka_unit_test(autocomplete_complete__is__case_insensitive),
         cmocka_unit_test(autocomplete_complete__returns__previous),
+        cmocka_unit_test(autocomplete_complete__returns__greek),
+        cmocka_unit_test(autocomplete_complete__returns__greek_false_match),
+        cmocka_unit_test(autocomplete_complete__returns__greek_case_insensitive),
+        cmocka_unit_test(autocomplete_complete__returns__russian),
+        cmocka_unit_test(autocomplete_complete__returns__chinese),
+        cmocka_unit_test(autocomplete_complete__returns__transliterated),
+        cmocka_unit_test(autocomplete_complete__returns__regular_ascii),
 
         cmocka_unit_test(jid_create__returns__null_from_null),
         cmocka_unit_test(jid_create__returns__null_from_empty_string),
@@ -240,6 +247,7 @@ main(int argc, char* argv[])
         cmocka_unit_test(roster_contact_autocomplete__returns__second_when_two_match),
         cmocka_unit_test(roster_contact_autocomplete__returns__fifth_when_multiple_match),
         cmocka_unit_test(roster_contact_autocomplete__returns__first_when_two_match_and_reset),
+        cmocka_unit_test(roster_contact_autocomplete__returns__utf8),
         cmocka_unit_test(roster_get_groups__returns__empty_for_no_group),
         cmocka_unit_test(roster_get_groups__returns__one_group),
         cmocka_unit_test(roster_get_groups__returns__two_groups),

tests/unittests/xmpp/test_roster_list.c

@@ -305,8 +305,30 @@ roster_contact_autocomplete__returns__first_when_two_match_and_reset(void** stat
     roster_destroy();
 }
 
+void
+roster_contact_autocomplete__returns__utf8(void** state)
+{
+    roster_create();
+    roster_add("Σωκράτης", NULL, NULL, NULL, FALSE);
+    roster_add("Πλάτων", NULL, NULL, NULL, FALSE);
+
+    // Byte-wise (strcmp): Πλάτων (CE A0...) < Σωκράτης (CE A3...)
+    char* result = roster_contact_autocomplete("Π", FALSE, NULL);
+    assert_string_equal("Πλάτων", result);
+    g_free(result);
+
+    roster_reset_search_attempts();
+
+    result = roster_contact_autocomplete("σω", FALSE, NULL);
+    assert_string_equal("Σωκράτης", result);
+    g_free(result);
+
+    roster_destroy();
+}
+
 void
 roster_get_groups__returns__empty_for_no_group(void** state)
+
 {
     roster_create();
     roster_add("person@server.org", NULL, NULL, NULL, FALSE);

tests/unittests/xmpp/test_roster_list.h

@@ -19,6 +19,7 @@ void roster_contact_autocomplete__returns__null_on_empty_roster(void** state);
 void roster_contact_autocomplete__returns__second_when_two_match(void** state);
 void roster_contact_autocomplete__returns__fifth_when_multiple_match(void** state);
 void roster_contact_autocomplete__returns__first_when_two_match_and_reset(void** state);
+void roster_contact_autocomplete__returns__utf8(void** state);
 void roster_get_groups__returns__empty_for_no_group(void** state);
 void roster_get_groups__returns__one_group(void** state);
 void roster_get_groups__returns__two_groups(void** state);