fix(ai): properly decode \uXXXX JSON escape sequences as UTF-8

Add helper functions to parse hex digits and encode UTF-8 characters. Update buffer allocation to account for UTF-8 expansion and implement full surrogate pair support for characters outside the BMP. Previously, \uXXXX sequences were passed through verbatim; they are now correctly decoded into proper UTF-8 strings.

src/ai/ai_client.c

@@ -186,14 +186,65 @@ ai_json_escape(const gchar* str)
     return g_string_free(result, FALSE);
 }
 
+/**
+ * Convert a 4-hex-digit sequence at @p to a 16-bit code point.
+ * Returns -1 on invalid hex.
+ */
+static int
+_hex4_to_int(const gchar* p)
+{
+    int val = 0;
+    for (int i = 0; i < 4; i++) {
+        char c = p[i];
+        val <<= 4;
+        if (c >= '0' && c <= '9')
+            val |= c - '0';
+        else if (c >= 'a' && c <= 'f')
+            val |= c - 'a' + 10;
+        else if (c >= 'A' && c <= 'F')
+            val |= c - 'A' + 10;
+        else
+            return -1;
+    }
+    return val;
+}
+
+/**
+ * Write a UTF-8 encoding of @cp to @w, advancing @w.
+ * Returns the updated pointer.
+ */
+static gchar*
+_write_utf8(gchar* w, unsigned int cp)
+{
+    if (cp <= 0x7F) {
+        *w++ = (char)cp;
+    } else if (cp <= 0x7FF) {
+        *w++ = (char)(0xC0 | (cp >> 6));
+        *w++ = (char)(0x80 | (cp & 0x3F));
+    } else if (cp <= 0xFFFF) {
+        *w++ = (char)(0xE0 | (cp >> 12));
+        *w++ = (char)(0x80 | ((cp >> 6) & 0x3F));
+        *w++ = (char)(0x80 | (cp & 0x3F));
+    } else if (cp <= 0x10FFFF) {
+        *w++ = (char)(0xF0 | (cp >> 18));
+        *w++ = (char)(0x80 | ((cp >> 12) & 0x3F));
+        *w++ = (char)(0x80 | ((cp >> 6) & 0x3F));
+        *w++ = (char)(0x80 | (cp & 0x3F));
+    }
+    /* Invalid code point: silently skip */
+    return w;
+}
+
 /* Decode JSON string escapes in [start, end) into a freshly allocated buffer.
- * Handles \" \\ \/ \n \t \r \b \f. \uXXXX sequences are passed through verbatim
- * since the rest of the AI pipeline already deals in UTF-8 byte streams. */
+ * Handles \" \\ \/ \n \t \r \b \f and \uXXXX (including surrogate pairs). */
 static gchar*
 _json_unescape_substring(const gchar* start, const gchar* end)
 {
-    gchar* out = g_new0(gchar, (end - start) + 1);
+    /* Estimate upper bound: each \uXXXX becomes up to 6 UTF-8 bytes. */
+    gsize max_len = (end - start) * 6 + 1;
+    gchar* out = g_new0(gchar, max_len);
     gchar* w = out;
+
     for (const gchar* in = start; in < end;) {
         if (*in == '\\' && in + 1 < end) {
             switch (in[1]) {
@@ -229,6 +280,64 @@ _json_unescape_substring(const gchar* start, const gchar* end)
                 *w++ = '\f';
                 in += 2;
                 break;
+            case 'u':
+                /* Handle \uXXXX or surrogate pair \uHHHH\uLLLL */
+                if (in + 6 > end) {
+                    /* Not enough characters — emit literal */
+                    *w++ = *in++;
+                    break;
+                }
+                if (in[2] != '\\' || in[3] != 'u') {
+                    /* Single \uXXXX */
+                    int cp = _hex4_to_int(in + 2);
+                    if (cp < 0) {
+                        /* Invalid hex — emit literal backslash */
+                        *w++ = *in++;
+                    } else if (cp < 0xD800 || cp > 0xDFFF) {
+                        /* Not a surrogate — encode directly */
+                        w = _write_utf8(w, (unsigned int)cp);
+                        in += 6;
+                    } else {
+                        /* High or low surrogate — expect pair */
+                        unsigned int high = (unsigned int)cp;
+                        if ((high >= 0xD800 && high <= 0xDBFF) && in + 12 <= end && in[6] == '\\' && in[7] == 'u') {
+                            /* High surrogate followed by \u */
+                            int low = _hex4_to_int(in + 8);
+                            if (low >= 0xDC00 && low <= 0xDFFF) {
+                                /* Valid surrogate pair */
+                                unsigned int cp2 = (0x10000 + ((high & 0x3FF) << 10)) | (low & 0x3FF);
+                                w = _write_utf8(w, cp2);
+                                in += 12;
+                            } else {
+                                /* Invalid low surrogate — emit replacement */
+                                w = _write_utf8(w, 0xFFFD);
+                                in += 6;
+                            }
+                        } else {
+                            /* Orphan high surrogate or lone low surrogate */
+                            w = _write_utf8(w, 0xFFFD);
+                            in += 6;
+                        }
+                    }
+                } else {
+                    /* \uHHHH\uLLLL — surrogate pair */
+                    int high = _hex4_to_int(in + 2);
+                    if (high < 0) {
+                        *w++ = *in++;
+                        break;
+                    }
+                    int low = _hex4_to_int(in + 8);
+                    if ((high >= 0xD800 && high <= 0xDBFF) && (low >= 0xDC00 && low <= 0xDFFF)) {
+                        unsigned int cp2 = (0x10000 + ((unsigned int)(high & 0x3FF) << 10)) | (unsigned int)(low & 0x3FF);
+                        w = _write_utf8(w, cp2);
+                        in += 12;
+                    } else {
+                        /* Invalid pair — emit replacement for high */
+                        w = _write_utf8(w, 0xFFFD);
+                        in += 6;
+                    }
+                }
+                break;
             default:
                 *w++ = *in++;
                 break;
@@ -238,7 +347,9 @@ _json_unescape_substring(const gchar* start, const gchar* end)
         }
     }
     *w = '\0';
-    return out;
+
+    /* Realloc to exact size */
+    return g_realloc(out, w - out + 1);
 }
 
 /* Locate "field":"...value..." in json and return the unescaped value.