From 36265dde2fd70f994bc8ee2b2cdff9a398ceff1d Mon Sep 17 00:00:00 2001 From: James Booth Date: Thu, 11 Jul 2013 22:57:35 +0100 Subject: [PATCH 1/8] Moved functions to parser.c, moved parser to tools --- Makefile.am | 8 ++-- src/command/command.c | 2 +- src/tools/autocomplete.c | 81 ++------------------------------- src/{command => tools}/parser.c | 76 +++++++++++++++++++++++++++++++ src/{command => tools}/parser.h | 2 + tests/test_parser.c | 2 +- 6 files changed, 88 insertions(+), 83 deletions(-) rename src/{command => tools}/parser.c (82%) rename src/{command => tools}/parser.h (92%) diff --git a/Makefile.am b/Makefile.am index f161b92a..f7c3f825 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,8 +14,8 @@ profanity_SOURCES = \ src/ui/titlebar.c src/ui/statusbar.c src/ui/inputwin.c \ src/ui/console.c src/ui/notifier.c src/ui/notifier.h \ src/command/command.h src/command/command.c src/command/history.c \ - src/command/history.h src/command/parser.c \ - src/command/parser.h \ + src/command/history.h src/tools/parser.c \ + src/tools/parser.h \ src/tools/autocomplete.c src/tools/autocomplete.h \ src/tools/history.c src/tools/history.h \ src/tools/tinyurl.c src/tools/tinyurl.h \ @@ -40,8 +40,8 @@ tests_testsuite_SOURCES = \ src/ui/titlebar.c src/ui/statusbar.c src/ui/inputwin.c \ src/ui/console.c src/ui/notifier.c src/ui/notifier.h \ src/command/command.h src/command/command.c src/command/history.c \ - src/command/history.h src/command/parser.c \ - src/command/parser.h \ + src/command/history.h src/tools/parser.c \ + src/tools/parser.h \ src/tools/autocomplete.c src/tools/autocomplete.h \ src/tools/history.c src/tools/history.h \ src/tools/tinyurl.c src/tools/tinyurl.h \ diff --git a/src/command/command.c b/src/command/command.c index 63d003f4..60fe6d6d 100644 --- a/src/command/command.c +++ b/src/command/command.c @@ -30,7 +30,6 @@ #include "chat_session.h" #include "command/command.h" #include "command/history.h" -#include "command/parser.h" #include "common.h" #include "config/accounts.h" #include "config/preferences.h" @@ -41,6 +40,7 @@ #include "muc.h" #include "profanity.h" #include "tools/autocomplete.h" +#include "tools/parser.h" #include "tools/tinyurl.h" #include "ui/ui.h" #include "xmpp/xmpp.h" diff --git a/src/tools/autocomplete.c b/src/tools/autocomplete.c index 849aaf1d..4b25edd6 100644 --- a/src/tools/autocomplete.c +++ b/src/tools/autocomplete.c @@ -24,7 +24,8 @@ #include #include -#include "autocomplete.h" +#include "tools/autocomplete.h" +#include "tools/parser.h" struct autocomplete_t { GSList *items; @@ -260,80 +261,6 @@ autocomplete_param_with_ac(char *input, int *size, char *command, return auto_msg; } -int -_count_tokens(char *string) -{ - int num_tokens = 0; - - // if no quotes, use glib - if (g_strrstr(string, "\"") == NULL) { - gchar **tokens = g_strsplit(string, " ", 0); - num_tokens = g_strv_length(tokens); - g_strfreev(tokens); - - // else count tokens including quoted - } else { - int length = strlen(string); - int i = 0; - gboolean in_quotes = FALSE; - - // include first token - num_tokens++; - - for (i = 0; i < length; i++) { - if (string[i] == ' ') { - if (!in_quotes) { - num_tokens++; - } - } else if (string[i] == '"') { - if (in_quotes) { - in_quotes = FALSE; - } else { - in_quotes = TRUE; - } - } - } - } - - return num_tokens; -} - -char * -_get_start(char *string, int tokens) -{ - char *result_str = NULL; - int num_tokens = 0; - int length = strlen(string); - int i = 0; - gboolean in_quotes = FALSE; - GString *result = g_string_new(""); - - // include first token - num_tokens++; - - for (i = 0; i < length; i++) { - if (num_tokens < tokens) { - g_string_append_c(result, string[i]); - } - if (string[i] == ' ') { - if (!in_quotes) { - num_tokens++; - } - } else if (string[i] == '"') { - if (in_quotes) { - in_quotes = FALSE; - } else { - in_quotes = TRUE; - } - } - } - - result_str = result->str; - g_string_free(result, FALSE); - - return result_str; -} - char * autocomplete_param_no_with_func(char *input, int *size, char *command, int arg_number, autocomplete_func func) @@ -353,11 +280,11 @@ autocomplete_param_no_with_func(char *input, int *size, char *command, g_strstrip(inp_cpy); // count tokens properly - int num_tokens = _count_tokens(inp_cpy); + int num_tokens = count_tokens(inp_cpy); // if correct number of tokens, then candidate for autocompletion of last param if (num_tokens == arg_number) { - gchar *start_str = _get_start(inp_cpy, arg_number); + gchar *start_str = get_start(inp_cpy, arg_number); gchar *comp_str = g_strdup(&inp_cpy[strlen(start_str)]); // autocomplete param diff --git a/src/command/parser.c b/src/tools/parser.c similarity index 82% rename from src/command/parser.c rename to src/tools/parser.c index d7dfebab..c3190a96 100644 --- a/src/command/parser.c +++ b/src/tools/parser.c @@ -271,3 +271,79 @@ parse_args_with_freetext(const char * const inp, int min, int max) return args; } } + +int +count_tokens(char *string) +{ + int num_tokens = 0; + + // if no quotes, use glib + if (g_strrstr(string, "\"") == NULL) { + gchar **tokens = g_strsplit(string, " ", 0); + num_tokens = g_strv_length(tokens); + g_strfreev(tokens); + + // else count tokens including quoted + } else { + int length = strlen(string); + int i = 0; + gboolean in_quotes = FALSE; + + // include first token + num_tokens++; + + for (i = 0; i < length; i++) { + if (string[i] == ' ') { + if (!in_quotes) { + num_tokens++; + } + } else if (string[i] == '"') { + if (in_quotes) { + in_quotes = FALSE; + } else { + in_quotes = TRUE; + } + } + } + } + + return num_tokens; +} + +char * +get_start(char *string, int tokens) +{ + char *result_str = NULL; + int num_tokens = 0; + int length = strlen(string); + int i = 0; + gboolean in_quotes = FALSE; + GString *result = g_string_new(""); + + // include first token + num_tokens++; + + for (i = 0; i < length; i++) { + if (num_tokens < tokens) { + g_string_append_c(result, string[i]); + } + if (string[i] == ' ') { + if (!in_quotes) { + num_tokens++; + } + } else if (string[i] == '"') { + if (in_quotes) { + in_quotes = FALSE; + } else { + in_quotes = TRUE; + } + } + } + + result_str = result->str; + g_string_free(result, FALSE); + + return result_str; +} + + diff --git a/src/command/parser.h b/src/tools/parser.h similarity index 92% rename from src/command/parser.h rename to src/tools/parser.h index f191ed15..6f00cc90 100644 --- a/src/command/parser.h +++ b/src/tools/parser.h @@ -27,5 +27,7 @@ gchar** parse_args(const char * const inp, int min, int max); gchar** parse_args_with_freetext(const char * const inp, int min, int max); +int count_tokens(char *string); +char* get_start(char *string, int tokens); #endif diff --git a/tests/test_parser.c b/tests/test_parser.c index fd1b1e1c..58e3f3e7 100644 --- a/tests/test_parser.c +++ b/tests/test_parser.c @@ -1,7 +1,7 @@ #include #include #include -#include "command/parser.h" +#include "tools/parser.h" void parse_null_returns_null(void) From 5233000498a30c61b22f68ac34ff10d60815f979 Mon Sep 17 00:00:00 2001 From: James Booth Date: Thu, 11 Jul 2013 23:03:20 +0100 Subject: [PATCH 2/8] Removed if clause in parser --- src/tools/parser.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tools/parser.c b/src/tools/parser.c index c3190a96..96c56628 100644 --- a/src/tools/parser.c +++ b/src/tools/parser.c @@ -205,8 +205,6 @@ parse_args_with_freetext(const char * const inp, int min, int max) token_start = ©[i+1]; } else { token_start = ©[i]; - } - if (copy[i] != '"') { token_size++; } } From bb550fed5547c7885e3f9c486051c742ac0b9b67 Mon Sep 17 00:00:00 2001 From: James Booth Date: Fri, 12 Jul 2013 00:46:33 +0100 Subject: [PATCH 3/8] Handle unicode chars in command parser --- src/tools/parser.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/tools/parser.c b/src/tools/parser.c index 96c56628..c8335879 100644 --- a/src/tools/parser.c +++ b/src/tools/parser.c @@ -58,7 +58,7 @@ parse_args(const char * const inp, int min, int max) char *copy = strdup(inp); g_strstrip(copy); - int inp_size = strlen(copy); + int inp_size = g_utf8_strlen(copy, -1); gboolean in_token = FALSE; gboolean in_quotes = FALSE; char *token_start = ©[0]; @@ -67,43 +67,48 @@ parse_args(const char * const inp, int min, int max) // add tokens to GSList int i; - for (i = 0; i <= inp_size; i++) { + for (i = 0; i < inp_size; i++) { + gchar *curr_ch = g_utf8_offset_to_pointer(copy, i); + gunichar curr_uni = g_utf8_get_char(curr_ch); if (!in_token) { - if (copy[i] == ' ') { + if (curr_uni == ' ') { continue; } else { in_token = TRUE; - if (copy[i] == '"') { + if (curr_uni == '"') { in_quotes = TRUE; i++; } - token_start = ©[i]; - token_size++; + token_start = curr_ch; + token_size += g_unichar_to_utf8(curr_uni, NULL); } } else { if (in_quotes) { - if ((copy[i] == '\0') || (copy[i] == '"')) { + if (curr_uni == '"') { tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); token_size = 0; in_token = FALSE; in_quotes = FALSE; } else { - token_size++; + token_size += g_unichar_to_utf8(curr_uni, NULL); } } else { - if (copy[i] == ' ' || copy[i] == '\0') { + if (curr_uni == ' ') { tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); token_size = 0; in_token = FALSE; } else { + token_size += g_unichar_to_utf8(curr_uni, NULL); token_size++; } } } } + tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); + int num = g_slist_length(tokens) - 1; // if num args not valid return NULL From 1d05a7047314fa3c669358114dc6ce36bbc08386 Mon Sep 17 00:00:00 2001 From: James Booth Date: Sat, 13 Jul 2013 22:59:42 +0100 Subject: [PATCH 4/8] Down arrow adds current line to history and shows empty line --- src/tools/history.c | 4 ++++ src/ui/inputwin.c | 4 ++++ tests/test_history.c | 6 +++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/tools/history.c b/src/tools/history.c index 5ac41969..b80a9555 100644 --- a/src/tools/history.c +++ b/src/tools/history.c @@ -146,6 +146,10 @@ history_next(History history, char *item) return NULL; } + if (g_list_next(history->session.sess_curr) == NULL) { + return NULL; + } + char *copied = ""; if (item != NULL) { copied = strdup(item); diff --git a/src/ui/inputwin.c b/src/ui/inputwin.c index 3f3d2ab5..feb4a0ba 100644 --- a/src/ui/inputwin.c +++ b/src/ui/inputwin.c @@ -490,6 +490,10 @@ _handle_edit(int result, const wint_t ch, char *input, int *size) next = cmd_history_next(input, size); if (next) { inp_replace_input(input, next, size); + } else if (*size != 0) { + input[*size] = '\0'; + cmd_history_append(input); + inp_replace_input(input, "", size); } return 1; diff --git a/tests/test_history.c b/tests/test_history.c index 91e5aa99..6af96946 100644 --- a/tests/test_history.c +++ b/tests/test_history.c @@ -93,7 +93,7 @@ void prev_with_val_then_next_returns_val(void) assert_string_equals("Oioi", item2); } -void prev_with_val_then_next_twice_returns_val(void) +void prev_with_val_then_next_twice_returns_null(void) { History history = history_new(10); history_append(history, "Hello"); @@ -102,7 +102,7 @@ void prev_with_val_then_next_twice_returns_val(void) char *item2 = history_next(history, item1); char *item3 = history_next(history, item2); - assert_string_equals("Oioi", item3); + assert_is_null(item3); } void navigate_then_append_new(void) @@ -225,7 +225,7 @@ void register_history_tests(void) TEST(previous_goes_to_correct_element); TEST(prev_then_next_returns_empty); TEST(prev_with_val_then_next_returns_val); - TEST(prev_with_val_then_next_twice_returns_val); + TEST(prev_with_val_then_next_twice_returns_null); TEST(navigate_then_append_new); TEST(edit_item_mid_history); TEST(edit_previous_and_append); From 51786f67a6f453ff5ac96aee08cf814eb3bcfd4d Mon Sep 17 00:00:00 2001 From: James Booth Date: Sun, 14 Jul 2013 00:14:36 +0100 Subject: [PATCH 5/8] Implemented parse_args with unicode compatibility --- src/tools/parser.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/tools/parser.c b/src/tools/parser.c index c8335879..42b1165f 100644 --- a/src/tools/parser.c +++ b/src/tools/parser.c @@ -65,11 +65,18 @@ parse_args(const char * const inp, int min, int max) int token_size = 0; GSList *tokens = NULL; + // add tokens to GSList int i; for (i = 0; i < inp_size; i++) { gchar *curr_ch = g_utf8_offset_to_pointer(copy, i); gunichar curr_uni = g_utf8_get_char(curr_ch); + + gchar *character = malloc(7); + gint num_written = 0; + num_written = g_unichar_to_utf8(curr_uni, character); + character[num_written] = '\0'; + if (!in_token) { if (curr_uni == ' ') { continue; @@ -78,9 +85,14 @@ parse_args(const char * const inp, int min, int max) if (curr_uni == '"') { in_quotes = TRUE; i++; + gchar *next_ch = g_utf8_next_char(curr_ch); + gunichar next_uni = g_utf8_get_char(next_ch); + token_start = next_ch; + token_size += g_unichar_to_utf8(next_uni, NULL); + } else { + token_start = curr_ch; + token_size += g_unichar_to_utf8(curr_uni, NULL); } - token_start = curr_ch; - token_size += g_unichar_to_utf8(curr_uni, NULL); } } else { if (in_quotes) { @@ -101,13 +113,14 @@ parse_args(const char * const inp, int min, int max) in_token = FALSE; } else { token_size += g_unichar_to_utf8(curr_uni, NULL); - token_size++; } } } } - tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); + if (in_token) { + tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); + } int num = g_slist_length(tokens) - 1; From 4d35031cb077a3aa03620d9372747dd69229b7da Mon Sep 17 00:00:00 2001 From: James Booth Date: Sun, 14 Jul 2013 00:24:57 +0100 Subject: [PATCH 6/8] Implemented parse_args_with_freetext with unicode compatibility --- src/tools/parser.c | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/tools/parser.c b/src/tools/parser.c index 42b1165f..b679b9f2 100644 --- a/src/tools/parser.c +++ b/src/tools/parser.c @@ -65,18 +65,12 @@ parse_args(const char * const inp, int min, int max) int token_size = 0; GSList *tokens = NULL; - // add tokens to GSList int i; for (i = 0; i < inp_size; i++) { gchar *curr_ch = g_utf8_offset_to_pointer(copy, i); gunichar curr_uni = g_utf8_get_char(curr_ch); - gchar *character = malloc(7); - gint num_written = 0; - num_written = g_unichar_to_utf8(curr_uni, character); - character[num_written] = '\0'; - if (!in_token) { if (curr_uni == ' ') { continue; @@ -195,7 +189,7 @@ parse_args_with_freetext(const char * const inp, int min, int max) char *copy = strdup(inp); g_strstrip(copy); - int inp_size = strlen(copy); + int inp_size = g_utf8_strlen(copy, -1); gboolean in_token = FALSE; gboolean in_freetext = FALSE; gboolean in_quotes = FALSE; @@ -206,54 +200,66 @@ parse_args_with_freetext(const char * const inp, int min, int max) // add tokens to GSList int i; - for (i = 0; i <= inp_size; i++) { + for (i = 0; i < inp_size; i++) { + gchar *curr_ch = g_utf8_offset_to_pointer(copy, i); + gunichar curr_uni = g_utf8_get_char(curr_ch); + if (!in_token) { - if (copy[i] == ' ') { + if (curr_uni == ' ') { continue; } else { in_token = TRUE; num_tokens++; if (num_tokens == max + 1) { in_freetext = TRUE; - } else if (copy[i] == '"') { + } else if (curr_uni == '"') { in_quotes = TRUE; i++; + gchar *next_ch = g_utf8_next_char(curr_ch); + gunichar next_uni = g_utf8_get_char(next_ch); + token_start = next_ch; + token_size += g_unichar_to_utf8(next_uni, NULL); } - if (copy[i] == '"') { - token_start = ©[i+1]; + if (curr_uni == '"') { + gchar *next_ch = g_utf8_next_char(curr_ch); + token_start = next_ch; } else { - token_start = ©[i]; - token_size++; + token_start = curr_ch; + token_size += g_unichar_to_utf8(curr_uni, NULL); } } } else { if (in_quotes) { - if ((copy[i] == '\0') || (copy[i] == '"')) { + if (curr_uni == '"') { tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); token_size = 0; in_token = FALSE; in_quotes = FALSE; } else { - if (copy[i] != '"') { - token_size++; + if (curr_uni != '"') { + token_size += g_unichar_to_utf8(curr_uni, NULL); } } } else { - if ((!in_freetext && copy[i] == ' ') || copy[i] == '\0') { + if (!in_freetext && curr_uni == ' ') { tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); token_size = 0; in_token = FALSE; } else { - if (copy[i] != '"') { - token_size++; + if (curr_uni != '"') { + token_size += g_unichar_to_utf8(curr_uni, NULL); } } } } } + if (in_token) { + tokens = g_slist_append(tokens, g_strndup(token_start, token_size)); + } + int num = g_slist_length(tokens) - 1; // if num args not valid return NULL From e7478d8cb8c55db91628a059ff9f1065bfb9cf0e Mon Sep 17 00:00:00 2001 From: James Booth Date: Sun, 14 Jul 2013 00:46:56 +0100 Subject: [PATCH 7/8] Added parser tests --- src/tools/parser.c | 2 - tests/test_parser.c | 131 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/src/tools/parser.c b/src/tools/parser.c index b679b9f2..595032c3 100644 --- a/src/tools/parser.c +++ b/src/tools/parser.c @@ -367,5 +367,3 @@ get_start(char *string, int tokens) return result_str; } - - diff --git a/tests/test_parser.c b/tests/test_parser.c index 58e3f3e7..e442baee 100644 --- a/tests/test_parser.c +++ b/tests/test_parser.c @@ -279,6 +279,124 @@ parse_cmd_freetext_with_many_quoted_and_many_spaces(void) assert_string_equals("and heres the free text", result[2]); g_strfreev(result); } + +void +count_one_token(void) +{ + char *inp = "one"; + int result = count_tokens(inp); + + assert_int_equals(1, result); +} + +void +count_one_token_quoted_no_whitespace(void) +{ + char *inp = "\"one\""; + int result = count_tokens(inp); + + assert_int_equals(1, result); +} + +void +count_one_token_quoted_with_whitespace(void) +{ + char *inp = "\"one two\""; + int result = count_tokens(inp); + + assert_int_equals(1, result); +} + +void +count_two_tokens(void) +{ + char *inp = "one two"; + int result = count_tokens(inp); + + assert_int_equals(2, result); +} + +void +count_two_tokens_first_quoted(void) +{ + char *inp = "\"one and\" two"; + int result = count_tokens(inp); + + assert_int_equals(2, result); +} + +void +count_two_tokens_second_quoted(void) +{ + char *inp = "one \"two and\""; + int result = count_tokens(inp); + + assert_int_equals(2, result); +} + +void +count_two_tokens_both_quoted(void) +{ + char *inp = "\"one and then\" \"two and\""; + int result = count_tokens(inp); + + assert_int_equals(2, result); +} + +void +get_first_of_one(void) +{ + char *inp = "one"; + char *result = get_start(inp, 2); + + assert_string_equals("one", result); +} + +void +get_first_of_two(void) +{ + char *inp = "one two"; + char *result = get_start(inp, 2); + + assert_string_equals("one ", result); +} + +void +get_first_two_of_three(void) +{ + char *inp = "one two three"; + char *result = get_start(inp, 3); + + assert_string_equals("one two ", result); +} + +void +get_first_two_of_three_first_quoted(void) +{ + char *inp = "\"one\" two three"; + char *result = get_start(inp, 3); + + assert_string_equals("\"one\" two ", result); +} + +void +get_first_two_of_three_second_quoted(void) +{ + char *inp = "one \"two\" three"; + char *result = get_start(inp, 3); + + assert_string_equals("one \"two\" ", result); +} + +void +get_first_two_of_three_first_and_second_quoted(void) +{ + char *inp = "\"one\" \"two\" three"; + char *result = get_start(inp, 3); + + assert_string_equals("\"one\" \"two\" ", result); +} + void register_parser_tests(void) { @@ -307,4 +425,17 @@ register_parser_tests(void) TEST(parse_cmd_freetext_with_quoted_and_space); TEST(parse_cmd_freetext_with_quoted_and_many_spaces); TEST(parse_cmd_freetext_with_many_quoted_and_many_spaces); + TEST(count_one_token); + TEST(count_one_token_quoted_no_whitespace); + TEST(count_one_token_quoted_with_whitespace); + TEST(count_two_tokens); + TEST(count_two_tokens_first_quoted); + TEST(count_two_tokens_second_quoted); + TEST(count_two_tokens_both_quoted); + TEST(get_first_of_one); + TEST(get_first_of_two); + TEST(get_first_two_of_three); + TEST(get_first_two_of_three_first_quoted); + TEST(get_first_two_of_three_second_quoted); + TEST(get_first_two_of_three_first_and_second_quoted); } From 7f82dc42f593f6410e4d0058add4b91112047e63 Mon Sep 17 00:00:00 2001 From: James Booth Date: Sun, 14 Jul 2013 01:00:11 +0100 Subject: [PATCH 8/8] Remaining parser function unicode compatible --- src/tools/parser.c | 59 +++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/tools/parser.c b/src/tools/parser.c index 595032c3..f4cfc3d2 100644 --- a/src/tools/parser.c +++ b/src/tools/parser.c @@ -297,34 +297,27 @@ parse_args_with_freetext(const char * const inp, int min, int max) int count_tokens(char *string) { + int length = g_utf8_strlen(string, -1); + gboolean in_quotes = FALSE; int num_tokens = 0; + int i = 0; - // if no quotes, use glib - if (g_strrstr(string, "\"") == NULL) { - gchar **tokens = g_strsplit(string, " ", 0); - num_tokens = g_strv_length(tokens); - g_strfreev(tokens); + // include first token + num_tokens++; - // else count tokens including quoted - } else { - int length = strlen(string); - int i = 0; - gboolean in_quotes = FALSE; + for (i = 0; i < length; i++) { + gchar *curr_ch = g_utf8_offset_to_pointer(string, i); + gunichar curr_uni = g_utf8_get_char(curr_ch); - // include first token - num_tokens++; - - for (i = 0; i < length; i++) { - if (string[i] == ' ') { - if (!in_quotes) { - num_tokens++; - } - } else if (string[i] == '"') { - if (in_quotes) { - in_quotes = FALSE; - } else { - in_quotes = TRUE; - } + if (curr_uni == ' ') { + if (!in_quotes) { + num_tokens++; + } + } else if (curr_uni == '"') { + if (in_quotes) { + in_quotes = FALSE; + } else { + in_quotes = TRUE; } } } @@ -335,25 +328,31 @@ count_tokens(char *string) char * get_start(char *string, int tokens) { + GString *result = g_string_new(""); + int length = g_utf8_strlen(string, -1); + gboolean in_quotes = FALSE; char *result_str = NULL; int num_tokens = 0; - int length = strlen(string); int i = 0; - gboolean in_quotes = FALSE; - GString *result = g_string_new(""); // include first token num_tokens++; for (i = 0; i < length; i++) { + gchar *curr_ch = g_utf8_offset_to_pointer(string, i); + gunichar curr_uni = g_utf8_get_char(curr_ch); + if (num_tokens < tokens) { - g_string_append_c(result, string[i]); + gchar *uni_char = malloc(7); + int len = g_unichar_to_utf8(curr_uni, uni_char); + uni_char[len] = '\0'; + g_string_append(result, uni_char); } - if (string[i] == ' ') { + if (curr_uni == ' ') { if (!in_quotes) { num_tokens++; } - } else if (string[i] == '"') { + } else if (curr_uni == '"') { if (in_quotes) { in_quotes = FALSE; } else {