Unicode support via normalisation step

.github/workflows/apitests.yml

@@ -5,8 +5,7 @@ on:
     branches:
       - main
   push:
-    branches:
-      - main
+    branches: [ "*" ]
 
 jobs:
   build:

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "frontend/public/manamoji"]
 	path = frontend/public/manamoji
 	url = https://github.com/scryfall/manamoji-discord.git
+[submodule "backend/utf8proc"]
+	path = backend/utf8proc
+	url = https://github.com/JuliaStrings/utf8proc

backend/CMakeLists.txt

@@ -142,8 +142,10 @@ set(ABSL_LIBS
 
 add_subdirectory(./re2)
 
+add_subdirectory(./utf8proc/)
+
 option(USE_JEMALLOC OFF)
-set(LIBS jansson CURL::libcurl pthread ${ABSL_LIBS} re2)
+set(LIBS jansson CURL::libcurl pthread ${ABSL_LIBS} re2 utf8proc)
 if(USE_JEMALLOC)
   set(LIBS ${LIBS} jemalloc)
 endif()
@@ -205,6 +207,8 @@ set(SRC
     ./mse/query_parser_internals.c
     ./mse/levenshtein_difference.h
     ./mse/levenshtein_difference.c
+    ./mse/utf8_normalisation.h
+    ./mse/utf8_normalisation.c
     ./strptime/strptime.h
     ./strptime/strptime.c
     ${PARSER_SRC}
@@ -256,6 +260,8 @@ set(TESTS
     ./tests/test_formats.c
     ./tests/test_levenshtein_difference.h
     ./tests/test_levenshtein_difference.c
+    ./tests/test_utf8_normalisation.h
+    ./tests/test_utf8_normalisation.c
     ./tests/system_test.h
     ./tests/system_test.c)
 set(WEB_API

backend/mse/card.c

@@ -1,9 +1,10 @@
 #include "./card.h"
-#include "./io_utils.h"
 #include "../testing_h/testing.h"
-#include <string.h>
-#include <stdlib.h>
+#include "./io_utils.h"
+#include "./utf8_normalisation.h"
 #include <math.h>
+#include <stdlib.h>
+#include <string.h>
 
 mse_colour_enum_t mse_parse_colour(char colour)
 {
@@ -26,7 +27,6 @@ mse_colour_enum_t mse_parse_colour(char colour)
     default:
         return 0;
     }
-
 }
 
 mse_colour_enum_t mse_parse_colours(const char *colours)
@@ -63,22 +63,34 @@ int mse_parse_card_json(json_t *json, mse_card_t *card)
     ASSERT(json_is_string(name_o));
     ASSERT(card->name = strdup(json_string_value(name_o)));
 
+    char *normalised = mse_normalise_utf8(card->name);
+    ASSERT(normalised != NULL);
+    free(card->name);
+    card->name = normalised;
+
     // Read oracle
     json_t *oracle_o = json_object_get(json, "text");
     if (oracle_o != NULL) {
         ASSERT(json_is_string(oracle_o));
         ASSERT(card->oracle_text = strdup(json_string_value(oracle_o)));
-        ASSERT(card->oracle_text_lower = mse_to_lower(card->oracle_text));
     } else {
         oracle_o = json_object_get(json, "originalText");
 
         if (oracle_o != NULL) {
             ASSERT(json_is_string(oracle_o));
             ASSERT(card->oracle_text = strdup(json_string_value(oracle_o)));
-            ASSERT(card->oracle_text_lower = mse_to_lower(card->oracle_text));
         }
     }
 
+    if (card->oracle_text != NULL) {
+        char *normalised = mse_normalise_utf8(card->oracle_text);
+        ASSERT(normalised != NULL);
+        free(card->oracle_text);
+        card->oracle_text= normalised;
+
+        ASSERT(card->oracle_text_lower = mse_to_lower(card->oracle_text));
+    }
+
     // Read mana cost
     json_t *mana_cost_o = json_object_get(json, "manaCost");
     if (mana_cost_o != NULL) {
@@ -99,7 +111,8 @@ int mse_parse_card_json(json_t *json, mse_card_t *card)
             ASSERT(card->types = malloc(sizeof(*card->types)));
         }
 
-        ASSERT(card->types = realloc(card->types, sizeof(*card->types) * (1 + card->types_count)));
+        ASSERT(card->types = realloc(card->types, sizeof(*card->types) *
+                                     (1 + card->types_count)));
 
         char *tmp;
         ASSERT(tmp = strdup(json_string_value(value)));
@@ -118,7 +131,8 @@ int mse_parse_card_json(json_t *json, mse_card_t *card)
             ASSERT(card->types = malloc(sizeof(*card->types)));
         }
 
-        ASSERT(card->types = realloc(card->types, sizeof(*card->types) * (1 + card->types_count)));
+        ASSERT(card->types = realloc(card->types, sizeof(*card->types) *
+                                     (1 + card->types_count)));
 
         char *tmp;
         ASSERT(tmp = strdup(json_string_value(value)));
@@ -137,7 +151,8 @@ int mse_parse_card_json(json_t *json, mse_card_t *card)
             ASSERT(card->types = malloc(sizeof(*card->types)));
         }
 
-        ASSERT(card->types = realloc(card->types, sizeof(*card->types) * (1 + card->types_count)));
+        ASSERT(card->types = realloc(card->types, sizeof(*card->types) *
+                                     (1 + card->types_count)));
 
         char *tmp;
         ASSERT(tmp = strdup(json_string_value(value)));
@@ -196,17 +211,20 @@ int mse_parse_card_json(json_t *json, mse_card_t *card)
             ASSERT(card->set_codes = malloc(sizeof(*card->set_codes)));
         }
 
-        ASSERT(card->set_codes = realloc(card->set_codes,
-                                         sizeof(*card->set_codes) * (1 + card->set_codes_count)));
+        ASSERT(card->set_codes =
+                   realloc(card->set_codes,
+                           sizeof(*card->set_codes) * (1 + card->set_codes_count)));
 
-        ASSERT(mse_get_set_code(json_string_value(value), &card->set_codes[card->set_codes_count]));
+        ASSERT(mse_get_set_code(json_string_value(value),
+                                &card->set_codes[card->set_codes_count]));
         card->set_codes_count++;
     }
 
     ASSERT(card->name_lower = mse_to_lower(card->name));
 
     // Read the format legalities
-    ASSERT(mse_card_formats_legalities_t_from_json(json, &card->format_legalities));
+    ASSERT(
+        mse_card_formats_legalities_t_from_json(json, &card->format_legalities));
     return 1;
 }
 
@@ -262,7 +280,8 @@ int mse_read_card(FILE *f, mse_card_t *card)
     card->colour_identity = tmp;
 
     ASSERT(mse_read_size_t(f, &card->set_codes_count));
-    ASSERT(card->set_codes = malloc(sizeof(*card->set_codes) * card->set_codes_count));
+    ASSERT(card->set_codes =
+               malloc(sizeof(*card->set_codes) * card->set_codes_count));
     for (size_t i = 0; i < card->set_codes_count; i++) {
         ASSERT(mse_read_set_code(f, &card->set_codes[i]));
     }
@@ -318,32 +337,32 @@ void mse_free_card(mse_card_t *card)
     memset(card, 0, sizeof(*card));
 }
 
-int mse_avl_cmp_card(void * restrict a, void * restrict b)
+int mse_avl_cmp_card(void *restrict a, void *restrict b)
 {
-    mse_card_t *ca = (mse_card_t *) a;
-    mse_card_t *cb = (mse_card_t *) b;
+    mse_card_t *ca = (mse_card_t *)a;
+    mse_card_t *cb = (mse_card_t *)b;
     return mse_uuid_cmp(ca->id, cb->id);
 }
 
-int mse_avl_cmp_card_name(void * restrict a, void * restrict b)
+int mse_avl_cmp_card_name(void *restrict a, void *restrict b)
 {
-    mse_card_t *ca = (mse_card_t *) a;
-    mse_card_t *cb = (mse_card_t *) b;
+    mse_card_t *ca = (mse_card_t *)a;
+    mse_card_t *cb = (mse_card_t *)b;
     return strcmp(ca->name, cb->name);
 }
 
-#define MSE_CARD_DOUBLE_AVL_CMP(a, b, field) \
-    mse_card_t *ca = (mse_card_t *) a; \
-    mse_card_t *cb = (mse_card_t *) b; \
-    if ((int) ca->field == (int) cb->field) { \
-        return mse_avl_cmp_card(a, b); \
-    } \
-    double cmp = ca->field - cb->field; \
-    if (cmp < 0) { \
-        return -1; \
-    } else { \
-        return 1; \
-    } \
+#define MSE_CARD_DOUBLE_AVL_CMP(a, b, field)                                   \
+  mse_card_t *ca = (mse_card_t *)a;                                            \
+  mse_card_t *cb = (mse_card_t *)b;                                            \
+  if ((int)ca->field == (int)cb->field) {                                      \
+    return mse_avl_cmp_card(a, b);                                             \
+  }                                                                            \
+  double cmp = ca->field - cb->field;                                          \
+  if (cmp < 0) {                                                               \
+    return -1;                                                                 \
+  } else {                                                                     \
+    return 1;                                                                  \
+  }
 
 int mse_avl_cmp_card_power(void *a, void *b)
 {

backend/mse/query_parser.peg

@@ -7,6 +7,7 @@
 #include "mse/interpretor.h"
 #include "mse/query_parser.h"
 #include "mse_query_parser.h"
+#include "mse/utf8_normalisation.h"
 }
 
 %source{
@@ -152,12 +153,15 @@ REGEX_STRING <- '/' ( '\/' / [^/] )+ '/'
 %%
 int mse_parse_input_string(const char* input_string, mse_interp_node_t **root)
 {
+    char *normalised = mse_normalise_utf8(input_string);
+    ASSERT(normalised != NULL);
+
     *root = NULL;
 
     mse_parser_status_t ret;
     memset(&ret, 0, sizeof(ret));
     ret.input_buffer = input_string;
-    ret.input_buffer_len = strlen(input_string);
+    ret.input_buffer_len = strlen(normalised);
 
     mse_context_t *ctx = mse_create(&ret);
     while (ret.error == 0 && mse_parse(ctx, NULL));
@@ -167,8 +171,9 @@ int mse_parse_input_string(const char* input_string, mse_interp_node_t **root)
     __mse_free_parser_status(&ret);
 
     if (ret.error) {
-      lprintf(LOG_ERROR, "Cannot parse '%s'\n", input_string);
+      lprintf(LOG_ERROR, "Cannot parse '%s'\n", normalised);
     }
+    free(normalised);
 
     *root = ret.root;
     ASSERT(root != NULL);

backend/mse/utf8_normalisation.c

@@ -0,0 +1,10 @@
+#include "./utf8_normalisation.h"
+#include <utf8proc.h>
+#include <string.h>
+
+char *mse_normalise_utf8(const char *utf8)
+{
+    utf8proc_uint8_t *fold_str = NULL;
+    utf8proc_map((unsigned char *) utf8, 0, &fold_str, UTF8PROC_NULLTERM | UTF8PROC_STRIPMARK | UTF8PROC_DECOMPOSE);
+    return (char *) fold_str;
+}

backend/mse/utf8_normalisation.h

@@ -0,0 +1,3 @@
+#pragma once
+
+char * mse_normalise_utf8(const char *utf8);

backend/tests/main.c

@@ -20,6 +20,7 @@
 #include "./test_re2.h"
 #include "./test_formats.h"
 #include "./test_levenshtein_difference.h"
+#include "./test_utf8_normalisation.h"
 
 static int sanity_test()
 {
@@ -29,6 +30,7 @@ static int sanity_test()
 SUB_TEST(tests, {&sanity_test, "Sanity Test"},
 {&test_re2, "Test re2 wrapper"},
 {&test_levenshtein_difference, "Test Levenshtein difference"},
+{&test_utf8_normalisation, "Test UTF8 normalisation"},
 /* Test common funcs and types */
 {&test_uuid, "Test UUID"},
 {&test_io_utils, "Test IO utils"},

backend/tests/test_utf8_normalisation.c

@@ -0,0 +1,37 @@
+#include "./test_utf8_normalisation.h"
+#include "../testing_h/testing.h"
+#include "../mse/utf8_normalisation.h"
+#include <string.h>
+
+#define TEST_STR_NORMAL "Testing 123."
+
+static int test_utf8_normalisation_normal_input()
+{
+    char *normalised = mse_normalise_utf8(TEST_STR_NORMAL);
+    ASSERT(normalised != NULL);
+    ASSERT(strcmp(TEST_STR_NORMAL, normalised) == 0);
+    free(normalised);
+    return 1;
+}
+
+static int test_utf8_normalisation_special_input_1()
+{
+    char *normalised = mse_normalise_utf8("Mein Code ist sheiße");
+    ASSERT(normalised != NULL);
+    ASSERT(strcmp("Mein Code ist sheiße", normalised) == 0);
+    free(normalised);
+    return 1;
+}
+
+static int test_utf8_normalisation_special_input_2()
+{
+    char *normalised = mse_normalise_utf8("Ça-va?");
+    ASSERT(normalised != NULL);
+    ASSERT(strcmp("Ca-va?", normalised) == 0);
+    free(normalised);
+    return 1;
+}
+
+SUB_TEST(test_utf8_normalisation, {&test_utf8_normalisation_normal_input, "Test normalisation with normal input"},
+{&test_utf8_normalisation_special_input_1, "Test normalisation with special input 1"},
+{&test_utf8_normalisation_special_input_2, "Test normalisation with special input 2"})

backend/tests/test_utf8_normalisation.h

@@ -0,0 +1,3 @@
+#pragma once
+
+int test_utf8_normalisation();