[[ Bug 20504 ]] Fix __MCStringCheck function

runrevmark · runrevmark · commit b319634dc85a · 2017-10-31T08:31:40.000Z
This function corrects the __MCStringCheck function when the
input string contains a decomposed char sequence which can
be mapped to a composed char in the native encoding.

Additionally, it corrects the ISO8850-1 'native pair' mapping
table ensuring that decomposable native chars in the encoding
map back to their composed counterparts.
diff --git a/docs/notes/bugfix-20504.md b/docs/notes/bugfix-20504.md
@@ -0,0 +1 @@
+# Fetching a char of a string can cause incorrect comparisons later on
diff --git a/libfoundation/src/foundation-string.cpp b/libfoundation/src/foundation-string.cpp
@@ -6841,9 +6841,26 @@ static void __MCStringCheck(MCStringRef self)
         
         if (!MCUnicodeIsGraphemeClusterBoundary(self -> chars[i], self -> chars[i + 1]))
         {
-            __MCStringSetFlags(self, kMCStringFlagNoChange, false, false);
-            t_can_be_native = false;
-            break;
+            /* There is no boundary between i and i+1, so check that the pair of
+             * codeunits won't map to native (e.g. e,acute).
+             * If they do map to native, check that there is a boundary after
+             * the second codeunit - otherwise it is more than 2 cu sequence which
+             * cannot be native. */
+            char_t t_native_char;
+            if (!MCUnicodeMapToNative(self->chars+i, 2, t_native_char) ||
+                (i+1 < self->char_count &&
+                 !MCUnicodeIsGraphemeClusterBoundary(self->chars[i+1], self->chars[i+2])))
+            {
+                __MCStringSetFlags(self, kMCStringFlagNoChange, false, false);
+                t_can_be_native = false;
+                break;
+            }
+            
+            /* At this point i is the first codeunit of a 2-codeunit combining
+             * sequence which maps to a native, so we can skip the combiner. */
+            i++;
+            
+            continue;
         }
         
         char_t t_native;
diff --git a/libfoundation/src/foundation-unicode.cpp b/libfoundation/src/foundation-unicode.cpp
@@ -1743,15 +1743,27 @@ bool MCUnicodeMapToNativePair_MacRoman(uinteger_t x, uinteger_t y, char_t& r_cha
 
 bool MCUnicodeMapToNativePair_ISO8859_1(uinteger_t x, uinteger_t y, char_t& r_char)
 {
-	static const uinteger_t s_pairs[] =
-	{
-		/* S */ 0xA653030C,
-		/* Y */ 0xBE590308,
-		/* Z */ 0x8E5A030C,
-		/* s */ 0xA873030C,
-		/* z */ 0xB87A030C
-	};
-	
+    static const uinteger_t s_pairs[] =
+    {
+        /* A */ 0xC0410300, 0xC1410301, 0xC2410302, 0xC3410303, 0xC4410308, 0xC541030A,
+        /* C */ 0xC7430327,
+        /* E */ 0xC8450300, 0xC9450301, 0xCA450302, 0xCB450308,
+        /* I */ 0xCC490300, 0xCD490301, 0xCE490302, 0xCF490308,
+        /* N */ 0xD14E0303,
+        /* O */ 0xD24F0300, 0xD34F0301, 0xD44F0302, 0xD54F0303, 0xD64F0308,
+        /* U */ 0xD9550300, 0xDA550301, 0xDB550302, 0xDC550308,
+        /* Y */ 0xDD590301,
+
+        /* a */ 0xE0610300, 0xE1610301, 0xE2610302, 0xE3610303, 0xE4610308, 0xE561030A,
+        /* c */ 0xE7630327,
+        /* e */ 0xE8650300, 0xE9650301, 0xEA650302, 0xEB650308,
+        /* i */ 0xEC690300, 0xED690301, 0xEE690302, 0xEF690308,
+        /* n */ 0xF16E0303,
+        /* i */ 0xF26F0300, 0xF36F0301, 0xF46F0302, 0xF56F0303, 0xF66F0308,
+        /* u */ 0xF9750300, 0xFA750301, 0xFB750302, 0xFC750308,
+        /* y */ 0xFD790301, 0xFF790308,
+    };
+
 	uinteger_t z;
 	z = (x << 16) | y;
 	
diff --git a/libfoundation/test/test_string.cpp b/libfoundation/test/test_string.cpp
@@ -174,3 +174,19 @@ TEST(string, surrogate_unicode_props)
     const int kSPUA_B_Upper = 0x10FFFD + 1; // non-inclusive
     check_bidi_of_surrogate_range(kSPUA_B_Lower, kSPUA_B_Upper);
 }
+
+TEST(string, normalize_compare)
+{
+    MCAutoStringRef t_decomposed;
+    MCStringCreateWithWString((const unichar_t*)u"\u0065\u0301\u0065\u0301\u0065\u0301", &t_decomposed);
+    
+    MCAutoStringRef t_composed;
+    MCStringCreateWithWString((const unichar_t*)u"\u00e9\u00e9\u00e9", &t_composed);
+    
+    ASSERT_TRUE(MCStringIsEqualTo(*t_decomposed, *t_composed, kMCStringOptionCompareCaseless));
+    
+    MCRange t_range;
+    MCStringMapGraphemeIndices(*t_decomposed, MCRangeMake(0, 1), t_range);
+    
+    ASSERT_TRUE(MCStringIsEqualTo(*t_decomposed, *t_composed, kMCStringOptionCompareCaseless));
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+# Fetching a char of a string can cause incorrect comparisons later on`