Implement codecs.ignore_errors (#711)

Author: BCSharp

Src/IronPython/Runtime/Operations/StringOps.cs

@@ -1948,9 +1948,12 @@ internal static Dictionary<string, object> MakeErrorHandlersDict() {
                     ReflectionUtils.GetMethodInfos(typeof(StringOps).GetMember(nameof(StrictErrors), BindingFlags.Static | BindingFlags.NonPublic)), 
                     typeof(StringOps));
 
-                // TODO: Implement remaining error handlers
-                d["ignore"] = null;
+                d["ignore"] = BuiltinFunction.MakeFunction(
+                    "ignore_errors", 
+                    ReflectionUtils.GetMethodInfos(typeof(StringOps).GetMember(nameof(IgnoreErrors), BindingFlags.Static | BindingFlags.NonPublic)), 
+                    typeof(StringOps));
 
+                // TODO: Implement remaining error handlers
                 d["replace"] = null;
 
                 d["xmlcharrefreplace"] = null;
@@ -2627,6 +2630,20 @@ private static object StrictErrors(object unicodeError) {
                 default: throw PythonOps.TypeError("codec must pass exception instance");
             }
         }
+
+        private static object IgnoreErrors(object unicodeError) {
+            switch (unicodeError) {
+                case PythonExceptions._UnicodeDecodeError ude:
+                    return PythonTuple.MakeTuple(string.Empty, ude.end);
+                case PythonExceptions._UnicodeEncodeError uee:
+                    return PythonTuple.MakeTuple(string.Empty, uee.end);
+                case DecoderFallbackException dfe: 
+                    return PythonTuple.MakeTuple(string.Empty, dfe.Index + dfe.BytesUnknown.Length);
+                case EncoderFallbackException efe: 
+                    return PythonTuple.MakeTuple(string.Empty, efe.Index + (efe.CharUnknownHigh != '\0' ? 2 : 1));
+                default: throw PythonOps.TypeError("codec must pass exception instance");
+            }
+        }
 #endif
 
         #endregion

Tests/modules/io_related/test_codecs.py

@@ -460,7 +460,7 @@ def test_latin_1_encode(self):
     def test_error_handlers(self):
         ude = UnicodeDecodeError('dummy', b"abcdefgh", 3, 5, "decoding testing purposes")
         uee = UnicodeEncodeError('dummy', "abcdefgh", 2, 6, "encoding testing purposes")
-        unicode_data = "ab\xff\u20ac\U0001f40d"
+        unicode_data = "ab\xff\u20ac\U0001f40d\0z"
         uee_unicode = UnicodeEncodeError('dummy', unicode_data, 2, len(unicode_data), "encoding testing purposes")
 
         strict = codecs.lookup_error('strict')
@@ -476,28 +476,31 @@ def test_error_handlers(self):
         self.assertRaisesRegex(TypeError, "\w+\(\) takes exactly (one|1) argument \(2 given\)", strict, ude, uee)
         self.assertRaises(LookupError, codecs.lookup_error, "STRICT")
 
-        return # TODO: Implement remaining error handlers
-
         ignore = codecs.lookup_error('ignore')
         self.assertEqual(ignore, codecs.ignore_errors)
         self.assertEqual(ignore(ude), ("", 5))
         self.assertEqual(ignore(uee), ("", 6))
+        self.assertEqual(ignore(uee_unicode), ("", uee_unicode.end))
+
+        return # TODO: Implement remaining error handlers
 
         replace = codecs.lookup_error('replace')
         self.assertEqual(replace, codecs.replace_errors)
         self.assertEqual(replace(ude), ("�", 5))
         self.assertEqual(replace(uee), ("????", 6))
+        self.assertEqual(replace(uee_unicode), ("?" * (uee_unicode.end - uee_unicode.start), uee_unicode.end))
 
         backslashreplace = codecs.lookup_error('backslashreplace')
         self.assertEqual(backslashreplace, codecs.backslashreplace_errors)
         self.assertRaisesRegex(TypeError, "don't know how to handle UnicodeDecodeError in error callback", backslashreplace, ude)
         self.assertEqual(backslashreplace(uee), (r"\x63\x64\x65\x66", 6))
-        self.assertEqual(backslashreplace(uee_unicode), (r"\xff\u20ac\U0001f40d", uee_unicode.end))
+        self.assertEqual(backslashreplace(uee_unicode), (r"\xff\u20ac\U0001f40d\x00\x7a", uee_unicode.end))
 
         xmlcharrefreplace = codecs.lookup_error('xmlcharrefreplace')
         self.assertEqual(xmlcharrefreplace, codecs.xmlcharrefreplace_errors)
         self.assertRaisesRegex(TypeError, "don't know how to handle UnicodeDecodeError in error callback", xmlcharrefreplace, ude)
         self.assertEqual(xmlcharrefreplace(uee), ("cdef", 6))
+        self.assertEqual(xmlcharrefreplace(uee_unicode), ("ÿ€🐍�z", uee_unicode.end))
 
     #TODO: @skip("multiple_execute")
     def test_lookup_error(self):