Fixed a bug where characters in the upper part of the BMP would not be escaped.

And made such that ParseString whould throw an error if it fails to parse a string

Author: jstroem

escape.go

@@ -11,6 +11,9 @@ const supplementalPlanesOffset = 0x10000
 const highSurrogateOffset = 0xD800
 const lowSurrogateOffset = 0xDC00
 
+const basicMultilingualPlaneReservedOffset = 0xDFFF
+const basicMultilingualPlaneOffset = 0xFFFF
+
 func combineUTF16Surrogates(high, low rune) rune {
 	return supplementalPlanesOffset + (high-highSurrogateOffset)<<10 + (low - lowSurrogateOffset)
 }
@@ -49,11 +52,18 @@ func decodeSingleUnicodeEscape(in []byte) (rune, bool) {
 	return rune(h1<<12 + h2<<8 + h3<<4 + h4), true
 }
 
+// isUTF16EncodedRune checks if a rune is in the range for non-BMP characters,
+// which is used to describe UTF16 chars.
+// Source: https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane
+func isUTF16EncodedRune(r rune) bool {
+	return highSurrogateOffset <= r && r <= basicMultilingualPlaneReservedOffset
+}
+
 func decodeUnicodeEscape(in []byte) (rune, int) {
 	if r, ok := decodeSingleUnicodeEscape(in); !ok {
 		// Invalid Unicode escape
 		return utf8.RuneError, -1
-	} else if r < highSurrogateOffset {
+	} else if r <= basicMultilingualPlaneOffset && !isUTF16EncodedRune(r) {
 		// Valid Unicode escape in Basic Multilingual Plane
 		return r, 6
 	} else if r2, ok := decodeSingleUnicodeEscape(in[6:]); !ok { // Note: previous decodeSingleUnicodeEscape success guarantees at least 6 bytes remain
@@ -66,7 +76,6 @@ func decodeUnicodeEscape(in []byte) (rune, int) {
 		// Valid UTF16 surrogate pair
 		return combineUTF16Surrogates(r, r2), 12
 	}
-
 }
 
 // backslashCharEscapeTable: when '\X' is found for some byte X, it is to be replaced with backslashCharEscapeTable[X]

escape_test.go

@@ -44,12 +44,14 @@ var singleUnicodeEscapeTests = append([]escapedUnicodeRuneTest{
 	{in: `\uD83D`, out: 0xD83D, len: 6},
 	{in: `\uDE03`, out: 0xDE03, len: 6},
 	{in: `\uFFFF`, out: 0xFFFF, len: 6},
+	{in: `\uFF11`, out: '１', len: 6},
 }, commonUnicodeEscapeTests...)
 
 var multiUnicodeEscapeTests = append([]escapedUnicodeRuneTest{
 	{in: `\uD83D`, isErr: true},
 	{in: `\uDE03`, isErr: true},
-	{in: `\uFFFF`, isErr: true},
+	{in: `\uFFFF`, out: '\uFFFF', len: 6},
+	{in: `\uFF11`, out: '１', len: 6},
 
 	{in: `\uD83D\uDE03`, out: '\U0001F603', len: 12},
 	{in: `\uD800\uDC00`, out: '\U00010000', len: 12},
@@ -109,13 +111,14 @@ var unescapeTests = []unescapeTest{
 	{in: `ab\\de`, out: `ab\de`, canAlloc: true},
 	{in: `ab\"de`, out: `ab"de`, canAlloc: true},
 	{in: `ab \u00B0 de`, out: `ab ° de`, canAlloc: true},
+	{in: `ab \uFF11 de`, out: `ab １ de`, canAlloc: true},
+	{in: `\uFFFF`, out: "\uFFFF", canAlloc: true},
 	{in: `ab \uD83D\uDE03 de`, out: "ab \U0001F603 de", canAlloc: true},
 	{in: `\u0000\u0000\u0000\u0000\u0000`, out: "\u0000\u0000\u0000\u0000\u0000", canAlloc: true},
 	{in: `\u0000 \u0000 \u0000 \u0000 \u0000`, out: "\u0000 \u0000 \u0000 \u0000 \u0000", canAlloc: true},
 	{in: ` \u0000 \u0000 \u0000 \u0000 \u0000 `, out: " \u0000 \u0000 \u0000 \u0000 \u0000 ", canAlloc: true},
 
 	{in: `\uD800`, isErr: true},
-	{in: `\uFFFF`, isErr: true},
 	{in: `abcde\`, isErr: true},
 	{in: `abcde\x`, isErr: true},
 	{in: `abcde\u`, isErr: true},

parser.go

@@ -832,7 +832,7 @@ func ParseBoolean(b []byte) (bool, error) {
 func ParseString(b []byte) (string, error) {
 	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
 	if bU, err := Unescape(b, stackbuf[:]); err != nil {
-		return "", nil
+		return "", MalformedValueError
 	} else {
 		return string(bU), nil
 	}

parser_test.go

@@ -358,17 +358,17 @@ var getTests = []GetTest{
 		isFound: false,
 	},
 	{ // Issue #81
-		desc:	`missing key in object in array`,
-		json: 	`{"p":{"a":[{"u":"abc","t":"th"}]}}`,
-		path: 	[]string{"p", "a", "[0]", "x"},
+		desc:    `missing key in object in array`,
+		json:    `{"p":{"a":[{"u":"abc","t":"th"}]}}`,
+		path:    []string{"p", "a", "[0]", "x"},
 		isFound: false,
 	},
 	{ // Issue #81 counter test
-		desc:	`existing key in object in array`,
-		json: 	`{"p":{"a":[{"u":"abc","t":"th"}]}}`,
-		path: 	[]string{"p", "a", "[0]", "u"},
+		desc:    `existing key in object in array`,
+		json:    `{"p":{"a":[{"u":"abc","t":"th"}]}}`,
+		path:    []string{"p", "a", "[0]", "u"},
 		isFound: true,
-		data:	"abc",
+		data:    "abc",
 	},
 	{ // This test returns not found instead of a parse error, as checking for the malformed JSON would reduce performance
 		desc:    "malformed key (followed by comma followed by colon)",
@@ -1183,3 +1183,33 @@ func TestParseFloat(t *testing.T) {
 		},
 	)
 }
+
+var parseStringTest = []ParseTest{
+	{
+		in:     `\uFF11`,
+		intype: String,
+		out:    "\uFF11",
+	},
+	{
+		in:     `\uFFFF`,
+		intype: String,
+		out:    "\uFFFF",
+	},
+	{
+		in:     `\uDF00`,
+		intype: String,
+		isErr:  true,
+	},
+}
+
+func TestParseString(t *testing.T) {
+	runParseTests(t, "ParseString()", parseStringTest,
+		func(test ParseTest) (value interface{}, err error) {
+			return ParseString([]byte(test.in))
+		},
+		func(test ParseTest, obtained interface{}) (bool, interface{}) {
+			expected := test.out.(string)
+			return obtained.(string) == expected, expected
+		},
+	)
+}