added handling of invalide UTF-8 byte sequence exceptions

Zaid Zawaideh · Zaid Zawaideh · commit 0f46921a9767 · 2013-02-11T14:17:32.000-05:00
diff --git a/lib/openid/consumer/html_parse.rb b/lib/openid/consumer/html_parse.rb
@@ -34,7 +34,11 @@ def OpenID.unescape_hash(h)
 
 
   def OpenID.parse_link_attrs(html)
-    stripped = html.gsub(REMOVED_RE,'')
+    begin
+      stripped = html.gsub(REMOVED_RE,'')
+    rescue ArgumentError
+      stripped = html.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(REMOVED_RE,'')
+    end
     parser = HTMLTokenizer.new(stripped)
 
     links = []
diff --git a/test/test_linkparse.rb b/test/test_linkparse.rb
@@ -84,7 +84,8 @@ def test_linkparse
           assert(false, "datafile parsing error: bad header #{h}")
         end
       }
-      links = OpenID::parse_link_attrs(html)
+
+      links = OpenID::parse_link_attrs(html.force_encoding('UTF-8'))
       
       found = links.dup
       expected = expected_links.dup
@@ -97,5 +98,12 @@ def test_linkparse
       end
     }
     assert_equal(numtests, testnum, "Number of tests")
+
+    # test handling of invalid UTF-8 byte sequences
+    html = "<html><body>hello joel\255</body></html>".force_encoding("UTF-8")
+    assert_nothing_raised do 
+      OpenID::parse_link_attrs(html)
+    end
+
   end
 end
