Merge pull request #46 from zawaideh/master

dennisreimann · dennisreimann · commit 0498ee1de87a · 2013-02-13T13:21:12.000-08:00
Fix 'invalid byte sequence in UTF-8' error in parse_link_attrs
diff --git a/lib/openid/consumer/html_parse.rb b/lib/openid/consumer/html_parse.rb
@@ -34,7 +34,15 @@ def OpenID.unescape_hash(h)
 
 
   def OpenID.parse_link_attrs(html)
-    stripped = html.gsub(REMOVED_RE,'')
+    begin
+      stripped = html.gsub(REMOVED_RE,'')
+    rescue ArgumentError
+      begin
+        stripped = html.encode('UTF-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '').gsub(REMOVED_RE,'')
+      rescue Encoding::UndefinedConversionError #needed for a problem in JRuby where it can't handle the conversion
+        stripped = html.encode('UTF-8', 'ASCII', :invalid => :replace, :undef => :replace, :replace => '').gsub(REMOVED_RE,'')
+      end
+    end
     parser = HTMLTokenizer.new(stripped)
 
     links = []
diff --git a/test/test_linkparse.rb b/test/test_linkparse.rb
@@ -84,6 +84,7 @@ def test_linkparse
           assert(false, "datafile parsing error: bad header #{h}")
         end
       }
+      html = html.force_encoding('UTF-8') if html.respond_to? :force_encoding
       links = OpenID::parse_link_attrs(html)
       
       found = links.dup
@@ -97,5 +98,16 @@ def test_linkparse
       end
     }
     assert_equal(numtests, testnum, "Number of tests")
+
+    # test handling of invalid UTF-8 byte sequences
+    if "".respond_to? :force_encoding
+      html = "<html><body>hello joel\255</body></html>".force_encoding('UTF-8') 
+    else
+      html = "<html><body>hello joel\255</body></html>"
+    end
+    assert_nothing_raised do 
+      OpenID::parse_link_attrs(html)
+    end
+
   end
 end
