Merge pull request #41 from grosser/encoding

use default-external encoding instead of ascii for badly encoded pages

Author: dennisreimann

lib/openid/fetchers.rb

@@ -238,15 +238,15 @@ def fetch(url, body=nil, headers=nil, redirect_limit=REDIRECT_LIMIT)
 
     private
     def setup_encoding(response)
-      return unless defined?(::Encoding::ASCII_8BIT)
-      charset = response.type_params["charset"]
-      return if charset.nil?
-      encoding = nil
+      return unless defined?(Encoding.default_external)
+      return unless charset = response.type_params["charset"]
+
       begin
         encoding = Encoding.find(charset)
       rescue ArgumentError
       end
-      encoding ||= Encoding::ASCII_8BIT
+      encoding ||= Encoding.default_external
+
       body = response.body
       if body.respond_to?(:force_encoding)
         body.force_encoding(encoding)

test/test_fetchers.rb

@@ -126,7 +126,28 @@ def _utf8_page
     }
   end
 
+  def _unencoded_page
+    lambda { |req, resp|
+      resp['Content-Type'] = "text/html"
+      body = "unencoded-body"
+      body.force_encoding("ASCII-8BIT") if body.respond_to?(:force_encoding)
+      resp.body = body
+    }
+  end
+
+  def _badly_encoded_page
+    lambda { |req, resp|
+      resp['Content-Type'] = "text/html; charset=wtf"
+      body = "badly-encoded-body"
+      body.force_encoding("ASCII-8BIT") if body.respond_to?(:force_encoding)
+      resp.body = body
+    }
+  end
+
   def setup
+    if defined?(Encoding.default_external)
+      @encoding_was = Encoding.default_external
+    end
     @fetcher = OpenID::StandardFetcher.new
     @logfile = StringIO.new
     @weblog = WEBrick::Log.new(logfile=@logfile)
@@ -152,6 +173,8 @@ def setup
       @server.mount_proc('/post', _require_post)
       @server.mount_proc('/redirect_loop', _redirect_loop)
       @server.mount_proc('/utf8_page', _utf8_page)
+      @server.mount_proc('/unencoded_page', _unencoded_page)
+      @server.mount_proc('/badly_encoded_page', _badly_encoded_page)
       @server.start
     }
     @uri = _uri_build
@@ -168,6 +191,9 @@ def _uri_build(path='/')
   end
 
   def teardown
+    if defined?(Encoding.default_external)
+      Encoding.default_external = @encoding_was
+    end
     @server.shutdown
     # Sleep a little because sometimes this blocks forever.
     @server_thread.join
@@ -234,6 +260,30 @@ def test_utf8_page
     end
   end
 
+  def test_unencoded_page
+    if defined?(Encoding.default_external)
+      Encoding.default_external = Encoding::SHIFT_JIS
+    end
+    uri = _uri_build('/unencoded_page')
+    response = @fetcher.fetch(uri)
+    assert_equal("unencoded-body", response.body)
+    if defined?(Encoding.default_external)
+      assert_equal(Encoding::US_ASCII, response.body.encoding)
+    end
+  end
+
+  def test_badly_encoded_page
+    if defined?(Encoding.default_external)
+      Encoding.default_external = Encoding::SHIFT_JIS
+    end
+    uri = _uri_build('/badly_encoded_page')
+    response = @fetcher.fetch(uri)
+    assert_equal("badly-encoded-body", response.body)
+    if defined?(Encoding.default_external)
+      assert_equal(Encoding::SHIFT_JIS, response.body.encoding)
+    end
+  end
+
   def test_cases
     for path, expected_code, expected_url in @@cases
       uri = _uri_build(path)