[Zope3-checkins] SVN: Zope3/trunk/ Fixed so that a UnicodeDecodeError isn't produced when Internet

[Bjorn Tillenius]

Log message for revision 25068:
Fixed so that a UnicodeDecodeError isn't produced when Internet
Explorer requests a page containing non-ISO-8859-1 characters.



-=-
Modified: Zope3/trunk/doc/CHANGES.txt
===================================================================
--- Zope3/trunk/doc/CHANGES.txt	2004-05-27 19:47:49 UTC (rev 25067)
+++ Zope3/trunk/doc/CHANGES.txt	2004-05-27 20:26:45 UTC (rev 25068)
@@ -12,7 +12,7 @@
     Much thanks to everyone who contributed to this release:
 
       Jim Fulton, Marius Gedminas, Fred Drake, Philipp von Weitershausen,
-      Stephan Richter, Dmitry Vasiliev, Scott Pascoe
+      Stephan Richter, Dmitry Vasiliev, Scott Pascoe, Bjorn Tillenius
 
       Note: If you are not listed and contributed, please add yourself. This
       note will be deleted before the release.
@@ -28,6 +28,9 @@
 
     Bug fixes
 
+      - Fixed so that a UnicodeDecodeError isn't produced when Internet
+        Explorer requests a page containing non-ISO-8859-1 characters.
+
     Restructuring
 
       - The event system was completely reimplemented:

Modified: Zope3/trunk/src/zope/app/publication/tests/test_browserpublication.py
===================================================================
--- Zope3/trunk/src/zope/app/publication/tests/test_browserpublication.py	2004-05-27 19:47:49 UTC (rev 25067)
+++ Zope3/trunk/src/zope/app/publication/tests/test_browserpublication.py	2004-05-27 20:26:45 UTC (rev 25068)
@@ -288,7 +288,7 @@
             output.getvalue(),
             'Status: 200 Ok\r\n'
             'Content-Length: 4\r\n'
-            'Content-Type: text/plain;charset=iso-8859-1\r\n'
+            'Content-Type: text/plain;charset=utf-8\r\n'
             'X-Powered-By: Zope (www.zope.org), Python (www.python.org)\r\n'
             '\r\nspam'
             )
@@ -305,12 +305,26 @@
             output.getvalue(),
             'Status: 200 Ok\r\n'
             'Content-Length: 0\r\n'
-            'Content-Type: text/plain;charset=iso-8859-1\r\n'
+            'Content-Type: text/plain;charset=utf-8\r\n'
             'X-Powered-By: Zope (www.zope.org), Python (www.python.org)\r\n'
             '\r\n'
             )
 
+    def testUnicode_NO_HTTP_CHARSET(self):
+        # Test so that a unicode body doesn't cause a UnicodeEncodeError
+        output = StringIO()
+        request = TestRequest(StringIO(''), output, {})
+        request.response.setBody(u"\u0442\u0435\u0441\u0442")
+        request.response.outputBody()
+        self.assertEqual(
+            output.getvalue(),
+            'Status: 200 Ok\r\n'
+            'Content-Length: 8\r\n'
+            'Content-Type: text/plain;charset=utf-8\r\n'
+            'X-Powered-By: Zope (www.zope.org), Python (www.python.org)\r\n'
+            '\r\n\xd1\x82\xd0\xb5\xd1\x81\xd1\x82')
 
+
 def test_suite():
     t2 = unittest.makeSuite(BrowserPublicationTests, 'test')
     t3 = unittest.makeSuite(BrowserDefaultTests, 'test')

Modified: Zope3/trunk/src/zope/publisher/http.py
===================================================================
--- Zope3/trunk/src/zope/publisher/http.py	2004-05-27 19:47:49 UTC (rev 25067)
+++ Zope3/trunk/src/zope/publisher/http.py	2004-05-27 20:26:45 UTC (rev 25068)
@@ -992,6 +992,7 @@
         '''See interface IUserPreferredCharsets'''
         charsets = []
         sawstar = sawiso88591 = 0
+        header_present = 'HTTP_ACCEPT_CHARSET' in self.request
         for charset in self.request.get('HTTP_ACCEPT_CHARSET', '').split(','):
             charset = charset.strip().lower()
             if charset:
@@ -1018,7 +1019,9 @@
         # field, then all character sets not explicitly mentioned get a
         # quality value of 0, except for ISO-8859-1, which gets a quality
         # value of 1 if not explicitly mentioned.
-        if not sawstar and not sawiso88591:
+        # And quoting RFC 2616, $14.2: "If no Accept-Charset header is
+        # present, the default is that any character set is acceptable."
+        if not sawstar and not sawiso88591 and header_present:
             charsets.append((1.0, 'iso-8859-1'))
         # UTF-8 is **always** preferred over anything else.
         # Reason: UTF-8 is not specific and can encode the entire unicode

Modified: Zope3/trunk/src/zope/publisher/tests/test_httpcharsets.py
===================================================================
--- Zope3/trunk/src/zope/publisher/tests/test_httpcharsets.py	2004-05-27 19:47:49 UTC (rev 25067)
+++ Zope3/trunk/src/zope/publisher/tests/test_httpcharsets.py	2004-05-27 20:26:45 UTC (rev 25068)
@@ -53,7 +53,15 @@
         self.assertEqual(list(browser_charsets.getPreferredCharsets()),
                          ['iso-8859-1', 'utf-16'])
 
+    def testNoHTTP_ACCEPT_CHARSET(self):
+        # If the client doesn't provide a HTTP_ACCEPT_CHARSET, it should
+        # accept any charset
+        request = {}
+        browser_charsets = HTTPCharsets(request)
+        self.assertEqual(list(browser_charsets.getPreferredCharsets()),
+                         [])
 
+
 def test_suite():
     loader=unittest.TestLoader()
     return loader.loadTestsFromTestCase(HTTPCharsetTest)