[Zope3-checkins] SVN: Zope3/branches/3.3/src/zope/ zope.tal.xmlparser.XMLParser couldn't deal with unicode strings, which meant that

[Philipp von Weitershausen]

Log message for revision 72022:
  zope.tal.xmlparser.XMLParser couldn't deal with unicode strings, which meant that
  PageTemplates in XML mode whose source code was available as a unicode string failed.
  Fixed the problem and added a test that exercises a PT w/ unicode source in XML mode
  (HTML mode already worked).
  

Changed:
  U   Zope3/branches/3.3/src/zope/pagetemplate/tests/test_basictemplate.py
  U   Zope3/branches/3.3/src/zope/tal/tests/test_xmlparser.py
  U   Zope3/branches/3.3/src/zope/tal/xmlparser.py

-=-
Modified: Zope3/branches/3.3/src/zope/pagetemplate/tests/test_basictemplate.py
===================================================================
--- Zope3/branches/3.3/src/zope/pagetemplate/tests/test_basictemplate.py	2007-01-14 12:48:31 UTC (rev 72021)
+++ Zope3/branches/3.3/src/zope/pagetemplate/tests/test_basictemplate.py	2007-01-14 13:45:26 UTC (rev 72022)
@@ -164,7 +164,17 @@
         self.t.write(text)
         self.t()
 
+    def test_unicode_html(self):
+        text = u'<p>\xe4\xf6\xfc\xdf</p>'
 
+        # test with HTML parser
+        self.t.pt_edit(text, 'text/html')
+        self.assertEquals(self.t().strip(), text)
+
+        # test with XML parser
+        self.t.pt_edit(text, 'text/xml')
+        self.assertEquals(self.t().strip(), text)
+
 def test_suite():
     return unittest.makeSuite(BasicTemplateTests)
 

Modified: Zope3/branches/3.3/src/zope/tal/tests/test_xmlparser.py
===================================================================
--- Zope3/branches/3.3/src/zope/tal/tests/test_xmlparser.py	2007-01-14 12:48:31 UTC (rev 72021)
+++ Zope3/branches/3.3/src/zope/tal/tests/test_xmlparser.py	2007-01-14 13:45:26 UTC (rev 72022)
@@ -249,7 +249,13 @@
     def test_declaration_junk_chars(self):
         self._parse_error("<!DOCTYPE foo $ >")
 
+    def test_unicode_string(self):
+        output = [('starttag', u'p', []),
+                  ('data', u'\xe4\xf6\xfc\xdf'),
+                  ('endtag', u'p')]
+        self._run_check(u'<p>\xe4\xf6\xfc\xdf</p>', output)
 
+
 # Support for the Zope regression test framework:
 def test_suite(skipxml=utils.skipxml):
     if skipxml:

Modified: Zope3/branches/3.3/src/zope/tal/xmlparser.py
===================================================================
--- Zope3/branches/3.3/src/zope/tal/xmlparser.py	2007-01-14 12:48:31 UTC (rev 72021)
+++ Zope3/branches/3.3/src/zope/tal/xmlparser.py	2007-01-14 13:45:26 UTC (rev 72022)
@@ -74,6 +74,11 @@
         self.parseStream(open(filename))
 
     def parseString(self, s):
+        if isinstance(s, unicode):
+            # Expat cannot deal with unicode strings, only with
+            # encoded ones.  Also, its range of encodings is rather
+            # limited, UTF-8 is the safest bet here.
+            s = s.encode('utf-8')
         self.parser.Parse(s, 1)
 
     def parseURL(self, url):