[Zope-Checkins] CVS: Zope/lib/python/ZPublisher - Converters.py:1.14.14.1 HTTPRequest.py:1.62.4.1 HTTPResponse.py:1.53.14.2

[Toby Dickenson]

Update of /cvs-repository/Zope/lib/python/ZPublisher
In directory cvs.zope.org:/tmp/cvs-serv25817/lib/python/ZPublisher

Modified Files:
      Tag: toby-unicode-branch
	Converters.py HTTPRequest.py HTTPResponse.py 
Log Message:
second phase of unicode patch merge

=== Zope/lib/python/ZPublisher/Converters.py 1.14 => 1.14.14.1 ===
     return v
 
+class _unicode_converter:
+    def __call__(self,v):
+        # Convert a regular python string. This probably doesnt do what you want,
+        # whatever that might be. If you are getting exceptions below, you
+        # probably missed the encoding tag from a form field name. Use:
+        #       <input name="description:utf8:ustring" .....
+        # rather than
+        #       <input name="description:ustring" .....
+        if hasattr(v,'read'): v=v.read()
+        v = unicode(v)
+        return self.convert_unicode(v)
+
+    def convert_unicode(self,v):
+        raise NotImplementedError('convert_unicode')
+
+class field2ustring(_unicode_converter):
+    def convert_unicode(self,v):
+        return v
+field2ustring = field2ustring()
+
+class field2utokens(_unicode_converter):
+    def convert_unicode(self,v):
+        return v.split()
+field2utokens = field2utokens()
+
+class field2utext(_unicode_converter):
+    def convert_unicode(self,v):
+        return unicode(field2text(v.encode('utf8')),'utf8')
+field2utext = field2utext()
+
+class field2ulines(_unicode_converter):
+    def convert_unicode(self,v):
+        return field2utext.convert_unicode(v).split('\n')
+field2ulines = field2ulines()
+
 type_converters = {
     'float':    field2float,
     'int':      field2int,
@@ -123,7 +158,11 @@
     'tokens':   field2tokens,
     'lines':    field2lines,
     'text':     field2text,
-    'boolean':     field2boolean,
+    'boolean':  field2boolean,
+    'ustring':  field2ustring,
+    'utokens':  field2utokens,
+    'ulines':   field2ulines,
+    'utext':    field2utext,
     }
 
 get_converter=type_converters.get


=== Zope/lib/python/ZPublisher/HTTPRequest.py 1.62 => 1.62.4.1 ===
 __version__='$Revision$'[11:-2]
 
-import re, sys, os,  urllib, time, whrandom, cgi
+import re, sys, os,  urllib, time, whrandom, cgi, codecs
 from BaseRequest import BaseRequest
 from HTTPResponse import HTTPResponse
 from cgi import FieldStorage, escape
@@ -384,6 +384,7 @@
                         item=item.value
 
                 flags=0
+                character_encoding = ''
 
                 # Loop through the different types and set
                 # the appropriate flags
@@ -431,6 +432,8 @@
                             flags=flags|RECORDS
                         elif type_name == 'ignore_empty':
                             if not item: flags=flags|EMPTY
+                        elif has_codec(type_name):
+                            character_encoding = type_name
     
                         l=key.rfind(':')
                         if l < 0: break
@@ -456,7 +459,17 @@
                     # defer conversion
                     if flags&CONVERTED:
                         try:
-                            item=converter(item)
+                            if character_encoding:
+                                # We have a string with a specified character encoding.
+                                # This gets passed to the converter either as unicode, if it can
+                                # handle it, or crunched back down to latin-1 if it can not.
+                                item = unicode(item,character_encoding)
+                                if hasattr(converter,'convert_unicode'):
+                                    item = converter.convert_unicode(item)
+                                else:
+                                    item = converter(item.encode('latin1'))
+                            else:
+                                item=converter(item)
                         except:
                             if (not item and not (flags&DEFAULT) and
                                 defaults.has_key(key)):
@@ -965,6 +978,13 @@
                 return name, password
 
 
+def has_codec(x):
+    try:
+        codecs.lookup(x)
+    except LookupError:
+        return 0
+    else:
+        return 1
 
 
 base64=None


=== Zope/lib/python/ZPublisher/HTTPResponse.py 1.53.14.1 => 1.53.14.2 ===
                 body=body.asHTML()
 
-        body=ustr(body)
-
         if type(body) is UnicodeType:
             body = self._encode_unicode(body)
+        elif type(body) is StringType:
+            pass
+        else:
+            try:
+                body = str(body)
+            except UnicodeError:
+                body = _encode_unicode(unicode(body))
 
         l=len(body)
         if ((l < 200) and body[:1]=='<' and body.find('>')==l-1 and 
@@ -545,7 +550,7 @@
             # Try to capture exception info for bci calls
             et = translate(str(t), nl2sp)
             self.setHeader('bobo-exception-type', et)
-            ev = translate(ustr(v), nl2sp)
+            ev = translate(str(v), nl2sp)
             if ev.find( '<html>') >= 0:
                 ev = 'bobo exception'
             self.setHeader('bobo-exception-value', ev[:255])
@@ -588,7 +593,7 @@
         b = v
         if isinstance(b, Exception):
             try:
-                b = ustr(b)
+                b = str(b)
             except:
                 b = '<unprintable %s object>' % type(b).__name__
 
@@ -714,22 +719,3 @@
         self.stdout.write(data)
 
 
-# Duplicated from DocumentTemplate, to avoid the dependency.
-# Is there a better place for it?
-def ustr(v):
-    """convert an object to a string or unicode string
-    """
-    string_types = (StringType,UnicodeType)
-    if type(v) in string_types:
-        return v
-    else:
-        try:
-            fn = v.__str__
-        except AttributeError:
-            return str(v)
-        else:
-            v = fn()
-            if type(v) in string_types:
-                return v
-            else:
-                raise ValueError('__str__ returned wrong type')