[Zope-Checkins] CVS: Zope/lib/python/ZPublisher - Converters.py:1.15 HTTPRequest.py:1.63 HTTPResponse.py:1.55

Toby Dickenson tdickenson@geminidataloggers.com
Wed, 27 Mar 2002 05:14:05 -0500


Update of /cvs-repository/Zope/lib/python/ZPublisher
In directory cvs.zope.org:/tmp/cvs-serv10497/lib/python/ZPublisher

Modified Files:
	Converters.py HTTPRequest.py HTTPResponse.py 
Log Message:
merged toby-stiff-cache-branch and toby-unicode-branch

=== Zope/lib/python/ZPublisher/Converters.py 1.14 => 1.15 ===
     return v
 
+class _unicode_converter:
+    def __call__(self,v):
+        # Convert a regular python string. This probably doesnt do what you want,
+        # whatever that might be. If you are getting exceptions below, you
+        # probably missed the encoding tag from a form field name. Use:
+        #       <input name="description:utf8:ustring" .....
+        # rather than
+        #       <input name="description:ustring" .....
+        if hasattr(v,'read'): v=v.read()
+        v = unicode(v)
+        return self.convert_unicode(v)
+
+    def convert_unicode(self,v):
+        raise NotImplementedError('convert_unicode')
+
+class field2ustring(_unicode_converter):
+    def convert_unicode(self,v):
+        return v
+field2ustring = field2ustring()
+
+class field2utokens(_unicode_converter):
+    def convert_unicode(self,v):
+        return v.split()
+field2utokens = field2utokens()
+
+class field2utext(_unicode_converter):
+    def convert_unicode(self,v):
+        return unicode(field2text(v.encode('utf8')),'utf8')
+field2utext = field2utext()
+
+class field2ulines(_unicode_converter):
+    def convert_unicode(self,v):
+        return field2utext.convert_unicode(v).split('\n')
+field2ulines = field2ulines()
+
 type_converters = {
     'float':    field2float,
     'int':      field2int,
@@ -123,7 +158,11 @@
     'tokens':   field2tokens,
     'lines':    field2lines,
     'text':     field2text,
-    'boolean':     field2boolean,
+    'boolean':  field2boolean,
+    'ustring':  field2ustring,
+    'utokens':  field2utokens,
+    'ulines':   field2ulines,
+    'utext':    field2utext,
     }
 
 get_converter=type_converters.get


=== Zope/lib/python/ZPublisher/HTTPRequest.py 1.62 => 1.63 ===
 __version__='$Revision$'[11:-2]
 
-import re, sys, os,  urllib, time, whrandom, cgi
+import re, sys, os,  urllib, time, whrandom, cgi, codecs
 from BaseRequest import BaseRequest
 from HTTPResponse import HTTPResponse
 from cgi import FieldStorage, escape
@@ -384,6 +384,7 @@
                         item=item.value
 
                 flags=0
+                character_encoding = ''
 
                 # Loop through the different types and set
                 # the appropriate flags
@@ -431,6 +432,8 @@
                             flags=flags|RECORDS
                         elif type_name == 'ignore_empty':
                             if not item: flags=flags|EMPTY
+                        elif has_codec(type_name):
+                            character_encoding = type_name
     
                         l=key.rfind(':')
                         if l < 0: break
@@ -456,7 +459,17 @@
                     # defer conversion
                     if flags&CONVERTED:
                         try:
-                            item=converter(item)
+                            if character_encoding:
+                                # We have a string with a specified character encoding.
+                                # This gets passed to the converter either as unicode, if it can
+                                # handle it, or crunched back down to latin-1 if it can not.
+                                item = unicode(item,character_encoding)
+                                if hasattr(converter,'convert_unicode'):
+                                    item = converter.convert_unicode(item)
+                                else:
+                                    item = converter(item.encode('latin1'))
+                            else:
+                                item=converter(item)
                         except:
                             if (not item and not (flags&DEFAULT) and
                                 defaults.has_key(key)):
@@ -965,6 +978,13 @@
                 return name, password
 
 
+def has_codec(x):
+    try:
+        codecs.lookup(x)
+    except LookupError:
+        return 0
+    else:
+        return 1
 
 
 base64=None


=== Zope/lib/python/ZPublisher/HTTPResponse.py 1.54 => 1.55 ===
 import types, os, sys, re
 from string import translate, maketrans
-from types import StringType, InstanceType, LongType
+from types import StringType, InstanceType, LongType, UnicodeType
 from BaseResponse import BaseResponse
 from zExceptions import Unauthorized
 
@@ -241,7 +241,16 @@
             if hasattr(body,'asHTML'):
                 body=body.asHTML()
 
-        body=str(body)
+        if type(body) is UnicodeType:
+            body = self._encode_unicode(body)
+        elif type(body) is StringType:
+            pass
+        else:
+            try:
+                body = str(body)
+            except UnicodeError:
+                body = _encode_unicode(unicode(body))
+
         l=len(body)
         if ((l < 200) and body[:1]=='<' and body.find('>')==l-1 and 
             bogus_str_search(body) is not None):
@@ -276,6 +285,16 @@
         self.insertBase()
         return self
 
+    def _encode_unicode(self,body,charset_re=re.compile(r'text/[0-9a-z]+\s*;\s*charset=([-_0-9a-z]+)(?:(?:\s*;)|\Z)',re.IGNORECASE)):
+        # Encode the Unicode data as requested
+        if self.headers.has_key('content-type'):
+            match = charset_re.match(self.headers['content-type'])
+            if match:
+                encoding = match.group(1)
+                return body.encode(encoding)
+        # Use the default character encoding
+        return body.encode('latin1','replace')
+
     def setBase(self,base):
         'Set the base URL for the returned document.'
         if base[-1:] != '/':
@@ -594,23 +613,28 @@
                 (str(t),
                  'Zope has exited normally.<p>' + self._traceback(t, v, tb)),
                 is_error=1)
-        elif type(b) is not types.StringType or tag_search(b) is None:
-            body = self.setBody(
-                (str(t),
-                'Sorry, a site error occurred.<p>'
-                 + self._traceback(t, v, tb)),
-                 is_error=1)
-        elif b.strip().lower()[:6]=='<html>' or \
-              b.strip().lower()[:14]=='<!doctype html':
-            # error is an HTML document, not just a snippet of html
-            body = self.setBody(b + self._traceback(t, '(see above)', tb),
-                              is_error=1)
         else:
-            body = self.setBody((str(t),
-                               b + self._traceback(t,'(see above)', tb)),
-                              is_error=1)
-        del tb
-        return body
+            try:
+                match = tag_search(b)
+            except TypeError:
+                match = None
+            if match is None:
+                body = self.setBody(
+                    (str(t),
+                    'Sorry, a site error occurred.<p>'
+                     + self._traceback(t, v, tb)),
+                     is_error=1)
+            elif b.strip().lower()[:6]=='<html>' or \
+                  b.strip().lower()[:14]=='<!doctype html':
+                # error is an HTML document, not just a snippet of html
+                body = self.setBody(b + self._traceback(t, '(see above)', tb),
+                                  is_error=1)
+            else:
+                body = self.setBody((str(t),
+                                   b + self._traceback(t,'(see above)', tb)),
+                                  is_error=1)
+            del tb
+            return body
 
     _wrote=None
 
@@ -704,4 +728,5 @@
             self.stdout.flush()
 
         self.stdout.write(data)
+