[Zope-Checkins] SVN: Zope/branches/dc-large_file-branch/ Create a
directive for configuring the large file threshold where temp
files are created and use it for NullResource.PUT.
Sidnei da Silva
sidnei at awkly.org
Wed Nov 3 15:27:26 EST 2004
Log message for revision 28335:
Create a directive for configuring the large file threshold where temp files are created and use it for NullResource.PUT.
Changed:
U Zope/branches/dc-large_file-branch/lib/python/ZServer/HTTPServer.py
U Zope/branches/dc-large_file-branch/lib/python/ZServer/__init__.py
U Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/handlers.py
U Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/zopeschema.xml
U Zope/branches/dc-large_file-branch/lib/python/webdav/NullResource.py
U Zope/branches/dc-large_file-branch/skel/etc/zope.conf.in
-=-
Modified: Zope/branches/dc-large_file-branch/lib/python/ZServer/HTTPServer.py
===================================================================
--- Zope/branches/dc-large_file-branch/lib/python/ZServer/HTTPServer.py 2004-11-03 20:17:35 UTC (rev 28334)
+++ Zope/branches/dc-large_file-branch/lib/python/ZServer/HTTPServer.py 2004-11-03 20:27:25 UTC (rev 28335)
@@ -53,7 +53,7 @@
from asyncore import compact_traceback, dispatcher
from ZServer import CONNECTION_LIMIT, ZOPE_VERSION, ZSERVER_VERSION
-from ZServer import requestCloseOnExec
+from ZServer import requestCloseOnExec, LARGE_FILE_THRESHOLD
from zLOG import LOG, register_subsystem, BLATHER, INFO, WARNING, ERROR
import DebugLogger
from medusa import logger
@@ -76,7 +76,7 @@
def __init__(self, handler, request, size):
self.handler = handler
self.request = request
- if size > 524288:
+ if size > LARGE_FILE_THRESHOLD:
# write large upload data to a file
from tempfile import TemporaryFile
self.data = TemporaryFile('w+b')
Modified: Zope/branches/dc-large_file-branch/lib/python/ZServer/__init__.py
===================================================================
--- Zope/branches/dc-large_file-branch/lib/python/ZServer/__init__.py 2004-11-03 20:17:35 UTC (rev 28334)
+++ Zope/branches/dc-large_file-branch/lib/python/ZServer/__init__.py 2004-11-03 20:27:25 UTC (rev 28335)
@@ -21,13 +21,18 @@
exit_code = 0
# the ZServer version number
-ZSERVER_VERSION='1.1'
+ZSERVER_VERSION = '1.1'
# the maximum number of incoming connections to ZServer
-CONNECTION_LIMIT=1000 # may be reset by max_listen_sockets handler in Zope
+CONNECTION_LIMIT = 1000 # may be reset by max_listen_sockets handler in Zope
+# request bigger than this size get saved into a
+# temporary file instead of being read completely into memory
+LARGE_FILE_THRESHOLD = 1 << 19 # may be reset by large_file_threshold
+ # handler in Zope
+
# the Zope version string
-ZOPE_VERSION=utils.getZopeVersion()
+ZOPE_VERSION = utils.getZopeVersion()
# backwards compatibility aliases
from utils import requestCloseOnExec
Modified: Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/handlers.py
===================================================================
--- Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/handlers.py 2004-11-03 20:17:35 UTC (rev 28334)
+++ Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/handlers.py 2004-11-03 20:27:25 UTC (rev 28335)
@@ -91,6 +91,10 @@
value and _setenv('REST_LANGUAGE_CODE' , value)
return value
+def large_file_threshold(value):
+ import ZServer
+ ZServer.LARGE_FILE_THRESHOLD = value
+
# server handlers
def root_handler(config):
Modified: Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/zopeschema.xml
===================================================================
--- Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/zopeschema.xml 2004-11-03 20:17:35 UTC (rev 28334)
+++ Zope/branches/dc-large_file-branch/lib/python/Zope/Startup/zopeschema.xml 2004-11-03 20:27:25 UTC (rev 28335)
@@ -734,8 +734,9 @@
</description>
</section>
- <!-- max-listen-sockets should really go into the ZServer package, but
- I can't quite figure out how to put it there -->
+ <!-- max-listen-sockets and large-file-threshold should really go
+ into the ZServer package, but I can't quite figure out how to
+ put it there -->
<key name="max-listen-sockets" datatype="integer"
default="1000">
@@ -745,6 +746,14 @@
</description>
</key>
+ <key name="large-file-threshold" datatype="byte-size"
+ handler="large_file_threshold" default="512KB">
+ <description>
+ Requests bigger than this size get saved into a temporary file
+ instead of being read completely into memory.
+ </description>
+ </key>
+
<multisection type="ZServer.server" name="*" attribute="servers"/>
<key name="port-base" datatype="integer" default="0">
<description>
Modified: Zope/branches/dc-large_file-branch/lib/python/webdav/NullResource.py
===================================================================
--- Zope/branches/dc-large_file-branch/lib/python/webdav/NullResource.py 2004-11-03 20:17:35 UTC (rev 28334)
+++ Zope/branches/dc-large_file-branch/lib/python/webdav/NullResource.py 2004-11-03 20:27:25 UTC (rev 28335)
@@ -81,13 +81,16 @@
ob=File(name, '', body, content_type=typ)
return ob
- PUT__roles__=('Anonymous',)
+ PUT__roles__ = ('Anonymous',)
def PUT(self, REQUEST, RESPONSE):
- """Create a new non-collection resource."""
+ """Create a new non-collection resource.
+ """
+ from ZServer import LARGE_FILE_THRESHOLD
+
self.dav__init(REQUEST, RESPONSE)
- name=self.__name__
- parent=self.__parent__
+ name = self.__name__
+ parent = self.__parent__
ifhdr = REQUEST.get_header('If', '')
if WriteLockInterface.isImplementedBy(parent) and parent.wl_isLocked():
@@ -101,17 +104,40 @@
# There was an If header, but the parent is not locked
raise PreconditionFailed
- body=REQUEST.get('BODY', '')
+ # SDS: Only use BODY if the file size is smaller than
+ # LARGE_FILE_THRESHOLD, otherwise read LARGE_FILE_THRESHOLD
+ # bytes from the file which should be enough to trigger
+ # content_type detection, and possibly enough for CMF's
+ # content_type_registry too.
+ #
+ # Note that body here is really just used for detecting the
+ # content type and figuring out the correct factory. The correct
+ # file content will be uploaded on ob.PUT(REQUEST, RESPONSE) after
+ # the object has been created.
+ #
+ # A problem I could see is content_type_registry predicates
+ # that do depend on the whole file being passed here as an
+ # argument. There's none by default that does this though. If
+ # they really do want to look at the file, they should use
+ # REQUEST['BODYFILE'] directly and try as much as possible not
+ # to read the whole file into memory.
+
+ if int(REQUEST.get('CONTENT_LENGTH') or 0) > LARGE_FILE_THRESHOLD:
+ file = REQUEST['BODYFILE']
+ body = file.read(LARGE_FILE_THRESHOLD)
+ file.seek(0)
+ else:
+ body = REQUEST.get('BODY', '')
+
typ=REQUEST.get_header('content-type', None)
if typ is None:
typ, enc=OFS.content_types.guess_content_type(name, body)
factory = getattr(parent, 'PUT_factory', self._default_PUT_factory )
ob = factory(name, typ, body)
- ob = (ob is None and
- self._default_PUT_factory(name, typ, body) or
- ob
- )
+ if ob is None:
+ ob = self._default_PUT_factory(name, typ, body)
+
# We call _verifyObjectPaste with verify_src=0, to see if the
# user can create this type of object (and we don't need to
# check the clipboard.
@@ -122,9 +148,11 @@
except:
raise Forbidden, sys.exc_info()[1]
- # Delegate actual PUT handling to the new object.
+ # Delegate actual PUT handling to the new object,
+ # SDS: But just *after* it has been stored.
+ self.__parent__._setObject(name, ob)
+ ob = self.__parent__._getOb(name)
ob.PUT(REQUEST, RESPONSE)
- self.__parent__._setObject(name, ob)
RESPONSE.setStatus(201)
RESPONSE.setBody('')
Modified: Zope/branches/dc-large_file-branch/skel/etc/zope.conf.in
===================================================================
--- Zope/branches/dc-large_file-branch/skel/etc/zope.conf.in 2004-11-03 20:17:35 UTC (rev 28334)
+++ Zope/branches/dc-large_file-branch/skel/etc/zope.conf.in 2004-11-03 20:27:25 UTC (rev 28335)
@@ -781,7 +781,19 @@
# max-listen-sockets 500
+# Directive: large-file-threshold
+#
+# Description:
+# Requests bigger than this size get saved into a temporary file
+# instead of being read completely into memory.
+#
+# Default: 512K
+#
+# Example:
+#
+# large-file-threshold 1Mb
+
# Directives: servers
#
# Description:
More information about the Zope-Checkins
mailing list