[Zope-Coders] Large Files and WebDAV
Sidnei da Silva
sidnei at awkly.org
Wed Nov 3 15:29:10 EST 2004
Hello,
I've made some changes to webdav.NullResource and added a ZConfig
directive to control the threshold where temporary files are created
for big requests. NullResource uses this threshold to decide between
reading a chunk of the file and reading the whole file when calling
the PUT_Factory.
If anyone could quickly review the changes, the patch is attached and
I've created branches from 2.7 and trunk which are ready to merge,
just pending a OK from the powers-that-be.
(CC'ing Chris as he told me he was suffering from the same bug)
--
Sidnei da Silva <sidnei at awkly.org>
http://awkly.org - dreamcatching :: making your dreams come true
http://www.enfoldsystems.com
http://plone.org/about/team#dreamcatcher
I've never been canoeing before, but I imagine there must be just a few
simple heuristics you have to remember...
Yes, don't fall out, and don't hit rocks.
-------------- next part --------------
Index: lib/python/ZServer/__init__.py
===================================================================
--- lib/python/ZServer/__init__.py (.../trunk) (revision 28336)
+++ lib/python/ZServer/__init__.py (.../branches/dc-large_file-branch) (revision 28336)
@@ -21,13 +21,18 @@
exit_code = 0
# the ZServer version number
-ZSERVER_VERSION='1.1'
+ZSERVER_VERSION = '1.1'
# the maximum number of incoming connections to ZServer
-CONNECTION_LIMIT=1000 # may be reset by max_listen_sockets handler in Zope
+CONNECTION_LIMIT = 1000 # may be reset by max_listen_sockets handler in Zope
+# request bigger than this size get saved into a
+# temporary file instead of being read completely into memory
+LARGE_FILE_THRESHOLD = 1 << 19 # may be reset by large_file_threshold
+ # handler in Zope
+
# the Zope version string
-ZOPE_VERSION=utils.getZopeVersion()
+ZOPE_VERSION = utils.getZopeVersion()
# backwards compatibility aliases
from utils import requestCloseOnExec
Index: lib/python/ZServer/FCGIServer.py
===================================================================
--- lib/python/ZServer/FCGIServer.py (.../trunk) (revision 28336)
+++ lib/python/ZServer/FCGIServer.py (.../branches/dc-large_file-branch) (revision 28336)
@@ -32,7 +32,7 @@
from medusa.counter import counter
from medusa.http_server import compute_timezone_for_log
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
from PubCore import handle
from PubCore.ZEvent import Wakeup
@@ -642,6 +642,7 @@
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return len(asyncore.socket_map) < CONNECTION_LIMIT
Index: lib/python/ZServer/HTTPServer.py
===================================================================
--- lib/python/ZServer/HTTPServer.py (.../trunk) (revision 28336)
+++ lib/python/ZServer/HTTPServer.py (.../branches/dc-large_file-branch) (revision 28336)
@@ -52,7 +52,7 @@
from medusa.default_handler import unquote
from asyncore import compact_traceback, dispatcher
-from ZServer import CONNECTION_LIMIT, ZOPE_VERSION, ZSERVER_VERSION
+from ZServer import ZOPE_VERSION, ZSERVER_VERSION
from ZServer import requestCloseOnExec
from zLOG import LOG, register_subsystem, BLATHER, INFO, WARNING, ERROR
import DebugLogger
@@ -74,9 +74,10 @@
class zhttp_collector:
def __init__(self, handler, request, size):
+ from ZServer import LARGE_FILE_THRESHOLD
self.handler = handler
self.request = request
- if size > 524288:
+ if size > LARGE_FILE_THRESHOLD:
# write large upload data to a file
from tempfile import TemporaryFile
self.data = TemporaryFile('w+b')
@@ -407,8 +408,9 @@
requestCloseOnExec(self.socket)
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return self.accepting and \
- len(asyncore.socket_map) < CONNECTION_LIMIT
+ len(asyncore.socket_map) < CONNECTION_LIMIT
def listen(self, num):
# override asyncore limits for nt's listen queue size
Index: lib/python/ZServer/PCGIServer.py
===================================================================
--- lib/python/ZServer/PCGIServer.py (.../trunk) (revision 28336)
+++ lib/python/ZServer/PCGIServer.py (.../branches/dc-large_file-branch) (revision 28336)
@@ -35,7 +35,7 @@
from asyncore import compact_traceback
import ZServer
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
from PubCore import handle
from PubCore.ZEvent import Wakeup
@@ -327,6 +327,7 @@
self.channel_class(self, conn, addr)
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return len(asyncore.socket_map) < CONNECTION_LIMIT
def writable (self):
Index: lib/python/ZServer/FTPServer.py
===================================================================
--- lib/python/ZServer/FTPServer.py (.../trunk) (revision 28336)
+++ lib/python/ZServer/FTPServer.py (.../branches/dc-large_file-branch) (revision 28336)
@@ -72,7 +72,7 @@
from FTPResponse import make_response
from FTPRequest import FTPRequest
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
from cStringIO import StringIO
import os
@@ -653,6 +653,7 @@
self.ftp_channel_class (self, conn, addr, self.module)
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return len(asyncore.socket_map) < CONNECTION_LIMIT
def listen(self, num):
Index: lib/python/Zope/Startup/handlers.py
===================================================================
--- lib/python/Zope/Startup/handlers.py (.../trunk) (revision 28336)
+++ lib/python/Zope/Startup/handlers.py (.../branches/dc-large_file-branch) (revision 28336)
@@ -91,6 +91,10 @@
value and _setenv('REST_LANGUAGE_CODE' , value)
return value
+def large_file_threshold(value):
+ import ZServer
+ ZServer.LARGE_FILE_THRESHOLD = value
+
# server handlers
def root_handler(config):
Index: lib/python/Zope/Startup/zopeschema.xml
===================================================================
--- lib/python/Zope/Startup/zopeschema.xml (.../trunk) (revision 28336)
+++ lib/python/Zope/Startup/zopeschema.xml (.../branches/dc-large_file-branch) (revision 28336)
@@ -734,8 +734,9 @@
</description>
</section>
- <!-- max-listen-sockets should really go into the ZServer package, but
- I can't quite figure out how to put it there -->
+ <!-- max-listen-sockets and large-file-threshold should really go
+ into the ZServer package, but I can't quite figure out how to
+ put it there -->
<key name="max-listen-sockets" datatype="integer"
default="1000">
@@ -745,6 +746,14 @@
</description>
</key>
+ <key name="large-file-threshold" datatype="byte-size"
+ handler="large_file_threshold" default="512KB">
+ <description>
+ Requests bigger than this size get saved into a temporary file
+ instead of being read completely into memory.
+ </description>
+ </key>
+
<multisection type="ZServer.server" name="*" attribute="servers"/>
<key name="port-base" datatype="integer" default="0">
<description>
Index: lib/python/webdav/NullResource.py
===================================================================
--- lib/python/webdav/NullResource.py (.../trunk) (revision 28336)
+++ lib/python/webdav/NullResource.py (.../branches/dc-large_file-branch) (revision 28336)
@@ -81,13 +81,16 @@
ob=File(name, '', body, content_type=typ)
return ob
- PUT__roles__=('Anonymous',)
+ PUT__roles__ = ('Anonymous',)
def PUT(self, REQUEST, RESPONSE):
- """Create a new non-collection resource."""
+ """Create a new non-collection resource.
+ """
+ from ZServer import LARGE_FILE_THRESHOLD
+
self.dav__init(REQUEST, RESPONSE)
- name=self.__name__
- parent=self.__parent__
+ name = self.__name__
+ parent = self.__parent__
ifhdr = REQUEST.get_header('If', '')
if WriteLockInterface.isImplementedBy(parent) and parent.wl_isLocked():
@@ -101,17 +104,40 @@
# There was an If header, but the parent is not locked
raise PreconditionFailed
- body=REQUEST.get('BODY', '')
+ # SDS: Only use BODY if the file size is smaller than
+ # LARGE_FILE_THRESHOLD, otherwise read LARGE_FILE_THRESHOLD
+ # bytes from the file which should be enough to trigger
+ # content_type detection, and possibly enough for CMF's
+ # content_type_registry too.
+ #
+ # Note that body here is really just used for detecting the
+ # content type and figuring out the correct factory. The correct
+ # file content will be uploaded on ob.PUT(REQUEST, RESPONSE) after
+ # the object has been created.
+ #
+ # A problem I could see is content_type_registry predicates
+ # that do depend on the whole file being passed here as an
+ # argument. There's none by default that does this though. If
+ # they really do want to look at the file, they should use
+ # REQUEST['BODYFILE'] directly and try as much as possible not
+ # to read the whole file into memory.
+
+ if int(REQUEST.get('CONTENT_LENGTH') or 0) > LARGE_FILE_THRESHOLD:
+ file = REQUEST['BODYFILE']
+ body = file.read(LARGE_FILE_THRESHOLD)
+ file.seek(0)
+ else:
+ body = REQUEST.get('BODY', '')
+
typ=REQUEST.get_header('content-type', None)
if typ is None:
typ, enc=OFS.content_types.guess_content_type(name, body)
factory = getattr(parent, 'PUT_factory', self._default_PUT_factory )
ob = factory(name, typ, body)
- ob = (ob is None and
- self._default_PUT_factory(name, typ, body) or
- ob
- )
+ if ob is None:
+ ob = self._default_PUT_factory(name, typ, body)
+
# We call _verifyObjectPaste with verify_src=0, to see if the
# user can create this type of object (and we don't need to
# check the clipboard.
@@ -122,9 +148,11 @@
except:
raise Forbidden, sys.exc_info()[1]
- # Delegate actual PUT handling to the new object.
+ # Delegate actual PUT handling to the new object,
+ # SDS: But just *after* it has been stored.
+ self.__parent__._setObject(name, ob)
+ ob = self.__parent__._getOb(name)
ob.PUT(REQUEST, RESPONSE)
- self.__parent__._setObject(name, ob)
RESPONSE.setStatus(201)
RESPONSE.setBody('')
Index: skel/etc/zope.conf.in
===================================================================
--- skel/etc/zope.conf.in (.../trunk) (revision 28336)
+++ skel/etc/zope.conf.in (.../branches/dc-large_file-branch) (revision 28336)
@@ -781,7 +781,19 @@
# max-listen-sockets 500
+# Directive: large-file-threshold
+#
+# Description:
+# Requests bigger than this size get saved into a temporary file
+# instead of being read completely into memory.
+#
+# Default: 512K
+#
+# Example:
+#
+# large-file-threshold 1Mb
+
# Directives: servers
#
# Description:
More information about the Zope-Coders
mailing list