[Zope-Checkins] SVN: Zope/trunk/ - Added a ZConfig directive
'large-file-threshold' to control
Sidnei da Silva
sidnei at awkly.org
Thu Nov 4 12:00:54 EST 2004
Log message for revision 28338:
- Added a ZConfig directive 'large-file-threshold' to control
the request content-size threshold at which a temporary file
gets created. Use the same value for deciding between reading
the whole request in memory or just a chunk inside
webdav.NullResource.PUT().
- Stitch newly-created object into it's container *before*
calling it's PUT() method. This fixes an issue with
OFS.File/OFS.Image that would result into reading the whole
file in memory and wrapping it into a *single* Pdata object.
- Import ZServer.CONNECTION_LIMIT variable *inside* the method
that uses it. Before this, the variable was imported at the
module level, thus binding it too early which would cause the
ZConfig handler to have no real effect.
Changed:
U Zope/trunk/doc/CHANGES.txt
U Zope/trunk/lib/python/ZServer/FCGIServer.py
U Zope/trunk/lib/python/ZServer/FTPServer.py
U Zope/trunk/lib/python/ZServer/HTTPServer.py
U Zope/trunk/lib/python/ZServer/PCGIServer.py
U Zope/trunk/lib/python/ZServer/__init__.py
U Zope/trunk/lib/python/Zope/Startup/handlers.py
U Zope/trunk/lib/python/Zope/Startup/zopeschema.xml
U Zope/trunk/lib/python/webdav/NullResource.py
U Zope/trunk/skel/etc/zope.conf.in
-=-
Modified: Zope/trunk/doc/CHANGES.txt
===================================================================
--- Zope/trunk/doc/CHANGES.txt 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/doc/CHANGES.txt 2004-11-04 17:00:49 UTC (rev 28338)
@@ -24,6 +24,19 @@
after Zope 2.8a1
+ Features added
+
+ - Added a ZConfig directive 'large-file-threshold' to control
+ the request content-size threshold at which a temporary file
+ gets created. Use the same value for deciding between reading
+ the whole request in memory or just a chunk inside
+ webdav.NullResource.PUT().
+
+ - Stitch newly-created object into it's container *before*
+ calling it's PUT() method. This fixes an issue with
+ OFS.File/OFS.Image that would result into reading the whole
+ file in memory and wrapping it into a *single* Pdata object.
+
Bugs fixed
- docutils: updated to V 0.3.5. The Zope core now contains a full copy of
@@ -69,10 +82,10 @@
a more verbose error message is issued, the same way it's done
on attribute/item traversal.
-
- - Collector #1523: replace the text field for importing .zexp/.xml
+ - Collector #1523: replace the text field for importing .zexp/.xml
files with a selection list
+
Zope 2.8a1
Modified: Zope/trunk/lib/python/ZServer/FCGIServer.py
===================================================================
--- Zope/trunk/lib/python/ZServer/FCGIServer.py 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/ZServer/FCGIServer.py 2004-11-04 17:00:49 UTC (rev 28338)
@@ -32,7 +32,7 @@
from medusa.counter import counter
from medusa.http_server import compute_timezone_for_log
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
from PubCore import handle
from PubCore.ZEvent import Wakeup
@@ -642,6 +642,7 @@
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return len(asyncore.socket_map) < CONNECTION_LIMIT
Modified: Zope/trunk/lib/python/ZServer/FTPServer.py
===================================================================
--- Zope/trunk/lib/python/ZServer/FTPServer.py 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/ZServer/FTPServer.py 2004-11-04 17:00:49 UTC (rev 28338)
@@ -72,7 +72,7 @@
from FTPResponse import make_response
from FTPRequest import FTPRequest
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
from cStringIO import StringIO
import os
@@ -653,6 +653,7 @@
self.ftp_channel_class (self, conn, addr, self.module)
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return len(asyncore.socket_map) < CONNECTION_LIMIT
def listen(self, num):
Modified: Zope/trunk/lib/python/ZServer/HTTPServer.py
===================================================================
--- Zope/trunk/lib/python/ZServer/HTTPServer.py 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/ZServer/HTTPServer.py 2004-11-04 17:00:49 UTC (rev 28338)
@@ -52,7 +52,7 @@
from medusa.default_handler import unquote
from asyncore import compact_traceback, dispatcher
-from ZServer import CONNECTION_LIMIT, ZOPE_VERSION, ZSERVER_VERSION
+from ZServer import ZOPE_VERSION, ZSERVER_VERSION
from ZServer import requestCloseOnExec
from zLOG import LOG, register_subsystem, BLATHER, INFO, WARNING, ERROR
import DebugLogger
@@ -74,9 +74,10 @@
class zhttp_collector:
def __init__(self, handler, request, size):
+ from ZServer import LARGE_FILE_THRESHOLD
self.handler = handler
self.request = request
- if size > 524288:
+ if size > LARGE_FILE_THRESHOLD:
# write large upload data to a file
from tempfile import TemporaryFile
self.data = TemporaryFile('w+b')
@@ -407,8 +408,9 @@
requestCloseOnExec(self.socket)
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return self.accepting and \
- len(asyncore.socket_map) < CONNECTION_LIMIT
+ len(asyncore.socket_map) < CONNECTION_LIMIT
def listen(self, num):
# override asyncore limits for nt's listen queue size
Modified: Zope/trunk/lib/python/ZServer/PCGIServer.py
===================================================================
--- Zope/trunk/lib/python/ZServer/PCGIServer.py 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/ZServer/PCGIServer.py 2004-11-04 17:00:49 UTC (rev 28338)
@@ -35,7 +35,7 @@
from asyncore import compact_traceback
import ZServer
-from ZServer import CONNECTION_LIMIT, requestCloseOnExec
+from ZServer import requestCloseOnExec
from PubCore import handle
from PubCore.ZEvent import Wakeup
@@ -327,6 +327,7 @@
self.channel_class(self, conn, addr)
def readable(self):
+ from ZServer import CONNECTION_LIMIT
return len(asyncore.socket_map) < CONNECTION_LIMIT
def writable (self):
Modified: Zope/trunk/lib/python/ZServer/__init__.py
===================================================================
--- Zope/trunk/lib/python/ZServer/__init__.py 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/ZServer/__init__.py 2004-11-04 17:00:49 UTC (rev 28338)
@@ -21,13 +21,18 @@
exit_code = 0
# the ZServer version number
-ZSERVER_VERSION='1.1'
+ZSERVER_VERSION = '1.1'
# the maximum number of incoming connections to ZServer
-CONNECTION_LIMIT=1000 # may be reset by max_listen_sockets handler in Zope
+CONNECTION_LIMIT = 1000 # may be reset by max_listen_sockets handler in Zope
+# request bigger than this size get saved into a
+# temporary file instead of being read completely into memory
+LARGE_FILE_THRESHOLD = 1 << 19 # may be reset by large_file_threshold
+ # handler in Zope
+
# the Zope version string
-ZOPE_VERSION=utils.getZopeVersion()
+ZOPE_VERSION = utils.getZopeVersion()
# backwards compatibility aliases
from utils import requestCloseOnExec
Modified: Zope/trunk/lib/python/Zope/Startup/handlers.py
===================================================================
--- Zope/trunk/lib/python/Zope/Startup/handlers.py 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/Zope/Startup/handlers.py 2004-11-04 17:00:49 UTC (rev 28338)
@@ -91,6 +91,10 @@
value and _setenv('REST_LANGUAGE_CODE' , value)
return value
+def large_file_threshold(value):
+ import ZServer
+ ZServer.LARGE_FILE_THRESHOLD = value
+
# server handlers
def root_handler(config):
Modified: Zope/trunk/lib/python/Zope/Startup/zopeschema.xml
===================================================================
--- Zope/trunk/lib/python/Zope/Startup/zopeschema.xml 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/Zope/Startup/zopeschema.xml 2004-11-04 17:00:49 UTC (rev 28338)
@@ -734,8 +734,9 @@
</description>
</section>
- <!-- max-listen-sockets should really go into the ZServer package, but
- I can't quite figure out how to put it there -->
+ <!-- max-listen-sockets and large-file-threshold should really go
+ into the ZServer package, but I can't quite figure out how to
+ put it there -->
<key name="max-listen-sockets" datatype="integer"
default="1000">
@@ -745,6 +746,14 @@
</description>
</key>
+ <key name="large-file-threshold" datatype="byte-size"
+ handler="large_file_threshold" default="512KB">
+ <description>
+ Requests bigger than this size get saved into a temporary file
+ instead of being read completely into memory.
+ </description>
+ </key>
+
<multisection type="ZServer.server" name="*" attribute="servers"/>
<key name="port-base" datatype="integer" default="0">
<description>
Modified: Zope/trunk/lib/python/webdav/NullResource.py
===================================================================
--- Zope/trunk/lib/python/webdav/NullResource.py 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/lib/python/webdav/NullResource.py 2004-11-04 17:00:49 UTC (rev 28338)
@@ -81,13 +81,16 @@
ob=File(name, '', body, content_type=typ)
return ob
- PUT__roles__=('Anonymous',)
+ PUT__roles__ = ('Anonymous',)
def PUT(self, REQUEST, RESPONSE):
- """Create a new non-collection resource."""
+ """Create a new non-collection resource.
+ """
+ from ZServer import LARGE_FILE_THRESHOLD
+
self.dav__init(REQUEST, RESPONSE)
- name=self.__name__
- parent=self.__parent__
+ name = self.__name__
+ parent = self.__parent__
ifhdr = REQUEST.get_header('If', '')
if WriteLockInterface.isImplementedBy(parent) and parent.wl_isLocked():
@@ -101,17 +104,40 @@
# There was an If header, but the parent is not locked
raise PreconditionFailed
- body=REQUEST.get('BODY', '')
+ # SDS: Only use BODY if the file size is smaller than
+ # LARGE_FILE_THRESHOLD, otherwise read LARGE_FILE_THRESHOLD
+ # bytes from the file which should be enough to trigger
+ # content_type detection, and possibly enough for CMF's
+ # content_type_registry too.
+ #
+ # Note that body here is really just used for detecting the
+ # content type and figuring out the correct factory. The correct
+ # file content will be uploaded on ob.PUT(REQUEST, RESPONSE) after
+ # the object has been created.
+ #
+ # A problem I could see is content_type_registry predicates
+ # that do depend on the whole file being passed here as an
+ # argument. There's none by default that does this though. If
+ # they really do want to look at the file, they should use
+ # REQUEST['BODYFILE'] directly and try as much as possible not
+ # to read the whole file into memory.
+
+ if int(REQUEST.get('CONTENT_LENGTH') or 0) > LARGE_FILE_THRESHOLD:
+ file = REQUEST['BODYFILE']
+ body = file.read(LARGE_FILE_THRESHOLD)
+ file.seek(0)
+ else:
+ body = REQUEST.get('BODY', '')
+
typ=REQUEST.get_header('content-type', None)
if typ is None:
typ, enc=OFS.content_types.guess_content_type(name, body)
factory = getattr(parent, 'PUT_factory', self._default_PUT_factory )
ob = factory(name, typ, body)
- ob = (ob is None and
- self._default_PUT_factory(name, typ, body) or
- ob
- )
+ if ob is None:
+ ob = self._default_PUT_factory(name, typ, body)
+
# We call _verifyObjectPaste with verify_src=0, to see if the
# user can create this type of object (and we don't need to
# check the clipboard.
@@ -122,9 +148,11 @@
except:
raise Forbidden, sys.exc_info()[1]
- # Delegate actual PUT handling to the new object.
+ # Delegate actual PUT handling to the new object,
+ # SDS: But just *after* it has been stored.
+ self.__parent__._setObject(name, ob)
+ ob = self.__parent__._getOb(name)
ob.PUT(REQUEST, RESPONSE)
- self.__parent__._setObject(name, ob)
RESPONSE.setStatus(201)
RESPONSE.setBody('')
Modified: Zope/trunk/skel/etc/zope.conf.in
===================================================================
--- Zope/trunk/skel/etc/zope.conf.in 2004-11-03 22:15:41 UTC (rev 28337)
+++ Zope/trunk/skel/etc/zope.conf.in 2004-11-04 17:00:49 UTC (rev 28338)
@@ -5,9 +5,20 @@
# This is the Zope configuration file. The Zope configuration file
# shows what the default configuration directives are, and show
# examples for each directive. To declare a directive, make sure that
-# you add it to a line that does not begin with '#'. Note that comments
+# you add it to a line that does not begin with '#'. Note that comments
# are only allowed at the beginning of a line: you may not add comments
# after directive text on the same line.
+#
+# Note for Developers
+# ===================
+#
+# This file is *not* auto-generated. If you create a new directive you
+# very likely want to include an example of how to use the new
+# directive in this file.
+#
+# You shouldn't modify 'zope.conf.in' to change
+# configuration. Instead, you should make a copy into 'zope.conf' and
+# modify that to avoid checking in changes to this file by mistake.
# ZConfig "defines" used for later textual substitution
@@ -151,7 +162,7 @@
# this directive set to "on" or you will receive an error. If using ZEO,
# at least one ZEO client must be run with this directive set to "on"
# once, the others can have it turned off.
-# NOTE: If your main storage is mounted read-only,
+# NOTE: If your main storage is mounted read-only,
# you must set this directive to "off".
#
# Default: on
@@ -781,7 +792,19 @@
# max-listen-sockets 500
+# Directive: large-file-threshold
+#
+# Description:
+# Requests bigger than this size get saved into a temporary file
+# instead of being read completely into memory.
+#
+# Default: 512K
+#
+# Example:
+#
+# large-file-threshold 1Mb
+
# Directives: servers
#
# Description:
More information about the Zope-Checkins
mailing list