[Zope-CVS] CVS: Packages/PartitionedFileStorage - FileStorage_py_1_129.patch:1.1 PartitionedFile.py:1.1
Shane Hathaway
shane@zope.com
Wed, 23 Apr 2003 14:25:41 -0400
Update of /cvs-repository/Packages/PartitionedFileStorage
In directory cvs.zope.org:/tmp/cvs-serv8282
Added Files:
FileStorage_py_1_129.patch PartitionedFile.py
Log Message:
Checked in latest version of PartitionedFileStorage.
=== Added File Packages/PartitionedFileStorage/FileStorage_py_1_129.patch ===
Index: FileStorage.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/FileStorage.py,v
retrieving revision 1.129
diff -u -u -r1.129 FileStorage.py
--- FileStorage.py 17 Mar 2003 18:58:30 -0000 1.129
+++ FileStorage.py 23 Apr 2003 18:14:01 -0000
@@ -203,6 +203,43 @@
packed_version='FS21'
+
+class FileOperations:
+ '''This class implements some basic operations on files.
+ PartitionedFileOperations provides an alternate implementation.
+ '''
+ def open(self, name, mode='r', bufsize=0):
+ global open
+ return open(name, mode, bufsize)
+
+ def exists(self, name):
+ return os.path.exists(name)
+
+ def rename(self, oldname, newname):
+ os.rename(oldname, newname)
+
+ def remove(self, name):
+ os.remove(name)
+
+ def fsync(self, file):
+ global fsync
+ if fsync is not None:
+ fsync(file.fileno())
+
+defaultFops = FileOperations()
+
+
+# Set the PARTITIONED_FILE_SIZE environment variable to use
+# PartitionedFile support. Note: To convert a partitioned file back
+# to a normal file, just 'cat' (Unix) or 'copy' (DOS/Windows) the
+# partitions together in order.
+PARTITIONED_FILE_SIZE = long(os.environ.get('PARTITIONED_FILE_SIZE', 0))
+if PARTITIONED_FILE_SIZE > 0:
+ from PartitionedFile import PartitionedFileOperations
+ defaultFops = PartitionedFileOperations(partlen=PARTITIONED_FILE_SIZE)
+
+
+
class FileStorage(BaseStorage.BaseStorage,
ConflictResolution.ConflictResolvingStorage):
# default pack time is 0
@@ -211,7 +248,10 @@
_records_before_save = 10000
def __init__(self, file_name, create=0, read_only=0, stop=None,
- quota=None):
+ quota=None, fops=None):
+ if fops is None:
+ fops = defaultFops
+ self.fops = fops
if read_only:
self._is_read_only = 1
@@ -243,7 +283,8 @@
self._file = None
if not create:
try:
- self._file = open(file_name, read_only and 'rb' or 'r+b')
+ self._file = self.fops.open(
+ file_name, read_only and 'rb' or 'r+b')
except IOError, exc:
if exc.errno == errno.EFBIG:
# The file is too big to open. Fail visibly.
@@ -254,15 +295,15 @@
# If something else went wrong, it's hard to guess
# what the problem was. If the file does not exist,
# create it. Otherwise, fail.
- if os.path.exists(file_name):
+ if self.fops.exists(file_name):
raise
else:
create = 1
if self._file is None and create:
- if os.path.exists(file_name):
- os.remove(file_name)
- self._file = open(file_name, 'w+b')
+ if self.fops.exists(file_name):
+ self.fops.remove(file_name)
+ self._file = self.fops.open(file_name, 'w+b')
self._file.write(packed_version)
r = self._restore_index()
@@ -1010,7 +1051,7 @@
file.write(self._tstatus)
file.flush()
- if fsync is not None: fsync(file.fileno())
+ self.fops.fsync(file)
self._pos = nextpos
@@ -1445,7 +1486,7 @@
_commit_lock_release=self._commit_lock_release
index, vindex, tindex, tvindex = self._newIndexes()
name=self.__name__
- file=open(name, 'rb')
+ file=self.fops.open(name, 'r+b')
stop=`apply(TimeStamp, time.gmtime(t)[:5]+(t%60,))`
if stop==z64: raise FileStorageError, 'Invalid pack time'
@@ -1502,7 +1543,7 @@
# Step 2, copy data and compute new index based on new positions.
index, vindex, tindex, tvindex = self._newIndexes()
- ofile=open(name+'.pack', 'w+b')
+ ofile=self.fops.open(name+'.pack', 'w+b')
# Index for non-version data. This is a temporary structure
# to reduce I/O during packing
@@ -1546,7 +1587,7 @@
# continuing
ofile.close()
file.close()
- os.remove(name+'.pack')
+ self.fops.remove(name+'.pack')
return
packing=0
@@ -1784,17 +1825,17 @@
file.close()
self._file.close()
try:
- if os.path.exists(name+'.old'):
- os.remove(name+'.old')
- os.rename(name, name+'.old')
+ if self.fops.exists(name+'.old'):
+ self.fops.remove(name+'.old')
+ self.fops.rename(name, name+'.old')
except:
# Waaa
- self._file=open(name,'r+b')
+ self._file=self.fops.open(name, 'r+b')
raise
# OK, we're beyond the point of no return
- os.rename(name+'.pack', name)
- self._file=open(name,'r+b')
+ self.fops.rename(name+'.pack', name)
+ self._file=self.fops.open(name, 'r+b')
self._initIndex(index, vindex, tindex, tvindex)
self._pos=opos
self._save_index()
@@ -1975,8 +2016,10 @@
return p, s
-def recover(file_name):
- file=open(file_name, 'r+b')
+def recover(file_name, fops=None):
+ if fops is None:
+ fops = defaultFops
+ file=fops.open(file_name, 'r+b')
index={}
vindex={}
tindex={}
@@ -2224,7 +2267,9 @@
# seek to transaction header, where tid is first 8 bytes
return file.read(8)
-def _truncate(file, name, pos):
+def _truncate(file, name, pos, fops=None):
+ if fops is None:
+ fops = defaultFops
seek=file.seek
seek(0,2)
file_size=file.tell()
@@ -2232,11 +2277,11 @@
i=0
while 1:
oname='%s.tr%s' % (name, i)
- if os.path.exists(oname):
+ if fops.exists(oname):
i=i+1
else:
warn("Writing truncated data from %s to %s", name, oname)
- o=open(oname,'wb')
+ o = fops.open(oname, 'wb')
seek(pos)
cp(file, o, file_size-pos)
o.close()
@@ -2271,9 +2316,11 @@
_ltid = z64
_file = None
- def __init__(self, file, start=None, stop=None):
+ def __init__(self, file, start=None, stop=None, fops=None):
+ if fops is None:
+ fops = defaultFops
if isinstance(file, StringType):
- file = open(file, 'rb')
+ file = fops.open(file, 'rb')
self._file = file
if file.read(4) != packed_version:
raise FileStorageFormatError, file.name
=== Added File Packages/PartitionedFileStorage/PartitionedFile.py === (564/664 lines abridged)
##############################################################################
#
# Zope Public License (ZPL) Version 1.0
# -------------------------------------
#
# Copyright (c) Digital Creations. All rights reserved.
#
# This license has been certified as Open Source(tm).
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions in source code must retain the above copyright
# notice, this list of conditions, and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions, and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# 3. Digital Creations requests that attribution be given to Zope
# in any manner possible. Zope includes a "Powered by Zope"
# button that is installed by default. While it is not a license
# violation to remove this button, it is requested that the
# attribution remain. A significant investment has been put
# into Zope, and this effort will continue if the Zope community
# continues to grow. This is one way to assure that growth.
#
# 4. All advertising materials and documentation mentioning
# features derived from or use of this software must display
# the following acknowledgement:
#
# "This product includes software developed by Digital Creations
# for use in the Z Object Publishing Environment
# (http://www.zope.org/)."
#
# In the event that the product being advertised includes an
# intact Zope distribution (with copyright and license included)
# then this clause is waived.
#
# 5. Names associated with Zope or Digital Creations must not be used to
# endorse or promote products derived from this software without
# prior written permission from Digital Creations.
#
# 6. Modified redistributions of any form whatsoever must retain
# the following acknowledgment:
#
# "This product includes software developed by Digital Creations
# for use in the Z Object Publishing Environment
[-=- -=- -=- 564 lines omitted -=- -=- -=-]
self.write(line) # According to the spec, doesn't add a newline.
def fsync(self):
'''
Requests an immediate flush of all data to physical media.
'''
if posixfsync is not None:
state = self.__state
for file in state.files:
posixfsync(file.fileno())
if __name__ == '__main__':
# Test the seek, write, and read functions.
import whrandom
generator = whrandom.whrandom()
while 1:
fops = PartitionedFileOperations(partlen=991)
try: os.mkdir('test')
except: pass
# Notice how easy it is to open a PartitionedFile:
file = fops.open('test/parts', 'wb')
# Here's how you do the same thing without partitions:
# file = open('test/parts', 'wb')
# Randomly shove data in the file, storing the same
# data in a string at the same time. Then read
# the file contents and see if it compares.
# Do this repeatedly.
MAXPOS = 12349
compare = ''
blank = '\0' * MAXPOS
for n in range(500):
s = ('%04d' % n) * 495
location = int(generator.random() * MAXPOS)
compare = (compare + blank)[:location] + s + \
compare[location + len(s):]
file.seek(location)
file.write(s)
file.close()
file = fops.open('test/parts', 'r')
# file = open('test/parts', 'r')
if file.read() == compare:
print 'Passed.'
else:
print 'FAILED! ' * 10
file.close()