[Zope-CVS] CVS: Products/AdaptableStorage/gateway_fs - FSConnection.py:1.15 FSSecurityAttributes.py:1.2
Shane Hathaway
shane@zope.com
Tue, 4 Mar 2003 23:25:40 -0500
Update of /cvs-repository/Products/AdaptableStorage/gateway_fs
In directory cvs.zope.org:/tmp/cvs-serv27335/gateway_fs
Modified Files:
FSConnection.py FSSecurityAttributes.py
Log Message:
Several changes to FSConnection:
- Moved pickled remainders to their own file. This is important for
placing ZODB data under version control, since property files are
text, while remainder pickles need to be stored as binary files (even
text pickles should be stored as binary).
- Property files are now opened in text mode. Next we need to make it
possible to read/write the data stream in text mode.
- Renamed all semi-private FSConnection methods to start with an
underscore.
- Dotted filenames are now allowed, but only if they don't look like
property or remainder files.
- Expanded the security attributes test (which revealed the SQL
gateway int/long OID bug)
- Don't write the [security] section if there are no security
declarations for an object.
=== Products/AdaptableStorage/gateway_fs/FSConnection.py 1.14 => 1.15 ===
--- Products/AdaptableStorage/gateway_fs/FSConnection.py:1.14 Mon Feb 10 21:30:09 2003
+++ Products/AdaptableStorage/gateway_fs/FSConnection.py Tue Mar 4 23:25:01 2003
@@ -37,6 +37,15 @@
DATA_SECTION = '@data' # Data is a string (file) or list of names (directory)
SUGGESTED_EXTENSION_SECTION = '@s_ext' # The suggested filename extension
OBJECT_NAMES_SECTION = 'object_names' # For directories
+REMAINDER_SECTION = 'remainder'
+
+PROPERTIES_EXTENSION = 'properties'
+REMAINDER_EXTENSION = 'remainder'
+
+# Match 'foo.properties', 'foo.remainder', 'properties', or 'remainder'.
+# This is for filtering out metadata filenames.
+metadata_re = re.compile('(|.+[.])(%s|%s)$' % (
+ PROPERTIES_EXTENSION, REMAINDER_EXTENSION))
class FSConnection:
@@ -65,32 +74,34 @@
self._dir_cache = ShortLivedCache()
- def computeDirectoryContents(self, path, ignore_error=0):
+ def _isLegalFilename(self, fn):
+ if (not fn or
+ (fn.startswith(self.metadata_prefix) and metadata_re.match(fn, 1))
+ or self.hidden_re.match(fn) is not None):
+ return 0
+ return 1
+
+
+ def _computeDirectoryContents(self, path, ignore_error=0):
"""Computes and returns intermediate directory contents info.
- Returns (filenames, object_names, translations).
+ Returns (filenames, object_names, translations). The results
+ are cached for a short time.
"""
res = self._dir_cache.get(path)
if res is not None:
return res
- filenames = []
obj_names = []
trans = {} # { base name -> filename with extension or None }
try:
fns = os.listdir(path)
except OSError:
if ignore_error:
- return (filenames, obj_names, trans)
- metadata_prefix = self.metadata_prefix
- for fn in fns:
- if (not fn
- or fn.startswith(metadata_prefix)
- or self.hidden_re.match(fn) is not None):
- continue
- filenames.append(fn)
+ return ([], obj_names, trans)
- props = self.getPropertiesFromFile(path)
+ filenames = filter(self._isLegalFilename, fns)
+ props = self._getPropertiesFromFile(path)
text = props.get(OBJECT_NAMES_SECTION)
if text:
# Prepare a dictionary of translations.
@@ -116,7 +127,7 @@
return res
- def listDirectoryAsMapping(self, path, ignore_error=0):
+ def _listDirectoryAsMapping(self, path, ignore_error=0):
"""Returns the translated filenames at path.
The ignore_error flag makes this method return an empty
@@ -124,7 +135,7 @@
Returns {filename -> obj_name}.
"""
- filenames, obj_names, trans = self.computeDirectoryContents(
+ filenames, obj_names, trans = self._computeDirectoryContents(
path, ignore_error)
res = {}
for fn in filenames:
@@ -137,7 +148,7 @@
return res
- def expandPath(self, subpath):
+ def _expandPath(self, subpath):
if self.basepath:
while subpath.startswith('/') or subpath.startswith('\\'):
subpath = subpath[1:]
@@ -149,7 +160,7 @@
dir_path, obj_name = os.path.split(path)
if '.' not in obj_name:
# This object might have an automatic filename extension.
- filenames, obj_names, trans = self.computeDirectoryContents(
+ filenames, obj_names, trans = self._computeDirectoryContents(
dir_path, 1)
fn = trans.get(obj_name)
if fn is not None:
@@ -158,7 +169,7 @@
return path
- def checkSectionName(self, section_name):
+ def _checkSectionName(self, section_name):
if (not isinstance(section_name, StringType)
or not section_name
or '[' in section_name
@@ -170,43 +181,43 @@
def writeSection(self, subpath, section_name, data):
- self.checkSectionName(section_name)
- self.queue(subpath, section_name, data)
+ self._checkSectionName(section_name)
+ self._queue(subpath, section_name, data)
def writeNodeType(self, subpath, data):
- self.queue(subpath, NODE_TYPE_SECTION, data)
+ self._queue(subpath, NODE_TYPE_SECTION, data)
def writeData(self, subpath, data):
- self.queue(subpath, DATA_SECTION, data)
+ self._queue(subpath, DATA_SECTION, data)
def suggestExtension(self, subpath, ext):
- self.queue(subpath, SUGGESTED_EXTENSION_SECTION, ext)
+ self._queue(subpath, SUGGESTED_EXTENSION_SECTION, ext)
def readSection(self, subpath, section_name, default=None):
- self.checkSectionName(section_name)
- path = self.expandPath(subpath)
- sections = self.getPropertiesFromFile(path)
+ self._checkSectionName(section_name)
+ path = self._expandPath(subpath)
+ sections = self._getPropertiesFromFile(path)
return sections.get(section_name, default)
def readNodeType(self, subpath):
- path = self.expandPath(subpath)
+ path = self._expandPath(subpath)
if not os.path.exists(path):
raise NoStateFoundError(subpath)
return os.path.isdir(path) and 'd' or 'f'
def readData(self, subpath, allow_missing=0):
- path = self.expandPath(subpath)
+ path = self._expandPath(subpath)
isdir = os.path.isdir(path)
# Read either the directory listing or the file contents.
if isdir:
# Return a sequence of object names.
- return self.listDirectoryAsMapping(path).values()
+ return self._listDirectoryAsMapping(path).values()
else:
# Return a string.
try:
@@ -222,44 +233,58 @@
def getExtension(self, subpath):
- path = self.expandPath(subpath)
+ path = self._expandPath(subpath)
stuff, ext = os.path.splitext(path)
return ext
- def getPropertiesPath(self, path):
+ def _getPropertyPaths(self, path):
+ """Returns the property and remainder paths for a path."""
if os.path.isdir(path):
- props_fn = os.path.join(path, self.metadata_prefix +
- 'properties')
+ base_fn = os.path.join(path, self.metadata_prefix)
else:
dirname, filename = os.path.split(path)
- props_fn = os.path.join(dirname, self.metadata_prefix +
- ('%s.properties' % filename))
- return props_fn
+ base_fn = os.path.join(dirname, '%s%s.' % (
+ self.metadata_prefix, filename))
+ return (base_fn + PROPERTIES_EXTENSION, base_fn + REMAINDER_EXTENSION)
- def getPropertiesFromFile(self, path):
- """Reads a properties file next to path."""
+ def _getPropertiesFromFile(self, path):
+ """Reads the properties and remainder for a path."""
res = self._props_cache.get(path)
if res is not None:
return res
- props_fn = self.getPropertiesPath(path)
+ props_fn, rem_fn = self._getPropertyPaths(path)
+ res = {}
try:
- f = open(props_fn, 'rb')
+ f = open(rem_fn, 'rb')
except IOError:
- # The file is presumably nonexistent
- res = {}
+ # The remainder file apparently does not exist
+ pass
+ else:
+ try:
+ data = f.read()
+ finally:
+ f.close()
+ res[REMAINDER_SECTION] = data
+ # Note that the remainder can be overridden by the properties
+ # file. Perhaps that should be prevented in the future.
+
+ try:
+ f = open(props_fn, 'rt')
+ except IOError:
+ # The properties file apparently does not exist
self._props_cache.set(path, res)
return res
try:
data = f.read()
finally:
f.close()
+
pos = 0
prev_section_name = None
- res = {}
while 1:
match = section_re.search(data, pos)
if match is None:
@@ -280,9 +305,10 @@
return res
- def writeFinal(self, subpath, sections):
+ def _writeFinal(self, subpath, sections):
+ """Performs an actual write of a file or directory to disk."""
# sections is a mapping.
- path = self.expandPath(subpath)
+ path = self._expandPath(subpath)
t = sections[NODE_TYPE_SECTION]
if not os.path.exists(path):
if t == 'd':
@@ -301,10 +327,11 @@
# No file is in the way.
# Use the suggested extension.
path = p
- props_fn = self.getPropertiesPath(path)
+ props_fn, rem_fn = self._getPropertyPaths(path)
+ props_f = None
+ rem_f = None
items = sections.items()
items.sort()
- props_f = open(props_fn, 'wb')
try:
for name, data in items:
if name == NODE_TYPE_SECTION:
@@ -312,10 +339,12 @@
elif name == DATA_SECTION:
if t == 'd':
# Change the list of subobjects.
- self.removeUnlinkedItems(path, data)
- writeSection(props_f, OBJECT_NAMES_SECTION,
- '\n'.join(data))
- self.disableConflictingExtensions(subpath, data)
+ self._removeUnlinkedItems(path, data)
+ if props_f is None:
+ props_f = open(props_fn, 'wt')
+ self._writeToProperties(props_f, OBJECT_NAMES_SECTION,
+ '\n'.join(data))
+ self._disableConflictingExtensions(subpath, data)
self._dir_cache.invalidate(path)
else:
# Change the file contents.
@@ -327,38 +356,68 @@
elif name == SUGGESTED_EXTENSION_SECTION:
# This doesn't need to be written.
pass
+ elif name == REMAINDER_SECTION:
+ # Write to the remainder file.
+ if rem_f is None:
+ rem_f = open(rem_fn, 'wb')
+ rem_f.write(data)
else:
- writeSection(props_f, name, data)
+ # Write a metadata section.
+ if props_f is None:
+ props_f = open(props_fn, 'wt')
+ self._writeToProperties(props_f, name, data)
finally:
- props_f.close()
+ self._closeOrDelete(props_f, props_fn)
+ self._closeOrDelete(rem_f, rem_fn)
self._props_cache.invalidate(path)
# The file might be new, so invalidate the directory.
self._dir_cache.invalidate(os.path.dirname(path))
- def removeUnlinkedItems(self, path, names):
+ def _writeToProperties(self, props_f, name, data):
+ props_f.write('[%s]\n' % name)
+ props_f.write(data.replace('[', '[['))
+ if data.endswith('\n'):
+ props_f.write('\n')
+ else:
+ props_f.write('\n\n')
+
+
+ def _closeOrDelete(self, f, fn):
+ if f is not None:
+ f.close()
+ else:
+ try:
+ os.remove(fn)
+ except OSError:
+ pass
+
+
+ def _removeUnlinkedItems(self, path, names):
"""Removes unused files/subtrees from a directory."""
linked = {}
for name in names:
linked[name] = 1
- for fn, obj_name in self.listDirectoryAsMapping(path).items():
+ for fn, obj_name in self._listDirectoryAsMapping(path).items():
if not linked.get(obj_name):
item_fn = os.path.join(path, fn)
if os.path.isdir(item_fn):
rmtree(item_fn)
else:
os.remove(item_fn)
- item_pfn = self.getPropertiesPath(item_fn)
- if os.path.exists(item_pfn):
- os.remove(item_pfn)
+ props_fn, rem_fn = self._getPropertyPaths(item_fn)
+ if os.path.exists(props_fn):
+ os.remove(props_fn)
+ if os.path.exists(rem_fn):
+ os.remove(rem_fn)
- def disableConflictingExtensions(self, subpath, obj_names):
+ def _disableConflictingExtensions(self, subpath, obj_names):
"""Fixes collisions before writing files in a directory.
Enforces the rule: if 'foo.*' is in the
database, 'foo' may not have an automatic extension.
- Enforces by removing suggested extensions.
+ Enforces by un-queuing suggested extensions.
"""
reserved = {} # { object name without extension -> 1 }
for obj_name in obj_names:
@@ -375,18 +434,19 @@
if reserved.has_key(obj_name):
# Prevent obj_name from using an automatic extension.
child_subpath = '%s/%s' % (subpath, obj_name)
- self.queue(child_subpath, SUGGESTED_EXTENSION_SECTION,
- '', force=1)
+ self._queue(child_subpath, SUGGESTED_EXTENSION_SECTION,
+ '', force=1)
- def beforeWrite(self, items):
+ def _beforeWrite(self, items):
"""Does some early checking while it's easy to bail out.
- This avoids exceptions during the second phase of transaction commit.
+ This helps avoid exceptions during the second phase of
+ transaction commit.
"""
non_containers = {}
for subpath, sections in items:
- path = self.expandPath(subpath)
+ path = self._expandPath(subpath)
exists = os.path.exists(path)
if exists and not os.access(path, os.W_OK):
raise FSWriteError(
@@ -422,9 +482,7 @@
'Data for a directory must be a list or tuple at %s'
% subpath)
for item in items:
- if (not item
- or item.startswith(self.metadata_prefix)
- or self.hidden_re.match(item) is not None):
+ if not self._isLegalFilename(item):
raise FSWriteError(
'Not a legal object name: %s' % repr(item))
else:
@@ -432,7 +490,7 @@
'Node type must be "d" or "f" at %s' % subpath)
- def queue(self, subpath, section_name, data, force=0):
+ def _queue(self, subpath, section_name, data, force=0):
"""Queues data to be written at commit time"""
m = self._pending
sections = m.get(subpath)
@@ -473,7 +531,7 @@
"""
items = self._pending.items()
items.sort() # Ensure that base directories come first.
- self.beforeWrite(items)
+ self._beforeWrite(items)
self._final = 1
def reset(self):
@@ -491,19 +549,10 @@
items = self._pending.items()
items.sort() # Ensure that base directories come first.
for subpath, sections in items:
- self.writeFinal(subpath, sections)
+ self._writeFinal(subpath, sections)
finally:
self.reset()
def close(self):
- pass
-
-
+ self.reset()
-def writeSection(props_f, name, data):
- props_f.write('[%s]\n' % name)
- props_f.write(data.replace('[', '[['))
- if data.endswith('\n'):
- props_f.write('\n')
- else:
- props_f.write('\n\n')
=== Products/AdaptableStorage/gateway_fs/FSSecurityAttributes.py 1.1 => 1.2 ===
--- Products/AdaptableStorage/gateway_fs/FSSecurityAttributes.py:1.1 Sat Mar 1 15:42:59 2003
+++ Products/AdaptableStorage/gateway_fs/FSSecurityAttributes.py Tue Mar 4 23:25:01 2003
@@ -53,7 +53,10 @@
decl_type = params[0][0]
row = [decl_type, '', '', '']
for k, v in params[1:]:
- k = k.lower().split('_', 1)[0]
+ k = k.lower()
+ if '_' in k:
+ # temporary backward compatibility
+ k = k.split('_', 1)[0]
if k == 'role':
row[1] = v
elif k == 'permission':
@@ -81,9 +84,10 @@
params.append(('username', u))
s = paramsToString(params)
lines.append(s)
- lines.sort()
- text = '\n'.join(lines)
- self.fs_conn.writeSection(event.getKey(), self.section, text)
+ if lines:
+ lines.sort()
+ text = '\n'.join(lines)
+ self.fs_conn.writeSection(event.getKey(), self.section, text)
state = list(state)
state.sort()
return tuple(state)