[Zope-CVS] CVS: Products/AdaptableStorage/gateway_fs - FSClassificationSection.py:1.9 FSConnection.py:1.11
Shane Hathaway
shane@zope.com
Tue, 4 Feb 2003 23:59:50 -0500
Update of /cvs-repository/Products/AdaptableStorage/gateway_fs
In directory cvs.zope.org:/tmp/cvs-serv22769/gateway_fs
Modified Files:
FSClassificationSection.py FSConnection.py
Log Message:
Added an automatic filename extensions feature to FSConnection and
corresponding unit tests. Also cleaned up minor details in FSConnection.
=== Products/AdaptableStorage/gateway_fs/FSClassificationSection.py 1.8 => 1.9 ===
--- Products/AdaptableStorage/gateway_fs/FSClassificationSection.py:1.8 Thu Jan 9 09:33:58 2003
+++ Products/AdaptableStorage/gateway_fs/FSClassificationSection.py Tue Feb 4 23:59:16 2003
@@ -16,6 +16,8 @@
$Id$
"""
+import os
+
from mapper_public import IGateway, FieldSchema
@@ -50,17 +52,21 @@
if '=' in line:
k, v = line.split('=', 1)
classification[k.strip()] = v.strip()
- classification['filename'] = self.getIdFrom(event)
+ filename = self.getIdFrom(event)
+ classification['extension'] = os.path.splitext(filename)[1]
return classification, text.strip()
def store(self, event, state):
# state is a classification
+ p = event.getKeychain()[-1]
items = state.items()
items.sort()
text = []
for k, v in items:
- text.append('%s=%s' % (k, v))
+ if k == 'extension':
+ self.fs_conn.suggestExtension(p, v)
+ else:
+ text.append('%s=%s' % (k, v))
text = '\n'.join(text)
- p = event.getKeychain()[-1]
self.fs_conn.writeSection(p, 'classification', text)
return text.strip()
=== Products/AdaptableStorage/gateway_fs/FSConnection.py 1.10 => 1.11 ===
--- Products/AdaptableStorage/gateway_fs/FSConnection.py:1.10 Thu Jan 9 09:33:58 2003
+++ Products/AdaptableStorage/gateway_fs/FSConnection.py Tue Feb 4 23:59:16 2003
@@ -34,6 +34,8 @@
NODE_TYPE_SECTION = '@node_type' # Data is 'f' (file) or 'd' (directory)
DATA_SECTION = '@data' # Data is a string (file) or list of names (directory)
+SUGGESTED_EXTENSION_SECTION = '@s_ext' # The suggested filename extension
+OBJECT_NAMES_SECTION = 'object_names' # For directories
class FSConnection:
@@ -59,6 +61,69 @@
self._pending = {}
+ def computeDirectoryContents(self, path, ignore_error=0):
+ """Computes and returns intermediate directory contents info.
+
+ Returns (filenames, object_names, translations).
+ """
+ filenames = []
+ obj_names = []
+ trans = {} # { base name -> filename with extension or None }
+ try:
+ fns = os.listdir(path)
+ except OSError:
+ if ignore_error:
+ return filenames, obj_names, trans
+ hidden_filename_prefix = self.hidden_filename_prefix
+ for fn in fns:
+ if fn.startswith(hidden_filename_prefix):
+ continue
+ filenames.append(fn)
+
+ props = self.getPropertiesFromFile(path)
+ text = props.get(OBJECT_NAMES_SECTION)
+ if text:
+ # Prepare a dictionary of translations.
+ for fn in filenames:
+ if '.' in fn:
+ base, ext = fn.split('.', 1)
+ if trans.has_key(base):
+ # Name collision: two or more files have the same base
+ # name. Don't use an extension for this name.
+ trans[base] = None
+ else:
+ trans[base] = fn
+ else:
+ trans[fn] = None
+ obj_names = [line.strip() for line in text.split('\n')]
+ for obj_name in obj_names:
+ if '.' in obj_name:
+ base, ext = obj_name.split('.', 1)
+ trans[base] = None
+ return filenames, obj_names, trans
+
+
+ def listDirectoryAsMapping(self, path, ignore_error=0):
+ """Returns the translated filenames at path.
+
+ The ignore_error flag makes this method return an empty
+ dictionary if the directory is not found.
+
+ Returns {filename -> obj_name}.
+ """
+ filenames, obj_names, trans = self.computeDirectoryContents(
+ path, ignore_error)
+ res = {}
+ for fn in filenames:
+ res[fn] = fn
+ # Translate names.
+ for obj_name in obj_names:
+ fn = trans.get(obj_name)
+ if fn:
+ res[fn] = obj_name
+ return res
+
+
def expandPath(self, subpath):
if self.basepath:
while subpath.startswith('/') or subpath.startswith('\\'):
@@ -67,48 +132,46 @@
else:
# unchanged.
path = subpath
+ if not os.path.exists(path):
+ dir_path, obj_name = os.path.split(path)
+ if '.' not in obj_name:
+ # This object might have an automatic filename extension.
+ # XXX This is expensive.
+ filenames, obj_names, trans = self.computeDirectoryContents(
+ dir_path, 1)
+ fn = trans.get(obj_name)
+ if fn is not None:
+ # Use the filename with an extension.
+ path = os.path.join(dir_path, fn)
return path
def checkSectionName(self, section_name):
- assert isinstance(section_name, StringType)
- assert '[' not in section_name
- assert ']' not in section_name
- assert '\n' not in section_name
- assert section_name != NODE_TYPE_SECTION
- assert section_name != DATA_SECTION
-
-
- def _write(self, subpath, section_name, data):
- # XXX We should be checking for '..'
- path = self.expandPath(subpath)
- # Do some early checking.
- if os.path.exists(path):
- v = os.access(path, os.W_OK)
- if not v:
- raise FSWriteError(
- "Can't get write access to %s" % subpath)
- self.queue(subpath, section_name, data)
+ if (not isinstance(section_name, StringType)
+ or not section_name
+ or '[' in section_name
+ or ']' in section_name
+ or '\n' in section_name
+ or section_name.startswith('@')
+ or section_name == OBJECT_NAMES_SECTION):
+ raise ValueError, section_name
def writeSection(self, subpath, section_name, data):
self.checkSectionName(section_name)
- self._write(subpath, section_name, data)
+ self.queue(subpath, section_name, data)
def writeNodeType(self, subpath, data):
- path = self.expandPath(subpath)
- # Do some early checking.
- if os.path.exists(path):
- want_dir = (data == 'd')
- if (want_dir != (not not os.path.isdir(path))):
- raise FSWriteError(
- "Can't mix file and directory at %s" % subpath)
self.queue(subpath, NODE_TYPE_SECTION, data)
def writeData(self, subpath, data):
- self._write(subpath, DATA_SECTION, data)
+ self.queue(subpath, DATA_SECTION, data)
+
+
+ def suggestExtension(self, subpath, ext):
+ self.queue(subpath, SUGGESTED_EXTENSION_SECTION, ext)
def readSection(self, subpath, section_name, default=None):
@@ -130,14 +193,10 @@
isdir = os.path.isdir(path)
# Read either the directory listing or the file contents.
if isdir:
- names = []
- prefix = self.hidden_filename_prefix
- for name in os.listdir(path):
- if not name.startswith(self.hidden_filename_prefix):
- names.append(name)
- # Return a sequence instead of a string.
- return names
+ # Return a sequence of object names.
+ return self.listDirectoryAsMapping(path).values()
else:
+ # Return a string.
try:
f = open(path, 'rb')
except IOError:
@@ -162,7 +221,7 @@
def getPropertiesFromFile(self, path):
- """Read a properties file next to path."""
+ """Reads a properties file next to path."""
props_fn = self.getPropertiesPath(path)
try:
@@ -200,8 +259,23 @@
# sections is a mapping.
path = self.expandPath(subpath)
t = sections[NODE_TYPE_SECTION]
- if t == 'd' and not os.path.exists(path):
- os.mkdir(path)
+ if not os.path.exists(path):
+ if t == 'd':
+ os.mkdir(path)
+ else:
+ fn = os.path.split(path)[1]
+ if '.' not in fn:
+ # This object has no extension and doesn't yet exist.
+ ext = sections.get(SUGGESTED_EXTENSION_SECTION)
+ if ext:
+ # Try to use the suggested extension.
+ if not ext.startswith('.'):
+ ext = '.' + ext
+ p = path + ext
+ if not os.path.exists(p):
+ # No file is in the way.
+ # Use the suggested extension.
+ path = p
props_fn = self.getPropertiesPath(path)
items = sections.items()
items.sort()
@@ -213,36 +287,33 @@
elif name == DATA_SECTION:
if t == 'd':
# Change the list of subobjects.
- # Here we only have to delete.
- # Subobjects will be created later.
- # XXX we might check for dotted names here.
self.removeUnlinkedItems(path, data)
+ writeSection(props_f, OBJECT_NAMES_SECTION,
+ '\n'.join(data))
+ self.disableConflictingExtensions(subpath, data)
else:
- # Change file contents.
+ # Change the file contents.
f = open(path, 'wb')
try:
f.write(data)
finally:
f.close()
+ elif name == SUGGESTED_EXTENSION_SECTION:
+ # This doesn't need to be written.
+ pass
else:
- props_f.write('[%s]\n' % name)
- props_f.write(data.replace('[', '[['))
- if data.endswith('\n'):
- props_f.write('\n')
- else:
- props_f.write('\n\n')
+ writeSection(props_f, name, data)
finally:
props_f.close()
def removeUnlinkedItems(self, path, names):
+ """Removes unused files/subtrees from a directory."""
linked = {}
for name in names:
linked[name] = 1
- existing = os.listdir(path)
- prefix = self.hidden_filename_prefix
- for fn in existing:
- if not fn.startswith(prefix) and not linked.get(fn):
+ for fn, obj_name in self.listDirectoryAsMapping(path).items():
+ if not linked.get(obj_name):
item_fn = os.path.join(path, fn)
if os.path.isdir(item_fn):
rmtree(item_fn)
@@ -253,43 +324,88 @@
os.remove(item_pfn)
+ def disableConflictingExtensions(self, subpath, obj_names):
+ """Fixes collisions before writing files in a directory.
+
+ Enforces the rule: if 'foo.*' is in the
+ database, 'foo' may not have an automatic extension.
+ Enforces by removing suggested extensions.
+ """
+ reserved = {} # { object name without extension -> 1 }
+ for obj_name in obj_names:
+ if '.' in obj_name:
+ base, ext = obj_name.split('.', 1)
+ reserved[base] = 1
+ if not reserved:
+ # No objects have extensions.
+ return
+
+ while subpath.endswith('/'):
+ subpath = subpath[:-1]
+ for obj_name in obj_names:
+ if reserved.has_key(obj_name):
+ # obj_name has no extension and must not have an
+ # automatic extension.
+ child_subpath = '%s/%s' % (subpath, obj_name)
+ self.queue(child_subpath, SUGGESTED_EXTENSION_SECTION,
+ '', force=1)
+
+
def beforeWrite(self, items):
+ """Does some early checking while it's easy to bail out.
+
+ This avoids exceptions during the second phase of transaction commit.
+ """
non_containers = {}
for subpath, sections in items:
+ path = self.expandPath(subpath)
+ exists = os.path.exists(path)
+ if exists and not os.access(path, os.W_OK):
+ raise FSWriteError(
+ "Can't get write access to %s" % subpath)
# type must be provided and must always be either 'd' or 'f'.
if (not sections.has_key(NODE_TYPE_SECTION)
or not sections.has_key(DATA_SECTION)):
raise FSWriteError(
'Data or node type not specified for %s' % subpath)
t = sections[NODE_TYPE_SECTION]
- if t not in ('d', 'f'):
- raise FSWriteError(
- 'node type must be "d" or "f" at %s' % subpath)
dir = os.path.dirname(subpath)
if non_containers.get(dir):
raise FSWriteError(
"Not a directory: %s" % dir)
if t == 'f':
+ if exists and os.path.isdir(path):
+ raise FSWriteError(
+ "Can't write file data to directory at %s"
+ % subpath)
non_containers[subpath] = 1
if not isinstance(sections[DATA_SECTION], StringType):
raise FSWriteError(
'Data for a file must be a string at %s'
% subpath)
- else:
- if isinstance(sections[DATA_SECTION], StringType):
+ elif t == 'd':
+ if exists and not os.path.isdir(path):
+ raise FSWriteError(
+ "Can't write directory contents to file at %s"
+ % subpath)
+ items = sections[DATA_SECTION]
+ if isinstance(items, StringType):
raise FSWriteError(
'Data for a directory must be a list or tuple at %s'
% subpath)
+ else:
+ raise FSWriteError(
+ 'Node type must be "d" or "f" at %s' % subpath)
- def queue(self, subpath, section_name, data):
+ def queue(self, subpath, section_name, data, force=0):
"""Queues data to be written at commit time"""
m = self._pending
sections = m.get(subpath)
if sections is None:
sections = {}
m[subpath] = sections
- if sections.has_key(section_name):
+ if sections.has_key(section_name) and not force:
if sections[section_name] != data:
raise FSWriteError(
'Conflicting data storage at %s (%s)' %
@@ -345,3 +461,12 @@
def close(self):
pass
+
+
+def writeSection(props_f, name, data):
+ props_f.write('[%s]\n' % name)
+ props_f.write(data.replace('[', '[['))
+ if data.endswith('\n'):
+ props_f.write('\n')
+ else:
+ props_f.write('\n\n')