[Zope3-checkins] CVS: Zope3/src/zope/fssync - snarf.py:1.1 fssync.py:1.24
Guido van Rossum
guido@python.org
Tue, 20 May 2003 15:09:15 -0400
Update of /cvs-repository/Zope3/src/zope/fssync
In directory cvs.zope.org:/tmp/cvs-serv19576
Modified Files:
fssync.py
Added Files:
snarf.py
Log Message:
Don't use zip/unzip -- use a home-grown protocol that can read/write
directly from/to a stream, dubbed "snarf". This only does what we need.
=== Added File Zope3/src/zope/fssync/snarf.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Simple New ARchival Format (SNARF).
This is for transferring collections of files over HTTP where the key
need is for simple software.
The format is as follows:
- for directory entries:
'/ <pathname>\n'
- for file entries:
'<size> <pathname>\n' followed by exactly <size> bytes
Pathnames are always relative and always use '/' for delimiters, and
should not use '.' or '..' or '' as components. All files are read
and written in binary mode.
$Id: snarf.py,v 1.1 2003/05/20 19:09:15 gvanrossum Exp $
"""
import os
class Snarfer(object):
"""Snarfer -- write an archive to a stream."""
def __init__(self, ostr):
"""Constructor. The argument is the output stream."""
self.ostr = ostr
def add(self, fspath, path):
if os.path.isdir(fspath):
self.addtree(fspath, path + "/")
elif os.path.isfile(fspath):
self.addfile(fspath, path)
def addtree(self, root, prefix="", filter=None):
"""Snarf a directory tree.
root -- the root of the tree in the filesystem.
prefix -- optional snarf path prefix, either empty or ending in '/'.
filter -- optional filter predicate.
"""
if filter is None:
def filter(fspath):
return True
names = os.listdir(root)
for name in names:
fspath = os.path.join(root, name)
if not filter(fspath):
continue
if os.path.isdir(fspath):
self.addtree(fspath, prefix + name + "/", filter)
elif os.path.isfile(fspath):
self.addfile(fspath, prefix + name)
def addfile(self, fspath, path):
"""Snarf a single file given by fspath."""
f = open(fspath, "rb")
try:
f.seek(0, 2)
size = f.tell()
f.seek(0)
self.addstream(f, size, path)
finally:
f.close()
def addstream(self, istr, size, path):
"""Snarf a single file from a data stream.
istr -- the input stream;
size -- the number of bytes to read from istr;
path -- the snarf path.
Raises IOError if reading istr returns an EOF condition before
size bytes have been read.
"""
self.ostr.write("%d %s\n" % (size, path))
copybytes(size, istr, self.ostr)
class Unsnarfer(object):
"""Unsnarfer -- read an archive from a stream."""
def __init__(self, istr):
"""Constructor. The argument is the input stream."""
self.istr = istr
def unsnarf(self, root):
"""Unsnarf the entire archive into a directory tree.
root -- the root of the directory tree where to write.
"""
self.root = root
while True:
infoline = self.istr.readline()
if not infoline:
break
if not infoline.endswith("\n"):
raise IOError("incomplete info line %r" % infoline)
infoline = infoline[:-1]
what, path = infoline.split(" ", 1)
if what == "/":
self.makedir(path)
else:
size = int(what)
f = self.createfile(path)
try:
copybytes(size, self.istr, f)
finally:
f.close()
def makedir(self, path):
fspath = self.translatepath(path)
self.ensuredir(fspath)
def createfile(self, path):
fspath = self.translatepath(path)
self.ensuredir(os.path.dirname(fspath))
return open(fspath, "wb")
def ensuredir(self, fspath):
if not os.path.isdir(fspath):
os.makedirs(fspath)
def translatepath(self, path):
if ":" in path and os.name != "posix":
raise IOError("path cannot contain colons: $r" % path)
if "\\" in path and os.name != "posix":
raise IOError("path cannot contain backslashes: %r" % path)
parts = path.split("/")
for forbidden in "", ".", "..", os.curdir, os.pardir:
if forbidden in parts:
raise IOError("forbidden part %r in path: %r" %
(forbidden, path))
return os.path.join(self.root, *parts)
def copybytes(size, istr, ostr, bufsize=8192):
aim = size - bufsize
pos = 0
while pos < aim:
data = istr.read(bufsize)
if not data:
raise IOError("not enough data read from input stream")
pos += len(data)
ostr.write(data)
while pos < size:
data = istr.read(size - pos)
if not data:
raise IOError("not enough data read from input stream")
pos += len(data)
ostr.write(data)
=== Zope3/src/zope/fssync/fssync.py 1.23 => 1.24 ===
--- Zope3/src/zope/fssync/fssync.py:1.23 Thu May 15 18:22:24 2003
+++ Zope3/src/zope/fssync/fssync.py Tue May 20 15:09:15 2003
@@ -42,6 +42,7 @@
from zope.fssync.fsmerger import FSMerger
from zope.fssync.fsutil import Error
from zope.fssync import fsutil
+from zope.fssync.snarf import Snarfer, Unsnarfer
class Network(object):
@@ -49,7 +50,7 @@
This class has various methods for managing the root url (which is
stored in a file @@Zope/Root) and has a method to send an HTTP(S)
- request to the root URL, expecting a zip file back (that's all the
+ request to the root URL, expecting a snarf file back (that's all the
application needs).
Public instance variables:
@@ -157,7 +158,8 @@
finally:
f.close()
- def httpreq(self, path, view, datafp=None, content_type="application/zip"):
+ def httpreq(self, path, view, datafp=None,
+ content_type="application/x-snarf"):
"""Issue an HTTP or HTTPS request.
The request parameters are taken from the root url, except
@@ -168,18 +170,18 @@
seekable stream from which the input document for the request
is taken. In this case, a POST request is issued, and the
content-type header is set to the 'content_type' argument,
- defaulting to 'application/zip'. Otherwise (if datafp is
+ defaulting to 'application/x-snarf'. Otherwise (if datafp is
None), a GET request is issued and no input document is sent.
If the request succeeds and returns a document whose
- content-type is 'application/zip', the return value is a tuple
+ content-type is 'application/x-snarf', the return value is a tuple
(fp, headers) where fp is a non-seekable stream from which the
return document can be read, and headers is a case-insensitive
mapping giving the response headers.
If the request returns an HTTP error, the Error exception is
raised. If it returns success (error code 200) but the
- content-type of the result is not 'application/zip', the Error
+ content-type of the result is not 'application/x-snarf', the Error
exception is also raised. In these error cases, if the result
document's content-type is a text type (anything starting with
'text/'), the text of the result document is included in the
@@ -227,7 +229,7 @@
raise Error("HTTP error %s (%s); error document:\n%s",
errcode, errmsg,
self.slurptext(fp, headers))
- if headers["Content-type"] != "application/zip":
+ if headers["Content-type"] != "application/x-snarf":
raise Error(self.slurptext(fp, headers))
return fp, headers
@@ -275,9 +277,9 @@
i = rootpath.rfind("/")
tail = rootpath[i+1:]
tail = tail or "root"
- fp, headers = self.network.httpreq(rootpath, "@@toFS.zip")
+ fp, headers = self.network.httpreq(rootpath, "@@toFS.snarf")
try:
- self.merge_zipfile(fp, target, tail)
+ self.merge_snarffile(fp, target, tail)
finally:
fp.close()
self.network.saverooturl(target)
@@ -302,27 +304,28 @@
raise Error("nothing known about", target)
self.network.loadrooturl(target)
path = entry["path"]
- zipfile = tempfile.mktemp(".zip")
+ snarffile = tempfile.mktemp(".snf")
head, tail = split(realpath(target))
try:
- sts = os.system("cd %s; zip -q -r %s %s @@Zope" %
- (commands.mkarg(head),
- zipfile,
- commands.mkarg(tail)))
- if sts:
- raise Error("zip command failed")
- infp = open(zipfile, "rb")
- view = "@@fromFS.zip?note=%s" % urllib.quote(note)
+ f = open(snarffile, "wb")
+ try:
+ snf = Snarfer(f)
+ snf.add(join(head, tail), tail)
+ snf.addtree(join(head, "@@Zope"), "@@Zope/")
+ finally:
+ f.close()
+ infp = open(snarffile, "rb")
+ view = "@@fromFS.snarf?note=%s" % urllib.quote(note)
try:
outfp, headers = self.network.httpreq(path, view, infp)
finally:
infp.close()
finally:
pass
- if isfile(zipfile):
- os.remove(zipfile)
+ if isfile(snarffile):
+ os.remove(snarffile)
try:
- self.merge_zipfile(outfp, head, tail)
+ self.merge_snarffile(outfp, head, tail)
finally:
outfp.close()
@@ -333,36 +336,24 @@
self.network.loadrooturl(target)
head, tail = fsutil.split(target)
path = entry["path"]
- fp, headers = self.network.httpreq(path, "@@toFS.zip")
+ fp, headers = self.network.httpreq(path, "@@toFS.snarf")
try:
- self.merge_zipfile(fp, head, tail)
+ self.merge_snarffile(fp, head, tail)
finally:
fp.close()
- def merge_zipfile(self, fp, localdir, tail):
- zipfile = tempfile.mktemp(".zip")
+ def merge_snarffile(self, fp, localdir, tail):
+ uns = Unsnarfer(fp)
+ tmpdir = tempfile.mktemp()
try:
- tfp = open(zipfile, "wb")
- try:
- shutil.copyfileobj(fp, tfp)
- finally:
- tfp.close()
- tmpdir = tempfile.mktemp()
- try:
- os.mkdir(tmpdir)
- cmd = "cd %s; unzip -q %s" % (tmpdir, zipfile)
- sts, output = commands.getstatusoutput(cmd)
- if sts:
- raise Error("unzip failed:\n%s" % output)
- self.fsmerger.merge(join(localdir, tail), join(tmpdir, tail))
- self.metadata.flush()
- print "All done."
- finally:
- if isdir(tmpdir):
- shutil.rmtree(tmpdir)
+ os.mkdir(tmpdir)
+ uns.unsnarf(tmpdir)
+ self.fsmerger.merge(join(localdir, tail), join(tmpdir, tail))
+ self.metadata.flush()
+ print "All done."
finally:
- if isfile(zipfile):
- os.remove(zipfile)
+ if isdir(tmpdir):
+ shutil.rmtree(tmpdir)
def reporter(self, msg):
if msg[0] not in "/*":