[Zope3-checkins] CVS: Zope3/src/zope/fssync - snarf.py:1.1 fssync.py:1.24

Guido van Rossum guido@python.org
Tue, 20 May 2003 15:09:15 -0400


Update of /cvs-repository/Zope3/src/zope/fssync
In directory cvs.zope.org:/tmp/cvs-serv19576

Modified Files:
	fssync.py 
Added Files:
	snarf.py 
Log Message:
Don't use zip/unzip -- use a home-grown protocol that can read/write
directly from/to a stream, dubbed "snarf".  This only does what we need.


=== Added File Zope3/src/zope/fssync/snarf.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
# 
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
# 
##############################################################################
"""Simple New ARchival Format (SNARF).

This is for transferring collections of files over HTTP where the key
need is for simple software.

The format is as follows:

- for directory entries:
  '/ <pathname>\n'

- for file entries:
  '<size> <pathname>\n' followed by exactly <size> bytes

Pathnames are always relative and always use '/' for delimiters, and
should not use '.' or '..' or '' as components.  All files are read
and written in binary mode.

$Id: snarf.py,v 1.1 2003/05/20 19:09:15 gvanrossum Exp $
"""

import os

class Snarfer(object):

    """Snarfer -- write an archive to a stream."""

    def __init__(self, ostr):
        """Constructor.  The argument is the output stream."""
        self.ostr = ostr

    def add(self, fspath, path):
        if os.path.isdir(fspath):
            self.addtree(fspath, path + "/")
        elif os.path.isfile(fspath):
            self.addfile(fspath, path)

    def addtree(self, root, prefix="", filter=None):
        """Snarf a directory tree.

        root -- the root of the tree in the filesystem.
        prefix -- optional snarf path prefix, either empty or ending in '/'.
        filter -- optional filter predicate.
        """
        if filter is None:
            def filter(fspath):
                return True
        names = os.listdir(root)
        for name in names:
            fspath = os.path.join(root, name)
            if not filter(fspath):
                continue
            if os.path.isdir(fspath):
                self.addtree(fspath, prefix + name + "/", filter)
            elif os.path.isfile(fspath):
                self.addfile(fspath, prefix + name)

    def addfile(self, fspath, path):
        """Snarf a single file given by fspath."""
        f = open(fspath, "rb")
        try:
            f.seek(0, 2)
            size = f.tell()
            f.seek(0)
            self.addstream(f, size, path)
        finally:
            f.close()

    def addstream(self, istr, size, path):
        """Snarf a single file from a data stream.

        istr -- the input stream;
        size -- the number of bytes to read from istr;
        path -- the snarf path.

        Raises IOError if reading istr returns an EOF condition before
        size bytes have been read.
        """
        self.ostr.write("%d %s\n" % (size, path))
        copybytes(size, istr, self.ostr)


class Unsnarfer(object):
    """Unsnarfer -- read an archive from a stream."""

    def __init__(self, istr):
        """Constructor.  The argument is the input stream."""
        self.istr = istr

    def unsnarf(self, root):
        """Unsnarf the entire archive into a directory tree.

        root -- the root of the directory tree where to write.
        """
        self.root = root
        while True:
            infoline = self.istr.readline()
            if not infoline:
                break
            if not infoline.endswith("\n"):
                raise IOError("incomplete info line %r" % infoline)
            infoline = infoline[:-1]
            what, path = infoline.split(" ", 1)
            if what == "/":
                self.makedir(path)
            else:
                size = int(what)
                f = self.createfile(path)
                try:
                    copybytes(size, self.istr, f)
                finally:
                    f.close()

    def makedir(self, path):
        fspath = self.translatepath(path)
        self.ensuredir(fspath)

    def createfile(self, path):
        fspath = self.translatepath(path)
        self.ensuredir(os.path.dirname(fspath))
        return open(fspath, "wb")

    def ensuredir(self, fspath):
        if not os.path.isdir(fspath):
            os.makedirs(fspath)

    def translatepath(self, path):
        if ":" in path and os.name != "posix":
            raise IOError("path cannot contain colons: $r" % path)
        if "\\" in path and os.name != "posix":
            raise IOError("path cannot contain backslashes: %r" % path)
        parts = path.split("/")
        for forbidden in "", ".", "..", os.curdir, os.pardir:
            if forbidden in parts:
                raise IOError("forbidden part %r in path: %r" %
                              (forbidden, path))
        return os.path.join(self.root, *parts)

def copybytes(size, istr, ostr, bufsize=8192):
    aim = size - bufsize
    pos = 0
    while pos < aim:
        data = istr.read(bufsize)
        if not data:
            raise IOError("not enough data read from input stream")
        pos += len(data)
        ostr.write(data)
    while pos < size:
        data = istr.read(size - pos)
        if not data:
            raise IOError("not enough data read from input stream")
        pos += len(data)
        ostr.write(data)


=== Zope3/src/zope/fssync/fssync.py 1.23 => 1.24 ===
--- Zope3/src/zope/fssync/fssync.py:1.23	Thu May 15 18:22:24 2003
+++ Zope3/src/zope/fssync/fssync.py	Tue May 20 15:09:15 2003
@@ -42,6 +42,7 @@
 from zope.fssync.fsmerger import FSMerger
 from zope.fssync.fsutil import Error
 from zope.fssync import fsutil
+from zope.fssync.snarf import Snarfer, Unsnarfer
 
 class Network(object):
 
@@ -49,7 +50,7 @@
 
     This class has various methods for managing the root url (which is
     stored in a file @@Zope/Root) and has a method to send an HTTP(S)
-    request to the root URL, expecting a zip file back (that's all the
+    request to the root URL, expecting a snarf file back (that's all the
     application needs).
 
     Public instance variables:
@@ -157,7 +158,8 @@
         finally:
             f.close()
 
-    def httpreq(self, path, view, datafp=None, content_type="application/zip"):
+    def httpreq(self, path, view, datafp=None,
+                content_type="application/x-snarf"):
         """Issue an HTTP or HTTPS request.
 
         The request parameters are taken from the root url, except
@@ -168,18 +170,18 @@
         seekable stream from which the input document for the request
         is taken.  In this case, a POST request is issued, and the
         content-type header is set to the 'content_type' argument,
-        defaulting to 'application/zip'.  Otherwise (if datafp is
+        defaulting to 'application/x-snarf'.  Otherwise (if datafp is
         None), a GET request is issued and no input document is sent.
 
         If the request succeeds and returns a document whose
-        content-type is 'application/zip', the return value is a tuple
+        content-type is 'application/x-snarf', the return value is a tuple
         (fp, headers) where fp is a non-seekable stream from which the
         return document can be read, and headers is a case-insensitive
         mapping giving the response headers.
 
         If the request returns an HTTP error, the Error exception is
         raised.  If it returns success (error code 200) but the
-        content-type of the result is not 'application/zip', the Error
+        content-type of the result is not 'application/x-snarf', the Error
         exception is also raised.  In these error cases, if the result
         document's content-type is a text type (anything starting with
         'text/'), the text of the result document is included in the
@@ -227,7 +229,7 @@
             raise Error("HTTP error %s (%s); error document:\n%s",
                         errcode, errmsg,
                         self.slurptext(fp, headers))
-        if headers["Content-type"] != "application/zip":
+        if headers["Content-type"] != "application/x-snarf":
             raise Error(self.slurptext(fp, headers))
         return fp, headers
 
@@ -275,9 +277,9 @@
         i = rootpath.rfind("/")
         tail = rootpath[i+1:]
         tail = tail or "root"
-        fp, headers = self.network.httpreq(rootpath, "@@toFS.zip")
+        fp, headers = self.network.httpreq(rootpath, "@@toFS.snarf")
         try:
-            self.merge_zipfile(fp, target, tail)
+            self.merge_snarffile(fp, target, tail)
         finally:
             fp.close()
         self.network.saverooturl(target)
@@ -302,27 +304,28 @@
             raise Error("nothing known about", target)
         self.network.loadrooturl(target)
         path = entry["path"]
-        zipfile = tempfile.mktemp(".zip")
+        snarffile = tempfile.mktemp(".snf")
         head, tail = split(realpath(target))
         try:
-            sts = os.system("cd %s; zip -q -r %s %s @@Zope" %
-                            (commands.mkarg(head),
-                             zipfile,
-                             commands.mkarg(tail)))
-            if sts:
-                raise Error("zip command failed")
-            infp = open(zipfile, "rb")
-            view = "@@fromFS.zip?note=%s" % urllib.quote(note)
+            f = open(snarffile, "wb")
+            try:
+                snf = Snarfer(f)
+                snf.add(join(head, tail), tail)
+                snf.addtree(join(head, "@@Zope"), "@@Zope/")
+            finally:
+                f.close()
+            infp = open(snarffile, "rb")
+            view = "@@fromFS.snarf?note=%s" % urllib.quote(note)
             try:
                 outfp, headers = self.network.httpreq(path, view, infp)
             finally:
                 infp.close()
         finally:
             pass
-            if isfile(zipfile):
-                os.remove(zipfile)
+            if isfile(snarffile):
+                os.remove(snarffile)
         try:
-            self.merge_zipfile(outfp, head, tail)
+            self.merge_snarffile(outfp, head, tail)
         finally:
             outfp.close()
 
@@ -333,36 +336,24 @@
         self.network.loadrooturl(target)
         head, tail = fsutil.split(target)
         path = entry["path"]
-        fp, headers = self.network.httpreq(path, "@@toFS.zip")
+        fp, headers = self.network.httpreq(path, "@@toFS.snarf")
         try:
-            self.merge_zipfile(fp, head, tail)
+            self.merge_snarffile(fp, head, tail)
         finally:
             fp.close()
 
-    def merge_zipfile(self, fp, localdir, tail):
-        zipfile = tempfile.mktemp(".zip")
+    def merge_snarffile(self, fp, localdir, tail):
+        uns = Unsnarfer(fp)
+        tmpdir = tempfile.mktemp()
         try:
-            tfp = open(zipfile, "wb")
-            try:
-                shutil.copyfileobj(fp, tfp)
-            finally:
-                tfp.close()
-            tmpdir = tempfile.mktemp()
-            try:
-                os.mkdir(tmpdir)
-                cmd = "cd %s; unzip -q %s" % (tmpdir, zipfile)
-                sts, output = commands.getstatusoutput(cmd)
-                if sts:
-                    raise Error("unzip failed:\n%s" % output)
-                self.fsmerger.merge(join(localdir, tail), join(tmpdir, tail))
-                self.metadata.flush()
-                print "All done."
-            finally:
-                if isdir(tmpdir):
-                    shutil.rmtree(tmpdir)
+            os.mkdir(tmpdir)
+            uns.unsnarf(tmpdir)
+            self.fsmerger.merge(join(localdir, tail), join(tmpdir, tail))
+            self.metadata.flush()
+            print "All done."
         finally:
-            if isfile(zipfile):
-                os.remove(zipfile)
+            if isdir(tmpdir):
+                shutil.rmtree(tmpdir)
 
     def reporter(self, msg):
         if msg[0] not in "/*":