[Zodb-checkins] CVS: ZODB3/ZODB/zodb4 - conversion.py:1.1.2.1 main.py:1.1.2.1

[Fred L. Drake, Jr.]

Update of /cvs-repository/ZODB3/ZODB/zodb4
In directory cvs.zope.org:/tmp/cvs-serv22567

Added Files:
      Tag: zope3-zodb3-devel-branch
	conversion.py main.py 
Log Message:
ZODB4 -> ZODB3 conversion script; not yet complete.
Checkpointing so I can work on it from another machine/location.


=== Added File ZODB3/ZODB/zodb4/conversion.py ===
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Nitty-gritty conversion of a ZODB 4 FileStorage to a ZODB 3 FileStorage."""

from cPickle import dumps

from ZODB.FileStorage import FileStorage, format
from ZODB.zodb4 import z4format
from ZODB.zodb4 import z4iterator
from ZODB.zodb4 import z4utils


class Conversion:

    def __init__(self, input, output):
        """Initialize a ZODB4->ZODB3 FileStorage converter.

        input is an open file containing a ZODB 4 filestorage; it should
        be read-only.

        output is an open file to which a new ZODB 3 filestorage will be
        written.

        Both files must be opened in binary mode.
        """
        self.input = input
        self.instore = z4iterator.FileIterator()
        self.instore._file = input
        self.output = output
        self.outstore = format.FileStorageFormatter()
        self.outstore._file = output
        # ZODB 4 data position --> ZODB 3 data position
        self._data_pos_map = {}

    def run(self):
        self.convert_header()
        for txn in self.instore:
            # txn is a RecordIterator
            self.convert_transaction(txn)

    def convert_header(self):
        self.instore._read_metadata()
        self.output.write(FileStorage.packed_version)

    def convert_transaction(self, txn):
        """Convert one transaction."""
        data_records = []  # [(z4rec, z3rec), ...]
        extension_data = dumps(txn._extension)
        header = format.TxnHeader(txn.tid,
                                  None, # tlen
                                  txn.status,
                                  len(txn.user or ""),
                                  len(txn.description or ""),
                                  len(extension_data))
        header.user = txn.user
        header.descr = txn.description
        header.ext = extension_data
        for datarec in txn:
            assert datarec.serial == txn.tid
            assert not datarec.version
            z3rec = format.DataHeader(datarec.oid,
                                      datarec.serial, # == tid
                                      None, # prev
                                      None, # tloc
                                      0,    # vlen
                                      0,    # plen
                                      )
            # setVersion() not needed since Z3 doesn't use them
            data_records.append((datarec, z3rec))

        data = []  #
            if datarec.data is None:
                data = z3rec.asString() + z4utils.p64(???)
            else:
                data = z3rec.asString() + datarec.data
            
            pos = self.output.tell()
            self._data_pos_map[datarec.] = pos
            self.output.write(data)


=== Added File ZODB3/ZODB/zodb4/main.py ===
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Script to convert a ZODB 4 file storage to a ZODB 3 file storage.

This is needed since Zope 3 is being changed to use ZODB 3 instead of
ZODB 4.

"""

import getopt
import os
import sys

try:
    __file__
except NameError:
    __file__ = os.path.realpath(sys.argv[0])

here = os.path.dirname(__file__)
topdir = os.path.dirname(os.path.dirname(here))

# Make sure that if we're run as a script, we can import the ZODB
# package and our sibling modules.
try:
    import ZODB.zodb4
except ImportError:
    sys.path.append(topdir)
    import ZODB.zodb4

from ZODB.lock_file import LockFile
from ZODB.zodb4 import conversion


class ConversionApp:

    def __init__(self, name=None, args=None):
        if name is None:
            name = os.path.basename(sys.argv[0])
        if args is None:
            args = sys.argv[1:]
        self.name = name
        self.verbosity = 0
        self.dbfile = None
        self.parse_args(args)

    def run(self):
        if not os.path.exists(self.dbfile):
            self.error("input database does not exist: %s" % self.infile)
        base, ext = os.path.splitext(self.dbfile)
        if ext != ".fs":
            base = self.dbfile
        self.bakfile = base + ".fs4"
        self.bakindex = self.bakfile + ".index"
        if os.path.exists(self.bakfile):
            self.error("backup database already exists: %s\n"
                       "please move aside and try again" % self.bakfile)
        if os.path.exists(self.bakindex):
            self.error("backup database index already exists: %s\n"
                       "please move aside and try again" % self.bakindex)
        self.convert()

    def convert(self):
        infile_lock = self.bakfile + ".lock"
        outfile_index = self.dbfile + ".index"
        outfile_lock = self.dbfile + ".lock"
        open_files = [LockFile(outfile_lock)]
        try:
            open_files.append(LockFile(infile_lock))
            # move the ZODB 4 database to be the backup
            os.rename(self.dbfile, self.bakfile)
            try:
                os.rename(self.dbfile + ".index", self.bakindex)
            except:
                # we couldn't rename *both*, so try to make sure we
                # don't rename either
                os.rename(self.bakfile, self.dbfile)
                raise
            # open the data files:
            input = open(self.infile, "rb")
            open_files.append(input)
            output = open(self.outfile, "wb+")
            open_files.append(output)
            # go:
            converter = conversion.Conversion(input, output)
            converter.run()
        finally:
            open_files.reverse()
            for f in open_files:
                f.close()

    def parse_args(self, args):
        opts, args = getopt.getopt(args, "v", ["verbose"])
        for opt, arg in opts:
            if opt in ("-v", "--verbose"):
                self.verbosity += 1
        if len(args) == 0:
            # use default location for Data.fs
            self.dbfile = os.path.join(topdir, "Data.fs")
        elif len(args) == 1:
            self.dbfile = args[0]
        else:
            self.error("too many command-line arguments", rc=2)

    def error(self, message, rc=1):
        print >>sys.stderr, "%s: %s" % (self.name, message)
        sys.exit(rc)


def main():
    ConversionApp().run()

if __name__ == "__main__":
    main()