[svn.haxx.se] · SVN Dev · SVN Users · SVN Org · TSVN Dev · TSVN Users · Subclipse Dev · Subclipse Users · this month's index

Re: SCCS to Subversion Python script

From: Robert Zeh <rzeh_at_efs-us.com>
Date: 2004-12-30 00:00:19 CET

Here you go!

Robert Zeh

Max Bowsher wrote:

> Robert Zeh wrote:
>
>> Hi,
>>
>> A few months ago I posted information on the user's list about a script
>> I wrote to convert SCCS repositories to Subversion. I've received a
>> small but steady stream of requests for the script, which still contains
>> hard coded paths. If I cleaned up the hard coded paths, could it be
>> placed in the contrib directory?
>
>
> Yes, very much so.
>
> Thankyou in advance!
>
> Max.
>

#!/usr/bin/env python

""" Convert SCCS repositories into a subversion repository. This
    assumes that there are multiple SCCS repositories (directories)
    under a single directory; the structure is replicated in the
    subversion repository.

    All of the information in the SCCS repositories is preserved; we
    use the subversion repository layer so that we can use the SCCS dates
    and users (if we use the subversion client layer all of that gets set
    for us).

    Todo: Check the result of creating the repository by calling svnadmin create.

    Robert Allan Zeh (razeh@yahoo.com)
    Error Free Software

"""

from svn import util, fs, repos, core, client, delta
import svn
import os
import time
import re
import string
import shutil
import sys
from optparse import OptionParser

versions = []

def subversionTime(t):
    """ Converts a time tuple to what subversion expects. """
    return time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", t)

def isTextFilename(filename):
    """ Determines if we should consider the filename to be a text file. """
    return re.match(".*pm|.*pl$|.*C$|.*h$|.*c$", filename)

def keywordSubstitution(input):
    """ Switches the input string from SCCS key words to Subversion keywords."""
    replacement = re.sub("#ident .*",
                         "#ident \"$Id:$ (c) Archelon LLC 1990-2004\"", input)
    replacement = re.sub("%W%(\s+)%G%","$Id:$", replacement)
    replacement = re.sub("%W%", "$URL:$", replacement)
    replacement = re.sub("%G%", "$LastChangedDate:$", replacement)
    return replacement

class SCCSDelta:
    """ Represents a change in the SCCS repository. """
    def __init__(self, pathname, version, author, date, comment):
        self.pathname = pathname
        self.version = version
        self.author = author
        self.date = date
        self.comment = comment

    def __str__(self):
        return self.pathname + "\n" + self.version + "\n" + self.author \
               + "\n" + self.getDate() + "\n" + self.comment

    def match(self, otherDelta):
        """ Returns true if we should be in the same commit as the other delta """
        if self.author == otherDelta.author and \
           self.comment == otherDelta.comment and \
           self.pathname != otherDelta.pathname:
            if abs(time.mktime(self.date) - time.mktime(otherDelta.date)) < 10:
                return True

        return False

    def _getSourceSCCSDirectory(self):
        return os.path.split(self.pathname)[0]

    def getFilename(self):
        """ Returns the filename for this delta. """
        filename = os.path.basename(self.pathname)
        # strip off the leading ".s"
        return filename[2:]

    def getRepositoryName(self):
        """ Return the name for the file in the SVN repository """
        name = string.replace(self.pathname, "SCCS/s.", "")
        return string.replace(name, SCCSDelta.rootDirectory, "")

    def getDate(self):
        """ Return a subversion formatted date string for when this
        delta was done."""
        return subversionTime(self.date)

    def getDirectory(self):
        """ Returns the repository directory where this delta
        should be placed. """
        name = string.replace(self.pathname,
                              SCCSDelta.rootDirectory, "")
        return os.path.dirname(os.path.dirname(name))

    def getFileContents(self, getArguments=""):
        """ Returns, as a string, the contents of this delta. """
        command = "sccs -p " + self._getSourceSCCSDirectory() + \
                  " get " + getArguments + " -r" + self.version +\
                  " " + self.getFilename() + " -p 2> /dev/null "
        return os.popen(command).read()

class SVNInterface:
    """ Our interface to subversion """

    def __init__(self, repositoryPath, pool):
        self.context = client.svn_client_ctx_t()
        configDirectory = util.svn_config_ensure( '', pool)
        self.context.config = util.svn_config_get_config(configDirectory, pool)
        self.pool = pool
        self.repos_ptr = repos.svn_repos_open(repositoryPath, pool)
        self.fsob = repos.svn_repos_fs(self.repos_ptr)
        # Directories we've added to the repository.
        self.addedDirectories = {}
        self.addedDirectories["/"] = 1
        
    def _commit(self, rev, date, txn, subpool):
        """ Commit the supplied transaction to Subversion. """
        fs.change_rev_prop(self.fsob, rev,
                           core.SVN_PROP_ENTRY_COMMITTED_DATE,
                           date, subpool)

        fs.change_rev_prop(self.fsob, rev,
                           core.SVN_PROP_REVISION_DATE,
                           date, subpool)

        return repos.svn_repos_fs_commit_txn(self.repos_ptr, txn, subpool)

    def _revisionSetup(self, subpool, author, message):
        """ All of the setup for performing a revision. """
        revision = fs.youngest_rev(self.fsob, subpool)
        transaction = repos.svn_repos_fs_begin_txn_for_commit(self.repos_ptr,
                                                              revision,
                                                              author, message,
                                                              subpool)
        root = fs.txn_root(transaction, subpool)
        return (revision, transaction, root)

    def _directoriesToAdd(self, delta):
        """ Return a list of directories to add for delta. """
        directoryName = delta.getDirectory()
        directoriesToAdd = []
        while directoryName and directoryName != "/":
            if not directoryName in self.addedDirectories:
                directoriesToAdd.insert(0, directoryName)
            directoryName = os.path.dirname(directoryName)
        return directoriesToAdd

    def _addDirectories(self, delta):
        """ Make sure that all of the directories needed for delta are added. """
        directoriesToAdd = self._directoriesToAdd(delta)
        if len(directoriesToAdd) == 0:
            return
        subpool = core.svn_pool_create(self.pool)
        (revision, transaction, root) = \
                   self._revisionSetup(subpool,
                                       options.userid,
                                       "Automatic directory addition")
        for directory in directoriesToAdd:
            print "adding directory", directory
            print delta.getDate()
            fs.make_dir(root, directory, subpool)
            self.addedDirectories[directory] = 1

        self._commit(revision, delta.getDate(), transaction, subpool)
        core.svn_pool_destroy(subpool)
            
    def add(self, deltas):
        """ Add the supplied set of deltas to the repository. They will
        all be added with the same user name, date, and comment. """

        # Add all of the directories first, or we will be trying
        # to cross transactions, which is bad.
        for delta in deltas:
            self._addDirectories(delta)

        subpool = core.svn_pool_create(self.pool)
        (revision, transaction, root) = self._revisionSetup(subpool,
                                                            deltas[0].author,
                                                            deltas[0].comment)
        for delta in deltas:
            subversionPath = delta.getRepositoryName()
            kind = fs.check_path(root, subversionPath, subpool)
            if kind == core.svn_node_none:
                fs.make_file(root, subversionPath, subpool)
            elif kind == core.svn_node_dir:
                raise EnvironmentError(subversionPath +
                                       " already present as a directory.")
            handler, baton = fs.apply_textdelta(root, subversionPath,
                                                None, None, subpool)
            svn.delta.svn_txdelta_send_string(delta.getFileContents(),
                                              handler, baton, subpool)
            print "sending ", subversionPath, delta.getDate()

        print "committing version ",
        print self._commit(revision, delta.getDate(), transaction, subpool)
        core.svn_pool_destroy(subpool)

    def remove(self, filenames):
        """ Remove the supplied filenames file from the repository. """
        subpool = core.svn_pool_create(self.pool)
        (revision,
         transaction,
         root) = self._revisionSetup(subpool,
                                     "efsadm-svn",
                                     "Automated SCCS conversion removal")
        for file in filenames:
            print "removing ", file
            fs.delete(root, file, subpool)
        self._commit(revision, subversionTime(time.localtime()),
                     transaction, subpool)

        core.svn_pool_destroy(subpool)
        
    def propertyUpdate(self, filenames):
        """ Set the keywords property for the supplied filenames. """
        subpool = core.svn_pool_create(self.pool)
        (revision,
         transaction,
         root) = self._revisionSetup(subpool,
                                     "efsadm-svn",
                                     "Automated property set")
        for filename in filenames:
            if isTextFilename(filename):
                print "property set for ", filename
                fs.change_node_prop(root, filename,
                                    core.SVN_PROP_KEYWORDS,
                                    "LastChangedDate LastChangedRevision LastChangedBy HeadURL Id",
                                    subpool)
                fs.change_node_prop(root, filename,
                                    core.SVN_PROP_EOL_STYLE,
                                    "native",
                                    subpool)
            else:
                print "skipping property set for ", filename
                
        self._commit(revision, subversionTime(time.localtime()),
                             transaction, subpool)
        core.svn_pool_destroy(subpool)

    def idKeyUpdate(self, deltas):
        """ Convert the SCCS keywords inside of the supplied deltas to
        subversion keywords. """
        subpool = core.svn_pool_create(self.pool)
        (revision,
         transaction,
         root) = self._revisionSetup(subpool,
                                     "efsadm-svn",
                                     "Automated keyword replacement")
        for delta in deltas:
            if isTextFilename(delta.getFilename()):
                originalContents = delta.getFileContents("-k")
                updatedContents = keywordSubstitution(originalContents)
                if originalContents != updatedContents:
                    handler, baton = fs.apply_textdelta(root,
                                                        delta.getRepositoryName(),
                                                        None, None, subpool)
                    svn.delta.svn_txdelta_send_string(updatedContents, handler,
                                                      baton, subpool)
                    print "sending ", delta.getRepositoryName()

        print "committing version ",
        print self._commit(revision, delta.getDate(), transaction, subpool)
        core.svn_pool_destroy(subpool)

def deltaSort(deltaOne, deltaTwo):
    """ Sort two deltas based on their time. """
    if time.mktime(deltaOne.date) < time.mktime(deltaTwo.date):
        return -1
    if time.mktime(deltaOne.date) > time.mktime(deltaTwo.date):
        return 1
    return 0

def parseSCCSLog(filename):
    """ Parse the SCCS log for the filename, and add all of its
    deltas to the global array versions. """
    startOfDeltaMarker="start091283123"
    endOfCommentMarker="endofcomment9123klfdgdfg;kdfg\n"
    sccsPrsCommand = "sccs prs -e -d\"%s\\t:I:\\t:P:\\t:D:\t:T:\\n:C:\\n%s\" " % (startOfDeltaMarker, endOfCommentMarker)
    output = os.popen(sccsPrsCommand + filename)
    log = output.readlines()
    output.close()
    commentMode = 0
    comments = ""
    for i in log:
        if i[0:len(endOfCommentMarker)] == endOfCommentMarker:
            versions.append(SCCSDelta(filename, version, user, dateTime, comments))
            commentMode = 0
            comments = ""

        if commentMode:
            comments += i

        if i[0:len(startOfDeltaMarker)] == startOfDeltaMarker:
            commentMode = 1
            (dummy, version, user, date, ti) = i.split("\t",4)
            ti = ti.rstrip()
            dateTime = time.strptime(date + " " + ti, "%y/%m/%d %H:%M:%S")
            

def visitSCCSRepository(interface,dirname,names):
    """ Visit each file in the SCCS directory, calling parseSCCSLog for each."""
    if os.path.split(dirname)[-1] == "SCCS":
        print "Visiting ", dirname
        for i in names:
            if i[0:2] == "s.":
                filename = os.path.join(dirname, i)
                parseSCCSLog(filename)

def run(pool):

    SCCSDelta.rootDirectory = options.sccs_repository
    interface = SVNInterface(options.svn_repository, pool)
    os.path.walk(SCCSDelta.rootDirectory, visitSCCSRepository, interface)
    versions.sort(deltaSort)

    print "Read",
    print len(versions),
    print "versions."

    if len(versions) == 0:
        print "Unable to read any SCCS versions; nothing to convert"
        sys.exit(1)

    # Merge deltas together.
    mergedVersions = [[versions[0]]]
    i = 0
    while i < len(versions):
        if versions[i].match(mergedVersions[-1][-1]):
            mergedVersions[-1].append(versions[i])
        else:
            mergedVersions.append([versions[i]])
        i += 1

    print "consolidated length = ", len(mergedVersions)
    
    # Add each delta.
    for i in mergedVersions:
        interface.add(i)
    
    # Get all of the filenames.
    filenames = {}
    for i in versions:
        filenames[i.getRepositoryName()] = i

    # Update their properties and keywords.
    interface.propertyUpdate(filenames.keys())
    interface.idKeyUpdate(filenames.values())

    # Delete any file ending in '-'
    versionsToDelete = {}
    for i in versions:
        if i.pathname[-1] == '-':
            versionsToDelete[i.getRepositoryName()] = 1
    interface.remove(versionsToDelete.keys())

    # Delete any directories ending in '-'
    directoriesToDelete = {}
    for i in versions:
        if os.path.dirname(i.getRepositoryName())[-1] == '-':
            directoriesToDelete[os.path.dirname(i.getRepositoryName())] = 1
    print directoriesToDelete.keys()
    interface.remove(directoriesToDelete.keys())

if __name__ == '__main__':
    parser = OptionParser(usage="usage: %prog options")
    parser.add_option("-u", "--user", dest="userid", metavar="userid",
                      help="The user id used for sccs2svn generated changes")
    parser.add_option("-o", "--svn-repository", dest="svn_repository",
                      metavar="svn repository directory",
                      help="The location of the Subversion repository; this location will be destroyed!")
    parser.add_option("-i", "--sccs-repository", dest="sccs_repository",
                      metavar="sccs root directory",
                      help="The location of the SCCS repository")
    
    (options, args) = parser.parse_args()

    # Make sure that we have all of the options we need.
    if options.userid == None:
        parser.error("You must supply a user id with --user")

    if options.svn_repository == None:
        parser.error("You must supply a Subversion repository with --svn-repository")

    if options.sccs_repository == None:
        parser.error("You must supply a SCCS repository with --sccs-repository")

    if os.path.exists(options.svn_repository):
        print "Removing directory " + options.svn_repository
        shutil.rmtree(options.svn_repository)
    os.popen("svnadmin create " + options.svn_repository).readlines()

    core.run_app(run)
    

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@subversion.tigris.org
For additional commands, e-mail: dev-help@subversion.tigris.org
Received on Thu Dec 30 14:46:29 2004

This is an archived mail posted to the Subversion Dev mailing list.

This site is subject to the Apache Privacy Policy and the Apache Public Forum Archive Policy.