SVN and python scripts for building/merging

I mentioned in a previous post that I was using scripts to detect
changes in branches and to automatically merge changes from branches
into the trunk. We use SCONS as our cross platform build tool and
python for our build scripts. Python has a SVN module at
http://pysvn.tigris.org <http://pysvn.tigris.org/> that I hope to
convert these scripts to later, but for now, I just execute the SVN
commands and parse the output from those commands. Hopefully this will
be helpful to someone, and that this is not the wrong place to post such
a thing, and that people can give me feedback.

Our main repository layout looks like this:

branches/

tags/

trunk/

The process we go through kicks off automatically at midnight. The
first thing we do is get a list of the branches that we're going to
check for changes with GetBranches(). We then check each branch for
changes using HasChanges( branch ), if it has changes, then we go run
the build script for that branch in our working copy (we can switch to
the branch or keep multiple WCs). The build process tags the branch (or
trunk) with a tag in the form '<branch>-<tag type (build or
merge)>-<build rev>' i.e. 'trunk-bld-7415'. If we are not building the
trunk, then we merge changes into the trunk. This involves merging from
the last merge tag (or from the start of the branch if there is no merge
tag yet) and then making a new merge tag from the build tag we created
at the start of the build. SVN is supposed to be automating the merge
tracking soon so I can get rid of the merge tags. The main python
functions used in the process I just described are shown below. (I
can't wait to convert these to pysvn! I think these functions will be
much smaller! :)

Robert

To get a list of branches (including the trunk)

def GetBranches():

    list = os.popen( 'svn list svn://svn/src/branches' )

    branches = list.readlines()

    list.close()

    for i in range( len( branches ) ):

      branches[i] = branches[i][0:-2]

    branches += ['trunk']

    return branches

To detect changes in a branch (or in the trunk)

def HasChanges( branch ):

    diffCommand = ''

    subversionRoot = 'svn://svn/src/'

    subversionPath = 'trunk'

    if ( branch != 'trunk' ):

      subversionPath = 'branches/' + branch

    highestTag = HighestTag( GetTags( branch, 'bld' ) )

    if highestTag == '':

      diffCommand = 'svn diff -r' + GetBranchCreationRevision( branch )
+ ':HEAD ' + subversionRoot + subversionPath

    else:

      diffTag = 'tags/' + highestTag

      diffCommand = 'svn diff ' + subversionRoot + subversionPath + ' '
+ subversionRoot + diffTag

    diffResults = os.popen( diffCommand )

    stuff = diffResults.read( 10 )

    diffResults.close()

    return stuff != ''

To find the highest tag from a list of tags

def HighestTag( tags ):

    highestLibrev = 0

    highestTag = ''

    for tag in tags:

      librev = int( tag[tag.index( '-', tag.index( '-' ) + 1 ) + 1:] )

      if librev > highestLibrev:

          highestLibrev = librev

          highestTag = tag

    return highestTag

To get a list of tags for a branch

def GetTags( branch, type = '' ):

    list = os.popen( 'svn list svn://svn/src/tags' )

    tags = list.readlines()

    list.close()

    for i in range( len( tags ) ):

      tags[i] = tags[i][0:-2]

    # Filter the list of tags down to the given branch

    tags = [n for n in tags if '-' in n and n[:n.index('-')] == branch]

    # If they only want a certain type (bld or mrg) filter those out
here

    if type != '':

      tags = [n for n in tags if n[n.index('-') +
1:n.index('-',n.index('-') + 1)] == type]

    return tags

To merge a branch to the trunk

def MergeBranch( branch ):

    # doIt is used to do a dry run for testing purposes

    global doIt

    if branch == 'trunk':

      # fail here however you would like to

    # GetLibRev below just returns our current build revision from
librev.h as a number

    # logprint just prints something to the screen and logs it to the
build log file

    logprint( ' Merging ' + branch + '...' )

    dryRunText = ''

    subversionRoot = 'svn://svn/src/'

    branchPath = 'branches/' + branch

    highestBldTag = subversionRoot + 'tags/' + HighestTag( GetTags(
branch, 'bld' ) )

    highestMrgTag = subversionRoot + 'tags/' + HighestTag( GetTags(
branch, 'mrg' ) )

    newMrgTag = subversionRoot + 'tags/' + branch + '-mrg-' +
GetLibRev()

    if not doIt:

      dryRunText = ' --dry-run '

    # If we don't have a merge tag for this branch yet (i.e. we've never
merged it) then

    # just merge from the branch creation point to the head of this
branch

    # (I can't wait until SVN makes this automatic! :)

    if HighestTag( GetTags( branch, 'mrg' ) ) == '':

      mergeCommand = 'svn merge ' + dryRunText + subversionRoot +
branchPath + '@' + GetBranchCreationRevision( branch ) + ' ' +
highestBldTag

    else:

      mergeCommand = 'svn merge ' + dryRunText + highestMrgTag + ' ' +
highestBldTag

    # Update the trunk before we merge

    Execute( GetSourceTop() + '\\trunk', 'svn cleanup', doIt, 0 )

    Execute( GetSourceTop() + '\\trunk', 'svn update', doIt, 0 )

    logprint( mergeCommand )

    mergeResults = os.popen( 'cd ' + GetSourceTop() + '\\trunk && ' +
mergeCommand )

    mergeLines = mergeResults.readlines()

    mergeResults.close()

    # Detect conflicts

    conflictFound = 0

    for i in range( len( mergeLines ) ):

      if mergeLines[i][0] == 'C':

          conflictFound = 1

          logprint( mergeLines[i] )

    if conflictFound:

      SendEmail( buildGuy, 'Conflicts while merging ' + branch + ' to
the trunk.', 'Go resolve the conflicts.\n\n' + str( mergeLines ) + '\n'
+ GetEvents(), '', doIt )

      logprint( 'Conflicts found!' )

      Execute( GetSourceTop() + '\\trunk', 'svn status & echo Go resolve
the conflicts & cmd', 1, 0 )

    # Check it into the trunk

    Execute( GetSourceTop() + '\\trunk', 'svn ci -m"Checking in merged
changes from the ' + branch + ' branch that were merged with the
following command: ' + mergeCommand + '"', doIt, 0 )

    tagCommand = 'svn copy ' + highestBldTag + ' ' + newMrgTag + '
-m"Tagged by the build process."'

    Execute( GetTop(), tagCommand, doIt, 0 )

    AddEvent( 'Merge to trunk finished successfully' )

To get the branch creation revision

def GetBranchCreationRevision( branch ):

    if branch == 'trunk':

      # Fail here

    branchRoot = 0

    # librev.h contains our build revision and is just a known file we
can take a log of in this case

    branchLog = os.popen( 'svn -q log svn://svn/src/branches/' + branch
+ '/imglib/common/librev.h' )

    branchLines = branchLog.readlines()

    branchLog.close()

    trunkLog = os.popen( 'svn -q log
svn://svn/src/trunk/imglib/common/librev.h' )

    trunkLines = trunkLog.readlines()

    trunkLog.close()

    # Filter out all lines that don't start with 'r'. These lines are

    # revision lines.

    branchLines = [n for n in branchLines if n[0] == 'r']

    trunkLines = [n for n in trunkLines if n[0] == 'r']

    # Chop off everything but the revision number. The line format for
the

    # revision lines is: 'r819 | rcronk | 2004-08-17...'

    for i in range( len( branchLines ) ):

      branchLines[i] = branchLines[i][1:branchLines[i].index('|') - 1]

    for i in range( len( trunkLines ) ):

      trunkLines[i] = trunkLines[i][1:trunkLines[i].index('|') - 1]

    branchLines.reverse()

    trunkLines.reverse()

    found = 0

    for i in range( max( len( branchLines ), len( trunkLines ) ) ):

      if not found and ( i >= len( trunkLines ) or branchLines[i] !=
trunkLines[i] ):

          branchRoot = branchLines[i]

          found = 1

    if branchRoot == 0:

      # fail( 'Build script error: branchroot for "' + branch + '" was
not found!', 1, 1 )

    return branchRoot
Received on Tue Aug 23 20:16:11 2005