Attachment 8031 Details for Bug 9277 – Patches for v4-0-test

[patch] Patches for v4-0-test

tmp40.diff (text/plain), 37.27 KB, created by Stefan Metzmacher on 2012-10-10 06:31:23 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Stefan Metzmacher

Created: 2012-10-10 06:31:23 UTC

Size: 37.27 KB

patch

obsolete

>From 04b9bf63721ec7c3b7ce1d839740f6fd9c90edc6 Mon Sep 17 00:00:00 2001
>From: Stefan Metzmacher <metze@samba.org>
>Date: Sun, 23 Sep 2012 00:50:48 +0200
>Subject: [PATCH 1/3] script/autobuild.py: remove --rebase-master and
> --push-master options
>
>You should explicitly pass the urls to --rebase= and --pushto=
>
>metze
>
>Autobuild-User(master): Stefan Metzmacher <metze@samba.org>
>Autobuild-Date(master): Sun Sep 23 02:53:56 CEST 2012 on sn-devel-104
>---
> script/autobuild.py |   21 +++------------------
> 1 file changed, 3 insertions(+), 18 deletions(-)
>
>diff --git a/script/autobuild.py b/script/autobuild.py
>index 9fdea3b..12656c1 100755
>--- a/script/autobuild.py
>+++ b/script/autobuild.py
>@@ -10,9 +10,6 @@ import smtplib
> from email.mime.text import MIMEText
> from distutils.sysconfig import get_python_lib
> 
>-samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
>-samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
>-
> # This speeds up testing remarkably.
> os.environ['TDB_NO_FSYNC'] = '1'
> 
>@@ -467,12 +464,8 @@ parser.add_option("", "--verbose", help="show all commands as they are run",
>                   default=False, action="store_true")
> parser.add_option("", "--rebase", help="rebase on the given tree before testing",
>                   default=None, type='str')
>-parser.add_option("", "--rebase-master", help="rebase on %s before testing" % samba_master,
>-                  default=False, action='store_true')
> parser.add_option("", "--pushto", help="push to a git url on success",
>                   default=None, type='str')
>-parser.add_option("", "--push-master", help="push to %s on success" % samba_master_ssh,
>-                  default=False, action='store_true')
> parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
>                   default=False, action="store_true")
> parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
>@@ -575,7 +568,7 @@ The top commit for the tree that was built was:
> (options, args) = parser.parse_args()
> 
> if options.retry:
>-    if not options.rebase_master and options.rebase is None:
>+    if options.rebase is None:
>         raise Exception('You can only use --retry if you also rebase')
> 
> testbase = "%s/b%u" % (options.testbase, os.getpid())
>@@ -613,13 +606,7 @@ while True:
>     try:
>         try:
>             if options.rebase is not None:
>-                rebase_url = options.rebase
>-            elif options.rebase_master:
>-                rebase_url = samba_master
>-            else:
>-                rebase_url = None
>-            if rebase_url is not None:
>-                rebase_tree(rebase_url, rebase_branch=options.branch)
>+                rebase_tree(options.rebase, rebase_branch=options.branch)
>         except Exception:
>             cleanup_list.append(gitroot + "/autobuild.pid")
>             cleanup()
>@@ -627,7 +614,7 @@ while True:
>                           'rebase on %s failed' % options.branch,
>                           log_base=options.log_base)
>             sys.exit(1)
>-        blist = buildlist(tasks, args, rebase_url, rebase_branch=options.branch)
>+        blist = buildlist(tasks, args, options.rebase, rebase_branch=options.branch)
>         if options.tail:
>             blist.start_tail()
>         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
>@@ -652,8 +639,6 @@ if status == 0:
>         run_cmd(options.passcmd, dir=test_master)
>     if options.pushto is not None:
>         push_to(options.pushto, push_branch=options.branch)
>-    elif options.push_master:
>-        push_to(samba_master_ssh, push_branch=options.branch)
>     if options.keeplogs:
>         blist.tarlogs("logs.tar.gz")
>         print("Logs in logs.tar.gz")
>-- 
>1.7.9.5
>
>
>From 89fce82a7b50b501e2f1fb979846aacbbdaa77e9 Mon Sep 17 00:00:00 2001
>From: Stefan Metzmacher <metze@samba.org>
>Date: Mon, 24 Sep 2012 11:53:22 +0200
>Subject: [PATCH 2/3] script/autobuild.py: set the default for --log-base to
> the current gitroot
>
>metze
>
>Autobuild-User(master): Stefan Metzmacher <metze@samba.org>
>Autobuild-Date(master): Mon Sep 24 15:27:16 CEST 2012 on sn-devel-104
>---
> script/autobuild.py |   14 +++++++-------
> 1 file changed, 7 insertions(+), 7 deletions(-)
>
>diff --git a/script/autobuild.py b/script/autobuild.py
>index 12656c1..efef2f4 100755
>--- a/script/autobuild.py
>+++ b/script/autobuild.py
>@@ -453,6 +453,10 @@ def push_to(push_url, push_branch = "master"):
> 
> def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
> 
>+gitroot = find_git_root()
>+if gitroot is None:
>+    raise Exception("Failed to find git root")
>+
> parser = OptionParser()
> parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
> parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
>@@ -481,13 +485,13 @@ parser.add_option("", "--daemon", help="daemonize after initial setup",
> parser.add_option("", "--branch", help="the branch to work on (default=master)",
>                   default="master", type='str')
> parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
>-                  default=None, type='str')
>+                  default=gitroot, type='str')
> 
> def email_failure(status, failed_task, failed_stage, failed_tag, errstr, log_base=None):
>     '''send an email to options.email about the failure'''
>     user = os.getenv("USER")
>     if log_base is None:
>-        log_base = "http://git.samba.org/%s/samba-autobuild" % user
>+        log_base = gitroot
>     text = '''
> Dear Developer,
> 
>@@ -531,7 +535,7 @@ def email_success(log_base=None):
>     '''send an email to options.email about a successful build'''
>     user = os.getenv("USER")
>     if log_base is None:
>-        log_base = "http://git.samba.org/%s/samba-autobuild" % user
>+        log_base = gitroot
>     text = '''
> Dear Developer,
> 
>@@ -574,10 +578,6 @@ if options.retry:
> testbase = "%s/b%u" % (options.testbase, os.getpid())
> test_master = "%s/master" % testbase
> 
>-gitroot = find_git_root()
>-if gitroot is None:
>-    raise Exception("Failed to find git root")
>-
> # get the top commit message, for emails
> top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
> 
>-- 
>1.7.9.5
>
>
>From 813e31d38aa2ff4522b60ab17c32da3ac1a395d0 Mon Sep 17 00:00:00 2001
>From: Jelmer Vernooij <jelmer@samba.org>
>Date: Wed, 26 Sep 2012 00:47:02 +0200
>Subject: [PATCH 3/3] Remove obsolete land-remote.py and land.py scripts.
>
>---
> script/land-remote.py |  113 --------
> script/land.py        |  741 -------------------------------------------------
> 2 files changed, 854 deletions(-)
> delete mode 100755 script/land-remote.py
> delete mode 100755 script/land.py
>
>diff --git a/script/land-remote.py b/script/land-remote.py
>deleted file mode 100755
>index 1009e6b..0000000
>--- a/script/land-remote.py
>+++ /dev/null
>@@ -1,113 +0,0 @@
>-#!/usr/bin/python
>-# Ship a local branch to a remote host (sn-104?) over ssh and run autobuild in it.
>-# Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
>-# Published under the GPL, v3 or later
>-
>-import optparse
>-import os
>-import subprocess
>-import sys
>-
>-samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
>-
>-parser = optparse.OptionParser("autoland-remote [options] [trees...]")
>-parser.add_option("--remote-repo", help="Location of remote repository (default: temporary repository)", type=str, default=None)
>-parser.add_option("--host", help="Host to land on (SSH connection string)", type=str, default="sn-devel-104.sn.samba.org")
>-parser.add_option("--foreground", help="Don't daemonize", action="store_true", default=False)
>-parser.add_option("--email", help="Email address to send build/test output to", type=str, default=None, metavar="EMAIL")
>-parser.add_option("--always-email", help="always send email, even on success", action="store_true")
>-parser.add_option("--rebase-master", help="rebase on master before testing", default=False, action='store_true')
>-parser.add_option("--push-master", help="push to samba.org master on success",
>-                  default=False, action='store_true')
>-parser.add_option("--pushto", help="push to a git url on success",
>-                  default=None, type='str')
>-parser.add_option("--rebase", help="rebase on the given tree before testing", default=None, type='str')
>-parser.add_option("--passcmd", help="command to run on success", default=None)
>-parser.add_option("--tail", help="show output while running", default=False, action="store_true")
>-parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
>-parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
>-parser.add_option("--revision", help="revision to compile if not HEAD", default=None, type=str)
>-parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
>-                  default=False, action="store_true")
>-parser.add_option("--fail-slowly", help="continue running tests even after one has already failed",
>-                  action="store_true")
>-
>-(opts, extra_args) = parser.parse_args()
>-
>-if opts.email is None and os.getenv("EMAIL") is not None:
>-    opts.email = os.getenv("EMAIL")
>-
>-if opts.email:
>-    print "Sending email to %s" % opts.email
>-
>-if not opts.foreground and not opts.email:
>-    print "Not running in foreground and --email not specified."
>-    sys.exit(1)
>-
>-if not opts.foreground and opts.push_master:
>-    print "Pushing to master, forcing run in foreground."
>-    opts.foreground = True
>-
>-if not opts.remote_repo:
>-    print "%s$ mktemp -d" % opts.host
>-    f = subprocess.Popen(["ssh", opts.host, "mktemp", "-d"], stdout=subprocess.PIPE)
>-    (stdout, stderr) = f.communicate()
>-    if f.returncode != 0:
>-        sys.exit(1)
>-    remote_repo = stdout.rstrip()
>-    print "Remote tempdir: %s" % remote_repo
>-    # Bootstrap, git.samba.org is usually more easily accessible.
>-    #remote_args = ["git", "clone", samba_master, remote_repo]
>-    remote_args = ["if [ -d /data/git/samba.git ]; then git clone --shared /data/git/samba.git %s; else git clone --shared %s %s; fi" % (remote_repo, samba_master, remote_repo)]
>-    #remote_args = ["git", "init", remote_repo]
>-    print "%s$ %s" % (opts.host, " ".join(remote_args))
>-    subprocess.check_call(["ssh", opts.host] + remote_args)
>-else:
>-    remote_repo = opts.remote_repo
>-
>-print "Pushing local branch"
>-
>-if opts.revision is not None:
>-    revision = opts.revision
>-else:
>-    revision = "HEAD"
>-args = ["git", "push", "--force", "git+ssh://%s/%s" % (opts.host, remote_repo), "%s:land" % revision]
>-print "$ " + " ".join(args)
>-subprocess.check_call(args)
>-remote_args = ["cd", remote_repo, ";", "git", "checkout", "land", ";", "python", "-u", "./script/land.py", "--repository=%s" % remote_repo]
>-
>-if (opts.email and not (opts.foreground or opts.pushto or opts.push_master)):
>-    # Force always emailing if there's nothing else to do
>-    opts.always_email = True
>-
>-if opts.email:
>-    remote_args.append("--email=%s" % opts.email)
>-if opts.always_email:
>-    remote_args.append("--always-email")
>-if not opts.foreground:
>-    remote_args.append("--daemon")
>-if opts.nocleanup:
>-    remote_args.append("--nocleanup")
>-if opts.fix_whitespace:
>-    remote_args.append("--fix-whitespace")
>-if opts.tail:
>-    remote_args.append("--tail")
>-if opts.keeplogs:
>-    remote_args.append("--keeplogs")
>-if opts.rebase_master:
>-    remote_args.append("--rebase-master")
>-if opts.rebase:
>-    remote_args.append("--rebase=%s" % opts.rebase)
>-if opts.passcmd:
>-    remote_args.append("--passcmd=%s" % opts.passcmd)
>-if opts.pushto:
>-    remote_args.append("--pushto=%s" % opts.pushto)
>-if opts.push_master:
>-    remote_args.append("--push-master")
>-if opts.fail_slowly:
>-    remote_args.append("--fail-slowly")
>-
>-remote_args += extra_args
>-print "%s$ %s" % (opts.host, " ".join(remote_args))
>-args = ["ssh", "-A", opts.host] + remote_args
>-sys.exit(subprocess.call(args))
>diff --git a/script/land.py b/script/land.py
>deleted file mode 100755
>index 72bdd4b..0000000
>--- a/script/land.py
>+++ /dev/null
>@@ -1,741 +0,0 @@
>-#!/usr/bin/env python
>-# run tests on all Samba subprojects and push to a git tree on success
>-# Copyright Andrew Tridgell 2010
>-# Copyright Jelmer Vernooij 2010
>-# released under GNU GPL v3 or later
>-
>-from cStringIO import StringIO
>-import fcntl
>-from subprocess import call, check_call, Popen, PIPE
>-import os, tarfile, sys, time
>-from optparse import OptionParser
>-import smtplib
>-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../selftest"))
>-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
>-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
>-import subunit
>-import testtools
>-import subunithelper
>-import tempfile
>-from email.mime.application import MIMEApplication
>-from email.mime.text import MIMEText
>-from email.mime.multipart import MIMEMultipart
>-
>-samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
>-samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
>-
>-cleanup_list = []
>-
>-os.environ['CC'] = "ccache gcc"
>-
>-tasks = {
>-    "source3" : [ ("autogen", "./autogen.sh", "text/plain"),
>-                  ("configure", "./configure.developer ${PREFIX}", "text/plain"),
>-                  ("make basics", "make basics", "text/plain"),
>-                  ("make", "make -j 4 everything", "text/plain"), # don't use too many processes
>-                  ("install", "make install", "text/plain"),
>-                  ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
>-
>-    "source4" : [ ("configure", "./configure.developer ${PREFIX}", "text/plain"),
>-                  ("make", "make -j", "text/plain"),
>-                  ("install", "make install", "text/plain"),
>-                  ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
>-
>-    "lib/ldb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
>-                          ("make", "make -j", "text/plain"),
>-                          ("install", "make install", "text/plain"),
>-                          ("test", "make test", "text/plain") ],
>-
>-    "lib/tdb" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
>-                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
>-                  ("make", "make -j", "text/plain"),
>-                  ("install", "make install", "text/plain"),
>-                  ("test", "make test", "text/plain") ],
>-
>-    "lib/talloc" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
>-                     ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
>-                     ("make", "make -j", "text/plain"),
>-                     ("install", "make install", "text/plain"),
>-                     ("test", "make test", "text/x-subunit"), ],
>-
>-    "lib/replace" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
>-                      ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
>-                      ("make", "make -j", "text/plain"),
>-                      ("install", "make install", "text/plain"),
>-                      ("test", "make test", "text/plain"), ],
>-
>-    "lib/tevent" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
>-                     ("make", "make -j", "text/plain"),
>-                     ("install", "make install", "text/plain"),
>-                     ("test", "make test", "text/plain"), ],
>-}
>-
>-
>-def run_cmd(cmd, dir=None, show=None, output=False, checkfail=True, shell=False):
>-    if show is None:
>-        show = options.verbose
>-    if show:
>-        print("Running: '%s' in '%s'" % (cmd, dir))
>-    if output:
>-        return Popen(cmd, stdout=PIPE, cwd=dir, shell=shell).communicate()[0]
>-    elif checkfail:
>-        return check_call(cmd, cwd=dir, shell=shell)
>-    else:
>-        return call(cmd, cwd=dir, shell=shell)
>-
>-
>-def clone_gitroot(test_master, revision="HEAD"):
>-    run_cmd(["git", "clone", "--shared", gitroot, test_master])
>-    if revision != "HEAD":
>-        run_cmd(["git", "checkout", revision])
>-
>-
>-class RetryChecker(object):
>-    """Check whether it is necessary to retry."""
>-
>-    def __init__(self, dir):
>-        run_cmd(["git", "remote", "add", "-t", "master", "master", samba_master])
>-        run_cmd(["git", "fetch", "master"])
>-        cmd = '''set -e
>-                while :; do
>-                  sleep 60
>-                  git describe master/master > old_master.desc
>-                  git fetch master
>-                  git describe master/master > master.desc
>-                  diff old_master.desc master.desc
>-                done
>-               '''
>-        self.proc = Popen(cmd, shell=True, cwd=self.dir)
>-
>-    def poll(self):
>-        return self.proc.poll()
>-
>-    def kill(self):
>-        self.proc.terminate()
>-        self.proc.wait()
>-        self.retry.proc = None
>-
>-
>-class TreeStageBuilder(object):
>-    """Handle building of a particular stage for a tree.
>-    """
>-
>-    def __init__(self, tree, name, command, fail_quickly=False):
>-        self.tree = tree
>-        self.name = name
>-        self.command = command
>-        self.fail_quickly = fail_quickly
>-        self.exitcode = None
>-        self.stdin = open(os.devnull, 'r')
>-
>-    def start(self):
>-        raise NotImplementedError(self.start)
>-
>-    def poll(self):
>-        self.exitcode = self.proc.poll()
>-        return self.exitcode
>-
>-    def kill(self):
>-        if self.proc is not None:
>-            try:
>-                run_cmd(["killbysubdir", self.tree.sdir], checkfail=False)
>-            except OSError:
>-                # killbysubdir doesn't exist ?
>-                pass
>-            self.proc.terminate()
>-            self.proc.wait()
>-            self.proc = None
>-
>-    @property
>-    def failure_reason(self):
>-        raise NotImplementedError(self.failure_reason)
>-
>-    @property
>-    def failed(self):
>-        return (self.exitcode != 0)
>-
>-
>-class PlainTreeStageBuilder(TreeStageBuilder):
>-
>-    def start(self):
>-        print '%s: [%s] Running %s' % (self.name, self.name, self.command)
>-        self.proc = Popen(self.command, shell=True, cwd=self.tree.dir,
>-                          stdout=self.tree.stdout, stderr=self.tree.stderr,
>-                          stdin=self.stdin)
>-
>-    @property
>-    def failure_reason(self):
>-        return "failed '%s' with exit code %d" % (self.command, self.exitcode)
>-
>-
>-class AbortingTestResult(subunithelper.TestsuiteEnabledTestResult):
>-
>-    def __init__(self, stage):
>-        super(AbortingTestResult, self).__init__()
>-        self.stage = stage
>-
>-    def addError(self, test, details=None):
>-        self.stage.proc.terminate()
>-
>-    def addFailure(self, test, details=None):
>-        self.stage.proc.terminate()
>-
>-
>-class FailureTrackingTestResult(subunithelper.TestsuiteEnabledTestResult):
>-
>-    def __init__(self, stage):
>-        super(FailureTrackingTestResult, self).__init__()
>-        self.stage = stage
>-
>-    def addError(self, test, details=None):
>-        if self.stage.failed_test is None:
>-            self.stage.failed_test = ("error", test)
>-
>-    def addFailure(self, test, details=None):
>-        if self.stage.failed_test is None:
>-            self.stage.failed_test = ("failure", test)
>-
>-
>-class SubunitTreeStageBuilder(TreeStageBuilder):
>-
>-    def __init__(self, tree, name, command, fail_quickly=False):
>-        super(SubunitTreeStageBuilder, self).__init__(tree, name, command,
>-                fail_quickly)
>-        self.failed_test = None
>-        self.subunit_path = os.path.join(gitroot,
>-            "%s.%s.subunit" % (self.tree.tag, self.name))
>-        self.tree.logfiles.append(
>-            (self.subunit_path, os.path.basename(self.subunit_path),
>-             "text/x-subunit"))
>-        self.subunit = open(self.subunit_path, 'w')
>-
>-        formatter = subunithelper.PlainFormatter(False, True, {})
>-        clients = [formatter, subunit.TestProtocolClient(self.subunit),
>-                   FailureTrackingTestResult(self)]
>-        if fail_quickly:
>-            clients.append(AbortingTestResult(self))
>-        self.subunit_server = subunit.TestProtocolServer(
>-            testtools.MultiTestResult(*clients),
>-            self.subunit)
>-        self.buffered = ""
>-
>-    def start(self):
>-        print '%s: [%s] Running' % (self.tree.name, self.name)
>-        self.proc = Popen(self.command, shell=True, cwd=self.tree.dir,
>-            stdout=PIPE, stderr=self.tree.stderr, stdin=self.stdin)
>-        fd = self.proc.stdout.fileno()
>-        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
>-        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
>-
>-    def poll(self):
>-        try:
>-            data = self.proc.stdout.read()
>-        except IOError:
>-            return None
>-        else:
>-            self.buffered += data
>-            buffered = ""
>-            for l in self.buffered.splitlines(True):
>-                if l[-1] == "\n":
>-                    self.subunit_server.lineReceived(l)
>-                else:
>-                    buffered += l
>-            self.buffered = buffered
>-            self.exitcode = self.proc.poll()
>-            if self.exitcode is not None:
>-                self.subunit.close()
>-            return self.exitcode
>-
>-    @property
>-    def failure_reason(self):
>-        if self.failed_test:
>-            return "failed '%s' with %s in test %s" (self.command, self.failed_test[0], self.failed_test[1])
>-        else:
>-            return "failed '%s' with exit code %d in unknown test" % (self.command, self.exitcode)
>-
>-
>-class TreeBuilder(object):
>-    '''handle build of one directory'''
>-
>-    def __init__(self, name, sequence, fail_quickly=False):
>-        self.name = name
>-        self.fail_quickly = fail_quickly
>-
>-        self.tag = self.name.replace('/', '_')
>-        self.sequence = sequence
>-        self.next = 0
>-        self.stages = []
>-        self.stdout_path = os.path.join(gitroot, "%s.stdout" % (self.tag, ))
>-        self.stderr_path = os.path.join(gitroot, "%s.stderr" % (self.tag, ))
>-        self.logfiles = [
>-            (self.stdout_path, os.path.basename(self.stdout_path), "text/plain"),
>-            (self.stderr_path, os.path.basename(self.stderr_path), "text/plain"),
>-            ]
>-        if options.verbose:
>-            print("stdout for %s in %s" % (self.name, self.stdout_path))
>-            print("stderr for %s in %s" % (self.name, self.stderr_path))
>-        if os.path.exists(self.stdout_path):
>-            os.unlink(self.stdout_path)
>-        if os.path.exists(self.stderr_path):
>-            os.unlink(self.stderr_path)
>-        self.stdout = open(self.stdout_path, 'w')
>-        self.stderr = open(self.stderr_path, 'w')
>-        self.sdir = os.path.join(testbase, self.tag)
>-        if name in ['pass', 'fail', 'retry']:
>-            self.dir = self.sdir
>-        else:
>-            self.dir = os.path.join(self.sdir, self.name)
>-        self.prefix = os.path.join(testbase, "prefix", self.tag)
>-        run_cmd(["rm", "-rf", self.sdir])
>-        cleanup_list.append(self.sdir)
>-        cleanup_list.append(self.prefix)
>-        os.makedirs(self.sdir)
>-        run_cmd(["rm",  "-rf", self.sdir])
>-        clone_gitroot(self.sdir, revision)
>-        self.start_next()
>-        self.exitcode = None
>-
>-    def start_next(self):
>-        if self.next == len(self.sequence):
>-            print '%s: Completed OK' % self.name
>-            self.done = True
>-            self.stdout.close()
>-            self.stderr.close()
>-            return
>-        (stage_name, cmd, output_mime_type) = self.sequence[self.next]
>-        cmd = cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
>-        if output_mime_type == "text/plain":
>-            self.current_stage = PlainTreeStageBuilder(self, stage_name, cmd,
>-                self.fail_quickly)
>-        elif output_mime_type == "text/x-subunit":
>-            self.current_stage = SubunitTreeStageBuilder(self, stage_name, cmd,
>-                self.fail_quickly)
>-        else:
>-            raise Exception("Unknown output mime type %s" % output_mime_type)
>-        self.stages.append(self.current_stage)
>-        self.current_stage.start()
>-        self.next += 1
>-
>-    def remove_logs(self):
>-        for path, name, mime_type in self.logfiles:
>-            os.unlink(path)
>-
>-    def poll(self):
>-        self.exitcode = self.current_stage.poll()
>-        if self.exitcode is not None:
>-            self.current_stage = None
>-        return self.exitcode
>-
>-    def kill(self):
>-        if self.current_stage is not None:
>-            self.current_stage.kill()
>-            self.current_stage = None
>-
>-    @property
>-    def failed(self):
>-        return any([s.failed for s in self.stages])
>-
>-    @property
>-    def failed_stage(self):
>-        for s in self.stages:
>-            if s.failed:
>-                return s
>-        return s
>-
>-    @property
>-    def failure_reason(self):
>-        return "%s: [%s] %s" % (self.name, self.failed_stage.name,
>-            self.failed_stage.failure_reason)
>-
>-
>-class BuildList(object):
>-    '''handle build of multiple directories'''
>-
>-    def __init__(self, tasklist, tasknames):
>-        global tasks
>-        self.tlist = []
>-        self.tail_proc = None
>-        self.retry = None
>-        if tasknames == ['pass']:
>-            tasks = { 'pass' : [ ("pass", '/bin/true', "text/plain") ]}
>-        if tasknames == ['fail']:
>-            tasks = { 'fail' : [ ("fail", '/bin/false', "text/plain") ]}
>-        if tasknames == []:
>-            tasknames = tasklist
>-        for n in tasknames:
>-            b = TreeBuilder(n, tasks[n], not options.fail_slowly)
>-            self.tlist.append(b)
>-        if options.retry:
>-            self.retry = RetryChecker(self.sdir)
>-            self.need_retry = False
>-
>-    def kill_kids(self):
>-        if self.tail_proc is not None:
>-            self.tail_proc.terminate()
>-            self.tail_proc.wait()
>-            self.tail_proc = None
>-        if self.retry is not None:
>-            self.retry.kill()
>-        for b in self.tlist:
>-            b.kill()
>-
>-    def wait_one(self):
>-        while True:
>-            none_running = True
>-            for b in self.tlist:
>-                if b.current_stage is None:
>-                    continue
>-                none_running = False
>-                if b.poll() is None:
>-                    continue
>-                return b
>-            if options.retry:
>-                ret = self.retry.poll()
>-                if ret:
>-                    self.need_retry = True
>-                    self.retry = None
>-                    return None
>-            if none_running:
>-                return None
>-            time.sleep(0.1)
>-
>-    def run(self):
>-        while True:
>-            b = self.wait_one()
>-            if options.retry and self.need_retry:
>-                self.kill_kids()
>-                print("retry needed")
>-                return (0, None, None, None, "retry")
>-            if b is None:
>-                break
>-            if b.failed:
>-                self.kill_kids()
>-                return (b.exitcode, b.name, b.failed_stage, b.tag, b.failure_reason)
>-            b.start_next()
>-        self.kill_kids()
>-        return (0, None, None, None, "All OK")
>-
>-    def tarlogs(self, name=None, fileobj=None):
>-        tar = tarfile.open(name=name, fileobj=fileobj, mode="w:gz")
>-        for b in self.tlist:
>-            for (path, name, mime_type) in b.logfiles:
>-                tar.add(path, arcname=name)
>-        if os.path.exists("autobuild.log"):
>-            tar.add("autobuild.log")
>-        tar.close()
>-
>-    def attach_logs(self, outer):
>-        f = StringIO()
>-        self.tarlogs(fileobj=f)
>-        msg = MIMEApplication(f.getvalue(), "x-gzip")
>-        msg.add_header('Content-Disposition', 'attachment',
>-                       filename="logs.tar.gz")
>-        outer.attach(msg)
>-
>-    def remove_logs(self):
>-        for b in self.tlist:
>-            b.remove_logs()
>-
>-    def start_tail(self):
>-        cmd = "tail -f *.stdout *.stderr"
>-        self.tail_proc = Popen(cmd, shell=True, cwd=gitroot)
>-
>-
>-def cleanup():
>-    if options.nocleanup:
>-        return
>-    print("Cleaning up ....")
>-    for d in cleanup_list:
>-        run_cmd(["rm", "-rf", d])
>-
>-
>-def find_git_root(p):
>-    '''get to the top of the git repo'''
>-    while p != '/':
>-        if os.path.isdir(os.path.join(p, ".git")):
>-            return p
>-        p = os.path.abspath(os.path.join(p, '..'))
>-    return None
>-
>-
>-def daemonize(logfile):
>-    pid = os.fork()
>-    if pid == 0: # Parent
>-        os.setsid()
>-        pid = os.fork()
>-        if pid != 0: # Actual daemon
>-            os._exit(0)
>-    else: # Grandparent
>-        os._exit(0)
>-
>-    import resource      # Resource usage information.
>-    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
>-    if maxfd == resource.RLIM_INFINITY:
>-        maxfd = 1024 # Rough guess at maximum number of open file descriptors.
>-    for fd in range(0, maxfd):
>-        try:
>-            os.close(fd)
>-        except OSError:
>-            pass
>-    os.open(logfile, os.O_RDWR | os.O_CREAT)
>-    os.dup2(0, 1)
>-    os.dup2(0, 2)
>-
>-
>-def rebase_tree(url):
>-    print("Rebasing on %s" % url)
>-    run_cmd(["git", "remote", "add", "-t", "master", "master", url], show=True,
>-            dir=test_master)
>-    run_cmd(["git", "fetch", "master"], show=True, dir=test_master)
>-    if options.fix_whitespace:
>-        run_cmd(["git", "rebase", "--whitespace=fix", "master/master"],
>-                show=True, dir=test_master)
>-    else:
>-        run_cmd(["git", "rebase", "master/master"], show=True, dir=test_master)
>-    diff = run_cmd(["git", "--no-pager", "diff", "HEAD", "master/master"],
>-        dir=test_master, output=True)
>-    if diff == '':
>-        print("No differences between HEAD and master/master - exiting")
>-        sys.exit(0)
>-
>-def push_to(url):
>-    print("Pushing to %s" % url)
>-    if options.mark:
>-        run_cmd("EDITOR=script/commit_mark.sh git commit --amend -c HEAD",
>-            dir=test_master, shell=True)
>-        # the notes method doesn't work yet, as metze hasn't allowed
>-        # refs/notes/* in master
>-        # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD",
>-        #     dir=test_master)
>-    run_cmd(["git", "remote", "add", "-t", "master", "pushto", url], show=True,
>-        dir=test_master)
>-    run_cmd(["git", "push", "pushto", "+HEAD:master"], show=True,
>-        dir=test_master)
>-
>-def_testbase = os.getenv("AUTOBUILD_TESTBASE")
>-if def_testbase is None:
>-    if os.path.exists("/memdisk"):
>-        def_testbase = "/memdisk/%s" % os.getenv('USER')
>-    else:
>-        def_testbase = os.path.join(tempfile.gettempdir(), "autobuild-%s" % os.getenv("USER"))
>-
>-parser = OptionParser()
>-parser.add_option("--repository", help="repository to run tests for", default=None, type=str)
>-parser.add_option("--revision", help="revision to compile if not HEAD", default=None, type=str)
>-parser.add_option("--tail", help="show output while running", default=False, action="store_true")
>-parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
>-parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
>-parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
>-                  default=def_testbase)
>-parser.add_option("--passcmd", help="command to run on success", default=None)
>-parser.add_option("--verbose", help="show all commands as they are run",
>-                  default=False, action="store_true")
>-parser.add_option("--rebase", help="rebase on the given tree before testing",
>-                  default=None, type='str')
>-parser.add_option("--rebase-master", help="rebase on %s before testing" % samba_master,
>-                  default=False, action='store_true')
>-parser.add_option("--pushto", help="push to a git url on success",
>-                  default=None, type='str')
>-parser.add_option("--push-master", help="push to %s on success" % samba_master_ssh,
>-                  default=False, action='store_true')
>-parser.add_option("--mark", help="add a Tested-By signoff before pushing",
>-                  default=False, action="store_true")
>-parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
>-                  default=False, action="store_true")
>-parser.add_option("--retry", help="automatically retry if master changes",
>-                  default=False, action="store_true")
>-parser.add_option("--email", help="send email to the given address on failure",
>-                  type='str', default=None)
>-parser.add_option("--always-email", help="always send email, even on success",
>-                  action="store_true")
>-parser.add_option("--daemon", help="daemonize after initial setup",
>-                  action="store_true")
>-parser.add_option("--fail-slowly", help="continue running tests even after one has already failed",
>-                  action="store_true")
>-
>-
>-def email_failure(blist, exitcode, failed_task, failed_stage, failed_tag, errstr):
>-    '''send an email to options.email about the failure'''
>-    user = os.getenv("USER")
>-    text = '''
>-Dear Developer,
>-
>-Your autobuild failed when trying to test %s with the following error:
>-   %s
>-
>-the autobuild has been abandoned. Please fix the error and resubmit.
>-
>-You can see logs of the failed task here:
>-
>-  http://git.samba.org/%s/samba-autobuild/%s.stdout
>-  http://git.samba.org/%s/samba-autobuild/%s.stderr
>-
>-A summary of the autobuild process is here:
>-
>-  http://git.samba.org/%s/samba-autobuild/autobuild.log
>-
>-or you can get full logs of all tasks in this job here:
>-
>-  http://git.samba.org/%s/samba-autobuild/logs.tar.gz
>-
>-The top commit for the tree that was built was:
>-
>-%s
>-
>-''' % (failed_task, errstr, user, failed_tag, user, failed_tag, user, user,
>-       get_top_commit_msg(test_master))
>-
>-    msg = MIMEMultipart()
>-    msg['Subject'] = 'autobuild failure for task %s during %s' % (
>-        failed_task, failed_stage.name)
>-    msg['From'] = 'autobuild@samba.org'
>-    msg['To'] = options.email
>-
>-    main = MIMEText(text)
>-    msg.attach(main)
>-
>-    blist.attach_logs(msg)
>-
>-    s = smtplib.SMTP()
>-    s.connect()
>-    s.sendmail(msg['From'], [msg['To']], msg.as_string())
>-    s.quit()
>-
>-def email_success(blist):
>-    '''send an email to options.email about a successful build'''
>-    user = os.getenv("USER")
>-    text = '''
>-Dear Developer,
>-
>-Your autobuild has succeeded.
>-
>-'''
>-
>-    if options.keeplogs:
>-        text += '''
>-
>-you can get full logs of all tasks in this job here:
>-
>-  http://git.samba.org/%s/samba-autobuild/logs.tar.gz
>-
>-''' % user
>-
>-    text += '''
>-The top commit for the tree that was built was:
>-
>-%s
>-''' % (get_top_commit_msg(test_master),)
>-
>-    msg = MIMEMultipart()
>-    msg['Subject'] = 'autobuild success'
>-    msg['From'] = 'autobuild@samba.org'
>-    msg['To'] = options.email
>-
>-    main = MIMEText(text, 'plain')
>-    msg.attach(main)
>-
>-    blist.attach_logs(msg)
>-
>-    s = smtplib.SMTP()
>-    s.connect()
>-    s.sendmail(msg['From'], [msg['To']], msg.as_string())
>-    s.quit()
>-
>-
>-(options, args) = parser.parse_args()
>-
>-if options.retry:
>-    if not options.rebase_master and options.rebase is None:
>-        raise Exception('You can only use --retry if you also rebase')
>-
>-testbase = os.path.join(options.testbase, "b%u" % (os.getpid(),))
>-test_master = os.path.join(testbase, "master")
>-
>-if options.repository is not None:
>-    repository = options.repository
>-else:
>-    repository = os.getcwd()
>-
>-gitroot = find_git_root(repository)
>-if gitroot is None:
>-    raise Exception("Failed to find git root under %s" % repository)
>-
>-# get the top commit message, for emails
>-if options.revision is not None:
>-    revision = options.revision
>-else:
>-    revision = "HEAD"
>-
>-def get_top_commit_msg(reporoot):
>-    return run_cmd(["git", "log", "-1"], dir=reporoot, output=True)
>-
>-try:
>-    os.makedirs(testbase)
>-except Exception, reason:
>-    raise Exception("Unable to create %s : %s" % (testbase, reason))
>-cleanup_list.append(testbase)
>-
>-if options.daemon:
>-    logfile = os.path.join(testbase, "log")
>-    print "Forking into the background, writing progress to %s" % logfile
>-    daemonize(logfile)
>-
>-while True:
>-    try:
>-        run_cmd(["rm", "-rf", test_master])
>-        cleanup_list.append(test_master)
>-        clone_gitroot(test_master, revision)
>-    except:
>-        cleanup()
>-        raise
>-
>-    try:
>-        if options.rebase is not None:
>-            rebase_tree(options.rebase)
>-        elif options.rebase_master:
>-            rebase_tree(samba_master)
>-        blist = BuildList(tasks, args)
>-        if options.tail:
>-            blist.start_tail()
>-        (exitcode, failed_task, failed_stage, failed_tag, errstr) = blist.run()
>-        if exitcode != 0 or errstr != "retry":
>-            break
>-        cleanup()
>-    except:
>-        cleanup()
>-        raise
>-
>-blist.kill_kids()
>-if options.tail:
>-    print("waiting for tail to flush")
>-    time.sleep(1)
>-
>-if exitcode == 0:
>-    print errstr
>-    if options.passcmd is not None:
>-        print("Running passcmd: %s" % options.passcmd)
>-        run_cmd(options.passcmd, dir=test_master, shell=True)
>-    if options.pushto is not None:
>-        push_to(options.pushto)
>-    elif options.push_master:
>-        push_to(samba_master_ssh)
>-    if options.keeplogs:
>-        blist.tarlogs("logs.tar.gz")
>-        print("Logs in logs.tar.gz")
>-    if options.always_email:
>-        email_success(blist)
>-    blist.remove_logs()
>-    cleanup()
>-    print(errstr)
>-else:
>-    # something failed, gather a tar of the logs
>-    blist.tarlogs("logs.tar.gz")
>-
>-    if options.email is not None:
>-        email_failure(blist, exitcode, failed_task, failed_stage, failed_tag,
>-            errstr)
>-
>-    cleanup()
>-    print(errstr)
>-    print("Logs in logs.tar.gz")
>-sys.exit(exitcode)
>-- 
>1.7.9.5
>

Flags:

kseeger: review+

Actions: View

Attachments on bug 9277: 8031