The Samba-Bugzilla – Attachment 13146 Details for
Bug 12703
Backport autobuild improvements
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Patches for v4-6-test
tmp46.diff.txt (text/plain), 13.24 KB, created by
Stefan Metzmacher
on 2017-04-07 16:06:32 UTC
(
hide
)
Description:
Patches for v4-6-test
Filename:
MIME Type:
Creator:
Stefan Metzmacher
Created:
2017-04-07 16:06:32 UTC
Size:
13.24 KB
patch
obsolete
>From e19d9b0fd736f3df3ce8a70c8c8743a8d77b5c1f Mon Sep 17 00:00:00 2001 >From: Stefan Metzmacher <metze@samba.org> >Date: Wed, 11 Jan 2017 14:42:08 +0100 >Subject: [PATCH 1/6] script/autobuild.py: cleanup the task subdirs when > they're done. > >This hopefully reduces the used space on the memdisk. > >Signed-off-by: Stefan Metzmacher <metze@samba.org> >Reviewed-by: Andreas Schneider <asn@samba.org> >(cherry picked from commit f9e188747753225e77f254fe41aad95ff11fec53) >--- > script/autobuild.py | 4 ++++ > 1 file changed, 4 insertions(+) > >diff --git a/script/autobuild.py b/script/autobuild.py >index 3d76130..e04452c 100755 >--- a/script/autobuild.py >+++ b/script/autobuild.py >@@ -278,6 +278,7 @@ class builder(object): > self.sdir = "%s/%s" % (testbase, self.tag) > self.prefix = "%s/%s" % (test_prefix, self.tag) > run_cmd("rm -rf %s" % self.sdir) >+ run_cmd("rm -rf %s" % self.prefix) > if cp: > run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True) > else: >@@ -286,6 +287,9 @@ class builder(object): > > def start_next(self): > if self.next == len(self.sequence): >+ if not options.nocleanup: >+ run_cmd("rm -rf %s" % self.sdir) >+ run_cmd("rm -rf %s" % self.prefix) > print '%s: Completed OK' % self.name > self.done = True > return >-- >1.9.1 > > >From 1bd3c3abe88f1cfc0c1208ed7a74428468e5dd2f Mon Sep 17 00:00:00 2001 >From: Stefan Metzmacher <metze@samba.org> >Date: Wed, 11 Jan 2017 14:48:45 +0100 >Subject: [PATCH 2/6] script/autobuild.py: export PYTHONUNBUFFERED=1 > >Signed-off-by: Stefan Metzmacher <metze@samba.org> >Reviewed-by: Andreas Schneider <asn@samba.org> >(cherry picked from commit 5a8d7a5446c23985a7dd3a9cb4856481b94931db) >--- > script/autobuild.py | 2 ++ > 1 file changed, 2 insertions(+) > >diff --git a/script/autobuild.py b/script/autobuild.py >index e04452c..d68a437 100755 >--- a/script/autobuild.py >+++ b/script/autobuild.py >@@ -15,6 +15,8 @@ from email.mime.multipart import MIMEMultipart > from distutils.sysconfig import get_python_lib > import platform > >+os.environ["PYTHONUNBUFFERED"] = "1" >+ > # This speeds up testing remarkably. > os.environ['TDB_NO_FSYNC'] = '1' > >-- >1.9.1 > > >From 0cd0c0b385da35de1332097014a1de03e2f9d5a0 Mon Sep 17 00:00:00 2001 >From: Stefan Metzmacher <metze@samba.org> >Date: Wed, 11 Jan 2017 15:02:17 +0100 >Subject: [PATCH 3/6] script/autobuild.py: add a do_print() wrapper function > that flushes after each message > >Signed-off-by: Stefan Metzmacher <metze@samba.org> >Reviewed-by: Andreas Schneider <asn@samba.org> >(cherry picked from commit 96277a9f82379c7fedf36ca13644eb3493dcd1e2) >--- > script/autobuild.py | 41 +++++++++++++++++++++++------------------ > 1 file changed, 23 insertions(+), 18 deletions(-) > >diff --git a/script/autobuild.py b/script/autobuild.py >index d68a437..dcb407f 100755 >--- a/script/autobuild.py >+++ b/script/autobuild.py >@@ -245,11 +245,16 @@ tasks = { > 'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ] > } > >+def do_print(msg): >+ print "%s" % msg >+ sys.stdout.flush() >+ sys.stderr.flush() >+ > def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): > if show is None: > show = options.verbose > if show: >- print("Running: '%s' in '%s'" % (cmd, dir)) >+ do_print("Running: '%s' in '%s'" % (cmd, dir)) > if output: > return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0] > elif checkfail: >@@ -271,8 +276,8 @@ class builder(object): > self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag) > self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag) > if options.verbose: >- print("stdout for %s in %s" % (self.name, self.stdout_path)) >- print("stderr for %s in %s" % (self.name, self.stderr_path)) >+ do_print("stdout for %s in %s" % (self.name, self.stdout_path)) >+ do_print("stderr for %s in %s" % (self.name, self.stderr_path)) > run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path)) > self.stdout = open(self.stdout_path, 'w') > self.stderr = open(self.stderr_path, 'w') >@@ -292,7 +297,7 @@ class builder(object): > if not options.nocleanup: > run_cmd("rm -rf %s" % self.sdir) > run_cmd("rm -rf %s" % self.prefix) >- print '%s: Completed OK' % self.name >+ do_print('%s: Completed OK' % self.name) > self.done = True > return > (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next] >@@ -303,7 +308,7 @@ class builder(object): > self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests) > # if self.output_mime_type == "text/x-subunit": > # self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit")) >- print '%s: [%s] Running %s' % (self.name, self.stage, self.cmd) >+ do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd)) > cwd = os.getcwd() > os.chdir("%s/%s" % (self.sdir, self.dir)) > self.proc = Popen(self.cmd, shell=True, >@@ -401,7 +406,7 @@ class buildlist(object): > b = self.wait_one() > if options.retry and self.need_retry: > self.kill_kids() >- print("retry needed") >+ do_print("retry needed") > return (0, None, None, None, "retry") > if b is None: > break >@@ -449,7 +454,7 @@ class buildlist(object): > def cleanup(): > if options.nocleanup: > return >- print("Cleaning up ....") >+ do_print("Cleaning up ....") > for d in cleanup_list: > run_cmd("rm -rf %s" % d) > >@@ -496,7 +501,7 @@ def write_pidfile(fname): > > def rebase_tree(rebase_url, rebase_branch = "master"): > rebase_remote = "rebaseon" >- print("Rebasing on %s" % rebase_url) >+ do_print("Rebasing on %s" % rebase_url) > run_cmd("git describe HEAD", show=True, dir=test_master) > run_cmd("git remote add -t %s %s %s" % > (rebase_branch, rebase_remote, rebase_url), >@@ -514,7 +519,7 @@ def rebase_tree(rebase_url, rebase_branch = "master"): > (rebase_remote, rebase_branch), > dir=test_master, output=True) > if diff == '': >- print("No differences between HEAD and %s/%s - exiting" % >+ do_print("No differences between HEAD and %s/%s - exiting" % > (rebase_remote, rebase_branch)) > sys.exit(0) > run_cmd("git describe %s/%s" % >@@ -527,7 +532,7 @@ def rebase_tree(rebase_url, rebase_branch = "master"): > > def push_to(push_url, push_branch = "master"): > push_remote = "pushto" >- print("Pushing to %s" % push_url) >+ do_print("Pushing to %s" % push_url) > if options.mark: > run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master) > run_cmd("git commit --amend -c HEAD", dir=test_master) >@@ -729,7 +734,7 @@ cleanup_list.append(testbase) > > if options.daemon: > logfile = os.path.join(testbase, "log") >- print "Forking into the background, writing progress to %s" % logfile >+ do_print("Forking into the background, writing progress to %s" % logfile) > daemonize(logfile) > > write_pidfile(gitroot + "/autobuild.pid") >@@ -774,25 +779,25 @@ cleanup_list.append(gitroot + "/autobuild.pid") > > blist.kill_kids() > if options.tail: >- print("waiting for tail to flush") >+ do_print("waiting for tail to flush") > time.sleep(1) > > elapsed_time = time.time() - start_time > if status == 0: >- print errstr >+ do_print(errstr) > if options.passcmd is not None: >- print("Running passcmd: %s" % options.passcmd) >+ do_print("Running passcmd: %s" % options.passcmd) > run_cmd(options.passcmd, dir=test_master) > if options.pushto is not None: > push_to(options.pushto, push_branch=options.branch) > if options.keeplogs or options.attach_logs: > blist.tarlogs("logs.tar.gz") >- print("Logs in logs.tar.gz") >+ do_print("Logs in logs.tar.gz") > if options.always_email: > email_success(elapsed_time, log_base=options.log_base) > blist.remove_logs() > cleanup() >- print(errstr) >+ do_print(errstr) > sys.exit(0) > > # something failed, gather a tar of the logs >@@ -821,6 +826,6 @@ the autobuild has been abandoned. Please fix the error and resubmit. > ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr) > > cleanup() >-print(errstr) >-print("Logs in logs.tar.gz") >+do_print(errstr) >+do_print("Logs in logs.tar.gz") > sys.exit(status) >-- >1.9.1 > > >From a28bd9e6696027ad89622e24eb8503df038d34bb Mon Sep 17 00:00:00 2001 >From: Stefan Metzmacher <metze@samba.org> >Date: Wed, 11 Jan 2017 14:13:00 +0100 >Subject: [PATCH 4/6] script/autobuild.py: try to make TMPDIR handling more > verbose > >This hopefully gives some hints regarding flakey tests where >the tmpdir is not available. > >Signed-off-by: Stefan Metzmacher <metze@samba.org> >Reviewed-by: Andreas Schneider <asn@samba.org> >(cherry picked from commit 278c921263550c1473df8944260bbb4e62a0e0e6) >--- > script/autobuild.py | 11 +++++++---- > 1 file changed, 7 insertions(+), 4 deletions(-) > >diff --git a/script/autobuild.py b/script/autobuild.py >index dcb407f..06c9b70 100755 >--- a/script/autobuild.py >+++ b/script/autobuild.py >@@ -454,6 +454,8 @@ class buildlist(object): > def cleanup(): > if options.nocleanup: > return >+ run_cmd("stat %s" % test_tmpdir, show=True) >+ run_cmd("stat %s" % testbase, show=True) > do_print("Cleaning up ....") > for d in cleanup_list: > run_cmd("rm -rf %s" % d) >@@ -743,10 +745,10 @@ start_time = time.time() > > while True: > try: >- run_cmd("rm -rf %s" % test_master) >- run_cmd("rm -rf %s" % test_prefix) >- run_cmd("rm -rf %s" % test_tmpdir) >+ run_cmd("rm -rf %s" % test_tmpdir, show=True) > os.makedirs(test_tmpdir) >+ run_cmd("stat %s" % test_tmpdir, show=True) >+ run_cmd("stat %s" % testbase, show=True) > run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot) > except Exception: > cleanup() >@@ -777,6 +779,8 @@ while True: > > cleanup_list.append(gitroot + "/autobuild.pid") > >+do_print(errstr) >+ > blist.kill_kids() > if options.tail: > do_print("waiting for tail to flush") >@@ -784,7 +788,6 @@ if options.tail: > > elapsed_time = time.time() - start_time > if status == 0: >- do_print(errstr) > if options.passcmd is not None: > do_print("Running passcmd: %s" % options.passcmd) > run_cmd(options.passcmd, dir=test_master) >-- >1.9.1 > > >From afe35a566749370ab646db71c06e34440f43de81 Mon Sep 17 00:00:00 2001 >From: Stefan Metzmacher <metze@samba.org> >Date: Tue, 21 Feb 2017 17:05:08 +0100 >Subject: [PATCH 5/6] script/autobuild.py: ignore missing test_tmpdir > >It is still unknown what removes it... > >Signed-off-by: Stefan Metzmacher <metze@samba.org> >Reviewed-by: Ralph Boehme <slow@samba.org> >(cherry picked from commit cad23629ac48253e508fd9bead2bb79bfa7ee3b8) >--- > script/autobuild.py | 2 +- > 1 file changed, 1 insertion(+), 1 deletion(-) > >diff --git a/script/autobuild.py b/script/autobuild.py >index 06c9b70..b56f9133 100755 >--- a/script/autobuild.py >+++ b/script/autobuild.py >@@ -454,7 +454,7 @@ class buildlist(object): > def cleanup(): > if options.nocleanup: > return >- run_cmd("stat %s" % test_tmpdir, show=True) >+ run_cmd("stat %s || true" % test_tmpdir, show=True) > run_cmd("stat %s" % testbase, show=True) > do_print("Cleaning up ....") > for d in cleanup_list: >-- >1.9.1 > > >From 74087f8ba1054c2e7798c0ddfed4f924aa262031 Mon Sep 17 00:00:00 2001 >From: Martin Schwenke <martin@meltin.net> >Date: Mon, 20 Mar 2017 14:49:34 +1100 >Subject: [PATCH 6/6] autobuild: Stop waf uninstall from removing test_tmpdir > >Most of the autobuild tasks run "make distcheck", which does a >recursive "waf configure make install uninstall". "waf uninstall" >(via BuildContext.install() in Build.py) removes empty directories all >the way up the directory tree. This means that it removes >test_tmpdir, if it is empty, and any empty directories above it. > >While this is arguably a waf bug, the simplest solution is to make >test_tmpdir non-empty so it don't get removed. > >BUG: https://bugzilla.samba.org/show_bug.cgi?id=12703 > >Signed-off-by: Martin Schwenke <martin@meltin.net> >Reviewed-by: Stefan Metzmacher <metze@samba.org> > >Autobuild-User(master): Stefan Metzmacher <metze@samba.org> >Autobuild-Date(master): Tue Mar 21 10:37:08 CET 2017 on sn-devel-144 > >(cherry picked from commit 05b5af4ae5fbc9b59c857468512858f73e5dea1b) >--- > script/autobuild.py | 5 +++++ > 1 file changed, 5 insertions(+) > >diff --git a/script/autobuild.py b/script/autobuild.py >index b56f9133..e46869c 100755 >--- a/script/autobuild.py >+++ b/script/autobuild.py >@@ -747,6 +747,11 @@ while True: > try: > run_cmd("rm -rf %s" % test_tmpdir, show=True) > os.makedirs(test_tmpdir) >+ # The waf uninstall code removes empty directories all the way >+ # up the tree. Creating a file in test_tmpdir stops it from >+ # being removed. >+ run_cmd("touch %s" % os.path.join(test_tmpdir, >+ ".directory-is-not-empty"), show=True) > run_cmd("stat %s" % test_tmpdir, show=True) > run_cmd("stat %s" % testbase, show=True) > run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot) >-- >1.9.1 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Raw
Flags:
slow
:
review+
Actions:
View
Attachments on
bug 12703
: 13146