From a5508958f17e8608d8c734abfde95eb73a95d05e Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Thu, 26 Aug 2021 14:52:14 +0200 Subject: [PATCH 1/5] third_party: Add a script to update waf ./third_party/waf/update.sh Signed-off-by: Andreas Schneider Reviewed-by: Andrew Bartlett (cherry picked from commit e41bc0f43f6d86d554f37881263c43c356994726) --- third_party/update.sh | 5 --- third_party/waf/update.sh | 79 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) create mode 100755 third_party/waf/update.sh diff --git a/third_party/update.sh b/third_party/update.sh index a510e8a7042..29456991874 100755 --- a/third_party/update.sh +++ b/third_party/update.sh @@ -23,9 +23,4 @@ hg clone https://bitbucket.org/micktwomey/pyiso8601 "$WORKDIR/pyiso8601" rm -rf "$WORKDIR/pyiso8601/.hg" rsync -avz --delete "$WORKDIR/pyiso8601/" "$THIRD_PARTY_DIR/pyiso8601/" -echo "Updating waf..." -git clone git://git.samba.org/third_party/waf.waf15/ "$WORKDIR/waf" -rm -rf "$WORKDIR/waf/.git" -rsync -C -avz --delete "$WORKDIR/waf/" "$THIRD_PARTY_DIR/waf/" - rm -rf "$WORKDIR" diff --git a/third_party/waf/update.sh b/third_party/waf/update.sh new file mode 100755 index 00000000000..16bda84a3f0 --- /dev/null +++ b/third_party/waf/update.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +if [[ $# -lt 1 ]]; then + echo "Usage: update.sh VERSION" + exit 1 +fi + +WAF_VERSION="${1}" +WAF_GIT="https://gitlab.com/ita1024/waf.git" +WAF_UPDATE_SCRIPT="$(readlink -f "$0")" +WAF_SAMBA_DIR="$(dirname "${WAF_UPDATE_SCRIPT}")" +WAF_TMPDIR=$(mktemp --tmpdir -d waf-XXXXXXXX) + +echo "VERSION: ${WAF_VERSION}" +echo "GIT URL: ${WAF_GIT}" +echo "WAF SAMBA DIR: ${WAF_SAMBA_DIR}" +echo "WAF TMP DIR: ${WAF_TMPDIR}" + +cleanup_tmpdir() { + popd 2>/dev/null || true + rm -rf "$WAF_TMPDIR" +} +trap cleanup_tmpdir SIGINT + +cleanup_and_exit() { + cleanup_tmpdir + if test "$1" = 0 -o -z "$1" ; then + exit 0 + else + exit "$1" + fi +} + +# Checkout the git tree +mkdir -p "${WAF_TMPDIR}" +pushd "${WAF_TMPDIR}" || cleanup_and_exit 1 + +git clone "${WAF_GIT}" +ret=$? +if [ $ret -ne 0 ]; then + echo "ERROR: Failed to clone repository" + cleanup_and_exit 1 +fi + + +pushd waf || cleanup_and_exit 1 +git checkout -b "waf-${WAF_VERSION}" "waf-${WAF_VERSION}" +ret=$? +if [ $ret -ne 0 ]; then + echo "ERROR: Failed to checkout waf-${WAF_VERSION} repository" + cleanup_and_exit 1 +fi +popd || cleanup_and_exit 1 + +popd || cleanup_and_exit 1 + +# Update waflib +pushd "${WAF_SAMBA_DIR}" || cleanup_and_exit 1 +pwd + +rm -rf waflib/ +rsync -av "${WAF_TMPDIR}/waf/waflib" . +ret=$? +if [ $ret -ne 0 ]; then + echo "ERROR: Failed copy waflib" + cleanup_and_exit 1 +fi +chmod -x waflib/Context.py + +git add waflib + +popd || cleanup_and_exit 1 + +echo +echo "Now please change VERSION in buildtools/bin/waf and" +echo "Context.HEXVERSION in buildtools/wafsamba/wafsamba.py" +echo + +cleanup_and_exit 0 -- 2.36.1 From e8526dd34e58d27cbd89ea82e78daf322e97e06d Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Wed, 25 Aug 2021 15:34:58 +0200 Subject: [PATCH 2/5] third_party: Update waf to version 2.0.22 New in waf 2.0.22 * Fix stdin propagation with faulty vcvarsall scripts #2315 * Enable mixing Unix-style paths with destdir on Windows platforms #2337 * Fix shell escaping unit test parameters #2314 * Improve extras/clang_compilation_database and extras/swig compatibility #2336 * Propagate C++ flags to the Cuda compiler in extras/cuda #2311 * Fix detection of Qt 5.0.0 (preparation for Qt6) #2331 * Enable Haxe processing #2308 * Fix regression in MACOSX_DEPLOYMENT_TARGET caused by distutils #2330 * Fix extras/wafcache concurrent trimming issues #2312 * Fix extras/wafcache symlink handling #2327 The import was done like this: ./third_party/waf/update.sh Then changing buildtools/bin/waf and buildtools/wafsamba/wafsamba.py by hand. Pair-Programmed-With: Stefan Metzmacher Signed-off-by: Andreas Schneider Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Thu Sep 2 21:22:17 UTC 2021 on sn-devel-184 (cherry picked from commit 59ed09928541d40df72592419247add608a54aca) --- buildtools/bin/waf | 2 +- buildtools/wafsamba/wafsamba.py | 2 +- third_party/waf/waflib/Build.py | 4 +- third_party/waf/waflib/Context.py | 6 +- third_party/waf/waflib/Tools/msvc.py | 2 +- third_party/waf/waflib/Tools/python.py | 2 +- third_party/waf/waflib/Tools/qt5.py | 6 +- third_party/waf/waflib/Tools/waf_unit_test.py | 2 +- third_party/waf/waflib/Utils.py | 15 +- .../extras/clang_compilation_database.py | 28 ++-- third_party/waf/waflib/extras/haxe.py | 131 ++++++++++++++++++ third_party/waf/waflib/extras/wafcache.py | 59 ++++++-- 12 files changed, 215 insertions(+), 44 deletions(-) create mode 100644 third_party/waf/waflib/extras/haxe.py diff --git a/buildtools/bin/waf b/buildtools/bin/waf index 041450fc131..b0ccb09a877 100755 --- a/buildtools/bin/waf +++ b/buildtools/bin/waf @@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE. import os, sys, inspect -VERSION="2.0.21" +VERSION="2.0.22" REVISION="x" GIT="x" INSTALL="x" diff --git a/buildtools/wafsamba/wafsamba.py b/buildtools/wafsamba/wafsamba.py index 0427c90ca80..865975cb2d1 100644 --- a/buildtools/wafsamba/wafsamba.py +++ b/buildtools/wafsamba/wafsamba.py @@ -38,7 +38,7 @@ LIB_PATH="shared" os.environ['PYTHONUNBUFFERED'] = '1' -if Context.HEXVERSION not in (0x2001500,): +if Context.HEXVERSION not in (0x2001600,): Logs.error(''' Please use the version of waf that comes with Samba, not a system installed version. See http://wiki.samba.org/index.php/Waf diff --git a/third_party/waf/waflib/Build.py b/third_party/waf/waflib/Build.py index 52837618577..b49dd8302b1 100644 --- a/third_party/waf/waflib/Build.py +++ b/third_party/waf/waflib/Build.py @@ -1066,9 +1066,9 @@ class inst(Task.Task): else: dest = os.path.normpath(Utils.subst_vars(self.install_to, self.env)) if not os.path.isabs(dest): - dest = os.path.join(self.env.PREFIX, dest) + dest = os.path.join(self.env.PREFIX, dest) if destdir and Options.options.destdir: - dest = os.path.join(Options.options.destdir, os.path.splitdrive(dest)[1].lstrip(os.sep)) + dest = Options.options.destdir.rstrip(os.sep) + os.sep + os.path.splitdrive(dest)[1].lstrip(os.sep) return dest def copy_fun(self, src, tgt): diff --git a/third_party/waf/waflib/Context.py b/third_party/waf/waflib/Context.py index 0ce9df6e91f..07ee1201f03 100644 --- a/third_party/waf/waflib/Context.py +++ b/third_party/waf/waflib/Context.py @@ -18,13 +18,13 @@ else: import imp # the following 3 constants are updated on each new release (do not touch) -HEXVERSION=0x2001500 +HEXVERSION=0x2001600 """Constant updated on new releases""" -WAFVERSION="2.0.21" +WAFVERSION="2.0.22" """Constant updated on new releases""" -WAFREVISION="edde20a6425a5c3eb6b47d5f3f5c4fbc93fed5f4" +WAFREVISION="816d5bc48ba2abc4ac22f2b44d94d322bf992b9c" """Git revision when the waf version is updated""" WAFNAME="waf" diff --git a/third_party/waf/waflib/Tools/msvc.py b/third_party/waf/waflib/Tools/msvc.py index 37233be8242..0c4703aaee9 100644 --- a/third_party/waf/waflib/Tools/msvc.py +++ b/third_party/waf/waflib/Tools/msvc.py @@ -193,7 +193,7 @@ echo PATH=%%PATH%% echo INCLUDE=%%INCLUDE%% echo LIB=%%LIB%%;%%LIBPATH%% """ % (vcvars,target)) - sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()]) + sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()], stdin=getattr(Utils.subprocess, 'DEVNULL', None)) lines = sout.splitlines() if not lines[0]: diff --git a/third_party/waf/waflib/Tools/python.py b/third_party/waf/waflib/Tools/python.py index b1c8dd01285..07442561dff 100644 --- a/third_party/waf/waflib/Tools/python.py +++ b/third_party/waf/waflib/Tools/python.py @@ -327,7 +327,7 @@ def check_python_headers(conf, features='pyembed pyext'): dct = dict(zip(v, lst)) x = 'MACOSX_DEPLOYMENT_TARGET' if dct[x]: - env[x] = conf.environ[x] = dct[x] + env[x] = conf.environ[x] = str(dct[x]) env.pyext_PATTERN = '%s' + dct['SO'] # not a mistake diff --git a/third_party/waf/waflib/Tools/qt5.py b/third_party/waf/waflib/Tools/qt5.py index cff2028174f..82c83e18c8a 100644 --- a/third_party/waf/waflib/Tools/qt5.py +++ b/third_party/waf/waflib/Tools/qt5.py @@ -566,7 +566,7 @@ def find_qt5_binaries(self): # at the end, try to find qmake in the paths given # keep the one with the highest version cand = None - prev_ver = ['5', '0', '0'] + prev_ver = ['0', '0', '0'] for qmk in ('qmake-qt5', 'qmake5', 'qmake'): try: qmake = self.find_program(qmk, path_list=paths) @@ -580,7 +580,7 @@ def find_qt5_binaries(self): else: if version: new_ver = version.split('.') - if new_ver > prev_ver: + if new_ver[0] == '5' and new_ver > prev_ver: cand = qmake prev_ver = new_ver @@ -783,7 +783,7 @@ def set_qt5_libs_to_check(self): pat = self.env.cxxstlib_PATTERN if Utils.unversioned_sys_platform() == 'darwin': pat = r"%s\.framework" - re_qt = re.compile(pat%'Qt5?(?P.*)'+'$') + re_qt = re.compile(pat % 'Qt5?(?P\\D+)' + '$') for x in dirlst: m = re_qt.match(x) if m: diff --git a/third_party/waf/waflib/Tools/waf_unit_test.py b/third_party/waf/waflib/Tools/waf_unit_test.py index dc66fe9c184..8cff89bdeb9 100644 --- a/third_party/waf/waflib/Tools/waf_unit_test.py +++ b/third_party/waf/waflib/Tools/waf_unit_test.py @@ -206,7 +206,7 @@ class utest(Task.Task): self.ut_exec = getattr(self.generator, 'ut_exec', [self.inputs[0].abspath()]) ut_cmd = getattr(self.generator, 'ut_cmd', False) if ut_cmd: - self.ut_exec = shlex.split(ut_cmd % ' '.join(self.ut_exec)) + self.ut_exec = shlex.split(ut_cmd % Utils.shell_escape(self.ut_exec)) return self.exec_command(self.ut_exec) diff --git a/third_party/waf/waflib/Utils.py b/third_party/waf/waflib/Utils.py index fc64fa05154..669490ca908 100644 --- a/third_party/waf/waflib/Utils.py +++ b/third_party/waf/waflib/Utils.py @@ -11,7 +11,7 @@ through Python versions 2.5 to 3.X and across different platforms (win32, linux, from __future__ import with_statement -import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time +import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time, shlex try: import cPickle @@ -577,10 +577,13 @@ def quote_define_name(s): fu = fu.upper() return fu -re_sh = re.compile('\\s|\'|"') -""" -Regexp used for shell_escape below -""" +# shlex.quote didn't exist until python 3.3. Prior to that it was a non-documented +# function in pipes. +try: + shell_quote = shlex.quote +except AttributeError: + import pipes + shell_quote = pipes.quote def shell_escape(cmd): """ @@ -589,7 +592,7 @@ def shell_escape(cmd): """ if isinstance(cmd, str): return cmd - return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd) + return ' '.join(shell_quote(x) for x in cmd) def h_list(lst): """ diff --git a/third_party/waf/waflib/extras/clang_compilation_database.py b/third_party/waf/waflib/extras/clang_compilation_database.py index ff71f22ecfd..17f66949376 100644 --- a/third_party/waf/waflib/extras/clang_compilation_database.py +++ b/third_party/waf/waflib/extras/clang_compilation_database.py @@ -29,22 +29,9 @@ from waflib import Logs, TaskGen, Task, Build, Scripting Task.Task.keep_last_cmd = True -@TaskGen.feature('c', 'cxx') -@TaskGen.after_method('process_use') -def collect_compilation_db_tasks(self): - "Add a compilation database entry for compiled tasks" - if not isinstance(self.bld, ClangDbContext): - return - - tup = tuple(y for y in [Task.classes.get(x) for x in ('c', 'cxx')] if y) - for task in getattr(self, 'compiled_tasks', []): - if isinstance(task, tup): - self.bld.clang_compilation_database_tasks.append(task) - class ClangDbContext(Build.BuildContext): '''generates compile_commands.json by request''' cmd = 'clangdb' - clang_compilation_database_tasks = [] def write_compilation_database(self): """ @@ -78,6 +65,8 @@ class ClangDbContext(Build.BuildContext): Build dry run """ self.restore() + self.cur_tasks = [] + self.clang_compilation_database_tasks = [] if not self.all_envs: self.load_envs() @@ -103,8 +92,21 @@ class ClangDbContext(Build.BuildContext): lst = [tg] else: lst = tg.tasks for tsk in lst: + if tsk.__class__.__name__ == "swig": + tsk.runnable_status() + if hasattr(tsk, 'more_tasks'): + lst.extend(tsk.more_tasks) + # Not all dynamic tasks can be processed, in some cases + # one may have to call the method "run()" like this: + #elif tsk.__class__.__name__ == 'src2c': + # tsk.run() + # if hasattr(tsk, 'more_tasks'): + # lst.extend(tsk.more_tasks) + tup = tuple(y for y in [Task.classes.get(x) for x in ('c', 'cxx')] if y) if isinstance(tsk, tup): + self.clang_compilation_database_tasks.append(tsk) + tsk.nocache = True old_exec = tsk.exec_command tsk.exec_command = exec_command tsk.run() diff --git a/third_party/waf/waflib/extras/haxe.py b/third_party/waf/waflib/extras/haxe.py new file mode 100644 index 00000000000..cb3ba6a949c --- /dev/null +++ b/third_party/waf/waflib/extras/haxe.py @@ -0,0 +1,131 @@ +import os, re +from waflib import Utils, Task, Errors +from waflib.TaskGen import extension, taskgen_method, feature +from waflib.Configure import conf + +@conf +def libname_haxe(self, libname): + return libname + +@conf +def check_lib_haxe(self, libname, uselib_store=None): + haxe_libs = [node.name for node in self.root.find_node('haxe_libraries').ant_glob()] + changed = False + self.start_msg('Checking for library %s' % libname) + if libname + '.hxml' in haxe_libs: + self.end_msg('yes') + else: + changed = True + try: + cmd = self.env.LIX + ['+lib', libname] + res = self.cmd_and_log(cmd) + if (res): + raise Errors.WafError(res) + else: + self.end_msg('downloaded', color = 'YELLOW') + except Errors.WafError as e: + self.end_msg('no', color = 'RED') + self.fatal('Getting %s has failed' % libname) + + postfix = uselib_store if uselib_store else libname.upper() + self.env['LIB_' + postfix] += [self.libname_haxe(libname)] + return changed + +@conf +def check_libs_haxe(self, libnames, uselib_store=None): + changed = False + for libname in Utils.to_list(libnames): + if self.check_lib_haxe(libname, uselib_store): + changed = True + return changed + +@conf +def ensure_lix_pkg(self, *k, **kw): + if kw.get('compiler') == 'hx': + if isinstance(kw.get('libs'), list) and len(kw.get('libs')): + changed = self.check_libs_haxe(kw.get('libs'), kw.get('uselib_store')) + if changed: + try: + cmd = self.env.LIX + ['download'] + res = self.cmd_and_log(cmd) + if (res): + raise Errors.WafError(res) + except Errors.WafError as e: + self.fatal('lix download has failed') + else: + self.check_lib_haxe(kw.get('lib'), kw.get('uselib_store')) + +@conf +def haxe(bld, *k, **kw): + task_gen = bld(*k, **kw) + +class haxe(Task.Task): + vars = ['HAXE', 'HAXE_VERSION', 'HAXEFLAGS'] + ext_out = ['.hl', '.c', '.h'] + + def run(self): + cmd = self.env.HAXE + self.env.HAXEFLAGS + return self.exec_command(cmd, stdout = open(os.devnull, 'w')) + +@taskgen_method +def init_haxe_task(self, node): + def addflags(flags): + self.env.append_value('HAXEFLAGS', flags) + + if node.suffix() == '.hxml': + addflags(self.path.abspath() + '/' + node.name) + else: + addflags(['-main', node.name]) + addflags(['-hl', self.path.get_bld().make_node(self.target).abspath()]) + addflags(['-cp', self.path.abspath()]) + addflags(['-D', 'resourcesPath=%s' % getattr(self, 'res', '')]) + if hasattr(self, 'use'): + for dep in self.use: + if self.env['LIB_' + dep]: + for lib in self.env['LIB_' + dep]: addflags(['-lib', lib]) + +@extension('.hx', '.hxml') +def haxe_file(self, node): + if len(self.source) > 1: + self.bld.fatal('Use separate task generators for multiple files') + + try: + haxetask = self.haxetask + except AttributeError: + haxetask = self.haxetask = self.create_task('haxe') + self.init_haxe_task(node) + + haxetask.inputs.append(node) + haxetask.outputs.append(self.path.get_bld().make_node(self.target)) + +@conf +def find_haxe(self, min_version): + npx = self.env.NPX = self.find_program('npx') + self.env.LIX = npx + ['lix'] + npx_haxe = self.env.HAXE = npx + ['haxe'] + try: + output = self.cmd_and_log(npx_haxe + ['-version']) + except Errors.WafError: + haxe_version = None + else: + ver = re.search(r'\d+.\d+.\d+', output).group().split('.') + haxe_version = tuple([int(x) for x in ver]) + + self.msg('Checking for haxe version', + haxe_version, haxe_version and haxe_version >= min_version) + if npx_haxe and haxe_version < min_version: + self.fatal('haxe version %r is too old, need >= %r' % (haxe_version, min_version)) + + self.env.HAXE_VERSION = haxe_version + return npx_haxe + +@conf +def check_haxe(self, min_version=(4,1,4)): + if self.env.HAXE_MINVER: + min_version = self.env.HAXE_MINVER + find_haxe(self, min_version) + +def configure(self): + self.env.HAXEFLAGS = [] + self.check_haxe() + self.add_os_flags('HAXEFLAGS', dup = False) diff --git a/third_party/waf/waflib/extras/wafcache.py b/third_party/waf/waflib/extras/wafcache.py index 088fd0d098d..cc23fcd6673 100644 --- a/third_party/waf/waflib/extras/wafcache.py +++ b/third_party/waf/waflib/extras/wafcache.py @@ -31,6 +31,7 @@ The following environment variables may be set: gsutil cp gs://mybucket/bb/bbbbb/2 build/somefile * WAFCACHE_NO_PUSH: if set, disables pushing to the cache * WAFCACHE_VERBOSITY: if set, displays more detailed cache operations +* WAFCACHE_STATS: if set, displays cache usage statistics on exit File cache specific options: Files are copied using hard links by default; if the cache is located @@ -69,6 +70,7 @@ EVICT_INTERVAL_MINUTES = int(os.environ.get('WAFCACHE_EVICT_INTERVAL_MINUTES', 3 EVICT_MAX_BYTES = int(os.environ.get('WAFCACHE_EVICT_MAX_BYTES', 10**10)) WAFCACHE_NO_PUSH = 1 if os.environ.get('WAFCACHE_NO_PUSH') else 0 WAFCACHE_VERBOSITY = 1 if os.environ.get('WAFCACHE_VERBOSITY') else 0 +WAFCACHE_STATS = 1 if os.environ.get('WAFCACHE_STATS') else 0 OK = "ok" re_waf_cmd = re.compile('(?P%{SRC})|(?P%{TGT})') @@ -93,6 +95,9 @@ def can_retrieve_cache(self): sig = self.signature() ssig = Utils.to_hex(self.uid() + sig) + if WAFCACHE_STATS: + self.generator.bld.cache_reqs += 1 + files_to = [node.abspath() for node in self.outputs] err = cache_command(ssig, [], files_to) if err.startswith(OK): @@ -100,6 +105,8 @@ def can_retrieve_cache(self): Logs.pprint('CYAN', ' Fetched %r from cache' % files_to) else: Logs.debug('wafcache: fetched %r from cache', files_to) + if WAFCACHE_STATS: + self.generator.bld.cache_hits += 1 else: if WAFCACHE_VERBOSITY: Logs.pprint('YELLOW', ' No cache entry %s' % files_to) @@ -117,11 +124,17 @@ def put_files_cache(self): if WAFCACHE_NO_PUSH or getattr(self, 'cached', None) or not self.outputs: return + files_from = [] + for node in self.outputs: + path = node.abspath() + if not os.path.isfile(path): + return + files_from.append(path) + bld = self.generator.bld sig = self.signature() ssig = Utils.to_hex(self.uid() + sig) - files_from = [node.abspath() for node in self.outputs] err = cache_command(ssig, files_from, []) if err.startswith(OK): @@ -129,6 +142,8 @@ def put_files_cache(self): Logs.pprint('CYAN', ' Successfully uploaded %s to cache' % files_from) else: Logs.debug('wafcache: Successfully uploaded %r to cache', files_from) + if WAFCACHE_STATS: + self.generator.bld.cache_puts += 1 else: if WAFCACHE_VERBOSITY: Logs.pprint('RED', ' Error caching step results %s: %s' % (files_from, err)) @@ -193,6 +208,10 @@ def make_cached(cls): if getattr(cls, 'nocache', None) or getattr(cls, 'has_cache', False): return + full_name = "%s.%s" % (cls.__module__, cls.__name__) + if full_name in ('waflib.Tools.ccroot.vnum', 'waflib.Build.inst'): + return + m1 = getattr(cls, 'run', None) def run(self): if getattr(self, 'nocache', False): @@ -208,9 +227,6 @@ def make_cached(cls): return m2(self) ret = m2(self) self.put_files_cache() - if hasattr(self, 'chmod'): - for node in self.outputs: - os.chmod(node.abspath(), self.chmod) return ret cls.post_run = post_run cls.has_cache = True @@ -257,6 +273,19 @@ def build(bld): for x in reversed(list(Task.classes.values())): make_cached(x) + if WAFCACHE_STATS: + # Init counter for statistics and hook to print results at the end + bld.cache_reqs = bld.cache_hits = bld.cache_puts = 0 + + def printstats(bld): + hit_ratio = 0 + if bld.cache_reqs > 0: + hit_ratio = (bld.cache_hits / bld.cache_reqs) * 100 + Logs.pprint('CYAN', ' wafcache stats: requests: %s, hits, %s, ratio: %.2f%%, writes %s' % + (bld.cache_reqs, bld.cache_hits, hit_ratio, bld.cache_puts) ) + + bld.add_post_fun(printstats) + def cache_command(sig, files_from, files_to): """ Create a command for cache worker processes, returns a pickled @@ -320,7 +349,10 @@ def lru_trim(): size = 0 for fname in os.listdir(path): - size += os.lstat(os.path.join(path, fname)).st_size + try: + size += os.lstat(os.path.join(path, fname)).st_size + except OSError: + pass lst.append((os.stat(path).st_mtime, size, path)) lst.sort(key=lambda x: x[0]) @@ -331,7 +363,7 @@ def lru_trim(): _, tmp_size, path = lst.pop() tot -= tmp_size - tmp = path + '.tmp' + tmp = path + '.remove' try: shutil.rmtree(tmp) except OSError: @@ -339,12 +371,12 @@ def lru_trim(): try: os.rename(path, tmp) except OSError: - sys.stderr.write('Could not rename %r to %r' % (path, tmp)) + sys.stderr.write('Could not rename %r to %r\n' % (path, tmp)) else: try: shutil.rmtree(tmp) except OSError: - sys.stderr.write('Could not remove %r' % tmp) + sys.stderr.write('Could not remove %r\n' % tmp) sys.stderr.write("Cache trimmed: %r bytes in %r folders left\n" % (tot, len(lst))) @@ -371,8 +403,8 @@ def lru_evict(): try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except EnvironmentError: - sys.stderr.write('another process is running!\n') - pass + if WAFCACHE_VERBOSITY: + sys.stderr.write('wafcache: another cleaning process is running\n') else: # now dow the actual cleanup lru_trim() @@ -443,7 +475,10 @@ class fcache(object): else: # attempt trimming if caching was successful: # we may have things to trim! - lru_evict() + try: + lru_evict() + except Exception: + return traceback.format_exc() return OK def copy_from_cache(self, sig, files_from, files_to): @@ -481,7 +516,7 @@ class bucket_cache(object): out, err = proc.communicate() if proc.returncode: raise OSError('Error copy %r to %r using: %r (exit %r):\n out:%s\n err:%s' % ( - source, target, cmd, proc.returncode, out.decode(), err.decode())) + source, target, cmd, proc.returncode, out.decode(errors='replace'), err.decode(errors='replace'))) def copy_to_cache(self, sig, files_from, files_to): try: -- 2.36.1 From b6ca86941bdb00fda0dd1cdcc575ff44251f7ae3 Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Thu, 17 Feb 2022 15:40:38 +0100 Subject: [PATCH 3/5] third_party:waf: Print the version of waf at the end of the update script Signed-off-by: Andreas Schneider Reviewed-by: Alexander Bokovoy (cherry picked from commit 6b8d30e0aa15fff7e831c0a3e34719e8cdc1ebec) --- third_party/waf/update.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/third_party/waf/update.sh b/third_party/waf/update.sh index 16bda84a3f0..95f57ed1d81 100755 --- a/third_party/waf/update.sh +++ b/third_party/waf/update.sh @@ -74,6 +74,8 @@ popd || cleanup_and_exit 1 echo echo "Now please change VERSION in buildtools/bin/waf and" echo "Context.HEXVERSION in buildtools/wafsamba/wafsamba.py" +grep WAFVERSION "${WAF_SAMBA_DIR}/waflib/Context.py" +grep HEXVERSION "${WAF_SAMBA_DIR}/waflib/Context.py" echo -cleanup_and_exit 0 +cleanup_and_exit 0 \ No newline at end of file -- 2.36.1 From 4783ece3bd6946615ce4e5aa08399caf1b27c874 Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Thu, 17 Feb 2022 15:40:20 +0100 Subject: [PATCH 4/5] third_party: Update waf to verison 2.0.23 Signed-off-by: Andreas Schneider Reviewed-by: Alexander Bokovoy Autobuild-User(master): Andreas Schneider Autobuild-Date(master): Mon Feb 21 10:06:27 UTC 2022 on sn-devel-184 (cherry picked from commit fb175576b698f43224dab815fd6c0763a12db2b2) --- buildtools/bin/waf | 3 +- buildtools/wafsamba/wafsamba.py | 2 +- third_party/waf/waflib/Context.py | 6 +- third_party/waf/waflib/Runner.py | 4 +- third_party/waf/waflib/TaskGen.py | 8 +- third_party/waf/waflib/Tools/c_config.py | 1 + third_party/waf/waflib/Tools/compiler_c.py | 25 +++--- third_party/waf/waflib/Tools/compiler_cxx.py | 25 +++--- third_party/waf/waflib/Tools/python.py | 7 +- third_party/waf/waflib/Tools/qt5.py | 4 +- third_party/waf/waflib/Tools/winres.py | 35 ++++++++ .../extras/clang_compilation_database.py | 2 +- .../waf/waflib/extras/classic_runner.py | 68 +++++++++++++++ third_party/waf/waflib/extras/color_gcc.py | 2 +- third_party/waf/waflib/extras/eclipse.py | 74 ++++++++++++++++- third_party/waf/waflib/extras/gccdeps.py | 82 ++++++++++--------- third_party/waf/waflib/extras/msvcdeps.py | 54 ++++++++---- third_party/waf/waflib/extras/msvs.py | 6 +- third_party/waf/waflib/extras/swig.py | 2 +- third_party/waf/waflib/extras/wafcache.py | 26 +++--- third_party/waf/waflib/fixpy2.py | 2 +- 21 files changed, 325 insertions(+), 113 deletions(-) create mode 100644 third_party/waf/waflib/extras/classic_runner.py diff --git a/buildtools/bin/waf b/buildtools/bin/waf index b0ccb09a877..2001ccdbd8a 100755 --- a/buildtools/bin/waf +++ b/buildtools/bin/waf @@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE. import os, sys, inspect -VERSION="2.0.22" +VERSION="2.0.23" REVISION="x" GIT="x" INSTALL="x" @@ -164,4 +164,3 @@ if __name__ == '__main__': from waflib import Scripting Scripting.waf_entry_point(cwd, VERSION, wafdir[0]) - diff --git a/buildtools/wafsamba/wafsamba.py b/buildtools/wafsamba/wafsamba.py index 865975cb2d1..e00e2704c9b 100644 --- a/buildtools/wafsamba/wafsamba.py +++ b/buildtools/wafsamba/wafsamba.py @@ -38,7 +38,7 @@ LIB_PATH="shared" os.environ['PYTHONUNBUFFERED'] = '1' -if Context.HEXVERSION not in (0x2001600,): +if Context.HEXVERSION not in (0x2001700,): Logs.error(''' Please use the version of waf that comes with Samba, not a system installed version. See http://wiki.samba.org/index.php/Waf diff --git a/third_party/waf/waflib/Context.py b/third_party/waf/waflib/Context.py index 07ee1201f03..36d1ca74fef 100644 --- a/third_party/waf/waflib/Context.py +++ b/third_party/waf/waflib/Context.py @@ -18,13 +18,13 @@ else: import imp # the following 3 constants are updated on each new release (do not touch) -HEXVERSION=0x2001600 +HEXVERSION=0x2001700 """Constant updated on new releases""" -WAFVERSION="2.0.22" +WAFVERSION="2.0.23" """Constant updated on new releases""" -WAFREVISION="816d5bc48ba2abc4ac22f2b44d94d322bf992b9c" +WAFREVISION="cc6b34cf555d354c34f554c41206134072588de7" """Git revision when the waf version is updated""" WAFNAME="waf" diff --git a/third_party/waf/waflib/Runner.py b/third_party/waf/waflib/Runner.py index 91d55479e20..350c86a22c0 100644 --- a/third_party/waf/waflib/Runner.py +++ b/third_party/waf/waflib/Runner.py @@ -71,7 +71,7 @@ class Consumer(Utils.threading.Thread): """Task to execute""" self.spawner = spawner """Coordinator object""" - self.setDaemon(1) + self.daemon = True self.start() def run(self): """ @@ -98,7 +98,7 @@ class Spawner(Utils.threading.Thread): """:py:class:`waflib.Runner.Parallel` producer instance""" self.sem = Utils.threading.Semaphore(master.numjobs) """Bounded semaphore that prevents spawning more than *n* concurrent consumers""" - self.setDaemon(1) + self.daemon = True self.start() def run(self): """ diff --git a/third_party/waf/waflib/TaskGen.py b/third_party/waf/waflib/TaskGen.py index f8f92bd57c1..89f63169910 100644 --- a/third_party/waf/waflib/TaskGen.py +++ b/third_party/waf/waflib/TaskGen.py @@ -631,12 +631,8 @@ def process_rule(self): cls.scan = self.scan elif has_deps: def scan(self): - nodes = [] - for x in self.generator.to_list(getattr(self.generator, 'deps', None)): - node = self.generator.path.find_resource(x) - if not node: - self.generator.bld.fatal('Could not find %r (was it declared?)' % x) - nodes.append(node) + deps = getattr(self.generator, 'deps', None) + nodes = self.generator.to_nodes(deps) return [nodes, []] cls.scan = scan diff --git a/third_party/waf/waflib/Tools/c_config.py b/third_party/waf/waflib/Tools/c_config.py index 03b6bf61bc0..f5ab19bf6ce 100644 --- a/third_party/waf/waflib/Tools/c_config.py +++ b/third_party/waf/waflib/Tools/c_config.py @@ -69,6 +69,7 @@ MACRO_TO_DEST_CPU = { '__sh__' : 'sh', '__xtensa__' : 'xtensa', '__e2k__' : 'e2k', +'__riscv' : 'riscv', } @conf diff --git a/third_party/waf/waflib/Tools/compiler_c.py b/third_party/waf/waflib/Tools/compiler_c.py index 931dc57efec..e033ce6c5c3 100644 --- a/third_party/waf/waflib/Tools/compiler_c.py +++ b/third_party/waf/waflib/Tools/compiler_c.py @@ -36,18 +36,19 @@ from waflib import Utils from waflib.Logs import debug c_compiler = { -'win32': ['msvc', 'gcc', 'clang'], -'cygwin': ['gcc', 'clang'], -'darwin': ['clang', 'gcc'], -'aix': ['xlc', 'gcc', 'clang'], -'linux': ['gcc', 'clang', 'icc'], -'sunos': ['suncc', 'gcc'], -'irix': ['gcc', 'irixcc'], -'hpux': ['gcc'], -'osf1V': ['gcc'], -'gnu': ['gcc', 'clang'], -'java': ['gcc', 'msvc', 'clang', 'icc'], -'default':['clang', 'gcc'], +'win32': ['msvc', 'gcc', 'clang'], +'cygwin': ['gcc', 'clang'], +'darwin': ['clang', 'gcc'], +'aix': ['xlc', 'gcc', 'clang'], +'linux': ['gcc', 'clang', 'icc'], +'sunos': ['suncc', 'gcc'], +'irix': ['gcc', 'irixcc'], +'hpux': ['gcc'], +'osf1V': ['gcc'], +'gnu': ['gcc', 'clang'], +'java': ['gcc', 'msvc', 'clang', 'icc'], +'gnukfreebsd': ['gcc', 'clang'], +'default': ['clang', 'gcc'], } """ Dict mapping platform names to Waf tools finding specific C compilers:: diff --git a/third_party/waf/waflib/Tools/compiler_cxx.py b/third_party/waf/waflib/Tools/compiler_cxx.py index 09fca7e4dc6..42658c5847e 100644 --- a/third_party/waf/waflib/Tools/compiler_cxx.py +++ b/third_party/waf/waflib/Tools/compiler_cxx.py @@ -37,18 +37,19 @@ from waflib import Utils from waflib.Logs import debug cxx_compiler = { -'win32': ['msvc', 'g++', 'clang++'], -'cygwin': ['g++', 'clang++'], -'darwin': ['clang++', 'g++'], -'aix': ['xlc++', 'g++', 'clang++'], -'linux': ['g++', 'clang++', 'icpc'], -'sunos': ['sunc++', 'g++'], -'irix': ['g++'], -'hpux': ['g++'], -'osf1V': ['g++'], -'gnu': ['g++', 'clang++'], -'java': ['g++', 'msvc', 'clang++', 'icpc'], -'default': ['clang++', 'g++'] +'win32': ['msvc', 'g++', 'clang++'], +'cygwin': ['g++', 'clang++'], +'darwin': ['clang++', 'g++'], +'aix': ['xlc++', 'g++', 'clang++'], +'linux': ['g++', 'clang++', 'icpc'], +'sunos': ['sunc++', 'g++'], +'irix': ['g++'], +'hpux': ['g++'], +'osf1V': ['g++'], +'gnu': ['g++', 'clang++'], +'java': ['g++', 'msvc', 'clang++', 'icpc'], +'gnukfreebsd': ['g++', 'clang++'], +'default': ['clang++', 'g++'] } """ Dict mapping the platform names to Waf tools finding specific C++ compilers:: diff --git a/third_party/waf/waflib/Tools/python.py b/third_party/waf/waflib/Tools/python.py index 07442561dff..fb641e5e20d 100644 --- a/third_party/waf/waflib/Tools/python.py +++ b/third_party/waf/waflib/Tools/python.py @@ -416,9 +416,14 @@ def check_python_headers(conf, features='pyembed pyext'): if not result: path = [os.path.join(dct['prefix'], "libs")] - conf.to_log("\n\n# try again with -L$prefix/libs, and pythonXY name rather than pythonX.Y (win32)\n") + conf.to_log("\n\n# try again with -L$prefix/libs, and pythonXY rather than pythonX.Y (win32)\n") result = conf.check(lib=name, uselib='PYEMBED', libpath=path, mandatory=False, msg='Checking for library %s in $prefix/libs' % name) + if not result: + path = [os.path.normpath(os.path.join(dct['INCLUDEPY'], '..', 'libs'))] + conf.to_log("\n\n# try again with -L$INCLUDEPY/../libs, and pythonXY rather than pythonX.Y (win32)\n") + result = conf.check(lib=name, uselib='PYEMBED', libpath=path, mandatory=False, msg='Checking for library %s in $INCLUDEPY/../libs' % name) + if result: break # do not forget to set LIBPATH_PYEMBED diff --git a/third_party/waf/waflib/Tools/qt5.py b/third_party/waf/waflib/Tools/qt5.py index 82c83e18c8a..b3e61325e50 100644 --- a/third_party/waf/waflib/Tools/qt5.py +++ b/third_party/waf/waflib/Tools/qt5.py @@ -783,8 +783,8 @@ def set_qt5_libs_to_check(self): pat = self.env.cxxstlib_PATTERN if Utils.unversioned_sys_platform() == 'darwin': pat = r"%s\.framework" - re_qt = re.compile(pat % 'Qt5?(?P\\D+)' + '$') - for x in dirlst: + re_qt = re.compile(pat % 'Qt5?(?P\\w+)' + '$') + for x in sorted(dirlst): m = re_qt.match(x) if m: self.qt5_vars.append("Qt5%s" % m.group('name')) diff --git a/third_party/waf/waflib/Tools/winres.py b/third_party/waf/waflib/Tools/winres.py index 9be1ed66009..73c0e95315b 100644 --- a/third_party/waf/waflib/Tools/winres.py +++ b/third_party/waf/waflib/Tools/winres.py @@ -4,10 +4,12 @@ "Process *.rc* files for C/C++: X{.rc -> [.res|.rc.o]}" +import os import re from waflib import Task from waflib.TaskGen import extension from waflib.Tools import c_preproc +from waflib import Utils @extension('.rc') def rc_file(self, node): @@ -61,6 +63,39 @@ class winrc(Task.Task): tmp.start(self.inputs[0], self.env) return (tmp.nodes, tmp.names) + def exec_command(self, cmd, **kw): + if self.env.WINRC_TGT_F == '/fo': + # Since winres include paths may contain spaces, they do not fit in + # response files and are best passed as environment variables + replace_cmd = [] + incpaths = [] + while cmd: + # filter include path flags + flag = cmd.pop(0) + if flag.upper().startswith('/I'): + if len(flag) == 2: + incpaths.append(cmd.pop(0)) + else: + incpaths.append(flag[2:]) + else: + replace_cmd.append(flag) + cmd = replace_cmd + if incpaths: + # append to existing environment variables in INCLUDE + env = kw['env'] = dict(kw.get('env') or self.env.env or os.environ) + pre_includes = env.get('INCLUDE', '') + env['INCLUDE'] = pre_includes + os.pathsep + os.pathsep.join(incpaths) + + return super(winrc, self).exec_command(cmd, **kw) + + def quote_flag(self, flag): + if self.env.WINRC_TGT_F == '/fo': + # winres does not support quotes around flags in response files + return flag + + return super(winrc, self).quote_flag(flag) + + def configure(conf): """ Detects the programs RC or windres, depending on the C/C++ compiler in use diff --git a/third_party/waf/waflib/extras/clang_compilation_database.py b/third_party/waf/waflib/extras/clang_compilation_database.py index 17f66949376..bd29db93fd5 100644 --- a/third_party/waf/waflib/extras/clang_compilation_database.py +++ b/third_party/waf/waflib/extras/clang_compilation_database.py @@ -126,7 +126,7 @@ def patch_execute(): Invoke clangdb command before build """ if self.cmd.startswith('build'): - Scripting.run_command('clangdb') + Scripting.run_command(self.cmd.replace('build','clangdb')) old_execute_build(self) diff --git a/third_party/waf/waflib/extras/classic_runner.py b/third_party/waf/waflib/extras/classic_runner.py new file mode 100644 index 00000000000..b08c794e880 --- /dev/null +++ b/third_party/waf/waflib/extras/classic_runner.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Thomas Nagy, 2021 (ita) + +from waflib import Utils, Runner + +""" +Re-enable the classic threading system from waf 1.x + +def configure(conf): + conf.load('classic_runner') +""" + +class TaskConsumer(Utils.threading.Thread): + """ + Task consumers belong to a pool of workers + + They wait for tasks in the queue and then use ``task.process(...)`` + """ + def __init__(self, spawner): + Utils.threading.Thread.__init__(self) + """ + Obtain :py:class:`waflib.Task.TaskBase` instances from this queue. + """ + self.spawner = spawner + self.daemon = True + self.start() + + def run(self): + """ + Loop over the tasks to execute + """ + try: + self.loop() + except Exception: + pass + + def loop(self): + """ + Obtain tasks from :py:attr:`waflib.Runner.TaskConsumer.ready` and call + :py:meth:`waflib.Task.TaskBase.process`. If the object is a function, execute it. + """ + master = self.spawner.master + while 1: + if not master.stop: + try: + tsk = master.ready.get() + if tsk: + tsk.log_display(tsk.generator.bld) + master.process_task(tsk) + else: + break + finally: + master.out.put(tsk) + +class Spawner(object): + """ + Daemon thread that consumes tasks from :py:class:`waflib.Runner.Parallel` producer and + spawns a consuming thread :py:class:`waflib.Runner.Consumer` for each + :py:class:`waflib.Task.Task` instance. + """ + def __init__(self, master): + self.master = master + """:py:class:`waflib.Runner.Parallel` producer instance""" + + self.pool = [TaskConsumer(self) for i in range(master.numjobs)] + +Runner.Spawner = Spawner diff --git a/third_party/waf/waflib/extras/color_gcc.py b/third_party/waf/waflib/extras/color_gcc.py index b68c5ebf2df..09729035fec 100644 --- a/third_party/waf/waflib/extras/color_gcc.py +++ b/third_party/waf/waflib/extras/color_gcc.py @@ -19,7 +19,7 @@ class ColorGCCFormatter(Logs.formatter): func = frame.f_code.co_name if func == 'exec_command': cmd = frame.f_locals.get('cmd') - if isinstance(cmd, list) and ('gcc' in cmd[0] or 'g++' in cmd[0]): + if isinstance(cmd, list) and (len(cmd) > 0) and ('gcc' in cmd[0] or 'g++' in cmd[0]): lines = [] for line in rec.msg.splitlines(): if 'warning: ' in line: diff --git a/third_party/waf/waflib/extras/eclipse.py b/third_party/waf/waflib/extras/eclipse.py index bb787416e9f..49ca9686b7b 100644 --- a/third_party/waf/waflib/extras/eclipse.py +++ b/third_party/waf/waflib/extras/eclipse.py @@ -10,6 +10,9 @@ Usage: def options(opt): opt.load('eclipse') +To add additional targets beside standard ones (configure, dist, install, check) +the environment ECLIPSE_EXTRA_TARGETS can be set (ie. to ['test', 'lint', 'docs']) + $ waf configure eclipse """ @@ -25,6 +28,8 @@ cdt_core = oe_cdt + '.core' cdt_bld = oe_cdt + '.build.core' extbuilder_dir = '.externalToolBuilders' extbuilder_name = 'Waf_Builder.launch' +settings_dir = '.settings' +settings_name = 'language.settings.xml' class eclipse(Build.BuildContext): cmd = 'eclipse' @@ -131,9 +136,11 @@ class eclipse(Build.BuildContext): path = p.path_from(self.srcnode) if (path.startswith("/")): - cpppath.append(path) + if path not in cpppath: + cpppath.append(path) else: - workspace_includes.append(path) + if path not in workspace_includes: + workspace_includes.append(path) if is_cc and path not in source_dirs: source_dirs.append(path) @@ -156,6 +163,61 @@ class eclipse(Build.BuildContext): project = self.impl_create_javaproject(javasrcpath, javalibpath) self.write_conf_to_xml('.classpath', project) + # Create editor language settings to have correct standards applied in IDE, as per project configuration + try: + os.mkdir(settings_dir) + except OSError: + pass # Ignore if dir already exists + + lang_settings = Document() + project = lang_settings.createElement('project') + + # Language configurations for C and C++ via cdt + if hasc: + configuration = self.add(lang_settings, project, 'configuration', + {'id' : 'org.eclipse.cdt.core.default.config.1', 'name': 'Default'}) + + extension = self.add(lang_settings, configuration, 'extension', {'point': 'org.eclipse.cdt.core.LanguageSettingsProvider'}) + + provider = self.add(lang_settings, extension, 'provider', + { 'copy-of': 'extension', + 'id': 'org.eclipse.cdt.ui.UserLanguageSettingsProvider'}) + + provider = self.add(lang_settings, extension, 'provider-reference', + { 'id': 'org.eclipse.cdt.core.ReferencedProjectsLanguageSettingsProvider', + 'ref': 'shared-provider'}) + + provider = self.add(lang_settings, extension, 'provider-reference', + { 'id': 'org.eclipse.cdt.managedbuilder.core.MBSLanguageSettingsProvider', + 'ref': 'shared-provider'}) + + # C and C++ are kept as separated providers so appropriate flags are used also in mixed projects + if self.env.CC: + provider = self.add(lang_settings, extension, 'provider', + { 'class': 'org.eclipse.cdt.managedbuilder.language.settings.providers.GCCBuiltinSpecsDetector', + 'console': 'false', + 'id': 'org.eclipse.cdt.managedbuilder.language.settings.providers.GCCBuiltinSpecsDetector.1', + 'keep-relative-paths' : 'false', + 'name': 'CDT GCC Built-in Compiler Settings', + 'parameter': '%s %s ${FLAGS} -E -P -v -dD "${INPUTS}"'%(self.env.CC[0],' '.join(self.env['CFLAGS'])), + 'prefer-non-shared': 'true' }) + + self.add(lang_settings, provider, 'language-scope', { 'id': 'org.eclipse.cdt.core.gcc'}) + + if self.env.CXX: + provider = self.add(lang_settings, extension, 'provider', + { 'class': 'org.eclipse.cdt.managedbuilder.language.settings.providers.GCCBuiltinSpecsDetector', + 'console': 'false', + 'id': 'org.eclipse.cdt.managedbuilder.language.settings.providers.GCCBuiltinSpecsDetector.2', + 'keep-relative-paths' : 'false', + 'name': 'CDT GCC Built-in Compiler Settings', + 'parameter': '%s %s ${FLAGS} -E -P -v -dD "${INPUTS}"'%(self.env.CXX[0],' '.join(self.env['CXXFLAGS'])), + 'prefer-non-shared': 'true' }) + self.add(lang_settings, provider, 'language-scope', { 'id': 'org.eclipse.cdt.core.g++'}) + + lang_settings.appendChild(project) + self.write_conf_to_xml('%s%s%s'%(settings_dir, os.path.sep, settings_name), lang_settings) + def impl_create_project(self, executable, appname, hasc, hasjava, haspython, waf_executable): doc = Document() projectDescription = doc.createElement('projectDescription') @@ -341,6 +403,8 @@ class eclipse(Build.BuildContext): addTargetWrap('dist', False) addTargetWrap('install', False) addTargetWrap('check', False) + for addTgt in self.env.ECLIPSE_EXTRA_TARGETS or []: + addTargetWrap(addTgt, False) storageModule = self.add(doc, cproject, 'storageModule', {'moduleId': 'cdtBuildSystem', @@ -348,6 +412,12 @@ class eclipse(Build.BuildContext): self.add(doc, storageModule, 'project', {'id': '%s.null.1'%appname, 'name': appname}) + storageModule = self.add(doc, cproject, 'storageModule', + {'moduleId': 'org.eclipse.cdt.core.LanguageSettingsProviders'}) + + storageModule = self.add(doc, cproject, 'storageModule', + {'moduleId': 'scannerConfiguration'}) + doc.appendChild(cproject) return doc diff --git a/third_party/waf/waflib/extras/gccdeps.py b/third_party/waf/waflib/extras/gccdeps.py index 1fc9373489a..9e9952f2f7d 100644 --- a/third_party/waf/waflib/extras/gccdeps.py +++ b/third_party/waf/waflib/extras/gccdeps.py @@ -29,13 +29,6 @@ if not c_preproc.go_absolute: # Third-party tools are allowed to add extra names in here with append() supported_compilers = ['gas', 'gcc', 'icc', 'clang'] -def scan(self): - if not self.__class__.__name__ in self.env.ENABLE_GCCDEPS: - return super(self.derived_gccdeps, self).scan() - nodes = self.generator.bld.node_deps.get(self.uid(), []) - names = [] - return (nodes, names) - re_o = re.compile(r"\.o$") re_splitter = re.compile(r'(? Node lookup PREPROCESSOR_FLAG = '/showIncludes' INCLUDE_PATTERN = 'Note: including file:' @@ -50,23 +49,47 @@ def apply_msvcdeps_flags(taskgen): if taskgen.env.get_flat(flag).find(PREPROCESSOR_FLAG) < 0: taskgen.env.append_value(flag, PREPROCESSOR_FLAG) + +def get_correct_path_case(base_path, path): + ''' + Return a case-corrected version of ``path`` by searching the filesystem for + ``path``, relative to ``base_path``, using the case returned by the filesystem. + ''' + components = Utils.split_path(path) + + corrected_path = '' + if os.path.isabs(path): + corrected_path = components.pop(0).upper() + os.sep + + for part in components: + part = part.lower() + search_path = os.path.join(base_path, corrected_path) + if part == '..': + corrected_path = os.path.join(corrected_path, part) + search_path = os.path.normpath(search_path) + continue + + for item in sorted(os.listdir(search_path)): + if item.lower() == part: + corrected_path = os.path.join(corrected_path, item) + break + else: + raise ValueError("Can't find %r in %r" % (part, search_path)) + + return corrected_path + + def path_to_node(base_node, path, cached_nodes): ''' Take the base node and the path and return a node Results are cached because searching the node tree is expensive The following code is executed by threads, it is not safe, so a lock is needed... ''' - # normalize the path because ant_glob() does not understand - # parent path components (..) + # normalize the path to remove parent path components (..) path = os.path.normpath(path) # normalize the path case to increase likelihood of a cache hit - path = os.path.normcase(path) - - # ant_glob interprets [] and () characters, so those must be replaced - path = path.replace('[', '?').replace(']', '?').replace('(', '[(]').replace(')', '[)]') - - node_lookup_key = (base_node, path) + node_lookup_key = (base_node, os.path.normcase(path)) try: node = cached_nodes[node_lookup_key] @@ -76,8 +99,8 @@ def path_to_node(base_node, path, cached_nodes): try: node = cached_nodes[node_lookup_key] except KeyError: - node_list = base_node.ant_glob([path], ignorecase=True, remove=False, quiet=True, regex=False) - node = cached_nodes[node_lookup_key] = node_list[0] if node_list else None + path = get_correct_path_case(base_node.abspath(), path) + node = cached_nodes[node_lookup_key] = base_node.find_node(path) return node @@ -89,9 +112,9 @@ def post_run(self): if getattr(self, 'cached', None): return Task.Task.post_run(self) - bld = self.generator.bld - unresolved_names = [] resolved_nodes = [] + unresolved_names = [] + bld = self.generator.bld # Dynamically bind to the cache try: @@ -124,11 +147,14 @@ def post_run(self): continue if id(node) == id(self.inputs[0]): - # Self-dependency + # ignore the source file, it is already in the dependencies + # this way, successful config tests may be retrieved from the cache continue resolved_nodes.append(node) + Logs.debug('deps: msvcdeps for %s returned %s', self, resolved_nodes) + bld.node_deps[self.uid()] = resolved_nodes bld.raw_deps[self.uid()] = unresolved_names diff --git a/third_party/waf/waflib/extras/msvs.py b/third_party/waf/waflib/extras/msvs.py index 8aa2db0b751..03b739f849c 100644 --- a/third_party/waf/waflib/extras/msvs.py +++ b/third_party/waf/waflib/extras/msvs.py @@ -787,8 +787,12 @@ class msvs_generator(BuildContext): self.collect_dirs() default_project = getattr(self, 'default_project', None) def sortfun(x): - if x.name == default_project: + # folders should sort to the top + if getattr(x, 'VS_GUID_SOLUTIONFOLDER', None): return '' + # followed by the default project + elif x.name == default_project: + return ' ' return getattr(x, 'path', None) and x.path.win32path() or x.name self.all_projects.sort(key=sortfun) diff --git a/third_party/waf/waflib/extras/swig.py b/third_party/waf/waflib/extras/swig.py index 740ab46d963..967caeb5a82 100644 --- a/third_party/waf/waflib/extras/swig.py +++ b/third_party/waf/waflib/extras/swig.py @@ -17,7 +17,7 @@ tasks have to be added dynamically: SWIG_EXTS = ['.swig', '.i'] -re_module = re.compile(r'%module(?:\s*\(.*\))?\s+(.+)', re.M) +re_module = re.compile(r'%module(?:\s*\(.*\))?\s+([^\r\n]+)', re.M) re_1 = re.compile(r'^%module.*?\s+([\w]+)\s*?$', re.M) re_2 = re.compile(r'[#%](?:include|import(?:\(module=".*"\))+|python(?:begin|code)) [<"](.*)[">]', re.M) diff --git a/third_party/waf/waflib/extras/wafcache.py b/third_party/waf/waflib/extras/wafcache.py index cc23fcd6673..2cef46c0e1c 100644 --- a/third_party/waf/waflib/extras/wafcache.py +++ b/third_party/waf/waflib/extras/wafcache.py @@ -258,6 +258,19 @@ def build(bld): """ Called during the build process to enable file caching """ + if WAFCACHE_STATS: + # Init counter for statistics and hook to print results at the end + bld.cache_reqs = bld.cache_hits = bld.cache_puts = 0 + + def printstats(bld): + hit_ratio = 0 + if bld.cache_reqs > 0: + hit_ratio = (bld.cache_hits / bld.cache_reqs) * 100 + Logs.pprint('CYAN', ' wafcache stats: requests: %s, hits, %s, ratio: %.2f%%, writes %s' % + (bld.cache_reqs, bld.cache_hits, hit_ratio, bld.cache_puts) ) + + bld.add_post_fun(printstats) + if process_pool: # already called once return @@ -273,19 +286,6 @@ def build(bld): for x in reversed(list(Task.classes.values())): make_cached(x) - if WAFCACHE_STATS: - # Init counter for statistics and hook to print results at the end - bld.cache_reqs = bld.cache_hits = bld.cache_puts = 0 - - def printstats(bld): - hit_ratio = 0 - if bld.cache_reqs > 0: - hit_ratio = (bld.cache_hits / bld.cache_reqs) * 100 - Logs.pprint('CYAN', ' wafcache stats: requests: %s, hits, %s, ratio: %.2f%%, writes %s' % - (bld.cache_reqs, bld.cache_hits, hit_ratio, bld.cache_puts) ) - - bld.add_post_fun(printstats) - def cache_command(sig, files_from, files_to): """ Create a command for cache worker processes, returns a pickled diff --git a/third_party/waf/waflib/fixpy2.py b/third_party/waf/waflib/fixpy2.py index 24176e06645..c99bff4b9ae 100644 --- a/third_party/waf/waflib/fixpy2.py +++ b/third_party/waf/waflib/fixpy2.py @@ -56,7 +56,7 @@ def r1(code): @subst('Runner.py') def r4(code): "generator syntax" - return code.replace('next(self.biter)', 'self.biter.next()') + return code.replace('next(self.biter)', 'self.biter.next()').replace('self.daemon = True', 'self.setDaemon(1)') @subst('Context.py') def r5(code): -- 2.36.1 From 44043df4ddcbdbc60d4c6b083e0c3d750fc904ad Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Mon, 23 May 2022 07:54:06 +0200 Subject: [PATCH 5/5] third_party: Update waf to version 2.0.24 This fixes building of python libraries with Python 3.11! BUG: https://bugzilla.samba.org/show_bug.cgi?id=15071 Signed-off-by: Andreas Schneider Reviewed-by: Stefan Metzmacher Autobuild-User(master): Andreas Schneider Autobuild-Date(master): Mon May 23 09:34:51 UTC 2022 on sn-devel-184 (cherry picked from commit d19dfe1efb2f6cb0dcf0a63b957df584d8ed5945) --- buildtools/bin/waf | 2 +- buildtools/wafsamba/wafsamba.py | 2 +- third_party/waf/waflib/Context.py | 8 ++++---- third_party/waf/waflib/Tools/ccroot.py | 1 + third_party/waf/waflib/Tools/msvc.py | 17 ++++++++++++++++- third_party/waf/waflib/Tools/python.py | 4 ++-- third_party/waf/waflib/Tools/tex.py | 1 + 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/buildtools/bin/waf b/buildtools/bin/waf index 2001ccdbd8a..d9cba343623 100755 --- a/buildtools/bin/waf +++ b/buildtools/bin/waf @@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE. import os, sys, inspect -VERSION="2.0.23" +VERSION="2.0.24" REVISION="x" GIT="x" INSTALL="x" diff --git a/buildtools/wafsamba/wafsamba.py b/buildtools/wafsamba/wafsamba.py index e00e2704c9b..8a83576f89e 100644 --- a/buildtools/wafsamba/wafsamba.py +++ b/buildtools/wafsamba/wafsamba.py @@ -38,7 +38,7 @@ LIB_PATH="shared" os.environ['PYTHONUNBUFFERED'] = '1' -if Context.HEXVERSION not in (0x2001700,): +if Context.HEXVERSION not in (0x2001800,): Logs.error(''' Please use the version of waf that comes with Samba, not a system installed version. See http://wiki.samba.org/index.php/Waf diff --git a/third_party/waf/waflib/Context.py b/third_party/waf/waflib/Context.py index 36d1ca74fef..4a0130b24a0 100644 --- a/third_party/waf/waflib/Context.py +++ b/third_party/waf/waflib/Context.py @@ -18,13 +18,13 @@ else: import imp # the following 3 constants are updated on each new release (do not touch) -HEXVERSION=0x2001700 +HEXVERSION=0x2001800 """Constant updated on new releases""" -WAFVERSION="2.0.23" +WAFVERSION="2.0.24" """Constant updated on new releases""" -WAFREVISION="cc6b34cf555d354c34f554c41206134072588de7" +WAFREVISION="1af97c71f5a6756abf36d0f78ed8fd551596d7cb" """Git revision when the waf version is updated""" WAFNAME="waf" @@ -144,7 +144,7 @@ class Context(ctx): :type fun: string .. inheritance-diagram:: waflib.Context.Context waflib.Build.BuildContext waflib.Build.InstallContext waflib.Build.UninstallContext waflib.Build.StepContext waflib.Build.ListContext waflib.Configure.ConfigurationContext waflib.Scripting.Dist waflib.Scripting.DistCheck waflib.Build.CleanContext - + :top-classes: waflib.Context.Context """ errors = Errors diff --git a/third_party/waf/waflib/Tools/ccroot.py b/third_party/waf/waflib/Tools/ccroot.py index 579d5b2b72b..76deff54dcb 100644 --- a/third_party/waf/waflib/Tools/ccroot.py +++ b/third_party/waf/waflib/Tools/ccroot.py @@ -128,6 +128,7 @@ class link_task(Task.Task): Base class for all link tasks. A task generator is supposed to have at most one link task bound in the attribute *link_task*. See :py:func:`waflib.Tools.ccroot.apply_link`. .. inheritance-diagram:: waflib.Tools.ccroot.stlink_task waflib.Tools.c.cprogram waflib.Tools.c.cshlib waflib.Tools.cxx.cxxstlib waflib.Tools.cxx.cxxprogram waflib.Tools.cxx.cxxshlib waflib.Tools.d.dprogram waflib.Tools.d.dshlib waflib.Tools.d.dstlib waflib.Tools.ccroot.fake_shlib waflib.Tools.ccroot.fake_stlib waflib.Tools.asm.asmprogram waflib.Tools.asm.asmshlib waflib.Tools.asm.asmstlib + :top-classes: waflib.Tools.ccroot.link_task """ color = 'YELLOW' diff --git a/third_party/waf/waflib/Tools/msvc.py b/third_party/waf/waflib/Tools/msvc.py index 0c4703aaee9..026a4c7fc48 100644 --- a/third_party/waf/waflib/Tools/msvc.py +++ b/third_party/waf/waflib/Tools/msvc.py @@ -109,6 +109,21 @@ def options(opt): opt.add_option('--msvc_targets', type='string', help = 'msvc targets, eg: "x64,arm"', default='') opt.add_option('--no-msvc-lazy', action='store_false', help = 'lazily check msvc target environments', default=True, dest='msvc_lazy') +class MSVCVersion(object): + def __init__(self, ver): + m = re.search('^(.*)\s+(\d+[.]\d+)', ver) + if m: + self.name = m.group(1) + self.number = float(m.group(2)) + else: + self.name = ver + self.number = 0. + + def __lt__(self, other): + if self.number == other.number: + return self.name < other.name + return self.number < other.number + @conf def setup_msvc(conf, versiondict): """ @@ -125,7 +140,7 @@ def setup_msvc(conf, versiondict): platforms=Utils.to_list(conf.env.MSVC_TARGETS) or [i for i,j in all_msvc_platforms+all_icl_platforms+all_wince_platforms] desired_versions = getattr(Options.options, 'msvc_version', '').split(',') if desired_versions == ['']: - desired_versions = conf.env.MSVC_VERSIONS or list(reversed(sorted(versiondict.keys()))) + desired_versions = conf.env.MSVC_VERSIONS or list(sorted(versiondict.keys(), key=MSVCVersion, reverse=True)) # Override lazy detection by evaluating after the fact. lazy_detect = getattr(Options.options, 'msvc_lazy', True) diff --git a/third_party/waf/waflib/Tools/python.py b/third_party/waf/waflib/Tools/python.py index fb641e5e20d..a23bd019335 100644 --- a/third_party/waf/waflib/Tools/python.py +++ b/third_party/waf/waflib/Tools/python.py @@ -315,7 +315,7 @@ def check_python_headers(conf, features='pyembed pyext'): conf.fatal('Could not find the python executable') # so we actually do all this for compatibility reasons and for obtaining pyext_PATTERN below - v = 'prefix SO LDFLAGS LIBDIR LIBPL INCLUDEPY Py_ENABLE_SHARED MACOSX_DEPLOYMENT_TARGET LDSHARED CFLAGS LDVERSION'.split() + v = 'prefix SO EXT_SUFFIX LDFLAGS LIBDIR LIBPL INCLUDEPY Py_ENABLE_SHARED MACOSX_DEPLOYMENT_TARGET LDSHARED CFLAGS LDVERSION'.split() try: lst = conf.get_python_variables(["get_config_var('%s') or ''" % x for x in v]) except RuntimeError: @@ -328,7 +328,7 @@ def check_python_headers(conf, features='pyembed pyext'): x = 'MACOSX_DEPLOYMENT_TARGET' if dct[x]: env[x] = conf.environ[x] = str(dct[x]) - env.pyext_PATTERN = '%s' + dct['SO'] # not a mistake + env.pyext_PATTERN = '%s' + (dct['EXT_SUFFIX'] or dct['SO']) # SO is deprecated in 3.5 and removed in 3.11 # Try to get pythonX.Y-config diff --git a/third_party/waf/waflib/Tools/tex.py b/third_party/waf/waflib/Tools/tex.py index eaf9fdb5802..b4792c3fe87 100644 --- a/third_party/waf/waflib/Tools/tex.py +++ b/third_party/waf/waflib/Tools/tex.py @@ -90,6 +90,7 @@ class tex(Task.Task): Compiles a tex/latex file. .. inheritance-diagram:: waflib.Tools.tex.latex waflib.Tools.tex.xelatex waflib.Tools.tex.pdflatex + :top-classes: waflib.Tools.tex.tex """ bibtex_fun, _ = Task.compile_fun('${BIBTEX} ${BIBTEXFLAGS} ${SRCFILE}', shell=False) -- 2.36.1