From 092dd57a6ed2adc85eb296217c0f74c0a0e0af67 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 10:38:49 +0100 Subject: [PATCH 01/11] Makefile: add support for 'make testonly' That skips any attempt to recompile before running the tests. Some times that's useful for debugging and we'll use it to split the build and test stages in autobuild and gitlab-ci later. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 1e4714940211b10ae6574770f15b7c6ed95f5f59) (cherry picked from commit 9fed2749c039164794faadef71aa83cfd360d130) --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 0b7b0ae88661..7f5960d5191c 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,9 @@ uninstall: test: $(WAF) test $(TEST_OPTIONS) +testonly: + $(WAF) testonly $(TEST_OPTIONS) + perftest: $(WAF) test --perf-test $(TEST_OPTIONS) -- 2.25.1 From f52eb92f492c32609f8caa8cef96b701435ccd19 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 19 Nov 2020 16:19:53 +0000 Subject: [PATCH 02/11] selftest: allow a prefix under /m/username/ We only want to match/replace only a '.' pathname component not any single character pathname compoment! BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 02301222386f2f08631d48d6e88c03cd1439325d) (cherry picked from commit f480161b754aade6c1af2d05f3ce742466b28026) --- selftest/selftest.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index d14df92a11c8..cda4c0e2f4bf 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -280,7 +280,7 @@ my $bindir_abs = abs_path($bindir); my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200); $prefix =~ s+//+/+; -$prefix =~ s+/./+/+; +$prefix =~ s+/\./+/+; $prefix =~ s+/$++; die("using an empty prefix isn't allowed") unless $prefix ne ""; -- 2.25.1 From e54d84699612df106edde3ee7f2fd6dfd62cecae Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 17 Dec 2020 10:42:03 +0100 Subject: [PATCH 03/11] selftest:Samba4: avoid File::Path 'make_path' in setup_dns_hub_internal() While spliting the build and test stages I hit strange permission problems, when a parent directory is missing, which can be avoided by using plain mkdir() on each level. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 719eccd445e9cc56a1c2988c4deeb39d301bcbff) (cherry picked from commit 56c2c0f651e1c038ecf87a14a7dbe478e5e58d8d) --- selftest/target/Samba4.pm | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index 649e923ff9a3..fa5b744c73be 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -17,7 +17,6 @@ use SocketWrapper; use target::Samba; use target::Samba3; use Archive::Tar; -use File::Path 'make_path'; sub new($$$$$) { my ($classname, $SambaCtx, $bindir, $srcdir, $server_maxtime) = @_; @@ -281,7 +280,7 @@ sub setup_dns_hub_internal($$$) my ($self, $hostname, $prefix) = @_; my $STDIN_READER; - unless(-d $prefix or make_path($prefix, 0777)) { + unless(-d $prefix or mkdir($prefix, 0777)) { warn("Unable to create $prefix"); return undef; } @@ -356,6 +355,10 @@ sub setup_dns_hub my $hostname = "rootdnsforwarder"; + unless(-d $prefix or mkdir($prefix, 0777)) { + warn("Unable to create $prefix"); + return undef; + } my $env = $self->setup_dns_hub_internal("$hostname", "$prefix/$hostname"); $self->{dns_hub_env} = $env; -- 2.25.1 From 29c6f0a65f5bdd7fca89400dc1f51ff549bd1dd2 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 11:35:33 +0100 Subject: [PATCH 04/11] selftest/Samba4: make more use of get_cmd_env_vars() This simplifies the code a lot and makes it much easier to add new environment variables in future. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 15b39160406c3ef49c5f074793d3a55b3bf12e0e) (cherry picked from commit 9d5f5e821cbe23cc2e64f201e7409aaec4b50387) --- selftest/target/Samba4.pm | 101 +++++++------------------------------- 1 file changed, 19 insertions(+), 82 deletions(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index fa5b744c73be..50103d5b2e0e 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -160,19 +160,7 @@ sub wait_for_start($$) my $max_wait = 60; # Add hosts file for name lookups - my $cmd = "NSS_WRAPPER_HOSTS='$testenv_vars->{NSS_WRAPPER_HOSTS}' "; - if (defined($testenv_vars->{RESOLV_WRAPPER_CONF})) { - $cmd .= "RESOLV_WRAPPER_CONF='$testenv_vars->{RESOLV_WRAPPER_CONF}' "; - } else { - $cmd .= "RESOLV_WRAPPER_HOSTS='$testenv_vars->{RESOLV_WRAPPER_HOSTS}' "; - } - $cmd .= "RESOLV_CONF='$testenv_vars->{RESOLV_CONF}' "; - if (defined($testenv_vars->{GNUTLS_FORCE_FIPS_MODE})) { - $cmd .= "GNUTLS_FORCE_FIPS_MODE=$testenv_vars->{GNUTLS_FORCE_FIPS_MODE} "; - } - if (defined($testenv_vars->{OPENSSL_FORCE_FIPS_MODE})) { - $cmd .= "OPENSSL_FORCE_FIPS_MODE=$testenv_vars->{OPENSSL_FORCE_FIPS_MODE} "; - } + my $cmd = $self->get_cmd_env_vars($testenv_vars); $cmd .= "$ldbsearch "; $cmd .= "$testenv_vars->{CONFIGURATION} "; @@ -925,11 +913,10 @@ sub provision_raw_step2($$$) return undef; } + my $cmd_env = $self->get_cmd_env_vars($ret); + my $testallowed_account = "testallowed"; - my $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + my $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} $testallowed_account $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -938,10 +925,7 @@ sub provision_raw_step2($$$) } my $srv_account = "srv_account"; - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} $srv_account $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -949,10 +933,7 @@ sub provision_raw_step2($$$) return undef; } - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " spn add HOST/$srv_account --configfile=$ctx->{smb_conf} $srv_account"; unless (system($samba_tool_cmd) == 0) { @@ -960,10 +941,7 @@ sub provision_raw_step2($$$) return undef; } - my $ldbmodify = ""; - $ldbmodify .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $ldbmodify .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $ldbmodify .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + my $ldbmodify = ${cmd_env}; $ldbmodify .= Samba::bindir_path($self, "ldbmodify"); $ldbmodify .= " --configfile=$ctx->{smb_conf}"; my $base_dn = "DC=".join(",DC=", split(/\./, $ctx->{realm})); @@ -994,10 +972,7 @@ servicePrincipalName: host/testallowed "; close(LDIF); - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} testdenied $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -1015,10 +990,7 @@ userPrincipalName: testdenied_upn\@$ctx->{realm}.upn "; close(LDIF); - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} testupnspn $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -1038,10 +1010,7 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} "; close(LDIF); - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " group addmembers --configfile=$ctx->{smb_conf} 'Allowed RODC Password Replication Group' '$testallowed_account'"; unless (system($samba_tool_cmd) == 0) { @@ -1053,11 +1022,8 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} my $user_account_array = ["alice", "bob", "jane", "joe"]; foreach my $user_account (@{$user_account_array}) { - my $samba_tool_cmd = ""; + my $samba_tool_cmd = ${cmd_env}; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} $user_account Secret007"; unless (system($samba_tool_cmd) == 0) { @@ -1069,10 +1035,8 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} my $group_array = ["Samba Users"]; foreach my $group (@{$group_array}) { - my $samba_tool_cmd = ""; + my $samba_tool_cmd = ${cmd_env}; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " group add --configfile=$ctx->{smb_conf} \"$group\""; unless (system($samba_tool_cmd) == 0) { @@ -1082,12 +1046,10 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} } # Add user joe to group "Samba Users" - $samba_tool_cmd = ""; my $group = "Samba Users"; my $user_account = "joe"; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " group addmembers --configfile=$ctx->{smb_conf} \"$group\" $user_account"; unless (system($samba_tool_cmd) == 0) { @@ -1095,12 +1057,10 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} return undef; } - $samba_tool_cmd = ""; $group = "Samba Users"; $user_account = "joe"; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user setprimarygroup --configfile=$ctx->{smb_conf} $user_account \"$group\""; unless (system($samba_tool_cmd) == 0) { @@ -1109,10 +1069,7 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} } # Change the userPrincipalName for jane - $ldbmodify = ""; - $ldbmodify .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $ldbmodify .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $ldbmodify .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $ldbmodify = ${cmd_env}; $ldbmodify .= Samba::bindir_path($self, "ldbmodify"); $ldbmodify .= " --configfile=$ctx->{smb_conf}"; $base_dn = "DC=".join(",DC=", split(/\./, $ctx->{realm})); @@ -1827,9 +1784,7 @@ sub provision_rodc($$$) # This ensures deterministic behaviour for tests that want to have the 'testallowed account' # user password verified on the RODC my $testallowed_account = "testallowed account"; - $cmd = "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; + $cmd = $self->get_cmd_env_vars($ret); $cmd .= "$samba_tool rodc preload '$testallowed_account' $ret->{CONFIGURATION}"; $cmd .= " --server=$dcvars->{DC_SERVER}"; @@ -2505,14 +2460,10 @@ sub setup_promoted_dc # force source and replicated DC to update repsTo/repsFrom # for vampired partitions my $samba_tool = Samba::bindir_path($self, "samba-tool"); - my $cmd = "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; + my $cmd = $self->get_cmd_env_vars($env); # as 'vampired' dc may add data in its local replica # we need to synchronize data between DCs my $base_dn = "DC=".join(",DC=", split(/\./, $dc_vars->{REALM})); - $cmd = "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\""; - $cmd .= " KRB5_CONFIG=\"$env->{KRB5_CONFIG}\""; - $cmd .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; $cmd .= " $samba_tool drs replicate $env->{DC_SERVER} $env->{SERVER}"; $cmd .= " $dc_vars->{CONFIGURATION}"; $cmd .= " -U$dc_vars->{DC_USERNAME}\%$dc_vars->{DC_PASSWORD}"; @@ -2548,14 +2499,9 @@ sub setup_rodc } my $samba_tool = Samba::bindir_path($self, "samba-tool"); - my $cmd = ""; + my $cmd = $self->get_cmd_env_vars($env); my $base_dn = "DC=".join(",DC=", split(/\./, $dc_vars->{REALM})); - $cmd .= "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; - $cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\""; - $cmd .= " KRB5_CONFIG=\"$env->{KRB5_CONFIG}\""; - $cmd .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; $cmd .= " $samba_tool drs replicate $env->{SERVER} $env->{DC_SERVER}"; $cmd .= " $dc_vars->{CONFIGURATION}"; $cmd .= " -U$dc_vars->{DC_USERNAME}\%$dc_vars->{DC_PASSWORD}"; @@ -2860,16 +2806,7 @@ sub setup_schema_pair_dc ""); my $samba_tool = Samba::bindir_path($self, "samba-tool"); - my $cmd_vars = "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; - $cmd_vars .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - if (defined($env->{RESOLV_WRAPPER_CONF})) { - $cmd_vars .= "RESOLV_WRAPPER_CONF=\"$env->{RESOLV_WRAPPER_CONF}\" "; - } else { - $cmd_vars .= "RESOLV_WRAPPER_HOSTS=\"$env->{RESOLV_WRAPPER_HOSTS}\" "; - } - $cmd_vars .= "KRB5_CONFIG=\"$env->{KRB5_CONFIG}\" "; - $cmd_vars .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; - $cmd_vars .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; + my $cmd_vars = $self->get_cmd_env_vars($env); my $join_cmd = $cmd_vars; $join_cmd .= "$samba_tool domain join $env->{CONFIGURATION} $dcvars->{REALM} DC --realm=$dcvars->{REALM}"; -- 2.25.1 From 65f6113b31122a68adae57fa3936565efde21fb3 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 11:35:33 +0100 Subject: [PATCH 05/11] selftest/Samba4: correctly pass KRB5CCNAME to provision BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit dce0bdc39ebb01ef4f5e35af0552451cfc29fd1b) (cherry picked from commit 85800df90358f3a76b6b86f8414582178fe50946) --- selftest/target/Samba4.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index 50103d5b2e0e..aab1fac612bd 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -600,7 +600,7 @@ sub provision_raw_prepare($$$$$$$$$$$$$$) my @provision_options = (); push (@provision_options, "KRB5_CONFIG=\"$ctx->{krb5_conf}\""); - push (@provision_options, "KRB5_CCACHE=\"$ctx->{krb5_ccache}\""); + push (@provision_options, "KRB5CCNAME=\"$ctx->{krb5_ccache}\""); push (@provision_options, "NSS_WRAPPER_PASSWD=\"$ctx->{nsswrap_passwd}\""); push (@provision_options, "NSS_WRAPPER_GROUP=\"$ctx->{nsswrap_group}\""); push (@provision_options, "NSS_WRAPPER_HOSTS=\"$ctx->{nsswrap_hosts}\""); -- 2.25.1 From 352c2a41bc5681fce001a1b57c3ecb40529f33de Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 11:35:33 +0100 Subject: [PATCH 06/11] selftest/Samba4: allow get_cmd_env_vars() to take an overwrite dictionary This way we can use it on even in some special cases, where we combine variables from multiple environments. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 568c7d38debaa5ccd90d6ea33c683de512de7005) (cherry picked from commit f1c7967b568034da2707ccc4bd1f64358d55eacc) --- selftest/target/Samba4.pm | 67 ++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index aab1fac612bd..dec83dbb02b9 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -366,10 +366,43 @@ sub get_dns_hub_env($) return undef; } +sub return_env_value +{ + my ($env, $overwrite, $key) = @_; + + if (defined($overwrite) and defined($overwrite->{$key})) { + return $overwrite->{$key}; + } + + if (defined($env->{$key})) { + return $env->{$key}; + } + + return undef; +} + # Returns the environmental variables that we pass to samba-tool commands sub get_cmd_env_vars { - my ($self, $localenv) = @_; + my ($self, $givenenv, $overwrite) = @_; + + my @keys = ( + "NSS_WRAPPER_HOSTS", + "SOCKET_WRAPPER_DEFAULT_IFACE", + "RESOLV_CONF", + "RESOLV_WRAPPER_CONF", + "RESOLV_WRAPPER_HOSTS", + "GNUTLS_FORCE_FIPS_MODE", + "OPENSSL_FORCE_FIPS_MODE", + "KRB5_CONFIG", + "KRB5_CCACHE", + ); + + my $localenv = undef; + foreach my $key (@keys) { + my $v = return_env_value($givenenv, $overwrite, $key); + $localenv->{$key} = $v if defined($v); + } my $cmd_env = "NSS_WRAPPER_HOSTS='$localenv->{NSS_WRAPPER_HOSTS}' "; $cmd_env .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$localenv->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; @@ -384,7 +417,7 @@ sub get_cmd_env_vars if (defined($localenv->{OPENSSL_FORCE_FIPS_MODE})) { $cmd_env .= "OPENSSL_FORCE_FIPS_MODE=$localenv->{OPENSSL_FORCE_FIPS_MODE} "; } - $cmd_env .= " KRB5_CONFIG=\"$localenv->{KRB5_CONFIG}\" "; + $cmd_env .= "KRB5_CONFIG=\"$localenv->{KRB5_CONFIG}\" "; $cmd_env .= "KRB5CCNAME=\"$localenv->{KRB5_CCACHE}\" "; $cmd_env .= "RESOLV_CONF=\"$localenv->{RESOLV_CONF}\" "; @@ -1369,12 +1402,13 @@ sub provision_rpc_proxy($$$) return undef; } + # Prepare a context of the DC, but using the local CCACHE. + my $overwrite = undef; + $overwrite->{KRB5_CCACHE} = $ret->{KRB5_CCACHE}; + my $dc_cmd_env = $self->get_cmd_env_vars($dcvars, $overwrite); + # Setting up delegation runs in the context of the DC for now - $cmd = ""; - $cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$dcvars->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - $cmd .= "KRB5_CONFIG=\"$dcvars->{KRB5_CONFIG}\" "; - $cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$dcvars->{RESOLV_CONF}\" "; + $cmd = $dc_cmd_env; $cmd .= "$samba_tool delegation for-any-protocol '$ret->{NETBIOSNAME}\$' on"; $cmd .= " $dcvars->{CONFIGURATION}"; print $cmd; @@ -1385,11 +1419,7 @@ sub provision_rpc_proxy($$$) } # Setting up delegation runs in the context of the DC for now - $cmd = ""; - $cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$dcvars->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - $cmd .= "KRB5_CONFIG=\"$dcvars->{KRB5_CONFIG}\" "; - $cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$dcvars->{RESOLV_CONF}\" "; + $cmd = $dc_cmd_env; $cmd .= "$samba_tool delegation add-service '$ret->{NETBIOSNAME}\$' cifs/$dcvars->{SERVER}"; $cmd .= " $dcvars->{CONFIGURATION}"; @@ -2948,17 +2978,10 @@ sub create_backup my ($self, $env, $dcvars, $backupdir, $backup_cmd) = @_; # get all the env variables we pass in with the samba-tool command - my $cmd_env = "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; - $cmd_env .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - if (defined($env->{RESOLV_WRAPPER_CONF})) { - $cmd_env .= "RESOLV_WRAPPER_CONF=\"$env->{RESOLV_WRAPPER_CONF}\" "; - } else { - $cmd_env .= "RESOLV_WRAPPER_HOSTS=\"$env->{RESOLV_WRAPPER_HOSTS}\" "; - } - $cmd_env .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; # Note: use the backupfrom-DC's krb5.conf to do the backup - $cmd_env .= " KRB5_CONFIG=\"$dcvars->{KRB5_CONFIG}\" "; - $cmd_env .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; + my $overwrite = undef; + $overwrite->{KRB5_CONFIG} = $dcvars->{KRB5_CONFIG}; + my $cmd_env = $self->get_cmd_env_vars($env, $overwrite); # use samba-tool to create a backup from the 'backupfromdc' DC my $cmd = ""; -- 2.25.1 From 7d875e6f83a2877d8879102c7060616182b64e8e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 17 Dec 2020 06:38:14 +0100 Subject: [PATCH 07/11] s3:selftest: run test_smbclient_tarmode.pl with a fixed subdirectory name $PREFIX is the the value from --with-selftest-prefix. The result of the test should not depend on --with-selftest-prefix, the 'long_path' test in particular. If the path is to long smbclient (via libarchive) will only put the full path into a PAX HEADER as 'path' keyword, that's fine in general, modern tools handle it just fine. But Perl's Archive::Tar don't handle it and only seems truncated file names. I have a fix for Archive::Tar, see: https://git.samba.org/?p=metze/samba/wip.git;a=shortlog;h=c75037d0a06a96cdaca3f3b20a6d237e768b075b But finishing that is a task for another day, for now I just want to remove the dependency to --with-selftest-prefix. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit e0d9b656452ba6277cdc7f0abb2a06d3d284ef3a) (cherry picked from commit 3eba14718dd6269fe1657de15a2f47c848b60518) --- source3/selftest/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index bbd249c3ccb0..47e914b10093 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -487,13 +487,13 @@ for env in ["fileserver"]: [os.path.join(samba3srcdir, "script/tests/test_smbclient_tarmode.pl"), '-n', '$SERVER', '-i', '$SERVER_IP', '-s', 'tarmode2', '-u', '$USERNAME', '-p', '$PASSWORD', '-l', '$LOCAL_PATH/tarmode2', - '-d', '$PREFIX', '-b', smbclient3, + '-d', 'smbclient_tar.NT1', '-b', smbclient3, '--subunit', '--', configuration, '-mNT1']) plantestsuite("samba3.blackbox.smbclient_tar.SMB3", env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_tarmode.pl"), '-n', '$SERVER', '-i', '$SERVER_IP', '-s', 'tarmode2', '-u', '$USERNAME', '-p', '$PASSWORD', '-l', '$LOCAL_PATH/tarmode2', - '-d', '$PREFIX', '-b', smbclient3, + '-d', 'smbclient_tar.SMB3', '-b', smbclient3, '--subunit', '--', configuration, '-mSMB3']) for env in ["fileserver:local"]: -- 2.25.1 From c299de6bd69d6c503f65f133c51b6ea75d17f8d8 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 22 Nov 2020 22:43:36 +0100 Subject: [PATCH 08/11] s4:selftest: use plansmbtorture4testsuite() for 'rpc.echo' This makes sure "--basedir=$SELFTEST_TMPDIR" is passed to smbtorture. Tests should not create files in the build nor the source directory! BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit d06f2c22d726a5ec7bd804d89154ee272ab1a679) (cherry picked from commit 81b36b389cb01eca9b2f0a2a452d290e21f31394) --- source4/selftest/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 3a903a7eee08..258c9122edc4 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -561,7 +561,8 @@ if have_gnutls_crypto_policies: plantestsuite("samba3.wbinfo_simple.fips.%s" % t, "ad_member_fips:local", [os.path.join(srcdir(), "nsswitch/tests/test_wbinfo_simple.sh"), t]) plantestsuite("samba4.wbinfo_name_lookup.fips", "ad_member_fips", [os.path.join(srcdir(), "nsswitch/tests/test_wbinfo_name_lookup.sh"), '$DOMAIN', '$REALM', '$DC_USERNAME']) -plantestsuite_loadlist("samba4.rpc.echo against NetBIOS alias", "ad_dc_ntvfs", [valgrindify(smbtorture4), "$LISTOPT", "$LOADLIST", 'ncacn_np:$NETBIOSALIAS', '-U$DOMAIN/$USERNAME%$PASSWORD', 'rpc.echo']) +plansmbtorture4testsuite('rpc.echo', "ad_dc_ntvfs", ['ncacn_np:$NETBIOSALIAS', '-U$DOMAIN/$USERNAME%$PASSWORD'], "samba4.rpc.echo against NetBIOS alias") + # json tests hook into ``chgdcpass'' to make them run in contributor CI on # gitlab planpythontestsuite("chgdcpass", "samba.tests.blackbox.netads_json") -- 2.25.1 From 67af56413fc3626600f7999d56a88e3034c48705 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 22 Nov 2020 23:28:31 +0100 Subject: [PATCH 09/11] selftest: make/use a copy of GNUPGHOME That makes it possible to run tests from a read only source tree. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 86343125a55d184c15aa94cd01f4c8893a5a0917) (cherry picked from commit c1a4cb97d1d71b974eed2ecb5f34bb1425f36294) --- selftest/selftest.pl | 5 ++++- selftest/target/Samba.pm | 26 ++++++++++++++++++++++++++ selftest/target/Samba4.pm | 6 ++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index cda4c0e2f4bf..258a8437922c 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -312,7 +312,6 @@ $ENV{PREFIX} = $prefix; $ENV{PREFIX_ABS} = $prefix_abs; $ENV{SRCDIR} = $srcdir; $ENV{SRCDIR_ABS} = $srcdir_abs; -$ENV{GNUPGHOME} = "$srcdir_abs/selftest/gnupg"; $ENV{BINDIR} = $bindir_abs; my $tls_enabled = not $opt_quick; @@ -693,6 +692,9 @@ $ENV{RESOLV_CONF} = "${selftest_resolv_conf_path}.global"; my $selftest_krbt_ccache_path = "$tmpdir_abs/selftest.krb5_ccache"; $ENV{KRB5CCNAME} = "FILE:${selftest_krbt_ccache_path}.global"; +my $selftest_gnupghome_path = "$tmpdir_abs/selftest.no.gnupg"; +$ENV{GNUPGHOME} = "${selftest_gnupghome_path}.global"; + my @available = (); foreach my $fn (@testlists) { foreach (read_testlist($fn)) { @@ -829,6 +831,7 @@ sub setup_env($$) $ENV{RESOLV_CONF} = "${selftest_resolv_conf_path}.${envname}/ignore"; $ENV{KRB5CCNAME} = "FILE:${selftest_krbt_ccache_path}.${envname}/ignore"; + $ENV{GNUPGHOME} = "${selftest_gnupghome_path}.${envname}/ignore"; if (defined(get_running_env($envname))) { $testenv_vars = get_running_env($envname); diff --git a/selftest/target/Samba.pm b/selftest/target/Samba.pm index 6118f2e243ad..d47f933376ef 100644 --- a/selftest/target/Samba.pm +++ b/selftest/target/Samba.pm @@ -280,6 +280,30 @@ EOF umask $oldumask; } +sub copy_gnupg_home($) +{ + my ($ctx) = @_; + + my $gnupg_srcdir = "$ENV{SRCDIR_ABS}/selftest/gnupg"; + my @files = ( + "gpg.conf", + "pubring.gpg", + "secring.gpg", + "trustdb.gpg", + ); + + my $oldumask = umask; + umask 0077; + mkdir($ctx->{gnupghome}, 0777); + umask 0177; + foreach my $file (@files) { + my $srcfile = "${gnupg_srcdir}/${file}"; + my $dstfile = "$ctx->{gnupghome}/${file}"; + copy_file_content(${srcfile}, ${dstfile}); + } + umask $oldumask; +} + sub mk_krb5_conf($$) { my ($ctx) = @_; @@ -672,6 +696,7 @@ sub get_env_for_process RESOLV_CONF => $env_vars->{RESOLV_CONF}, KRB5_CONFIG => $env_vars->{KRB5_CONFIG}, KRB5CCNAME => "$env_vars->{KRB5_CCACHE}.$proc_name", + GNUPGHOME => $env_vars->{GNUPGHOME}, SELFTEST_WINBINDD_SOCKET_DIR => $env_vars->{SELFTEST_WINBINDD_SOCKET_DIR}, NMBD_SOCKET_DIR => $env_vars->{NMBD_SOCKET_DIR}, NSS_WRAPPER_PASSWD => $env_vars->{NSS_WRAPPER_PASSWD}, @@ -857,6 +882,7 @@ my @exported_envvars = ( # misc stuff "KRB5_CONFIG", "KRB5CCNAME", + "GNUPGHOME", "SELFTEST_WINBINDD_SOCKET_DIR", "NMBD_SOCKET_DIR", "LOCAL_PATH", diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index dec83dbb02b9..77bd741d4766 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -396,6 +396,7 @@ sub get_cmd_env_vars "OPENSSL_FORCE_FIPS_MODE", "KRB5_CONFIG", "KRB5_CCACHE", + "GNUPGHOME", ); my $localenv = undef; @@ -420,6 +421,7 @@ sub get_cmd_env_vars $cmd_env .= "KRB5_CONFIG=\"$localenv->{KRB5_CONFIG}\" "; $cmd_env .= "KRB5CCNAME=\"$localenv->{KRB5_CCACHE}\" "; $cmd_env .= "RESOLV_CONF=\"$localenv->{RESOLV_CONF}\" "; + $cmd_env .= "GNUPGHOME=\"$localenv->{GNUPGHOME}\" "; return $cmd_env; } @@ -589,6 +591,7 @@ sub provision_raw_prepare($$$$$$$$$$$$$$) $ctx->{krb5_conf} = "$ctx->{etcdir}/krb5.conf"; $ctx->{krb5_ccache} = "$prefix_abs/krb5_ccache"; $ctx->{mitkdc_conf} = "$ctx->{etcdir}/mitkdc.conf"; + $ctx->{gnupghome} = "$prefix_abs/gnupg"; $ctx->{privatedir} = "$prefix_abs/private"; $ctx->{binddnsdir} = "$prefix_abs/bind-dns"; $ctx->{ncalrpcdir} = "$prefix_abs/ncalrpc"; @@ -632,6 +635,7 @@ sub provision_raw_prepare($$$$$$$$$$$$$$) $ctx->{smb_conf_extra_options} = ""; my @provision_options = (); + push (@provision_options, "GNUPGHOME=\"$ctx->{gnupghome}\""); push (@provision_options, "KRB5_CONFIG=\"$ctx->{krb5_conf}\""); push (@provision_options, "KRB5CCNAME=\"$ctx->{krb5_ccache}\""); push (@provision_options, "NSS_WRAPPER_PASSWD=\"$ctx->{nsswrap_passwd}\""); @@ -724,6 +728,7 @@ sub provision_raw_step1($$) return undef; } + Samba::copy_gnupg_home($ctx); Samba::prepare_keyblobs($ctx); my $crlfile = "$ctx->{tlsdir}/crl.pem"; $crlfile = "" unless -e ${crlfile}; @@ -867,6 +872,7 @@ nogroup:x:65534:nobody # Note that we have SERVER_X and DC_SERVER_X variables (which have the same # value initially). In a 2 DC setup, $DC_SERVER_X will always be the PDC. my $ret = { + GNUPGHOME => $ctx->{gnupghome}, KRB5_CONFIG => $ctx->{krb5_conf}, KRB5_CCACHE => $ctx->{krb5_ccache}, MITKDC_CONFIG => $ctx->{mitkdc_conf}, -- 2.25.1 From 55bbb796cb901f97ea435f18938136e0dcd5e953 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 20 Nov 2020 09:20:14 +0000 Subject: [PATCH 10/11] script/autobuild.py: split out a rmdir_force() helper function That also tries to re-add write permissions before removing. In future we'll have jobs changing there directory to read-only. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 7a5df2deaaf62a7edd7c64251f75ab15abe94c07) (cherry picked from commit c933135969be29072971f96481b05f499fd48b57) --- script/autobuild.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/script/autobuild.py b/script/autobuild.py index 0ab04eb7c267..7811ff8a7b06 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -4,7 +4,7 @@ # released under GNU GPL v3 or later from __future__ import print_function -from subprocess import call, check_call, check_output, Popen, PIPE +from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError import os import tarfile import sys @@ -860,6 +860,17 @@ def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): else: return call(cmd, shell=True, cwd=dir) +def rmdir_force(dirname, re_raise=True): + try: + run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % ( + dirname, dirname, dirname), output=True, show=True) + except CalledProcessError as e: + do_print("Failed: '%s'" % (str(e))) + run_cmd("tree %s" % dirname, output=True, show=True) + if re_raise: + raise + return False + return True class builder(object): '''handle build of one directory''' @@ -882,8 +893,8 @@ class builder(object): self.test_source_dir = "%s/%s" % (testbase, self.tag) self.cwd = "%s/%s" % (self.test_source_dir, self.dir) self.prefix = "%s/%s" % (test_prefix, self.tag) - run_cmd("rm -rf %s" % self.test_source_dir) - run_cmd("rm -rf %s" % self.prefix) + rmdir_force(self.test_source_dir) + rmdir_force(self.prefix) if cp: run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) else: @@ -893,8 +904,8 @@ class builder(object): def start_next(self): if self.next == len(self.sequence): if not options.nocleanup: - run_cmd("rm -rf %s" % self.test_source_dir) - run_cmd("rm -rf %s" % self.prefix) + rmdir_force(self.test_source_dir) + rmdir_force(self.prefix) do_print('%s: Completed OK' % self.name) self.done = True return @@ -1018,7 +1029,7 @@ class buildlist(object): 'df -m %s' % testbase]: try: out = run_cmd(cmd, output=True, checkfail=False) - except subprocess.CalledProcessError as e: + except CalledProcessError as e: out = "" % str(e) print('### %s' % cmd, file=f) print(out, file=f) @@ -1055,7 +1066,7 @@ def cleanup(): run_cmd("stat %s" % testbase, show=True) do_print("Cleaning up %r" % cleanup_list) for d in cleanup_list: - run_cmd("rm -rf %s" % d) + rmdir_force(d) def daemonize(logfile): -- 2.25.1 From 7f735d5b2ccceafd535b069b1d84b9c24c474a03 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 20 Nov 2020 09:20:14 +0000 Subject: [PATCH 11/11] script/autobuild.py: let cleanup() ignore errors from rmdir_force() by default It's not useful to generate a python backtrace from within the cleanup code. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 9883ac45939f253a63f3ff312fc3912c5f02cdac) Autobuild-User(v4-14-test): Karolin Seeger Autobuild-Date(v4-14-test): Tue Feb 2 10:29:44 UTC 2021 on sn-devel-184 (cherry picked from commit cc1568be4d4250390a9ad03c84f5e260fc7acffd) --- script/autobuild.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/script/autobuild.py b/script/autobuild.py index 7811ff8a7b06..0f837d0c1091 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -1059,14 +1059,23 @@ class buildlist(object): self.tail_proc = Popen(cmd, close_fds=True) -def cleanup(): +def cleanup(do_raise=False): if options.nocleanup: return run_cmd("stat %s || true" % test_tmpdir, show=True) run_cmd("stat %s" % testbase, show=True) do_print("Cleaning up %r" % cleanup_list) for d in cleanup_list: - rmdir_force(d) + ok = rmdir_force(d, re_raise=False) + if ok: + continue + if os.path.isdir(d): + do_print("Killing, waiting and retry") + run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False) + else: + do_print("Waiting and retry") + time.sleep(1) + rmdir_force(d, re_raise=do_raise) def daemonize(logfile): @@ -1332,7 +1341,7 @@ while True: (status, failed_task, failed_stage, failed_tag, errstr) = blist.run() if status != 0 or errstr != "retry": break - cleanup() + cleanup(do_raise=True) except Exception: cleanup() raise -- 2.25.1