From 4198b95cc0ab30f07366bda0385472868f6969c3 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 10:38:49 +0100 Subject: [PATCH 01/11] Makefile: add support for 'make testonly' That skips any attempt to recompile before running the tests. Some times that's useful for debugging and we'll use it to split the build and test stages in autobuild and gitlab-ci later. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 1e4714940211b10ae6574770f15b7c6ed95f5f59) --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 0b7b0ae8866..7f5960d5191 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,9 @@ uninstall: test: $(WAF) test $(TEST_OPTIONS) +testonly: + $(WAF) testonly $(TEST_OPTIONS) + perftest: $(WAF) test --perf-test $(TEST_OPTIONS) -- 2.30.0 From 4ab2d463ef2308eb89bb2941f94e3f82892a997d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 19 Nov 2020 16:19:53 +0000 Subject: [PATCH 02/11] selftest: allow a prefix under /m/username/ We only want to match/replace only a '.' pathname component not any single character pathname compoment! BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 02301222386f2f08631d48d6e88c03cd1439325d) --- selftest/selftest.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index 6ea21fa6bfe..3bb0c212667 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -281,7 +281,7 @@ my $bindir_abs = abs_path($bindir); my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200); $prefix =~ s+//+/+; -$prefix =~ s+/./+/+; +$prefix =~ s+/\./+/+; $prefix =~ s+/$++; die("using an empty prefix isn't allowed") unless $prefix ne ""; -- 2.30.0 From 65189356e29c20e6d3be7a616a5a883f9c16685d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 17 Dec 2020 10:42:03 +0100 Subject: [PATCH 03/11] selftest:Samba4: avoid File::Path 'make_path' in setup_dns_hub_internal() While spliting the build and test stages I hit strange permission problems, when a parent directory is missing, which can be avoided by using plain mkdir() on each level. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 719eccd445e9cc56a1c2988c4deeb39d301bcbff) --- selftest/target/Samba4.pm | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index 1ebdf2a5484..c743de0a981 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -17,7 +17,6 @@ use SocketWrapper; use target::Samba; use target::Samba3; use Archive::Tar; -use File::Path 'make_path'; sub new($$$$$) { my ($classname, $SambaCtx, $bindir, $srcdir, $server_maxtime) = @_; @@ -281,7 +280,7 @@ sub setup_dns_hub_internal($$$) my ($self, $hostname, $prefix) = @_; my $STDIN_READER; - unless(-d $prefix or make_path($prefix, 0777)) { + unless(-d $prefix or mkdir($prefix, 0777)) { warn("Unable to create $prefix"); return undef; } @@ -356,6 +355,10 @@ sub setup_dns_hub my $hostname = "rootdnsforwarder"; + unless(-d $prefix or mkdir($prefix, 0777)) { + warn("Unable to create $prefix"); + return undef; + } my $env = $self->setup_dns_hub_internal("$hostname", "$prefix/$hostname"); $self->{dns_hub_env} = $env; -- 2.30.0 From 119a3a779324ea6a4b10551f6d71df67c1258a7e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 11:35:33 +0100 Subject: [PATCH 04/11] selftest/Samba4: make more use of get_cmd_env_vars() This simplifies the code a lot and makes it much easier to add new environment variables in future. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 15b39160406c3ef49c5f074793d3a55b3bf12e0e) --- selftest/target/Samba4.pm | 101 +++++++------------------------------- 1 file changed, 19 insertions(+), 82 deletions(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index c743de0a981..7bce72b7ebf 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -160,19 +160,7 @@ sub wait_for_start($$) my $max_wait = 60; # Add hosts file for name lookups - my $cmd = "NSS_WRAPPER_HOSTS='$testenv_vars->{NSS_WRAPPER_HOSTS}' "; - if (defined($testenv_vars->{RESOLV_WRAPPER_CONF})) { - $cmd .= "RESOLV_WRAPPER_CONF='$testenv_vars->{RESOLV_WRAPPER_CONF}' "; - } else { - $cmd .= "RESOLV_WRAPPER_HOSTS='$testenv_vars->{RESOLV_WRAPPER_HOSTS}' "; - } - $cmd .= "RESOLV_CONF='$testenv_vars->{RESOLV_CONF}' "; - if (defined($testenv_vars->{GNUTLS_FORCE_FIPS_MODE})) { - $cmd .= "GNUTLS_FORCE_FIPS_MODE=$testenv_vars->{GNUTLS_FORCE_FIPS_MODE} "; - } - if (defined($testenv_vars->{OPENSSL_FORCE_FIPS_MODE})) { - $cmd .= "OPENSSL_FORCE_FIPS_MODE=$testenv_vars->{OPENSSL_FORCE_FIPS_MODE} "; - } + my $cmd = $self->get_cmd_env_vars($testenv_vars); $cmd .= "$ldbsearch "; $cmd .= "$testenv_vars->{CONFIGURATION} "; @@ -925,11 +913,10 @@ sub provision_raw_step2($$$) return undef; } + my $cmd_env = $self->get_cmd_env_vars($ret); + my $testallowed_account = "testallowed"; - my $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + my $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} $testallowed_account $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -938,10 +925,7 @@ sub provision_raw_step2($$$) } my $srv_account = "srv_account"; - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} $srv_account $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -949,10 +933,7 @@ sub provision_raw_step2($$$) return undef; } - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " spn add HOST/$srv_account --configfile=$ctx->{smb_conf} $srv_account"; unless (system($samba_tool_cmd) == 0) { @@ -960,10 +941,7 @@ sub provision_raw_step2($$$) return undef; } - my $ldbmodify = ""; - $ldbmodify .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $ldbmodify .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $ldbmodify .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + my $ldbmodify = ${cmd_env}; $ldbmodify .= Samba::bindir_path($self, "ldbmodify"); $ldbmodify .= " --configfile=$ctx->{smb_conf}"; my $base_dn = "DC=".join(",DC=", split(/\./, $ctx->{realm})); @@ -994,10 +972,7 @@ servicePrincipalName: host/testallowed "; close(LDIF); - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} testdenied $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -1015,10 +990,7 @@ userPrincipalName: testdenied_upn\@$ctx->{realm}.upn "; close(LDIF); - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} testupnspn $ctx->{password}"; unless (system($samba_tool_cmd) == 0) { @@ -1038,10 +1010,7 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} "; close(LDIF); - $samba_tool_cmd = ""; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " group addmembers --configfile=$ctx->{smb_conf} 'Allowed RODC Password Replication Group' '$testallowed_account' -d10"; unless (system($samba_tool_cmd) == 0) { @@ -1053,11 +1022,8 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} my $user_account_array = ["alice", "bob", "jane", "joe"]; foreach my $user_account (@{$user_account_array}) { - my $samba_tool_cmd = ""; + my $samba_tool_cmd = ${cmd_env}; - $samba_tool_cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user create --configfile=$ctx->{smb_conf} $user_account Secret007"; unless (system($samba_tool_cmd) == 0) { @@ -1069,10 +1035,8 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} my $group_array = ["Samba Users"]; foreach my $group (@{$group_array}) { - my $samba_tool_cmd = ""; + my $samba_tool_cmd = ${cmd_env}; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " group add --configfile=$ctx->{smb_conf} \"$group\""; unless (system($samba_tool_cmd) == 0) { @@ -1082,12 +1046,10 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} } # Add user joe to group "Samba Users" - $samba_tool_cmd = ""; my $group = "Samba Users"; my $user_account = "joe"; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " group addmembers --configfile=$ctx->{smb_conf} \"$group\" $user_account"; unless (system($samba_tool_cmd) == 0) { @@ -1095,12 +1057,10 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} return undef; } - $samba_tool_cmd = ""; $group = "Samba Users"; $user_account = "joe"; - $samba_tool_cmd .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $samba_tool_cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $samba_tool_cmd = ${cmd_env}; $samba_tool_cmd .= Samba::bindir_path($self, "samba-tool") . " user setprimarygroup --configfile=$ctx->{smb_conf} $user_account \"$group\""; unless (system($samba_tool_cmd) == 0) { @@ -1109,10 +1069,7 @@ servicePrincipalName: http/testupnspn.$ctx->{dnsname} } # Change the userPrincipalName for jane - $ldbmodify = ""; - $ldbmodify .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; - $ldbmodify .= "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $ldbmodify .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; + $ldbmodify = ${cmd_env}; $ldbmodify .= Samba::bindir_path($self, "ldbmodify"); $ldbmodify .= " --configfile=$ctx->{smb_conf}"; $base_dn = "DC=".join(",DC=", split(/\./, $ctx->{realm})); @@ -1827,9 +1784,7 @@ sub provision_rodc($$$) # This ensures deterministic behaviour for tests that want to have the 'testallowed account' # user password verified on the RODC my $testallowed_account = "testallowed account"; - $cmd = "KRB5_CONFIG=\"$ret->{KRB5_CONFIG}\" "; - $cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$ret->{RESOLV_CONF}\" "; + $cmd = $self->get_cmd_env_vars($ret); $cmd .= "$samba_tool rodc preload '$testallowed_account' $ret->{CONFIGURATION}"; $cmd .= " --server=$dcvars->{DC_SERVER}"; @@ -2505,14 +2460,10 @@ sub setup_promoted_dc # force source and replicated DC to update repsTo/repsFrom # for vampired partitions my $samba_tool = Samba::bindir_path($self, "samba-tool"); - my $cmd = "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; + my $cmd = $self->get_cmd_env_vars($env); # as 'vampired' dc may add data in its local replica # we need to synchronize data between DCs my $base_dn = "DC=".join(",DC=", split(/\./, $dc_vars->{REALM})); - $cmd = "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\""; - $cmd .= " KRB5_CONFIG=\"$env->{KRB5_CONFIG}\""; - $cmd .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; $cmd .= " $samba_tool drs replicate $env->{DC_SERVER} $env->{SERVER}"; $cmd .= " $dc_vars->{CONFIGURATION}"; $cmd .= " -U$dc_vars->{DC_USERNAME}\%$dc_vars->{DC_PASSWORD}"; @@ -2548,14 +2499,9 @@ sub setup_rodc } my $samba_tool = Samba::bindir_path($self, "samba-tool"); - my $cmd = ""; + my $cmd = $self->get_cmd_env_vars($env); my $base_dn = "DC=".join(",DC=", split(/\./, $dc_vars->{REALM})); - $cmd .= "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; - $cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\""; - $cmd .= " KRB5_CONFIG=\"$env->{KRB5_CONFIG}\""; - $cmd .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; $cmd .= " $samba_tool drs replicate $env->{SERVER} $env->{DC_SERVER}"; $cmd .= " $dc_vars->{CONFIGURATION}"; $cmd .= " -U$dc_vars->{DC_USERNAME}\%$dc_vars->{DC_PASSWORD}"; @@ -2860,16 +2806,7 @@ sub setup_schema_pair_dc ""); my $samba_tool = Samba::bindir_path($self, "samba-tool"); - my $cmd_vars = "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; - $cmd_vars .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - if (defined($env->{RESOLV_WRAPPER_CONF})) { - $cmd_vars .= "RESOLV_WRAPPER_CONF=\"$env->{RESOLV_WRAPPER_CONF}\" "; - } else { - $cmd_vars .= "RESOLV_WRAPPER_HOSTS=\"$env->{RESOLV_WRAPPER_HOSTS}\" "; - } - $cmd_vars .= "KRB5_CONFIG=\"$env->{KRB5_CONFIG}\" "; - $cmd_vars .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; - $cmd_vars .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; + my $cmd_vars = $self->get_cmd_env_vars($env); my $join_cmd = $cmd_vars; $join_cmd .= "$samba_tool domain join $env->{CONFIGURATION} $dcvars->{REALM} DC --realm=$dcvars->{REALM}"; -- 2.30.0 From da34b1a19e63b199b2b7980a90be91d46d749644 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 11:35:33 +0100 Subject: [PATCH 05/11] selftest/Samba4: correctly pass KRB5CCNAME to provision BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit dce0bdc39ebb01ef4f5e35af0552451cfc29fd1b) --- selftest/target/Samba4.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index 7bce72b7ebf..87b2391071e 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -600,7 +600,7 @@ sub provision_raw_prepare($$$$$$$$$$$$$$) my @provision_options = (); push (@provision_options, "KRB5_CONFIG=\"$ctx->{krb5_conf}\""); - push (@provision_options, "KRB5_CCACHE=\"$ctx->{krb5_ccache}\""); + push (@provision_options, "KRB5CCNAME=\"$ctx->{krb5_ccache}\""); push (@provision_options, "NSS_WRAPPER_PASSWD=\"$ctx->{nsswrap_passwd}\""); push (@provision_options, "NSS_WRAPPER_GROUP=\"$ctx->{nsswrap_group}\""); push (@provision_options, "NSS_WRAPPER_HOSTS=\"$ctx->{nsswrap_hosts}\""); -- 2.30.0 From c33b38fdd70164fccdee147fe3185dbbf2199277 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 23 Nov 2020 11:35:33 +0100 Subject: [PATCH 06/11] selftest/Samba4: allow get_cmd_env_vars() to take an overwrite dictionary This way we can use it on even in some special cases, where we combine variables from multiple environments. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit 568c7d38debaa5ccd90d6ea33c683de512de7005) --- selftest/target/Samba4.pm | 67 ++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index 87b2391071e..1c9f85217d8 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -366,10 +366,43 @@ sub get_dns_hub_env($) return undef; } +sub return_env_value +{ + my ($env, $overwrite, $key) = @_; + + if (defined($overwrite) and defined($overwrite->{$key})) { + return $overwrite->{$key}; + } + + if (defined($env->{$key})) { + return $env->{$key}; + } + + return undef; +} + # Returns the environmental variables that we pass to samba-tool commands sub get_cmd_env_vars { - my ($self, $localenv) = @_; + my ($self, $givenenv, $overwrite) = @_; + + my @keys = ( + "NSS_WRAPPER_HOSTS", + "SOCKET_WRAPPER_DEFAULT_IFACE", + "RESOLV_CONF", + "RESOLV_WRAPPER_CONF", + "RESOLV_WRAPPER_HOSTS", + "GNUTLS_FORCE_FIPS_MODE", + "OPENSSL_FORCE_FIPS_MODE", + "KRB5_CONFIG", + "KRB5_CCACHE", + ); + + my $localenv = undef; + foreach my $key (@keys) { + my $v = return_env_value($givenenv, $overwrite, $key); + $localenv->{$key} = $v if defined($v); + } my $cmd_env = "NSS_WRAPPER_HOSTS='$localenv->{NSS_WRAPPER_HOSTS}' "; $cmd_env .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$localenv->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; @@ -384,7 +417,7 @@ sub get_cmd_env_vars if (defined($localenv->{OPENSSL_FORCE_FIPS_MODE})) { $cmd_env .= "OPENSSL_FORCE_FIPS_MODE=$localenv->{OPENSSL_FORCE_FIPS_MODE} "; } - $cmd_env .= " KRB5_CONFIG=\"$localenv->{KRB5_CONFIG}\" "; + $cmd_env .= "KRB5_CONFIG=\"$localenv->{KRB5_CONFIG}\" "; $cmd_env .= "KRB5CCNAME=\"$localenv->{KRB5_CCACHE}\" "; $cmd_env .= "RESOLV_CONF=\"$localenv->{RESOLV_CONF}\" "; @@ -1369,12 +1402,13 @@ sub provision_rpc_proxy($$$) return undef; } + # Prepare a context of the DC, but using the local CCACHE. + my $overwrite = undef; + $overwrite->{KRB5_CCACHE} = $ret->{KRB5_CCACHE}; + my $dc_cmd_env = $self->get_cmd_env_vars($dcvars, $overwrite); + # Setting up delegation runs in the context of the DC for now - $cmd = ""; - $cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$dcvars->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - $cmd .= "KRB5_CONFIG=\"$dcvars->{KRB5_CONFIG}\" "; - $cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$dcvars->{RESOLV_CONF}\" "; + $cmd = $dc_cmd_env; $cmd .= "$samba_tool delegation for-any-protocol '$ret->{NETBIOSNAME}\$' on"; $cmd .= " $dcvars->{CONFIGURATION}"; print $cmd; @@ -1385,11 +1419,7 @@ sub provision_rpc_proxy($$$) } # Setting up delegation runs in the context of the DC for now - $cmd = ""; - $cmd .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$dcvars->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - $cmd .= "KRB5_CONFIG=\"$dcvars->{KRB5_CONFIG}\" "; - $cmd .= "KRB5CCNAME=\"$ret->{KRB5_CCACHE}\" "; - $cmd .= "RESOLV_CONF=\"$dcvars->{RESOLV_CONF}\" "; + $cmd = $dc_cmd_env; $cmd .= "$samba_tool delegation add-service '$ret->{NETBIOSNAME}\$' cifs/$dcvars->{SERVER}"; $cmd .= " $dcvars->{CONFIGURATION}"; @@ -2948,17 +2978,10 @@ sub create_backup my ($self, $env, $dcvars, $backupdir, $backup_cmd) = @_; # get all the env variables we pass in with the samba-tool command - my $cmd_env = "NSS_WRAPPER_HOSTS='$env->{NSS_WRAPPER_HOSTS}' "; - $cmd_env .= "SOCKET_WRAPPER_DEFAULT_IFACE=\"$env->{SOCKET_WRAPPER_DEFAULT_IFACE}\" "; - if (defined($env->{RESOLV_WRAPPER_CONF})) { - $cmd_env .= "RESOLV_WRAPPER_CONF=\"$env->{RESOLV_WRAPPER_CONF}\" "; - } else { - $cmd_env .= "RESOLV_WRAPPER_HOSTS=\"$env->{RESOLV_WRAPPER_HOSTS}\" "; - } - $cmd_env .= "RESOLV_CONF=\"$env->{RESOLV_CONF}\" "; # Note: use the backupfrom-DC's krb5.conf to do the backup - $cmd_env .= " KRB5_CONFIG=\"$dcvars->{KRB5_CONFIG}\" "; - $cmd_env .= "KRB5CCNAME=\"$env->{KRB5_CCACHE}\" "; + my $overwrite = undef; + $overwrite->{KRB5_CONFIG} = $dcvars->{KRB5_CONFIG}; + my $cmd_env = $self->get_cmd_env_vars($env, $overwrite); # use samba-tool to create a backup from the 'backupfromdc' DC my $cmd = ""; -- 2.30.0 From 502fd327e81f5dde96aa107a8b62fc601029bbe9 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 17 Dec 2020 06:38:14 +0100 Subject: [PATCH 07/11] s3:selftest: run test_smbclient_tarmode.pl with a fixed subdirectory name $PREFIX is the the value from --with-selftest-prefix. The result of the test should not depend on --with-selftest-prefix, the 'long_path' test in particular. If the path is to long smbclient (via libarchive) will only put the full path into a PAX HEADER as 'path' keyword, that's fine in general, modern tools handle it just fine. But Perl's Archive::Tar don't handle it and only seems truncated file names. I have a fix for Archive::Tar, see: https://git.samba.org/?p=metze/samba/wip.git;a=shortlog;h=c75037d0a06a96cdaca3f3b20a6d237e768b075b But finishing that is a task for another day, for now I just want to remove the dependency to --with-selftest-prefix. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit e0d9b656452ba6277cdc7f0abb2a06d3d284ef3a) --- source3/selftest/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index 6f65bf5ef9d..fe2fee610e5 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -518,13 +518,13 @@ for env in ["fileserver"]: [os.path.join(samba3srcdir, "script/tests/test_smbclient_tarmode.pl"), '-n', '$SERVER', '-i', '$SERVER_IP', '-s', 'tarmode2', '-u', '$USERNAME', '-p', '$PASSWORD', '-l', '$LOCAL_PATH/tarmode2', - '-d', '$PREFIX', '-b', smbclient3, + '-d', 'smbclient_tar.NT1', '-b', smbclient3, '--subunit', '--', configuration, '-mNT1']) plantestsuite("samba3.blackbox.smbclient_tar.SMB3", env, [os.path.join(samba3srcdir, "script/tests/test_smbclient_tarmode.pl"), '-n', '$SERVER', '-i', '$SERVER_IP', '-s', 'tarmode2', '-u', '$USERNAME', '-p', '$PASSWORD', '-l', '$LOCAL_PATH/tarmode2', - '-d', '$PREFIX', '-b', smbclient3, + '-d', 'smbclient_tar.SMB3', '-b', smbclient3, '--subunit', '--', configuration, '-mSMB3']) for env in ["fileserver:local"]: -- 2.30.0 From 6a648ee423545593f44981e2801df6edf767212d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 22 Nov 2020 22:43:36 +0100 Subject: [PATCH 08/11] s4:selftest: use plansmbtorture4testsuite() for 'rpc.echo' This makes sure "--basedir=$SELFTEST_TMPDIR" is passed to smbtorture. Tests should not create files in the build nor the source directory! BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider (cherry picked from commit d06f2c22d726a5ec7bd804d89154ee272ab1a679) --- source4/selftest/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index bc6bdb484ed..4c9ddccd01b 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -578,7 +578,8 @@ if have_gnutls_crypto_policies: plantestsuite("samba3.wbinfo_simple.fips.%s" % t, "ad_member_fips:local", [os.path.join(srcdir(), "nsswitch/tests/test_wbinfo_simple.sh"), t]) plantestsuite("samba4.wbinfo_name_lookup.fips", "ad_member_fips", [os.path.join(srcdir(), "nsswitch/tests/test_wbinfo_name_lookup.sh"), '$DOMAIN', '$REALM', '$DC_USERNAME']) -plantestsuite_loadlist("samba4.rpc.echo against NetBIOS alias", "ad_dc_ntvfs", [valgrindify(smbtorture4), "$LISTOPT", "$LOADLIST", 'ncacn_np:$NETBIOSALIAS', '-U$DOMAIN/$USERNAME%$PASSWORD', 'rpc.echo']) +plansmbtorture4testsuite('rpc.echo', "ad_dc_ntvfs", ['ncacn_np:$NETBIOSALIAS', '-U$DOMAIN/$USERNAME%$PASSWORD'], "samba4.rpc.echo against NetBIOS alias") + # json tests hook into ``chgdcpass'' to make them run in contributor CI on # gitlab planpythontestsuite("chgdcpass", "samba.tests.blackbox.netads_json") -- 2.30.0 From 1d0f0a19278ad46d4455a1dc6f578cc718b42f80 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 22 Nov 2020 23:28:31 +0100 Subject: [PATCH 09/11] selftest: make/use a copy of GNUPGHOME That makes it possible to run tests from a read only source tree. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 86343125a55d184c15aa94cd01f4c8893a5a0917) --- selftest/selftest.pl | 5 ++++- selftest/target/Samba.pm | 26 ++++++++++++++++++++++++++ selftest/target/Samba4.pm | 6 ++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index 3bb0c212667..4c27edd2969 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -313,7 +313,6 @@ $ENV{PREFIX} = $prefix; $ENV{PREFIX_ABS} = $prefix_abs; $ENV{SRCDIR} = $srcdir; $ENV{SRCDIR_ABS} = $srcdir_abs; -$ENV{GNUPGHOME} = "$srcdir_abs/selftest/gnupg"; $ENV{BINDIR} = $bindir_abs; my $tls_enabled = not $opt_quick; @@ -667,6 +666,9 @@ $ENV{RESOLV_CONF} = "${selftest_resolv_conf_path}.global"; my $selftest_krbt_ccache_path = "$tmpdir_abs/selftest.krb5_ccache"; $ENV{KRB5CCNAME} = "FILE:${selftest_krbt_ccache_path}.global"; +my $selftest_gnupghome_path = "$tmpdir_abs/selftest.no.gnupg"; +$ENV{GNUPGHOME} = "${selftest_gnupghome_path}.global"; + my @available = (); foreach my $fn (@testlists) { foreach (read_testlist($fn)) { @@ -803,6 +805,7 @@ sub setup_env($$) $ENV{RESOLV_CONF} = "${selftest_resolv_conf_path}.${envname}/ignore"; $ENV{KRB5CCNAME} = "FILE:${selftest_krbt_ccache_path}.${envname}/ignore"; + $ENV{GNUPGHOME} = "${selftest_gnupghome_path}.${envname}/ignore"; if (defined(get_running_env($envname))) { $testenv_vars = get_running_env($envname); diff --git a/selftest/target/Samba.pm b/selftest/target/Samba.pm index 0d7e13b7e66..5a7efa9c280 100644 --- a/selftest/target/Samba.pm +++ b/selftest/target/Samba.pm @@ -280,6 +280,30 @@ EOF umask $oldumask; } +sub copy_gnupg_home($) +{ + my ($ctx) = @_; + + my $gnupg_srcdir = "$ENV{SRCDIR_ABS}/selftest/gnupg"; + my @files = ( + "gpg.conf", + "pubring.gpg", + "secring.gpg", + "trustdb.gpg", + ); + + my $oldumask = umask; + umask 0077; + mkdir($ctx->{gnupghome}, 0777); + umask 0177; + foreach my $file (@files) { + my $srcfile = "${gnupg_srcdir}/${file}"; + my $dstfile = "$ctx->{gnupghome}/${file}"; + copy_file_content(${srcfile}, ${dstfile}); + } + umask $oldumask; +} + sub mk_krb5_conf($$) { my ($ctx) = @_; @@ -682,6 +706,7 @@ sub get_env_for_process RESOLV_CONF => $env_vars->{RESOLV_CONF}, KRB5_CONFIG => $env_vars->{KRB5_CONFIG}, KRB5CCNAME => "$env_vars->{KRB5_CCACHE}.$proc_name", + GNUPGHOME => $env_vars->{GNUPGHOME}, SELFTEST_WINBINDD_SOCKET_DIR => $env_vars->{SELFTEST_WINBINDD_SOCKET_DIR}, NMBD_SOCKET_DIR => $env_vars->{NMBD_SOCKET_DIR}, NSS_WRAPPER_PASSWD => $env_vars->{NSS_WRAPPER_PASSWD}, @@ -867,6 +892,7 @@ my @exported_envvars = ( # misc stuff "KRB5_CONFIG", "KRB5CCNAME", + "GNUPGHOME", "SELFTEST_WINBINDD_SOCKET_DIR", "NMBD_SOCKET_DIR", "LOCAL_PATH", diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index 1c9f85217d8..1ae9fb9d996 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -396,6 +396,7 @@ sub get_cmd_env_vars "OPENSSL_FORCE_FIPS_MODE", "KRB5_CONFIG", "KRB5_CCACHE", + "GNUPGHOME", ); my $localenv = undef; @@ -420,6 +421,7 @@ sub get_cmd_env_vars $cmd_env .= "KRB5_CONFIG=\"$localenv->{KRB5_CONFIG}\" "; $cmd_env .= "KRB5CCNAME=\"$localenv->{KRB5_CCACHE}\" "; $cmd_env .= "RESOLV_CONF=\"$localenv->{RESOLV_CONF}\" "; + $cmd_env .= "GNUPGHOME=\"$localenv->{GNUPGHOME}\" "; return $cmd_env; } @@ -589,6 +591,7 @@ sub provision_raw_prepare($$$$$$$$$$$$$$) $ctx->{krb5_conf} = "$ctx->{etcdir}/krb5.conf"; $ctx->{krb5_ccache} = "$prefix_abs/krb5_ccache"; $ctx->{mitkdc_conf} = "$ctx->{etcdir}/mitkdc.conf"; + $ctx->{gnupghome} = "$prefix_abs/gnupg"; $ctx->{privatedir} = "$prefix_abs/private"; $ctx->{binddnsdir} = "$prefix_abs/bind-dns"; $ctx->{ncalrpcdir} = "$prefix_abs/ncalrpc"; @@ -632,6 +635,7 @@ sub provision_raw_prepare($$$$$$$$$$$$$$) $ctx->{smb_conf_extra_options} = ""; my @provision_options = (); + push (@provision_options, "GNUPGHOME=\"$ctx->{gnupghome}\""); push (@provision_options, "KRB5_CONFIG=\"$ctx->{krb5_conf}\""); push (@provision_options, "KRB5CCNAME=\"$ctx->{krb5_ccache}\""); push (@provision_options, "NSS_WRAPPER_PASSWD=\"$ctx->{nsswrap_passwd}\""); @@ -724,6 +728,7 @@ sub provision_raw_step1($$) return undef; } + Samba::copy_gnupg_home($ctx); Samba::prepare_keyblobs($ctx); my $crlfile = "$ctx->{tlsdir}/crl.pem"; $crlfile = "" unless -e ${crlfile}; @@ -867,6 +872,7 @@ nogroup:x:65534:nobody # Note that we have SERVER_X and DC_SERVER_X variables (which have the same # value initially). In a 2 DC setup, $DC_SERVER_X will always be the PDC. my $ret = { + GNUPGHOME => $ctx->{gnupghome}, KRB5_CONFIG => $ctx->{krb5_conf}, KRB5_CCACHE => $ctx->{krb5_ccache}, MITKDC_CONFIG => $ctx->{mitkdc_conf}, -- 2.30.0 From f90b61ee9eef581cd1148e20bccb08a66d71b305 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 20 Nov 2020 09:20:14 +0000 Subject: [PATCH 10/11] script/autobuild.py: split out a rmdir_force() helper function That also tries to re-add write permissions before removing. In future we'll have jobs changing there directory to read-only. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 7a5df2deaaf62a7edd7c64251f75ab15abe94c07) --- script/autobuild.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/script/autobuild.py b/script/autobuild.py index 444bc156f48..c1d59b9c1a5 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -4,7 +4,7 @@ # released under GNU GPL v3 or later from __future__ import print_function -from subprocess import call, check_call, check_output, Popen, PIPE +from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError import os import tarfile import sys @@ -846,6 +846,17 @@ def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): else: return call(cmd, shell=True, cwd=dir) +def rmdir_force(dirname, re_raise=True): + try: + run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % ( + dirname, dirname, dirname), output=True, show=True) + except CalledProcessError as e: + do_print("Failed: '%s'" % (str(e))) + run_cmd("tree %s" % dirname, output=True, show=True) + if re_raise: + raise + return False + return True class builder(object): '''handle build of one directory''' @@ -868,8 +879,8 @@ class builder(object): self.test_source_dir = "%s/%s" % (testbase, self.tag) self.cwd = "%s/%s" % (self.test_source_dir, self.dir) self.prefix = "%s/%s" % (test_prefix, self.tag) - run_cmd("rm -rf %s" % self.test_source_dir) - run_cmd("rm -rf %s" % self.prefix) + rmdir_force(self.test_source_dir) + rmdir_force(self.prefix) if cp: run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) else: @@ -879,8 +890,8 @@ class builder(object): def start_next(self): if self.next == len(self.sequence): if not options.nocleanup: - run_cmd("rm -rf %s" % self.test_source_dir) - run_cmd("rm -rf %s" % self.prefix) + rmdir_force(self.test_source_dir) + rmdir_force(self.prefix) do_print('%s: Completed OK' % self.name) self.done = True return @@ -1004,7 +1015,7 @@ class buildlist(object): 'df -m %s' % testbase]: try: out = run_cmd(cmd, output=True, checkfail=False) - except subprocess.CalledProcessError as e: + except CalledProcessError as e: out = "" % str(e) print('### %s' % cmd, file=f) print(out, file=f) @@ -1041,7 +1052,7 @@ def cleanup(): run_cmd("stat %s" % testbase, show=True) do_print("Cleaning up %r" % cleanup_list) for d in cleanup_list: - run_cmd("rm -rf %s" % d) + rmdir_force(d) def daemonize(logfile): -- 2.30.0 From fc39514ca7bc4cd1b7e996c29be1acff4b234e1c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 20 Nov 2020 09:20:14 +0000 Subject: [PATCH 11/11] script/autobuild.py: let cleanup() ignore errors from rmdir_force() by default It's not useful to generate a python backtrace from within the cleanup code. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14628 Signed-off-by: Stefan Metzmacher Reviewed-by: Ralph Boehme (cherry picked from commit 9883ac45939f253a63f3ff312fc3912c5f02cdac) --- script/autobuild.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/script/autobuild.py b/script/autobuild.py index c1d59b9c1a5..dded5c9dec9 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -1045,14 +1045,23 @@ class buildlist(object): self.tail_proc = Popen(cmd, close_fds=True) -def cleanup(): +def cleanup(do_raise=False): if options.nocleanup: return run_cmd("stat %s || true" % test_tmpdir, show=True) run_cmd("stat %s" % testbase, show=True) do_print("Cleaning up %r" % cleanup_list) for d in cleanup_list: - rmdir_force(d) + ok = rmdir_force(d, re_raise=False) + if ok: + continue + if os.path.isdir(d): + do_print("Killing, waiting and retry") + run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False) + else: + do_print("Waiting and retry") + time.sleep(1) + rmdir_force(d, re_raise=do_raise) def daemonize(logfile): @@ -1318,7 +1327,7 @@ while True: (status, failed_task, failed_stage, failed_tag, errstr) = blist.run() if status != 0 or errstr != "retry": break - cleanup() + cleanup(do_raise=True) except Exception: cleanup() raise -- 2.30.0