(gdb) bt full #0 0x0000003d6f4ad7ee in waitpid () from /lib64/libc.so.6 No symbol table info available. #1 0x0000003d6f440049 in do_system () from /lib64/libc.so.6 No symbol table info available. #2 0x00000000006133aa in smb_panic (why=0xc94fc0 "internal error") at lib/util.c:1370 cmd = 0x1c32620 "sleep 99999" result = 50 __FUNCTION__ = "smb_panic" #3 0x00000000005f9d6d in fault_report (sig=6) at lib/fault.c:52 counter = 1 __FUNCTION__ = "fault_report" #4 0x00000000005f9d82 in sig_fault (sig=6) at lib/fault.c:75 No locals. #5 <signal handler called> No symbol table info available. #6 0x0000003d6f434085 in raise () from /lib64/libc.so.6 No symbol table info available. #7 0x0000003d6f435a36 in abort () from /lib64/libc.so.6 No symbol table info available. #8 0x0000003d6f42c8c5 in __assert_fail () from /lib64/libc.so.6 No symbol table info available. #9 0x00000032a5620a2e in ldap_get_values () from /usr/lib64/libldap-2.4.so.2 No symbol table info available. #10 0x0000000000b32b9d in ads_pull_string (ads=0x1c00420, mem_ctx=0x1bb5180, msg=0x0, field=0x1c2cf00 "unixHomeDirectory") at libads/ldap.c:2397 values = 0x1c00420 ret = 0x0 ux_string = 0x1bb5180 "" converted_size = 0 #11 0x00007f942b79ac1c in ?? () from /usr/lib64/samba/nss_info/rfc2307.so No symbol table info available. #12 0x0000000000b669b7 in nss_get_info (domain=0x1bf7930 "W2K8DOM", user_sid=0x1c307f8, ctx=0x1bb5180, ads=0x1c00420, msg=0x0, homedir=0x1c307e0, shell=0x1c307e8, gecos=0x7fffd186c5c8, p_gid=0x7fffd186c5fc) at winbindd/nss_info.c:327 p = 0x1bfdbf0 m = 0x7f942b99f200 __FUNCTION__ = "nss_get_info" #13 0x00000000004acd96 in nss_get_info_cached (domain=0x1bf7930, user_sid=0x1c307f8, ctx=0x1bb5180, ads=0x1c00420, msg=0x0, homedir=0x1c307e0, shell=0x1c307e8, gecos=0x7fffd186c5c8, p_gid=0x7fffd186c5fc) at winbindd/winbindd_cache.c:4678 cache = 0x1bf2da0 centry = 0x0 nt_status = {v = 63024} tmp = "S-1-5-21-2740362468-2820882270-1993041616-501\000\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361---Type <return> to continue, or q <return> to quit--- \361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361\361", <incomplete sequence \361> __FUNCTION__ = "nss_get_info_cached" #14 0x00000000004c91bb in query_user_list (domain=0x1bf7930, mem_ctx=0x1bb5180, num_entries=0x1c2df70, pinfo=0x1c2df78) at winbindd/winbindd_ads.c:253 info = 0x1c307d0 gecos = 0x0 primary_gid = 4294967295 ads = 0x1c00420 attrs = {0xc4c370 "*", 0x0} i = 0 count = 8 rc = {error_type = ENUM_ADS_ERROR_LDAP, err = {rc = 0, nt_status = {v = 0}}, minor_status = 0} res = 0x1bfe820 msg = 0x0 status = {v = 3221225473} __FUNCTION__ = "query_user_list" #15 0x00000000004a10b4 in query_user_list (domain=0x1bf7930, mem_ctx=0x1bb5180, num_entries=0x1c2df70, info=0x1c2df78) at winbindd/winbindd_cache.c:1477 cache = 0x1bf2da0 centry = 0x0 status = {v = 0} i = 3515279312 retry = 0 old_status = true __FUNCTION__ = "query_user_list" #16 0x00000000004d823d in _wbint_QueryUserList (p=0x7fffd186c7d0, r=0x1c2e900) at winbindd/winbindd_dual_srv.c:238 domain = 0x1bf7930 #17 0x00000000004e966c in api_wbint_QueryUserList (p=0x7fffd186c7d0) at librpc/gen_ndr/srv_wbint.c:1105 call = 0x10cdb10 pull = 0x1be7080 push = 0x5ca5fb ndr_err = NDR_ERR_SUCCESS r = 0x1c2e900 #18 0x00000000004d77a3 in winbindd_dual_ndrcmd (domain=0x1bf7930, state=0x7fffd186ca60) at winbindd/winbindd_dual_ndr.c:321 p = {next = 0x0, prev = 0x0, client_id = 0x0, server_info = 0x0, msg_ctx = 0x0, syntax = {uuid = {time_low = 0, time_mid = 0, time_hi_and_version = 0, clock_seq = "\000", node = "\000\000\000\000\000"}, if_version = 0}, contexts = 0x0, auth = {auth_type = DCERPC_AUTH_TYPE_NONE, auth_level = 0, auth_ctx = 0x0, domain = 0x0, user_name = 0x0, user_session_key = {data = 0x0, length = 0}}, pipe_bound = false, fault_state = false, bad_handle_fault_state = false, rng_fault_state = false, endian = false, in_data = {pdu = { data = 0x0, length = 0}, pdu_needed_len = 0, data = {data = 0x0, length = 0}}, out_data = {rdata = {data = 0x0, length = 0}, data_sent_length = 0, frag = {data = 0x0, length = 0}, current_pdu_sent = 0}, mem_ctx = 0x1bb5180, pipe_handles = 0x0, call_id = 0, opnum = 0, private_data = 0x0} fns = 0x10d7b20 num_fns = 21 ret = false __FUNCTION__ = "winbindd_dual_ndrcmd" #19 0x00000000004d1fca in child_process_request (child=0x1bf7eb8, state=0x7fffd186ca60) at winbindd/winbindd_dual.c:394 domain = 0x1bf7930 table = 0x10cdd20 ---Type <return> to continue, or q <return> to quit--- __FUNCTION__ = "child_process_request" #20 0x00000000004d67de in fork_domain_child (child=0x1bf7eb8) at winbindd/winbindd_dual.c:1465 r_fds = {fds_bits = {2097152, 0 <repeats 15 times>}} maxfd = 21 t = {tv_sec = 3592, tv_usec = 671809} iov_count = 2 ret = 1 frame = 0x1b7f320 status = {v = 0} w_fds = {fds_bits = {0 <repeats 16 times>}} tp = 0x7fffd186cb10 now = {tv_sec = 1291116357, tv_usec = 436161} iov = {{iov_base = 0x7fffd186cb70, iov_len = 3496}, {iov_base = 0x1c2e6f0, iov_len = 74}} fdpair = {21, 22} state = {prev = 0x0, next = 0x0, sock = 21, pid = 26870, last_access = 0, privileged = false, mem_ctx = 0x1b7f320, cmd_name = 0x0, recv_fn = 0, request = 0x7fffd186d920, out_queue = 0x0, response = 0x7fffd186cb70, getpwent_initialized = false, getgrent_initialized = false, pwent_state = 0x0, grent_state = 0x0} request = {length = 2096, cmd = WINBINDD_DUAL_NDRCMD, original_cmd = WINBINDD_INTERFACE_VERSION, pid = 0, wb_flags = 0, flags = 0, domain_name = '\000' <repeats 255 times>, data = { winsreq = "\016", '\000' <repeats 254 times>, username = "\016", '\000' <repeats 254 times>, groupname = "\016", '\000' <repeats 254 times>, uid = 14, gid = 14, ndrcmd = 14, auth = { user = "\016", '\000' <repeats 254 times>, pass = '\000' <repeats 255 times>, require_membership_of_sid = '\000' <repeats 1023 times>, krb5_cc_type = '\000' <repeats 255 times>, uid = 0}, auth_crap = {chal = "\016\000\000\000\000\000\000", logon_parameters = 0, user = '\000' <repeats 255 times>, domain = '\000' <repeats 255 times>, lm_resp = '\000' <repeats 255 times>, lm_resp_len = 0, nt_resp = '\000' <repeats 255 times>, nt_resp_len = 0, workstation = '\000' <repeats 255 times>, require_membership_of_sid = '\000' <repeats 255 times>}, chauthtok = {user = "\016", '\000' <repeats 254 times>, oldpass = '\000' <repeats 255 times>, newpass = '\000' <repeats 255 times>}, chng_pswd_auth_crap = {user = "\016", '\000' <repeats 254 times>, domain = '\000' <repeats 255 times>, new_nt_pswd = '\000' <repeats 515 times>, new_nt_pswd_len = 0, old_nt_hash_enc = '\000' <repeats 15 times>, old_nt_hash_enc_len = 0, new_lm_pswd = '\000' <repeats 515 times>, new_lm_pswd_len = 0, old_lm_hash_enc = '\000' <repeats 15 times>, old_lm_hash_enc_len = 0}, logoff = {user = "\016", '\000' <repeats 254 times>, krb5ccname = '\000' <repeats 255 times>, uid = 0}, sid = "\016", '\000' <repeats 254 times>, name = {dom_name = "\016", '\000' <repeats 254 times>, name = '\000' <repeats 255 times>}, num_entries = 14, acct_mgt = {username = "\016", '\000' <repeats 254 times>, groupname = '\000' <repeats 255 times>}, init_conn = { is_primary = 14, dcname = '\000' <repeats 255 times>}, dual_sid2id = {sid = "\016", '\000' <repeats 254 times>, name = '\000' <repeats 255 times>}, dual_idmapset = { sid = "\016", '\000' <repeats 254 times>, type = 0, id = 0}, list_all_domains = 14, ccache_ntlm_auth = {uid = 14, user = '\000' <repeats 255 times>, initial_blob_len = 0, challenge_blob_len = 0}, ccache_save = {uid = 14, user = '\000' <repeats 255 times>, pass = '\000' <repeats 255 times>}, dsgetdcname = { domain_name = "\016", '\000' <repeats 254 times>, domain_guid = '\000' <repeats 255 times>, site_name = '\000' <repeats 255 times>, flags = 0}, padding = "\016", '\000' <repeats 1798 times>}, extra_data = {padding = 0, data = 0x0}, extra_len = 0, null_term = 0 '\000'} response = {length = 3496, result = WINBINDD_ERROR, data = {interface_version = 0, winsresp = '\000' <repeats 255 times>, pw = {pw_name = '\000' <repeats 255 times>, pw_passwd = '\000' <repeats 255 times>, pw_uid = 0, pw_gid = 0, pw_gecos = '\000' <repeats 255 times>, pw_dir = '\000' <repeats 255 times>, pw_shell = '\000' <repeats 255 times>}, gr = {gr_name = '\000' <repeats 255 times>, gr_passwd = '\000' <repeats 255 times>, gr_gid = 0, num_gr_mem = 0, gr_mem_ofs = 0}, num_entries = 0, sid = { sid = '\000' <repeats 255 times>, type = 0}, name = {dom_name = '\000' <repeats 255 times>, name = '\000' <repeats 255 times>, type = 0}, uid = 0, gid = 0, info = { winbind_separator = 0 '\000', samba_version = '\000' <repeats 255 times>}, domain_name = '\000' <repeats 255 times>, netbios_name = '\000' <repeats 255 times>, dc_name = '\000' <repeats 255 times>, auth = {nt_status = 0, nt_status_string = '\000' <repeats 255 times>, error_string = '\000' <repeats 255 times>, pam_error = 0, user_session_key = '\000' <repeats 15 times>, first_8_lm_hash = "\000\000\000\000\000\000\000", krb5ccname = '\000' <repeats 255 times>, reject_reason = 0, padding = 0, policy = { min_length_password = 0, password_history = 0, password_properties = 0, padding = 0, expire = 0, min_passwordage = 0}, info3 = {logon_time = 0, logoff_time = 0, kickoff_time = 0, pass_last_set_time = 0, pass_can_change_time = 0, pass_must_change_time = 0, logon_count = 0, bad_pw_count = 0, user_rid = 0, group_rid = 0, num_groups = 0, user_flgs = 0, acct_flags = 0, num_other_sids = 0, dom_sid = '\000' <repeats 255 times>, user_name = '\000' <repeats 255 times>, full_name = '\000' <repeats 255 times>, logon_script = '\000' <repeats 255 times>, profile_path = '\000' <repeats 255 times>, home_dir = '\000' <repeats 255 times>, dir_drive = '\000' <repeats 255 times>, logon_srv = '\000' <repeats 255 times>, logon_dom = '\000' <repeats 255 times>}, unix_username = '\000' <repeats 255 times>}, domain_info = {name = '\000' <repeats 255 times>, ---Type <return> to continue, or q <return> to quit--- alt_name = '\000' <repeats 255 times>, sid = '\000' <repeats 255 times>, native_mode = false, active_directory = false, primary = false}, sequence_number = 0, user_info = { acct_name = '\000' <repeats 255 times>, full_name = '\000' <repeats 255 times>, homedir = '\000' <repeats 255 times>, shell = '\000' <repeats 255 times>, primary_gid = 0, group_rid = 0}, ccache_ntlm_auth = {session_key = '\000' <repeats 15 times>, auth_blob_len = 0}, dsgetdcname = {dc_unc = '\000' <repeats 255 times>, dc_address = '\000' <repeats 255 times>, dc_address_type = 0, domain_guid = '\000' <repeats 255 times>, domain_name = '\000' <repeats 255 times>, forest_name = '\000' <repeats 255 times>, dc_flags = 0, dc_site_name = '\000' <repeats 255 times>, client_site_name = '\000' <repeats 255 times>}}, extra_data = {padding = 0, data = 0x0}} primary_domain = 0x1bf7930 __FUNCTION__ = "fork_domain_child" #21 0x00000000004d12a1 in wb_child_request_trigger (req=0x1b8b980, private_data=0x0) at winbindd/winbindd_dual.c:141 state = 0x1b825c0 subreq = 0x0 #22 0x0000000000631d58 in tevent_queue_immediate_trigger (ev=0x1be6ce0, im=0x1bf8310, private_data=0x1bf81e0) at ../lib/tevent/tevent_queue.c:144 q = 0x1bf81e0 #23 0x0000000000630044 in tevent_common_loop_immediate (ev=0x1be6ce0) at ../lib/tevent/tevent_immediate.c:135 im = 0x1bf8310 handler = 0x631ce6 <tevent_queue_immediate_trigger> private_data = 0x1bf81e0 #24 0x000000000062cd15 in run_events (ev=0x1be6ce0, selrtn=0x7fffd186e438, read_fds=0x0, write_fds=0x0) at lib/events.c:81 fde = 0x7fffd186e2f0 now = {tv_sec = 1291116357, tv_usec = 192180} __FUNCTION__ = "run_events" #25 0x000000000062d348 in s3_event_loop_once (ev=0x1be6ce0, location=0xc40c14 "winbindd/winbindd.c:1335") at lib/events.c:181 now = {tv_sec = 13150272, tv_usec = 263876268833} to = {tv_sec = 9999, tv_usec = 0} r_fds = {fds_bits = {0 <repeats 16 times>}} w_fds = {fds_bits = {0 <repeats 16 times>}} maxfd = 0 ret = 0 #26 0x000000000062e6a6 in _tevent_loop_once (ev=0x1be6ce0, location=0xc40c14 "winbindd/winbindd.c:1335") at ../lib/tevent/tevent.c:493 ret = 0 nesting_stack_ptr = 0x0 #27 0x0000000000497441 in main (argc=1, argv=0x7fffd186e7d8, envp=0x7fffd186e7e8) at winbindd/winbindd.c:1335 is_daemon = false Fork = true log_stdout = false no_process_group = false long_options = {{longName = 0x0, shortName = 0 '\000', argInfo = 4, arg = 0x7f943294a420, val = 0, descrip = 0xc407fc "Help options:", argDescrip = 0x0}, {longName = 0xc4080a "stdout", shortName = 83 'S', argInfo = 0, arg = 0x0, val = 1003, descrip = 0xc40811 "Log to stdout", argDescrip = 0x0}, {longName = 0xc4081f "foreground", shortName = 70 'F', argInfo = 0, arg = 0x0, val = 1001, descrip = 0xc4082a "Daemon in foreground mode", argDescrip = 0x0}, {longName = 0xc40844 "no-process-group", shortName = 0 '\000', argInfo = 0, arg = 0x0, val = 1002, descrip = 0xc40858 "Don't create a new process group", argDescrip = 0x0}, {longName = 0xc40879 "daemon", shortName = 68 'D', argInfo = 0, arg = 0x0, val = 1000, descrip = 0xc40880 "Become a daemon (default)", argDescrip = 0x0}, {longName = 0xc4089a "interactive", shortName = 105 'i', argInfo = 0, arg = 0x0, val = 105, descrip = 0xc408a6 "Interactive mode", argDescrip = 0x0}, {longName = 0xc408b7 "no-caching", shortName = 110 'n', argInfo = 0, arg = 0x0, val = 110, descrip = 0xc408c2 "Disable caching", argDescrip = 0x0}, {longName = 0x0, shortName = 0 '\000', argInfo = 4, arg = 0x10dfac0, val = 0, descrip = 0xc408d2 "Common samba options:", argDescrip = 0x0}, {longName = 0x0, shortName = 0 '\000', argInfo = 0, arg = 0x0, val = 0, descrip = 0x0, argDescrip = 0x0}} ---Type <return> to continue, or q <return> to quit--- pc = 0x1b7f720 opt = -1 frame = 0x1b7f140 status = {v = 0} __FUNCTION__ = "main"
Created attachment 6095 [details] Patch Günther, can you give the attached patch a try? Thanks, Volker
yep, this fixes it, but feeding the idmap module with NULL msg in general looks wrong, doesnt it ?
Well, we have to convert the get_info calls to not use ldap messages at all. The problem is that every call to any ads_search like function can destroy the LDAP struct and thus make all ldap_msg structs that came from this LDAP struct invalid. We have to make separate copies of the user structs in the first loop and hand these copies into the nss_get_info_cached functions. But that is a larger change given the urgency of this bug I do not have the proper time for right now. Sorry, Volker
(In reply to comment #2) > yep, this fixes it, but feeding the idmap module with NULL msg in general looks > wrong, doesnt it ? It does, indeed. The rewrite of the ads query_user_list() to prevent a segfault (ads being accessed after a call to nss_get_info_cached()), explicitly clears the msg before calling nss_get_info_cached(). The corresponding patch 09a9cc32ee611c20c0e3384c404dd39f615b89ed to ads query_user() does precisely the same. So Volker's fix to comment out the if (ads) {} block in nss_ad_get_info() fixes both of these new segfaults. But I have to agree with Günther that this fix feels wrong. Why don't we rather call nss_get_info_cached() with the ads parameter == NULL? Or even better, check for msg == NULL in the if (ads) { ... } check? This will have the same effect. I'll attach a patchset that changes the fix to do exactly that. I think this is a cleaner temporary solution. But let me ask a dumb question: why can nss_get_info_cached destroy the ads struct (or the ldap stuct dangling off it) in the first place? nss_ad_get_info() uses ads_pull_string() and ads_pull_uint32() which call ldap_get_values() and ldap_free_values() on the result. Just trying to understand.
Created attachment 6127 [details] patchset with alternative fix Here is an alternative patchset that just checks whether msg == NULL, thus circumventing the segfault. The two optional additional commits just change the callers to explicitly hand in NULL for ads and msg, since it is no use anyway after the message has been NULLed.
(In reply to comment #4) > But let me ask a dumb question: why can nss_get_info_cached destroy the ads > struct (or the ldap stuct dangling off it) in the first place? > nss_ad_get_info() uses ads_pull_string() and ads_pull_uint32() which call > ldap_get_values() and ldap_free_values() on the result. > Just trying to understand. idmap_ad.c:900 (in 7a5e47b) I see a call to ads_search_retry(). This goes out to the DC. If the DC died in the wrong moment, this will take down the LDAP connection. Thus all LDAPmessage objects hanging off that will potentially become invalid. Regards, Volker
(In reply to comment #6) > (In reply to comment #4) > > But let me ask a dumb question: why can nss_get_info_cached destroy the ads > > struct (or the ldap stuct dangling off it) in the first place? > > nss_ad_get_info() uses ads_pull_string() and ads_pull_uint32() which call > > ldap_get_values() and ldap_free_values() on the result. > > Just trying to understand. > > idmap_ad.c:900 (in 7a5e47b) I see a call to ads_search_retry(). This goes out > to the DC. If the DC died in the wrong moment, this will take down the LDAP > connection. Thus all LDAPmessage objects hanging off that will potentially > become invalid. Right, but this does not use the ads connection handed in. The code block that uses the provided ads struct skips that ads_search with a "goto done:" (line 876 approx.). What is my misconception? Are the ldap connections in the various ads stucts the _same_ ldap connection? Cheers - Michael
Please revert my change of the loop. Probably I was just chasing ghosts. Volker
(In reply to comment #8) > Please revert my change of the loop. Probably I was just chasing ghosts. > > Volker No need to retreat right away. :-) I digged a little deeper and in commit 7cf04431594e09043b3b53144fc8511d20b088ee, I found the comment that the problem actually lies in following code path: nss_get_info_cached()--> wcache_fetch()--> refresh_sequence_number()--> (winbindd_ads.c:)sequence_number() which then calls ads_cached_connection() and does a ads_USN()->ads_do_search_retry() on the ads struct which is in fact the same as the one originally handed in from the caller (query_user() and friends). sequence_number() would not have been called always but just when the sequence number could not be obtained from the tdb (fetch_cache_seqnum). So you seem not to have been chasing gosts. :-) Still thinking of what a good fix is. Cheers - Michael
Hm, has this been resolved in the meantime ?
hello, hello ? would be good if in the turn of the idmap changes this could be verified alongside.
(In reply to comment #11) > hello, hello ? would be good if in the turn of the idmap changes this could be > verified alongside. Hello, hello! Thanks for the reminder GeeDee! :-)
Hello, helo, any progress on this in the meantime?
I assume this is fixed, do you agree, Günther?