Build environment:

AIX 6100-03-10-1119
Compiler:IBM XL C/C++ for AIX, V10.1 Version: 10.01.0000.0000

CFLAGS=-q64 -qmaxmem=-1 -DSYSV -D_AIX -D_AIX32 -D_AIX41 -D_AIX43 -D_AIX51 -D_AIX52 -D_AIX53 -D_AIX61 -D_ALL_SOURCE -DFUNCPROTO=15 -g -I/opt/pware/include -L/opt/pware/lib

CC=xlc_r -qcpluscmt -D_LARGE_FILES=1

LDFLAGS=-L/opt/pware/lib -Wl,-blibpath:/opt/pware/lib:/opt/pware/lib32:/usr/lib

Configure Parameters

./configure --prefix=/opt/pware/samba/3.6.12 --with-acl-support --with-libiconv=/opt/pware --enable-developer --with-pam --with-aio-support=yes --enable-shared

When samba is built without acl-support it works fine in 64 Bit, and it also works when built as 32 Bit with acl-support. In 64 Bit mode, with acl-support, the smbd daemon dumps core:



Here's the stack trace:
reading symbolic information ...
stopped in waitpid at 0x900000000116eb0 ($t1)
0x900000000116eb0 (waitpid+0x210) e8410028          ld   r2,0x28(r1)
(dbx) where
waitpid(??, ??, ??) at 0x900000000116eb0
system.system(??) at 0x9000000001e3b00
smb_panic(why = "===============================================================\n"), line 1123 in "util.c"
fault_report(sig = 1), line 53 in "fault.c"
sys_acl_get_entry(acl_d = 0x0fffffffffffd4a0, entry_id = 268435455, entry_p = 0x0fffffffffffd568), line 84 in "sysacls.c"
vfswrap_sys_acl_get_entry(handle = 0x00000001000a058c, theacl = 0x0fffffffffffd720, entry_id = 268435455, entry_p = 0x0000000000000001), line 1376 in "vfs_default.c"
smb_vfs_call_sys_acl_get_entry(handle = 0x000000011014430d, theacl = 0x0000000000000010, entry_id = 0, entry_p = 0x0fffffffffffd6d8), line 1785 in "vfs.c"
canonicalise_acl(conn = 0x8000002800000001, fname = "", posix_acl = 0x0000000100aae7dc, psbuf = (nil), powner = 0x0000000100aae7c4, pgroup = 0x0000000100aae6e8, pal = 0x3ffc000000000003, the_acl_type = 0), line 2462 in "posix_acls.c"
posix_get_nt_acl_common(conn = 0x0000000100aac438, name = ".", sbuf = 0x0000000100aac40c, pal = 0x0000000100aac424, posix_acl = 0x000000011005b738, def_acl = 0x0000000100908850, security_info = 1, ppdesc = 0x00000001100012e8), line 3177 in "posix_acls.c"
posix_get_nt_acl(conn = 0x000000011010dd80, name = "|rC\246\350c", security_info = 268435455, ppdesc = 0x0000000000000010), line 3467 in "posix_acls.c"
vfswrap_get_nt_acl(handle = 0x0000000000000007, name = "", security_info = 268435455, ppdesc = 0x0fffffffffffdcd8), line 1329 in "vfs_default.c"
smb_vfs_call_get_nt_acl(handle = 0x0000000000000007, name = "", security_info = 268435455, ppdesc = 0x09001000a00800b8), line 1754 in "vfs.c"
smbd_check_open_rights(conn = 0x000000011014ed00, smb_fname = 0x0000000110150500, access_mask = 16777216, access_granted = 0x0010000100100001), line 115 in "open.c"
open_directory(conn = 0x000000011014ed00, req = 0x00000001101503b0, smb_dname = 0x0000000110150500, access_mask = 1048577, share_access = 7, create_disposition = 1, create_options = 1, file_attributes = 16, pinfo = 0x0fffffffffffdd40, result = 0x0fffffffffffdd50), line 2706 in "open.c"
create_file_unixpath(conn = 0x0000000100a923a4, req = 0x0000000000100001, smb_fname = (nil), access_mask = 0, share_access = 0, create_disposition = 0, create_options = 0, file_attributes = 0, oplock_request = 0, allocation_size = 0, private_flags = 0, sd = (nil), ea_list = (nil), result = 0x0fffffffffffdea8, pinfo = 0x0fffffffffffdea0), line 3341 in "open.c"
create_file_default(conn = (nil), req = (nil), root_dir_fid = 4095, smb_fname = 0x2844b22800000100, access_mask = 1, share_access = 0, create_disposition = 0, create_options = 0, file_attributes = 0, oplock_request = 0, allocation_size = 0, private_flags = 0, sd = (nil), ea_list = (nil), result = 0x0fffffffffffe1c0, pinfo = 0x0fffffffffffe1a0), line 3689 in "open.c"
vfswrap_create_file(handle = 0x00000001100ddfa0, req = 0x0000000000000010, root_dir_fid = 0, smb_fname = 0x0000000000000010, access_mask = 1, share_access = 0, create_disposition = 268435455, create_options = 0, file_attributes = 0, oplock_request = 0, allocation_size = 0, private_flags = 0, sd = (nil), ea_list = (nil), result = 0x0fffffffffffe1c0, pinfo = 0x0fffffffffffe1a0), line 329 in "vfs_default.c"
smb_vfs_call_create_file(handle = 0x0000000100af5894, req = 0x0000000000000010, root_dir_fid = 0, smb_fname = (nil), access_mask = 0, share_access = 0, create_disposition = 0, create_options = 0, file_attributes = 0, oplock_request = 0, allocation_size = 0, private_flags = 0, sd = (nil), ea_list = (nil), result = 0x0fffffffffffe1c0, pinfo = 0x0fffffffffffe1a0), line 1312 in "vfs.c"
reply_ntcreate_and_X(req = 0x0000000100a8845c), line 563 in "nttrans.c"
switch_message(type = '^O', req = 0x0ffffffffffff8e9, size = 10), line 1574 in "process.c"
construct_reply(sconn = 0x00000001100acf20, inbuf = (nil), size = 90, unread_bytes = 0, seqnum = 0, encrypted = @0x0000000000000000, deferred_pcd = (nil)), line 1610 in "process.c"
process_smb(sconn = (nil), inbuf = "", nread = 1152921504606840368, unread_bytes = 4564111136, seqnum = 150999040, encrypted = @0x0000000000000001, deferred_pcd = 0x0fffffffffffe610), line 1688 in "process.c"
smbd_server_connection_read_handler(conn = 0x00000001008e7b58, fd = 1), line 2318 in "process.c"
smbd_server_connection_handler(ev = 0x00081c8b10150350, fde = 0x0000000110094260, flags = 6, private_data = 0x00000001100acf20), line 2335 in "process.c"
run_events_poll(ev = 0x0ffffffffffff8b9, pollrtn = 1, pfds = 0x00000001008e7b58, num_pfds = 0), line 286 in "events.c"
smbd_server_connection_loop_once(conn = 0x00000001100ace40), line 1017 in "process.c"
unnamed block in smbd_process(sconn = 0x09001000a00a5b38), line 3159 in "process.c"
smbd_process(sconn = 0x09001000a00a5b38), line 3159 in "process.c"
smbd_accept_connection(ev = 0x00000001008e7d3c, fde = 0x0000000110094260, flags = 4095, private_data = 0x2244422000000050), line 438 in "server.c"
run_events_poll(ev = 0x0ffffffffffff560, pollrtn = 150999040, pfds = 0x0ffffffffffff570, num_pfds = 0), line 286 in "events.c"
s3_event_loop_once(ev = 0x00000001000038b0, location = ""), line 349 in "events.c"
_tevent_loop_once(ev = 0x00000001008e7d20, location = (nil)), line 494 in "tevent.c"
unnamed block in smbd_parent_loop(parent = 0x00000001008e8214), line 820 in "server.c"
smbd_parent_loop(parent = 0x00000001008e8214), line 820 in "server.c"
main(argc = 0, argv = (nil)), line 1304 in "server.c"