From 2da5bcb1a14eedb6ff6937a2cf21cbcc0d011da1 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Sat, 11 Jan 2014 14:36:17 -0800 Subject: [PATCH 1/8] s3:dir - In the old SMB1 search code, rename offset to wire_offset to distinguish between wire and native offsets. Rename uint32 type to correct uint32_t. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider (cherry picked from commit 584de2078d3320ba2e232e5f504191616347d0d7) --- source3/smbd/dir.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c index 52bd6a1..6665bb3 100644 --- a/source3/smbd/dir.c +++ b/source3/smbd/dir.c @@ -912,16 +912,16 @@ bool dptr_fill(struct smbd_server_connection *sconn, { unsigned char *buf = (unsigned char *)buf1; struct dptr_struct *dptr = dptr_get(sconn, key, false); - uint32 offset; + uint32_t wire_offset; if (!dptr) { DEBUG(1,("filling null dirptr %d\n",key)); return(False); } - offset = (uint32)TellDir(dptr->dir_hnd); + wire_offset = (uint32_t)TellDir(dptr->dir_hnd); DEBUG(6,("fill on key %u dirptr 0x%lx now at %d\n",key, - (long)dptr->dir_hnd,(int)offset)); + (long)dptr->dir_hnd,(int)wire_offset)); buf[0] = key; - SIVAL(buf,1,offset); + SIVAL(buf,1,wire_offset); return(True); } @@ -934,7 +934,7 @@ struct dptr_struct *dptr_fetch(struct smbd_server_connection *sconn, { unsigned int key = *(unsigned char *)buf; struct dptr_struct *dptr = dptr_get(sconn, key, false); - uint32 offset; + uint32_t wire_offset; long seekoff; if (!dptr) { @@ -942,11 +942,11 @@ struct dptr_struct *dptr_fetch(struct smbd_server_connection *sconn, return(NULL); } *num = key; - offset = IVAL(buf,1); - if (offset == (uint32)-1) { + wire_offset = IVAL(buf,1); + if (wire_offset == (uint32_t)-1) { seekoff = END_OF_DIRECTORY_OFFSET; } else { - seekoff = (long)offset; + seekoff = (long)wire_offset; } SeekDir(dptr->dir_hnd,seekoff); DEBUG(3,("fetching dirptr %d for path %s at offset %d\n", -- 1.8.5.2 From e4f77b2d3a1a678ac8c023c3c06862021f8203cf Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Sat, 11 Jan 2014 14:48:00 -0800 Subject: [PATCH 2/8] s3:dir - Introduce a function to map a directory cookie to a 32-bit wire cookie. Make this an identity for now. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider (cherry picked from commit 5afc25eceb0c0e031bbe162617309178f3bcc425) --- source3/smbd/dir.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c index 6665bb3..db6e0e7 100644 --- a/source3/smbd/dir.c +++ b/source3/smbd/dir.c @@ -904,6 +904,15 @@ void dptr_init_search_op(struct dptr_struct *dptr) } /**************************************************************************** + Map a native directory offset to a 32-bit cookie. +****************************************************************************/ + +static uint32_t map_dir_offset_to_wire(struct dptr_struct *dptr, long offset) +{ + return (uint32_t)offset; +} + +/**************************************************************************** Fill the 5 byte server reserved dptr field. ****************************************************************************/ @@ -917,7 +926,7 @@ bool dptr_fill(struct smbd_server_connection *sconn, DEBUG(1,("filling null dirptr %d\n",key)); return(False); } - wire_offset = (uint32_t)TellDir(dptr->dir_hnd); + wire_offset = map_dir_offset_to_wire(dptr,TellDir(dptr->dir_hnd)); DEBUG(6,("fill on key %u dirptr 0x%lx now at %d\n",key, (long)dptr->dir_hnd,(int)wire_offset)); buf[0] = key; -- 1.8.5.2 From 9f60043f6892b84995cd6f586f795f8484960e31 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Sat, 11 Jan 2014 14:56:57 -0800 Subject: [PATCH 3/8] s3: dir - Introduce 32-bit wire versions of the 'special' values. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider (cherry picked from commit 51a115b62048735b4c8ec79211ce45600cfa5c01) --- source3/smbd/dir.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c index db6e0e7..2bae122 100644 --- a/source3/smbd/dir.c +++ b/source3/smbd/dir.c @@ -34,6 +34,11 @@ #define START_OF_DIRECTORY_OFFSET ((long)0) #define DOT_DOT_DIRECTORY_OFFSET ((long)0x80000000) +/* "Special" directory offsets in 32-bit wire format. */ +#define WIRE_END_OF_DIRECTORY_OFFSET ((uint32_t)0xFFFFFFFF) +#define WIRE_START_OF_DIRECTORY_OFFSET ((uint32_t)0) +#define WIRE_DOT_DOT_DIRECTORY_OFFSET ((uint32_t)0x80000000) + /* Make directory handle internals available. */ struct name_cache_entry { -- 1.8.5.2 From 4d4a421e05efdf42405f7ec407deb5d3dba3aea9 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Sat, 11 Jan 2014 14:59:00 -0800 Subject: [PATCH 4/8] s3:dir - Cope with fixed mapping of 'special' values. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider (cherry picked from commit 81df4123ca6fae6e9d901c59a12407f3f89dc335) --- source3/smbd/dir.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c index 2bae122..845af00 100644 --- a/source3/smbd/dir.c +++ b/source3/smbd/dir.c @@ -914,6 +914,13 @@ void dptr_init_search_op(struct dptr_struct *dptr) static uint32_t map_dir_offset_to_wire(struct dptr_struct *dptr, long offset) { + if (offset == END_OF_DIRECTORY_OFFSET) { + return WIRE_END_OF_DIRECTORY_OFFSET; + } else if(offset == START_OF_DIRECTORY_OFFSET) { + return WIRE_START_OF_DIRECTORY_OFFSET; + } else if (offset == DOT_DOT_DIRECTORY_OFFSET) { + return WIRE_DOT_DOT_DIRECTORY_OFFSET; + } return (uint32_t)offset; } -- 1.8.5.2 From 73132927836af966bd74c323f2effa970dc9e6f1 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Sat, 11 Jan 2014 15:04:38 -0800 Subject: [PATCH 5/8] s3:dir - Map wire offsets to native directory cookies. Take care of the special offsets. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider (cherry picked from commit 42c80358c83dca65cdde78f442056ec0f55ecbb1) --- source3/smbd/dir.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c index 845af00..9d79c84 100644 --- a/source3/smbd/dir.c +++ b/source3/smbd/dir.c @@ -947,6 +947,22 @@ bool dptr_fill(struct smbd_server_connection *sconn, } /**************************************************************************** + Map a 32-bit wire cookie to a native directory offset. +****************************************************************************/ + +static long map_wire_to_dir_offset(struct dptr_struct *dptr, uint32_t wire_offset) +{ + if (wire_offset == WIRE_END_OF_DIRECTORY_OFFSET) { + return END_OF_DIRECTORY_OFFSET; + } else if(wire_offset == WIRE_START_OF_DIRECTORY_OFFSET) { + return START_OF_DIRECTORY_OFFSET; + } else if (wire_offset == WIRE_DOT_DOT_DIRECTORY_OFFSET) { + return DOT_DOT_DIRECTORY_OFFSET; + } + return (long)wire_offset; +} + +/**************************************************************************** Fetch the dir ptr and seek it given the 5 byte server field. ****************************************************************************/ @@ -964,11 +980,7 @@ struct dptr_struct *dptr_fetch(struct smbd_server_connection *sconn, } *num = key; wire_offset = IVAL(buf,1); - if (wire_offset == (uint32_t)-1) { - seekoff = END_OF_DIRECTORY_OFFSET; - } else { - seekoff = (long)wire_offset; - } + seekoff = map_wire_to_dir_offset(dptr, wire_offset); SeekDir(dptr->dir_hnd,seekoff); DEBUG(3,("fetching dirptr %d for path %s at offset %d\n", key, dptr->path, (int)seekoff)); -- 1.8.5.2 From 19b5c4867a88ee90b8a8e8a22b8d25bef4cf1b4d Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Sat, 11 Jan 2014 13:58:46 -0800 Subject: [PATCH 6/8] s3:dir - Add a new memcache type (non-talloc) - SMB1_SEARCH_OFFSET_MAP. We will use this in mapping 64-bit directory offset cookies to a 32-bit counter. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider (cherry picked from commit 4e0c41a321b2683610748c8c176fc46aaa8d114d) --- source3/include/memcache.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source3/include/memcache.h b/source3/include/memcache.h index e0ac4af..9362483 100644 --- a/source3/include/memcache.h +++ b/source3/include/memcache.h @@ -40,7 +40,8 @@ enum memcache_number { MANGLE_HASH2_CACHE, PDB_GETPWSID_CACHE, /* talloc */ SINGLETON_CACHE_TALLOC, /* talloc */ - SINGLETON_CACHE + SINGLETON_CACHE, + SMB1_SEARCH_OFFSET_MAP }; /* -- 1.8.5.2 From 41506265a621cb2a910c2f485e88d868fd988d78 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Sat, 11 Jan 2014 15:45:48 -0800 Subject: [PATCH 7/8] s3:dir - Introduce a 64-bit directory offset <-> 32 bit wire offset map using memcache. Should fix the DOS clients against 64-bit smbd's bug. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider (cherry picked from commit 97cd9c6729a3821faa2dbc1588a40c5e03b9fd4f) --- source3/smbd/dir.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c index 9d79c84..c39c624 100644 --- a/source3/smbd/dir.c +++ b/source3/smbd/dir.c @@ -24,6 +24,7 @@ #include "smbd/globals.h" #include "libcli/security/security.h" #include "lib/util/bitmap.h" +#include "memcache.h" /* This module implements directory related functions for Samba. @@ -72,6 +73,8 @@ struct dptr_struct { bool has_wild; /* Set to true if the wcard entry has MS wildcard characters in it. */ bool did_stat; /* Optimisation for non-wcard searches. */ bool priv; /* Directory handle opened with privilege. */ + uint32_t counter; + struct memcache *dptr_cache; }; static struct smb_Dir *OpenDir_fsp(TALLOC_CTX *mem_ctx, connection_struct *conn, @@ -156,6 +159,8 @@ static void dptr_idle(struct dptr_struct *dptr) if (dptr->dir_hnd) { DEBUG(4,("Idling dptr dnum %d\n",dptr->dnum)); TALLOC_FREE(dptr->dir_hnd); + TALLOC_FREE(dptr->dptr_cache); + dptr->counter = 0; } } @@ -914,6 +919,9 @@ void dptr_init_search_op(struct dptr_struct *dptr) static uint32_t map_dir_offset_to_wire(struct dptr_struct *dptr, long offset) { + DATA_BLOB key; + DATA_BLOB val; + if (offset == END_OF_DIRECTORY_OFFSET) { return WIRE_END_OF_DIRECTORY_OFFSET; } else if(offset == START_OF_DIRECTORY_OFFSET) { @@ -921,7 +929,58 @@ static uint32_t map_dir_offset_to_wire(struct dptr_struct *dptr, long offset) } else if (offset == DOT_DOT_DIRECTORY_OFFSET) { return WIRE_DOT_DOT_DIRECTORY_OFFSET; } - return (uint32_t)offset; + if (sizeof(long) == 4) { + /* 32-bit machine. We can cheat... */ + return (uint32_t)offset; + } + if (dptr->dptr_cache == NULL) { + /* Lazy initialize cache. */ + dptr->dptr_cache = memcache_init(dptr, 0); + if (dptr->dptr_cache == NULL) { + return WIRE_END_OF_DIRECTORY_OFFSET; + } + } else { + /* Have we seen this offset before ? */ + key.data = (void *)&offset; + key.length = sizeof(offset); + if (memcache_lookup(dptr->dptr_cache, + SMB1_SEARCH_OFFSET_MAP, + key, + &val)) { + uint32_t wire_offset; + SMB_ASSERT(val.length == sizeof(wire_offset)); + memcpy(&wire_offset, val.data, sizeof(wire_offset)); + DEBUG(10,("found wire %u <-> offset %ld\n", + (unsigned int)wire_offset, + (long)offset)); + return wire_offset; + } + } + /* Allocate a new wire cookie. */ + do { + dptr->counter++; + } while (dptr->counter == WIRE_START_OF_DIRECTORY_OFFSET || + dptr->counter == WIRE_END_OF_DIRECTORY_OFFSET || + dptr->counter == WIRE_DOT_DOT_DIRECTORY_OFFSET); + /* Store it in the cache. */ + key.data = (void *)&offset; + key.length = sizeof(offset); + val.data = (void *)&dptr->counter; + val.length = sizeof(dptr->counter); /* MUST BE uint32_t ! */ + memcache_add(dptr->dptr_cache, + SMB1_SEARCH_OFFSET_MAP, + key, + val); + /* And the reverse mapping for lookup from + map_wire_to_dir_offset(). */ + memcache_add(dptr->dptr_cache, + SMB1_SEARCH_OFFSET_MAP, + val, + key); + DEBUG(10,("stored wire %u <-> offset %ld\n", + (unsigned int)dptr->counter, + (long)offset)); + return dptr->counter; } /**************************************************************************** @@ -952,6 +1011,9 @@ bool dptr_fill(struct smbd_server_connection *sconn, static long map_wire_to_dir_offset(struct dptr_struct *dptr, uint32_t wire_offset) { + DATA_BLOB key; + DATA_BLOB val; + if (wire_offset == WIRE_END_OF_DIRECTORY_OFFSET) { return END_OF_DIRECTORY_OFFSET; } else if(wire_offset == WIRE_START_OF_DIRECTORY_OFFSET) { @@ -959,7 +1021,30 @@ static long map_wire_to_dir_offset(struct dptr_struct *dptr, uint32_t wire_offse } else if (wire_offset == WIRE_DOT_DOT_DIRECTORY_OFFSET) { return DOT_DOT_DIRECTORY_OFFSET; } - return (long)wire_offset; + if (sizeof(long) == 4) { + /* 32-bit machine. We can cheat... */ + return (long)wire_offset; + } + if (dptr->dptr_cache == NULL) { + /* Logic error, cache should be initialized. */ + return END_OF_DIRECTORY_OFFSET; + } + key.data = (void *)&wire_offset; + key.length = sizeof(wire_offset); + if (memcache_lookup(dptr->dptr_cache, + SMB1_SEARCH_OFFSET_MAP, + key, + &val)) { + /* Found mapping. */ + long offset; + SMB_ASSERT(val.length == sizeof(offset)); + memcpy(&offset, val.data, sizeof(offset)); + DEBUG(10,("lookup wire %u <-> offset %ld\n", + (unsigned int)wire_offset, + (long)offset)); + return offset; + } + return END_OF_DIRECTORY_OFFSET; } /**************************************************************************** -- 1.8.5.2 From f90807ac0c7875edc9fa5c61001a966d283c7160 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 13 Jan 2014 10:20:25 -0800 Subject: [PATCH 8/8] s3:dir - We now pass the previously spinning directory tests on ext4. https://bugzilla.samba.org/show_bug.cgi?id=2662 Signed-off-by: Jeremy Allison Reviewed-by: Andreas Schneider Autobuild-User(master): Andreas Schneider Autobuild-Date(master): Wed Jan 15 11:39:12 CET 2014 on sn-devel-104 (cherry picked from commit 0f9a189e36d8e30dfd40e42130329a0984938ddd) --- selftest/skip | 2 -- 1 file changed, 2 deletions(-) diff --git a/selftest/skip b/selftest/skip index 48d9ba6..f1fc690 100644 --- a/selftest/skip +++ b/selftest/skip @@ -106,5 +106,3 @@ bench # don't run benchmarks in our selftest ^samba4.drs.delete_object.python # flakey test ^samba4.rpc.unixinfo # This contains a server-side getpwuid call which hangs the server when nss_winbindd is in use ^samba.tests.dcerpc.unix # This contains a server-side getpwuid call which hangs the server when nss_winbindd is in use -base.dir2 # This test spins on modern ext4, so we have to skip it -raw.search # This test spins on modern ext4, so we have to skip it -- 1.8.5.2