Attachment 16122 Details for Bug 13703 – WIP patch from my Persistent Handles branch

[patch] WIP patch from my Persistent Handles branch

look (text/plain), 14.18 KB, created by Ralph Böhme on 2020-07-08 09:45:43 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ralph Böhme

Created: 2020-07-08 09:45:43 UTC

Size: 14.18 KB

patch

obsolete

>From f3331fb6981def2bf23e062090cb5269c17c7f58 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Fri, 21 Sep 2018 10:05:34 -0700
>Subject: [PATCH 1/6] s3:smbXsrv: move the replay cache to the global state
>
>Still using an in-memory db, but in preperation of using a disk backed
>db. No change in behaviour so far, ignoring the fact that the db is now
>initialized earlier at startup and not just when a connection comes in.
>---
> source3/smbd/smbXsrv_open.c | 23 ++++++++++++++---------
> 1 file changed, 14 insertions(+), 9 deletions(-)
>
>diff --git a/source3/smbd/smbXsrv_open.c b/source3/smbd/smbXsrv_open.c
>index 477ca9bc64d..8bebca4afdf 100644
>--- a/source3/smbd/smbXsrv_open.c
>+++ b/source3/smbd/smbXsrv_open.c
>@@ -35,7 +35,6 @@
> struct smbXsrv_open_table {
> 	struct {
> 		struct db_context *db_ctx;
>-		struct db_context *replay_cache_db_ctx;
> 		uint32_t lowest_id;
> 		uint32_t highest_id;
> 		uint32_t max_opens;
>@@ -43,10 +42,12 @@ struct smbXsrv_open_table {
> 	} local;
> 	struct {
> 		struct db_context *db_ctx;
>+		struct db_context *replay_cache_db_ctx;
> 	} global;
> };
> 
> static struct db_context *smbXsrv_open_global_db_ctx = NULL;
>+static struct db_context *smbXsrv_replay_cache_db_ctx = NULL;
> 
> NTSTATUS smbXsrv_open_global_init(void)
> {
>@@ -81,6 +82,14 @@ NTSTATUS smbXsrv_open_global_init(void)
> 
> 	smbXsrv_open_global_db_ctx = db_ctx;
> 
>+	db_ctx = db_open_rbt(NULL);
>+	if (db_ctx == NULL) {
>+		TALLOC_FREE(smbXsrv_open_global_db_ctx);
>+		return NT_STATUS_NO_MEMORY;
>+	}
>+
>+	smbXsrv_replay_cache_db_ctx = db_ctx;
>+
> 	return NT_STATUS_OK;
> }
> 
>@@ -227,11 +236,6 @@ static NTSTATUS smbXsrv_open_table_init(struct smbXsrv_connection *conn,
> 		TALLOC_FREE(table);
> 		return NT_STATUS_NO_MEMORY;
> 	}
>-	table->local.replay_cache_db_ctx = db_open_rbt(table);
>-	if (table->local.replay_cache_db_ctx == NULL) {
>-		TALLOC_FREE(table);
>-		return NT_STATUS_NO_MEMORY;
>-	}
> 	table->local.lowest_id = lowest_id;
> 	table->local.highest_id = highest_id;
> 	table->local.max_opens = max_opens;
>@@ -243,6 +247,7 @@ static NTSTATUS smbXsrv_open_table_init(struct smbXsrv_connection *conn,
> 	}
> 
> 	table->global.db_ctx = smbXsrv_open_global_db_ctx;
>+	table->global.replay_cache_db_ctx = smbXsrv_replay_cache_db_ctx;
> 
> 	client->open_table = table;
> 	return NT_STATUS_OK;
>@@ -931,7 +936,7 @@ static NTSTATUS smbXsrv_open_set_replay_cache(struct smbXsrv_open *op)
> 	struct GUID *create_guid;
> 	struct GUID_txt_buf buf;
> 	char *guid_string;
>-	struct db_context *db = op->table->local.replay_cache_db_ctx;
>+	struct db_context *db = op->table->global.replay_cache_db_ctx;
> 	NTSTATUS status;
> 
> 	if (!(op->flags & SMBXSRV_OPEN_NEED_REPLAY_CACHE)) {
>@@ -974,7 +979,7 @@ static NTSTATUS smbXsrv_open_clear_replay_cache(struct smbXsrv_open *op)
> 		return NT_STATUS_OK;
> 	}
> 
>-	db = op->table->local.replay_cache_db_ctx;
>+	db = op->table->global.replay_cache_db_ctx;
> 
> 	if (!(op->flags & SMBXSRV_OPEN_HAVE_REPLAY_CACHE)) {
> 		return NT_STATUS_OK;
>@@ -1283,7 +1288,7 @@ NTSTATUS smb2srv_open_lookup_replay_cache(struct smbXsrv_connection *conn,
> 	struct GUID_txt_buf buf;
> 	uint32_t local_id = 0;
> 	struct smbXsrv_open_table *table = conn->client->open_table;
>-	struct db_context *db = table->local.replay_cache_db_ctx;
>+	struct db_context *db = table->global.replay_cache_db_ctx;
> 
> 	if (GUID_all_zero(create_guid)) {
> 		return NT_STATUS_NOT_FOUND;
>-- 
>2.26.2
>
>
>From 9b6aaa3759c4119931e94b086d179f8fa4991e61 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Fri, 21 Sep 2018 10:16:56 -0700
>Subject: [PATCH 2/6] s3:smbXsrv_open: make replay cache database disk backed
>
>Create replay may come in new connections and even on different nodes in
>a cluster. To support that we must make it a disk backed database.
>
>To go the full circle we'll also enable support for per-record
>persistency in a subsequent commit, so the replay cache entry survives a
>node failure and the replay is correctly detected when the client
>connects on a different node.
>---
> source3/smbd/smbXsrv_open.c | 16 +++++++++++++++-
> 1 file changed, 15 insertions(+), 1 deletion(-)
>
>diff --git a/source3/smbd/smbXsrv_open.c b/source3/smbd/smbXsrv_open.c
>index 8bebca4afdf..40436c71e37 100644
>--- a/source3/smbd/smbXsrv_open.c
>+++ b/source3/smbd/smbXsrv_open.c
>@@ -81,8 +81,22 @@ NTSTATUS smbXsrv_open_global_init(void)
> 	}
> 
> 	smbXsrv_open_global_db_ctx = db_ctx;
>+	TALLOC_FREE(global_path);
>+
>+	global_path = lock_path(talloc_tos(), "smbXsrv_persistent_global.tdb");
>+	if (global_path == NULL) {
>+		return NT_STATUS_NO_MEMORY;
>+	}
> 
>-	db_ctx = db_open_rbt(NULL);
>+	db_ctx = db_open(NULL, global_path,
>+			 0, /* hash_size */
>+			 TDB_DEFAULT |
>+			 TDB_CLEAR_IF_FIRST |
>+			 TDB_INCOMPATIBLE_HASH,
>+			 O_RDWR | O_CREAT, 0600,
>+			 DBWRAP_LOCK_ORDER_2,
>+			 DBWRAP_FLAG_NONE);
>+	TALLOC_FREE(global_path);
> 	if (db_ctx == NULL) {
> 		TALLOC_FREE(smbXsrv_open_global_db_ctx);
> 		return NT_STATUS_NO_MEMORY;
>-- 
>2.26.2
>
>
>From a0358469e776f4178af748b3fbd6c2dfeb476e62 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Fri, 21 Sep 2018 11:59:31 -0700
>Subject: [PATCH 3/6] s3:smbXsrv_open: use the global id as record key in the
> replay cache
>
>In future this allows create replay to work across session and cluster
>node failures.
>---
> source3/smbd/smbXsrv_open.c | 36 ++++++++++++++++++++++++++++--------
> 1 file changed, 28 insertions(+), 8 deletions(-)
>
>diff --git a/source3/smbd/smbXsrv_open.c b/source3/smbd/smbXsrv_open.c
>index 40436c71e37..f16363cb9ea 100644
>--- a/source3/smbd/smbXsrv_open.c
>+++ b/source3/smbd/smbXsrv_open.c
>@@ -971,7 +971,8 @@ static NTSTATUS smbXsrv_open_set_replay_cache(struct smbXsrv_open *op)
> 		return NT_STATUS_INVALID_PARAMETER;
> 	}
> 
>-	status = dbwrap_store_uint32_bystring(db, guid_string, op->local_id);
>+	status = dbwrap_store_uint32_bystring(db, guid_string,
>+					      op->global->open_global_id);
> 
> 	if (NT_STATUS_IS_OK(status)) {
> 		op->flags |= SMBXSRV_OPEN_HAVE_REPLAY_CACHE;
>@@ -1300,9 +1301,10 @@ NTSTATUS smb2srv_open_lookup_replay_cache(struct smbXsrv_connection *conn,
> 	NTSTATUS status;
> 	char *guid_string;
> 	struct GUID_txt_buf buf;
>-	uint32_t local_id = 0;
>+	uint32_t global_id = 0;
> 	struct smbXsrv_open_table *table = conn->client->open_table;
> 	struct db_context *db = table->global.replay_cache_db_ctx;
>+	struct smbXsrv_open_global0 *global = NULL;
> 
> 	if (GUID_all_zero(create_guid)) {
> 		return NT_STATUS_NOT_FOUND;
>@@ -1313,7 +1315,7 @@ NTSTATUS smb2srv_open_lookup_replay_cache(struct smbXsrv_connection *conn,
> 		return NT_STATUS_INVALID_PARAMETER;
> 	}
> 
>-	status = dbwrap_fetch_uint32_bystring(db, guid_string, &local_id);
>+	status = dbwrap_fetch_uint32_bystring(db, guid_string, &global_id);
> 	if (NT_STATUS_EQUAL(status, NT_STATUS_NOT_FOUND)) {
> 		return status;
> 	}
>@@ -1323,14 +1325,32 @@ NTSTATUS smb2srv_open_lookup_replay_cache(struct smbXsrv_connection *conn,
> 		return status;
> 	}
> 
>-	status = smbXsrv_open_local_lookup(table, local_id, 0, /* global_id */
>-					   now, _open);
>+	status = smbXsrv_open_global_lookup(table, global_id,
>+					    talloc_tos(), &global);
> 	if (!NT_STATUS_IS_OK(status)) {
>-		DBG_ERR("smbXsrv_open_local_lookup failed for local_id %u\n",
>-			(unsigned)local_id);
>+		DBG_ERR("failed to fetch global_id from replay cache: %s\n",
>+			nt_errstr(status));
>+		return status;
> 	}
> 
>-	return status;
>+	status = smbXsrv_open_local_lookup(table, global->open_volatile_id,
>+					   global_id, now, _open);
>+	TALLOC_FREE(global);
>+	if (NT_STATUS_IS_OK(status)) {
>+		return NT_STATUS_OK;
>+	}
>+
>+	DBG_DEBUG("smbXsrv_open_local_lookup failed for local_id %u, "
>+		  "replay on different node\n",
>+		  (unsigned)global->open_volatile_id);
>+
>+	/*
>+	 * TODO: implement open recreate.
>+	 *
>+	 * Better merge create-replay and create reconnect.
>+	 */
>+
>+	return NT_STATUS_OK;
> }
> 
> NTSTATUS smb2srv_open_recreate(struct smbXsrv_connection *conn,
>-- 
>2.26.2
>
>
>From 69093d31fd88a7303b426a3a262147696e8ce2af Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Fri, 21 Sep 2018 10:20:02 -0700
>Subject: [PATCH 4/6] s3:smbXsrv_open: enable per-record persistency on replay
> cache database
>
>Go the full circle and enable support for per-record persistency in a
>subsequent commit, so the replay cache entry survives a node failure and
>the replay is correctly detected when the client connects on a different
>node.
>---
> source3/smbd/smbXsrv_open.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/source3/smbd/smbXsrv_open.c b/source3/smbd/smbXsrv_open.c
>index f16363cb9ea..c9aa1dc8437 100644
>--- a/source3/smbd/smbXsrv_open.c
>+++ b/source3/smbd/smbXsrv_open.c
>@@ -95,7 +95,7 @@ NTSTATUS smbXsrv_open_global_init(void)
> 			 TDB_INCOMPATIBLE_HASH,
> 			 O_RDWR | O_CREAT, 0600,
> 			 DBWRAP_LOCK_ORDER_2,
>-			 DBWRAP_FLAG_NONE);
>+			 DBWRAP_FLAG_PER_REC_PERSISTENT);
> 	TALLOC_FREE(global_path);
> 	if (db_ctx == NULL) {
> 		TALLOC_FREE(smbXsrv_open_global_db_ctx);
>-- 
>2.26.2
>
>
>From 2066dc9998550d3f8dcef787f5ee00f91f3c747f Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Fri, 21 Sep 2018 12:44:19 -0700
>Subject: [PATCH 5/6] dbwrap: add dbwrap_fetch_uint32()
>
>---
> lib/dbwrap/dbwrap.h      |  2 ++
> lib/dbwrap/dbwrap_util.c | 26 ++++++++++++++++++++++++++
> 2 files changed, 28 insertions(+)
>
>diff --git a/lib/dbwrap/dbwrap.h b/lib/dbwrap/dbwrap.h
>index a69f0942c52..1764be23e44 100644
>--- a/lib/dbwrap/dbwrap.h
>+++ b/lib/dbwrap/dbwrap.h
>@@ -169,6 +169,8 @@ NTSTATUS dbwrap_fetch_bystring(struct db_context *db, TALLOC_CTX *mem_ctx,
> 
> NTSTATUS dbwrap_fetch_int32(struct db_context *db, TDB_DATA key,
> 			    int32_t *result);
>+NTSTATUS dbwrap_fetch_uint32(struct db_context *db, TDB_DATA key,
>+			     uint32_t *result);
> NTSTATUS dbwrap_fetch_int32_bystring(struct db_context *db, const char *keystr,
> 				     int32_t *result);
> NTSTATUS dbwrap_store_int32_bystring(struct db_context *db, const char *keystr,
>diff --git a/lib/dbwrap/dbwrap_util.c b/lib/dbwrap/dbwrap_util.c
>index ae0a2a741c4..62bfb17a18c 100644
>--- a/lib/dbwrap/dbwrap_util.c
>+++ b/lib/dbwrap/dbwrap_util.c
>@@ -144,6 +144,32 @@ NTSTATUS dbwrap_store_uint32_bystring(struct db_context *db,
> 	return status;
> }
> 
>+NTSTATUS dbwrap_fetch_uint32(struct db_context *db, TDB_DATA key,
>+			     uint32_t *result)
>+{
>+	struct dbwrap_fetch_int32_state state;
>+	NTSTATUS status;
>+
>+	if (result == NULL) {
>+		return NT_STATUS_INVALID_PARAMETER;
>+	}
>+
>+	state.status = NT_STATUS_INTERNAL_ERROR;
>+
>+	status = dbwrap_parse_record(db,
>+				     key,
>+				     dbwrap_fetch_uint32_parser,
>+				     &state);
>+	if (!NT_STATUS_IS_OK(status)) {
>+		return status;
>+	}
>+
>+	if (NT_STATUS_IS_OK(state.status)) {
>+		*result = state.result;
>+	}
>+	return state.status;
>+}
>+
> /**
>  * Atomic unsigned integer change (addition):
>  *
>-- 
>2.26.2
>
>
>From 3692f000fd1f8e38d0279c82f1b37e8f53e32a82 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Fri, 21 Sep 2018 10:39:34 -0700
>Subject: [PATCH 6/6] s3:smbXsrv_open: use binary guid as key for the replay
> cache, not string representation
>
>smbXsrv_open_clear_replay_cache() gets called on every open lookup, so
>we may want to avoid the conversion cost.
>---
> source3/smbd/smbXsrv_open.c | 52 ++++++++++++++++++-------------------
> 1 file changed, 25 insertions(+), 27 deletions(-)
>
>diff --git a/source3/smbd/smbXsrv_open.c b/source3/smbd/smbXsrv_open.c
>index c9aa1dc8437..aa98586c581 100644
>--- a/source3/smbd/smbXsrv_open.c
>+++ b/source3/smbd/smbXsrv_open.c
>@@ -947,10 +947,10 @@ NTSTATUS smbXsrv_open_create(struct smbXsrv_connection *conn,
> 
> static NTSTATUS smbXsrv_open_set_replay_cache(struct smbXsrv_open *op)
> {
>-	struct GUID *create_guid;
>-	struct GUID_txt_buf buf;
>-	char *guid_string;
> 	struct db_context *db = op->table->global.replay_cache_db_ctx;
>+	TDB_DATA key;
>+	TDB_DATA data;
>+	int flags = 0;
> 	NTSTATUS status;
> 
> 	if (!(op->flags & SMBXSRV_OPEN_NEED_REPLAY_CACHE)) {
>@@ -961,19 +961,20 @@ static NTSTATUS smbXsrv_open_set_replay_cache(struct smbXsrv_open *op)
> 		return NT_STATUS_OK;
> 	}
> 
>-	create_guid = &op->global->create_guid;
>-	if (GUID_all_zero(create_guid)) {
>+	if (GUID_all_zero(&op->global->create_guid)) {
> 		return NT_STATUS_OK;
> 	}
> 
>-	guid_string = GUID_buf_string(create_guid, &buf);
>-	if (guid_string == NULL) {
>-		return NT_STATUS_INVALID_PARAMETER;
>-	}
>-
>-	status = dbwrap_store_uint32_bystring(db, guid_string,
>-					      op->global->open_global_id);
>+	key = (TDB_DATA) {
>+		.dsize = sizeof(struct GUID),
>+		.dptr = (uint8_t *)&op->global->create_guid,
>+	};
>+	data = (TDB_DATA) {
>+		.dsize = sizeof(uint32_t),
>+		.dptr = (uint8_t *)&op->global->open_global_id,
>+	};
> 
>+	status = dbwrap_store(db, key, data, flags);
> 	if (NT_STATUS_IS_OK(status)) {
> 		op->flags |= SMBXSRV_OPEN_HAVE_REPLAY_CACHE;
> 		op->flags &= ~SMBXSRV_OPEN_NEED_REPLAY_CACHE;
>@@ -985,10 +986,9 @@ static NTSTATUS smbXsrv_open_set_replay_cache(struct smbXsrv_open *op)
> static NTSTATUS smbXsrv_open_clear_replay_cache(struct smbXsrv_open *op)
> {
> 	struct GUID *create_guid;
>-	struct GUID_txt_buf buf;
>-	char *guid_string;
> 	struct db_context *db;
> 	NTSTATUS status;
>+	TDB_DATA key;
> 
> 	if (op->table == NULL) {
> 		return NT_STATUS_OK;
>@@ -1005,12 +1005,11 @@ static NTSTATUS smbXsrv_open_clear_replay_cache(struct smbXsrv_open *op)
> 		return NT_STATUS_OK;
> 	}
> 
>-	guid_string = GUID_buf_string(create_guid, &buf);
>-	if (guid_string == NULL) {
>-		return NT_STATUS_INVALID_PARAMETER;
>-	}
>-
>-	status = dbwrap_purge_bystring(db, guid_string);
>+	key = (TDB_DATA) {
>+		.dptr = (uint8_t *)&create_guid,
>+		.dsize = sizeof(create_guid)
>+	};
>+	status = dbwrap_purge(db, key);
> 
> 	if (NT_STATUS_IS_OK(status)) {
> 		op->flags &= ~SMBXSRV_OPEN_HAVE_REPLAY_CACHE;
>@@ -1299,23 +1298,22 @@ NTSTATUS smb2srv_open_lookup_replay_cache(struct smbXsrv_connection *conn,
> 					  struct smbXsrv_open **_open)
> {
> 	NTSTATUS status;
>-	char *guid_string;
>-	struct GUID_txt_buf buf;
> 	uint32_t global_id = 0;
> 	struct smbXsrv_open_table *table = conn->client->open_table;
> 	struct db_context *db = table->global.replay_cache_db_ctx;
> 	struct smbXsrv_open_global0 *global = NULL;
>+	TDB_DATA key;
> 
> 	if (GUID_all_zero(create_guid)) {
> 		return NT_STATUS_NOT_FOUND;
> 	}
> 
>-	guid_string = GUID_buf_string(create_guid, &buf);
>-	if (guid_string == NULL) {
>-		return NT_STATUS_INVALID_PARAMETER;
>-	}
>+	key = (TDB_DATA) {
>+		.dptr = (uint8_t *)discard_const_p(struct GUID, create_guid),
>+		.dsize = sizeof(struct GUID)
>+	};
> 
>-	status = dbwrap_fetch_uint32_bystring(db, guid_string, &global_id);
>+	status = dbwrap_fetch_uint32(db, key, &global_id);
> 	if (NT_STATUS_EQUAL(status, NT_STATUS_NOT_FOUND)) {
> 		return status;
> 	}
>-- 
>2.26.2
>

Actions: View

Attachments on bug 13703: 16122