Attachment 12169 Details for Bug 11940 – Patches for v4-4 branch

[patch] Patches for v4-4 branch

v4-4.patches (text/plain), 179.08 KB, created by Amitay Isaacs on 2016-06-09 04:10:39 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Amitay Isaacs

Created: 2016-06-09 04:10:39 UTC

Size: 179.08 KB

patch

obsolete

>From b4a2b86dc975dade9005443b27e3805f5fffc694 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Thu, 11 Feb 2016 14:32:34 +1100
>Subject: [PATCH 01/29] ctdb-recovery: Create recovery databases in state dir
>
>This matches the behaviour during serial database recovery.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>
>Autobuild-User(master): Martin Schwenke <martins@samba.org>
>Autobuild-Date(master): Thu Feb 11 08:01:14 CET 2016 on sn-devel-144
>
>(cherry picked from commit 19a411f839c5c34fec2aa5e6cb095346be56d94e)
>---
> ctdb/server/ctdb_recoverd.c        | 2 ++
> ctdb/server/ctdb_recovery_helper.c | 9 ++++++++-
> 2 files changed, 10 insertions(+), 1 deletion(-)
>
>diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
>index c89649a..e42433d 100644
>--- a/ctdb/server/ctdb_recoverd.c
>+++ b/ctdb/server/ctdb_recoverd.c
>@@ -1781,6 +1781,8 @@ static int db_recovery_parallel(struct ctdb_recoverd *rec, TALLOC_CTX *mem_ctx)
> 		goto fail;
> 	}
> 
>+	setenv("CTDB_DBDIR_STATE", rec->ctdb->db_directory_state, 1);
>+
> 	if (!ctdb_vfork_with_logging(state, rec->ctdb, "recovery", prog, nargs,
> 				     args, NULL, NULL, &state->pid)) {
> 		DEBUG(DEBUG_ERR,
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 6d6a835..1ddddfc 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -80,6 +80,7 @@ static struct recdb_context *recdb_create(TALLOC_CTX *mem_ctx, uint32_t db_id,
> 					  const char *db_path,
> 					  uint32_t hash_size, bool persistent)
> {
>+	static char *db_dir_state = NULL;
> 	struct recdb_context *recdb;
> 	unsigned int tdb_flags;
> 
>@@ -88,10 +89,16 @@ static struct recdb_context *recdb_create(TALLOC_CTX *mem_ctx, uint32_t db_id,
> 		return NULL;
> 	}
> 
>+	if (db_dir_state == NULL) {
>+		db_dir_state = getenv("CTDB_DBDIR_STATE");
>+	}
>+
> 	recdb->db_name = db_name;
> 	recdb->db_id = db_id;
> 	recdb->db_path = talloc_asprintf(recdb, "%s/recdb.%s",
>-					 dirname(discard_const(db_path)),
>+					 db_dir_state != NULL ?
>+					    db_dir_state :
>+					    dirname(discard_const(db_path)),
> 					 db_name);
> 	if (recdb->db_path == NULL) {
> 		talloc_free(recdb);
>-- 
>2.5.5
>
>
>From 45f62ac8c8e50c3ff37359919bc3cf249aada1ce Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Wed, 10 Feb 2016 11:39:37 +1100
>Subject: [PATCH 02/29] ctdb-recovery: Add a log message when marshalling
> recovery database fails
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 157e19b9848c4745e2a9a2169e25698cfd4dcaa6)
>---
> ctdb/server/ctdb_recovery_helper.c | 2 ++
> 1 file changed, 2 insertions(+)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 1ddddfc..f0fa12d 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -270,6 +270,8 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb,
> 
> 	ret = tdb_traverse_read(recdb->db->tdb, recdb_traverse, &state);
> 	if (ret == -1 || state.failed) {
>+		LOG("Failed to marshall recovery records for %s\n",
>+		    recdb->db_name);
> 		TALLOC_FREE(state.recbuf);
> 		return NULL;
> 	}
>-- 
>2.5.5
>
>
>From edf6473236d4fbe545c74f417b33689132d756e8 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Wed, 18 Nov 2015 15:11:37 +1100
>Subject: [PATCH 03/29] ctdb-client: Add async version of set/remove message
> handler functions
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit d23c5a6c2f365b249233f9a1c6d42d220b2b7c67)
>---
> ctdb/client/client.h         |  18 +++++
> ctdb/client/client_message.c | 179 +++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 197 insertions(+)
>
>diff --git a/ctdb/client/client.h b/ctdb/client/client.h
>index bce0c6b..48b69b6 100644
>--- a/ctdb/client/client.h
>+++ b/ctdb/client/client.h
>@@ -76,6 +76,24 @@ int ctdb_client_message(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
> 			struct ctdb_client_context *client,
> 			uint32_t destnode, struct ctdb_req_message *message);
> 
>+struct tevent_req *ctdb_client_set_message_handler_send(
>+					TALLOC_CTX *mem_ctx,
>+					struct tevent_context *ev,
>+					struct ctdb_client_context *client,
>+					uint64_t srvid,
>+					srvid_handler_fn handler,
>+					void *private_data);
>+bool ctdb_client_set_message_handler_recv(struct tevent_req *req, int *perr);
>+
>+struct tevent_req *ctdb_client_remove_message_handler_send(
>+					TALLOC_CTX *mem_ctx,
>+					struct tevent_context *ev,
>+					struct ctdb_client_context *client,
>+					uint64_t srvid,
>+					void *private_data);
>+bool ctdb_client_remove_message_handler_recv(struct tevent_req *req,
>+					     int *perr);
>+
> int ctdb_client_set_message_handler(TALLOC_CTX *mem_ctx,
> 				    struct tevent_context *ev,
> 				    struct ctdb_client_context *client,
>diff --git a/ctdb/client/client_message.c b/ctdb/client/client_message.c
>index bfa9ba2..c316f42 100644
>--- a/ctdb/client/client_message.c
>+++ b/ctdb/client/client_message.c
>@@ -192,6 +192,185 @@ int ctdb_client_message(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
> 	return 0;
> }
> 
>+struct ctdb_client_set_message_handler_state {
>+	struct ctdb_client_context *client;
>+	uint64_t srvid;
>+	srvid_handler_fn handler;
>+	void *private_data;
>+};
>+
>+static void ctdb_client_set_message_handler_done(struct tevent_req *subreq);
>+
>+struct tevent_req *ctdb_client_set_message_handler_send(
>+					TALLOC_CTX *mem_ctx,
>+					struct tevent_context *ev,
>+					struct ctdb_client_context *client,
>+					uint64_t srvid,
>+					srvid_handler_fn handler,
>+					void *private_data)
>+{
>+	struct tevent_req *req, *subreq;
>+	struct ctdb_client_set_message_handler_state *state;
>+	struct ctdb_req_control request;
>+
>+	req = tevent_req_create(mem_ctx, &state,
>+				struct ctdb_client_set_message_handler_state);
>+	if (req == NULL) {
>+		return NULL;
>+	}
>+
>+	state->client = client;
>+	state->srvid = srvid;
>+	state->handler = handler;
>+	state->private_data = private_data;
>+
>+	ctdb_req_control_register_srvid(&request, srvid);
>+	subreq = ctdb_client_control_send(state, ev, client, client->pnn,
>+					  tevent_timeval_zero(), &request);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+	tevent_req_set_callback(subreq, ctdb_client_set_message_handler_done,
>+				req);
>+
>+	return req;
>+}
>+
>+static void ctdb_client_set_message_handler_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct ctdb_client_set_message_handler_state *state = tevent_req_data(
>+		req, struct ctdb_client_set_message_handler_state);
>+	struct ctdb_reply_control *reply;
>+	bool status;
>+	int ret;
>+
>+	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	ret = ctdb_reply_control_register_srvid(reply);
>+	talloc_free(reply);
>+	if (ret != 0) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	ret = srvid_register(state->client->srv, state->client, state->srvid,
>+			     state->handler, state->private_data);
>+	if (ret != 0) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	tevent_req_done(req);
>+}
>+
>+bool ctdb_client_set_message_handler_recv(struct tevent_req *req, int *perr)
>+{
>+	int err;
>+
>+	if (tevent_req_is_unix_error(req, &err)) {
>+		if (perr != NULL) {
>+			*perr = err;
>+		}
>+		return false;
>+	}
>+	return true;
>+}
>+
>+struct ctdb_client_remove_message_handler_state {
>+	struct ctdb_client_context *client;
>+	uint64_t srvid;
>+	void *private_data;
>+};
>+
>+static void ctdb_client_remove_message_handler_done(struct tevent_req *subreq);
>+
>+struct tevent_req *ctdb_client_remove_message_handler_send(
>+					TALLOC_CTX *mem_ctx,
>+					struct tevent_context *ev,
>+					struct ctdb_client_context *client,
>+					uint64_t srvid,
>+					void *private_data)
>+{
>+	struct tevent_req *req, *subreq;
>+	struct ctdb_client_remove_message_handler_state *state;
>+	struct ctdb_req_control request;
>+
>+	req = tevent_req_create(mem_ctx, &state,
>+				struct ctdb_client_remove_message_handler_state);
>+	if (req == NULL) {
>+		return NULL;
>+	}
>+
>+	state->client = client;
>+	state->srvid = srvid;
>+	state->private_data = private_data;
>+
>+	ctdb_req_control_deregister_srvid(&request, srvid);
>+	subreq = ctdb_client_control_send(state, ev, client, client->pnn,
>+					  tevent_timeval_zero(), &request);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+	tevent_req_set_callback(subreq,
>+				ctdb_client_remove_message_handler_done, req);
>+
>+	return req;
>+}
>+
>+static void ctdb_client_remove_message_handler_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct ctdb_client_remove_message_handler_state *state = tevent_req_data(
>+		req, struct ctdb_client_remove_message_handler_state);
>+	struct ctdb_reply_control *reply;
>+	bool status;
>+	int ret;
>+
>+	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	ret = ctdb_reply_control_deregister_srvid(reply);
>+	talloc_free(reply);
>+	if (ret != 0) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	ret = srvid_deregister(state->client->srv, state->srvid,
>+			       state->private_data);
>+	if (ret != 0) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	tevent_req_done(req);
>+}
>+
>+bool ctdb_client_remove_message_handler_recv(struct tevent_req *req, int *perr)
>+{
>+	int err;
>+
>+	if (tevent_req_is_unix_error(req, &err)) {
>+		if (perr != NULL) {
>+			*perr = err;
>+		}
>+		return false;
>+	}
>+	return true;
>+}
>+
> int ctdb_client_set_message_handler(TALLOC_CTX *mem_ctx,
> 				    struct tevent_context *ev,
> 				    struct ctdb_client_context *client,
>-- 
>2.5.5
>
>
>From b835d3b6b05a8ea5fe042a5dfe337435b6fca9b7 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Mon, 7 Mar 2016 17:05:11 +1100
>Subject: [PATCH 04/29] ctdb-doc: Sort the tunable variables in alphabetical
> order
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 4bf6cab4a1e1c11f79fde40fc8b9ae29b0f00e35)
>---
> ctdb/doc/ctdb-tunables.7.xml | 658 ++++++++++++++++++++++---------------------
> 1 file changed, 332 insertions(+), 326 deletions(-)
>
>diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml
>index 6c164f3..a6ef435 100644
>--- a/ctdb/doc/ctdb-tunables.7.xml
>+++ b/ctdb/doc/ctdb-tunables.7.xml
>@@ -29,39 +29,17 @@
>       <command>getvar</command> commands for more details.
>     </para>
> 
>-    <refsect2>
>-      <title>MaxRedirectCount</title>
>-      <para>Default: 3</para>
>-      <para>
>-	If we are not the DMASTER and need to fetch a record across the network
>-	we first send the request to the LMASTER after which the record
>-	is passed onto the current DMASTER. If the DMASTER changes before
>-	the request has reached that node, the request will be passed onto the
>-	"next" DMASTER. For very hot records that migrate rapidly across the
>-	cluster this can cause a request to "chase" the record for many hops
>-	before it catches up with the record.
>-
>-	this is how many hops we allow trying to chase the DMASTER before we
>-	switch back to the LMASTER again to ask for new directions.
>-      </para>
>-      <para>
>-	When chasing a record, this is how many hops we will chase the record
>-	for before going back to the LMASTER to ask for new guidance.
>-      </para>
>-    </refsect2>
>+    <para>
>+      The tunable variables are listed alphabetically.
>+    </para>
> 
>     <refsect2>
>-      <title>SeqnumInterval</title>
>-      <para>Default: 1000</para>
>-      <para>
>-	Some databases have seqnum tracking enabled, so that samba will be able
>-	to detect asynchronously when there has been updates to the database.
>-	Everytime a database is updated its sequence number is increased.
>-      </para>
>+      <title>AllowClientDBAttach</title>
>+      <para>Default: 1</para>
>       <para>
>-	This tunable is used to specify in 'ms' how frequently ctdb will
>-	send out updates to remote nodes to inform them that the sequence
>-	number is increased.
>+	When set to 0, clients are not allowed to attach to any databases.
>+	This can be used to temporarily block any new processes from
>+	attaching to and accessing the databases.
>       </para>
>     </refsect2>
> 
>@@ -69,98 +47,130 @@
>       <title>ControlTimeout</title>
>       <para>Default: 60</para>
>       <para>
>-	This is the default
>-	setting for timeout for when sending a control message to either the
>-	local or a remote ctdb daemon.
>+	This is the default setting for timeout for when sending a
>+	control message to either the local or a remote ctdb daemon.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>TraverseTimeout</title>
>-      <para>Default: 20</para>
>+      <title>DatabaseHashSize</title>
>+      <para>Default: 100001</para>
>       <para>
>-	This setting controls how long we allow a traverse process to run.
>-	After this timeout triggers, the main ctdb daemon will abort the
>-	traverse if it has not yet finished.
>+	Number of the hash chains for the local store of the tdbs that
>+	ctdb manages.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>KeepaliveInterval</title>
>+      <title>DatabaseMaxDead</title>
>       <para>Default: 5</para>
>       <para>
>-	How often in seconds should the nodes send keepalives to eachother.
>+	How many dead records per hashchain in the TDB database do we
>+	allow before the freelist needs to be processed.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>KeepaliveLimit</title>
>-      <para>Default: 5</para>
>+      <title>DBRecordCountWarn</title>
>+      <para>Default: 100000</para>
>       <para>
>-	After how many keepalive intervals without any traffic should a node
>-	wait until marking the peer as DISCONNECTED.
>+	When set to non-zero, ctdb will log a warning when we try
>+	to recover a database with more than this many records. This
>+	will produce a warning if a database grows uncontrollably with
>+	orphaned records.
>       </para>
>+    </refsect2>
>+
>+    <refsect2>
>+      <title>DBRecordSizeWarn</title>
>+      <para>Default: 10000000</para>
>       <para>
>-	If a node has hung, it can thus take KeepaliveInterval*(KeepaliveLimit+1)
>-	seconds before we determine that the node is DISCONNECTED and that we
>-	require a recovery. This limitshould not be set too high since we want
>-	a hung node to be detectec, and expunged from the cluster well before
>-	common CIFS timeouts (45-90 seconds) kick in.
>+	When set to non-zero, ctdb will log a warning when we try to
>+	recover a database where a single record is bigger than this. This
>+	will produce a warning if a database record grows uncontrollably
>+	with orphaned sub-records.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>RecoverTimeout</title>
>-      <para>Default: 20</para>
>+      <title>DBSizeWarn</title>
>+      <para>Default: 1000000000</para>
>       <para>
>-	This is the default setting for timeouts for controls when sent from the
>-	recovery daemon. We allow longer control timeouts from the recovery daemon
>-	than from normal use since the recovery dameon often use controls that 
>-	can take a lot longer than normal controls.
>+	When set to non-zero, ctdb will log a warning when we try to
>+	recover a database bigger than this. This will produce a warning
>+	if a database grows uncontrollably.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>RecoverInterval</title>
>-      <para>Default: 1</para>
>+      <title>DeferredAttachTO</title>
>+      <para>Default: 120</para>
>+      <para>
>+	When databases are frozen we do not allow clients to attach to
>+	the databases. Instead of returning an error immediately to the
>+	application the attach request from the client is deferred until
>+	the database becomes available again at which stage we respond
>+	to the client.
>+      </para>
>       <para>
>-	How frequently in seconds should the recovery daemon perform the
>-	consistency checks that determine if we need to perform a recovery or not.
>+	This timeout controls how long we will defer the request from the
>+	client before timing it out and returning an error to the client.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>ElectionTimeout</title>
>-      <para>Default: 3</para>
>+      <title>DeterministicIPs</title>
>+      <para>Default: 0</para>
>       <para>
>-	When electing a new recovery master, this is how many seconds we allow
>-	the election to take before we either deem the election finished
>-	or we fail the election and start a new one.
>+	When enabled, this tunable makes ctdb try to keep public IP
>+	addresses locked to specific nodes as far as possible. This makes
>+	it easier for debugging since you can know that as long as all
>+	nodes are healthy public IP X will always be hosted by node Y.
>+      </para>
>+      <para>
>+	The cost of using deterministic IP address assignment is that it
>+	disables part of the logic where ctdb tries to reduce the number
>+	of public IP assignment changes in the cluster. This tunable may
>+	increase the number of IP failover/failbacks that are performed
>+	on the cluster by a small margin.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>TakeoverTimeout</title>
>-      <para>Default: 9</para>
>+      <title>DisableIPFailover</title>
>+      <para>Default: 0</para>
>+      <para>
>+	When enabled, ctdb will not perform failover or failback. Even
>+	if a node fails while holding public IPs, ctdb will not recover
>+	the IPs or assign them to another node.
>+      </para>
>       <para>
>-	This is how many seconds we allow controls to take for IP failover events.
>+	When you enable this tunable, CTDB will no longer attempt
>+	to recover the cluster by failing IP addresses over to other
>+	nodes. This leads to a service outage until the administrator
>+	has manually performed failover to replacement nodes using the
>+	'ctdb moveip' command.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>MonitorInterval</title>
>-      <para>Default: 15</para>
>+      <title>ElectionTimeout</title>
>+      <para>Default: 3</para>
>       <para>
>-	How often should ctdb run the event scripts to check for a nodes health.
>+	When electing a new recovery master, this is how many seconds
>+	we allow the election to take before we either deem the election
>+	finished or we fail the election and start a new one.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>TickleUpdateInterval</title>
>-      <para>Default: 20</para>
>+      <title>EnableBans</title>
>+      <para>Default: 1</para>
>       <para>
>-	How often will ctdb record and store the "tickle" information used to
>-	kickstart stalled tcp connections after a recovery.
>+	When set to 0, this disables BANNING completely in the cluster
>+	and thus nodes can not get banned, even it they break. Don't
>+	set to 0 unless you know what you are doing.  You should set
>+	this to the same value on all nodes to avoid unexpected behaviour.
>       </para>
>     </refsect2>
> 
>@@ -172,7 +182,6 @@
> 	out.  This is the total time for all enabled scripts that are
> 	run for an event, not just a single event script.
>       </para>
>-
>       <para>
> 	Note that timeouts are ignored for some events ("takeip",
> 	"releaseip", "startrecovery", "recovered") and converted to
>@@ -182,162 +191,157 @@
>     </refsect2>
> 
>     <refsect2>
>-      <title>MonitorTimeoutCount</title>
>-      <para>Default: 20</para>
>+      <title>FetchCollapse</title>
>+      <para>Default: 1</para>
>       <para>
>-	How many monitor events in a row need to timeout before a node
>-	is flagged as UNHEALTHY.  This setting is useful if scripts
>-	can not be written so that they do not hang for benign
>-	reasons.
>+	When many clients across many nodes try to access the same record
>+	at the same time this can lead to a fetch storm where the record
>+	becomes very active and bounces between nodes very fast. This
>+	leads to high CPU utilization of the ctdbd daemon, trying to
>+	bounce that record around very fast, and poor performance.
>       </para>
>-    </refsect2>
>-
>-    <refsect2>
>-      <title>RecoveryGracePeriod</title>
>-      <para>Default: 120</para>
>       <para>
>-	During recoveries, if a node has not caused recovery failures during the
>-	last grace period, any records of transgressions that the node has caused
>-	recovery failures will be forgiven. This resets the ban-counter back to 
>-	zero for that node.
>+	This parameter is used to activate a fetch-collapse. A
>+	fetch-collapse is when we track which records we have requests in
>+	flight so that we only keep one request in flight from a certain
>+	node, even if multiple smbd processes are attemtping to fetch
>+	the record at the same time. This can improve performance and
>+	reduce CPU utilization for certain workloads.
>       </para>
>-    </refsect2>
>-
>-    <refsect2>
>-      <title>RecoveryBanPeriod</title>
>-      <para>Default: 300</para>
>       <para>
>-	If a node becomes banned causing repetitive recovery failures. The node will
>-	eventually become banned from the cluster.
>-	This controls how long the culprit node will be banned from the cluster
>-	before it is allowed to try to join the cluster again.
>-	Don't set to small. A node gets banned for a reason and it is usually due
>-	to real problems with the node.
>+	This timeout controls if we should collapse multiple fetch
>+	operations of the same record into a single request and defer
>+	all duplicates or not.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>DatabaseHashSize</title>
>-      <para>Default: 100001</para>
>+      <title>HopcountMakeSticky</title>
>+      <para>Default: 50</para>
>       <para>
>-	Size of the hash chains for the local store of the tdbs that ctdb manages.
>+	If the database is set to 'STICKY' mode, using the 'ctdb
>+	setdbsticky' command, any record that is seen as very hot and
>+	migrating so fast that hopcount surpasses 50 is set to become a
>+	STICKY record for StickyDuration seconds. This means that after
>+	each migration the record will be kept on the node and prevented
>+	from being migrated off the node.
>+       </para>
>+       <para>
>+	This setting allows one to try to identify such records and
>+	stop them from migrating across the cluster so fast. This will
>+	improve performance for certain workloads, such as locking.tdb
>+	if many clients are opening/closing the same file concurrently.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>DatabaseMaxDead</title>
>+      <title>KeepaliveInterval</title>
>       <para>Default: 5</para>
>       <para>
>-	How many dead records per hashchain in the TDB database do we allow before
>-	the freelist needs to be processed.
>+	How often in seconds should the nodes send keep-alive packets to
>+	each other.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>RerecoveryTimeout</title>
>-      <para>Default: 10</para>
>+      <title>KeepaliveLimit</title>
>+      <para>Default: 5</para>
>       <para>
>-	Once a recovery has completed, no additional recoveries are permitted
>-	until this timeout has expired.
>+	After how many keepalive intervals without any traffic should
>+	a node wait until marking the peer as DISCONNECTED.
>+       </para>
>+       <para>
>+	If a node has hung, it can thus take
>+	KeepaliveInterval*(KeepaliveLimit+1) seconds before we determine
>+	that the node is DISCONNECTED and that we require a recovery. This
>+	limitshould not be set too high since we want a hung node to be
>+	detectec, and expunged from the cluster well before common CIFS
>+	timeouts (45-90 seconds) kick in.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>EnableBans</title>
>+      <title>LCP2PublicIPs</title>
>       <para>Default: 1</para>
>       <para>
>-	When set to 0, this disables BANNING completely in the cluster and thus
>-	nodes can not get banned, even it they break. Don't set to 0 unless you
>-	know what you are doing.  You should set this to the same value on
>-	all nodes to avoid unexpected behaviour.
>+	When enabled this switches ctdb to use the LCP2 ip allocation
>+	algorithm.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>DeterministicIPs</title>
>+      <title>LogLatencyMs</title>
>       <para>Default: 0</para>
>       <para>
>-	When enabled, this tunable makes ctdb try to keep public IP addresses
>-	locked to specific nodes as far as possible. This makes it easier for
>-	debugging since you can know that as long as all nodes are healthy
>-	public IP X will always be hosted by node Y. 
>-      </para>
>-      <para>
>-	The cost of using deterministic IP address assignment is that it
>-	disables part of the logic where ctdb tries to reduce the number of
>-	public IP assignment changes in the cluster. This tunable may increase
>-	the number of IP failover/failbacks that are performed on the cluster
>-	by a small margin.
>+	When set to non-zero, this will make the main daemon log any
>+	operation that took longer than this value, in 'ms', to complete.
>+	These include "how long time a lockwait child process needed",
>+	"how long time to write to a persistent database" but also "how
>+	long did it take to get a response to a CALL from a remote node".
>       </para>
>-
>     </refsect2>
>+
>     <refsect2>
>-      <title>LCP2PublicIPs</title>
>-      <para>Default: 1</para>
>+      <title>MaxRedirectCount</title>
>+      <para>Default: 3</para>
>       <para>
>-	When enabled this switches ctdb to use the LCP2 ip allocation
>-	algorithm.
>+	If we are not the DMASTER and need to fetch a record across the
>+	network we first send the request to the LMASTER after which the
>+	record is passed onto the current DMASTER. If the DMASTER changes
>+	before the request has reached that node, the request will be
>+	passed onto the "next" DMASTER. For very hot records that migrate
>+	rapidly across the cluster this can cause a request to "chase"
>+	the record for many hops before it catches up with the record.
>+      </para>
>+      <para>
>+	When chasing a record, this is how many hops we will chase
>+	the record for before going back to the LMASTER to ask for
>+	new guidance.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>ReclockPingPeriod</title>
>-      <para>Default: x</para>
>+      <title>MonitorInterval</title>
>+      <para>Default: 15</para>
>       <para>
>-	Obsolete
>+	How often should ctdb run the event scripts in seconds to check
>+	for a node's health.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>NoIPFailback</title>
>-      <para>Default: 0</para>
>-      <para>
>-	When set to 1, ctdb will not perform failback of IP addresses when a node
>-	becomes healthy. Ctdb WILL perform failover of public IP addresses when a
>-	node becomes UNHEALTHY, but when the node becomes HEALTHY again, ctdb
>-	will not fail the addresses back.
>-      </para>
>-      <para>
>-	Use with caution! Normally when a node becomes available to the cluster
>-	ctdb will try to reassign public IP addresses onto the new node as a way
>-	to distribute the workload evenly across the clusternode. Ctdb tries to
>-	make sure that all running nodes have approximately the same number of
>-	public addresses it hosts.
>-      </para>
>+      <title>MonitorTimeoutCount</title>
>+      <para>Default: 20</para>
>       <para>
>-	When you enable this tunable, CTDB will no longer attempt to rebalance
>-	the cluster by failing IP addresses back to the new nodes. An unbalanced
>-	cluster will therefore remain unbalanced until there is manual
>-	intervention from the administrator. When this parameter is set, you can
>-	manually fail public IP addresses over to the new node(s) using the
>-	'ctdb moveip' command.
>+	How many monitor events in a row need to timeout before a node
>+	is flagged as UNHEALTHY.  This setting is useful if scripts can
>+	not be written so that they do not hang for benign reasons.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>DisableIPFailover</title>
>+      <title>NoIPFailback</title>
>       <para>Default: 0</para>
>       <para>
>-	When enabled, ctdb will not perform failover or failback. Even if a
>-	node fails while holding public IPs, ctdb will not recover the IPs or
>-	assign them to another node.
>+	When set to 1, ctdb will not perform failback of IP addresses
>+	when a node becomes healthy. Ctdb WILL perform failover of public
>+	IP addresses when a node becomes UNHEALTHY, but when the node
>+	becomes HEALTHY again, ctdb will not fail the addresses back.
>       </para>
>       <para>
>-	When you enable this tunable, CTDB will no longer attempt to recover
>-	the cluster by failing IP addresses over to other nodes. This leads to
>-	a service outage until the administrator has manually performed failover
>-	to replacement nodes using the 'ctdb moveip' command.
>+	Use with caution! Normally when a node becomes available to the
>+	cluster ctdb will try to reassign public IP addresses onto the
>+	new node as a way to distribute the workload evenly across the
>+	clusternode. Ctdb tries to make sure that all running nodes have
>+	approximately the same number of public addresses it hosts.
>       </para>
>-    </refsect2>
>-
>-    <refsect2>
>-      <title>NoIPTakeover</title>
>-      <para>Default: 0</para>
>       <para>
>-	When set to 1, ctdb will not allow IP addresses to be failed over
>-	onto this node. Any IP addresses that the node currently hosts
>-	will remain on the node but no new IP addresses can be failed over
>-	to the node.
>+	When you enable this tunable, CTDB will no longer attempt to
>+	rebalance the cluster by failing IP addresses back to the new
>+	nodes. An unbalanced cluster will therefore remain unbalanced
>+	until there is manual intervention from the administrator. When
>+	this parameter is set, you can manually fail public IP addresses
>+	over to the new node(s) using the 'ctdb moveip' command.
>       </para>
>     </refsect2>
> 
>@@ -347,7 +351,7 @@
>       <para>
> 	If no nodes are healthy then by default ctdb will happily host
> 	public IPs on disabled (unhealthy or administratively disabled)
>-	nodes.  This can cause problems, for example if the underlying
>+	nodes.	This can cause problems, for example if the underlying
> 	cluster filesystem is not mounted.  When set to 1 on a node and
> 	that node is disabled it, any IPs hosted by this node will be
> 	released and the node will not takeover any IPs until it is no
>@@ -356,85 +360,100 @@
>     </refsect2>
> 
>     <refsect2>
>-      <title>DBRecordCountWarn</title>
>-      <para>Default: 100000</para>
>+      <title>NoIPTakeover</title>
>+      <para>Default: 0</para>
>       <para>
>-	When set to non-zero, ctdb will log a warning when we try to recover a
>-	database with more than this many records. This will produce a warning
>-	if a database grows uncontrollably with orphaned records.
>+	When set to 1, ctdb will not allow IP addresses to be failed
>+	over onto this node. Any IP addresses that the node currently
>+	hosts will remain on the node but no new IP addresses can be
>+	failed over to the node.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>DBRecordSizeWarn</title>
>-      <para>Default: 10000000</para>
>+      <title>RecdFailCount</title>
>+      <para>Default: 10</para>
>       <para>
>-	When set to non-zero, ctdb will log a warning when we try to recover a
>-	database where a single record is bigger than this. This will produce
>-	a warning if a database record grows uncontrollably with orphaned
>-	sub-records.
>+	If the recovery daemon has failed to ping the main dameon for
>+	this many consecutive intervals, the main daemon will consider
>+	the recovery daemon as hung and will try to restart it to recover.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>DBSizeWarn</title>
>-      <para>Default: 1000000000</para>
>+      <title>RecdPingTimeout</title>
>+      <para>Default: 60</para>
>       <para>
>-	When set to non-zero, ctdb will log a warning when we try to recover a
>-	database bigger than this. This will produce
>-	a warning if a database grows uncontrollably.
>+	If the main dameon has not heard a "ping" from the recovery dameon
>+	for this many seconds, the main dameon will log a message that
>+	the recovery daemon is potentially hung.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>VerboseMemoryNames</title>
>-      <para>Default: 0</para>
>+      <title>RecLockLatencyMs</title>
>+      <para>Default: 1000</para>
>       <para>
>-	This feature consumes additional memory. when used the talloc library
>-	will create more verbose names for all talloc allocated objects.
>+	When using a reclock file for split brain prevention, if set
>+	to non-zero this tunable will make the recovery dameon log a
>+	message if the fcntl() call to lock/testlock the recovery file
>+	takes longer than this number of ms.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>RecdPingTimeout</title>
>-      <para>Default: 60</para>
>+      <title>RecoverInterval</title>
>+      <para>Default: 1</para>
>       <para>
>-	If the main dameon has not heard a "ping" from the recovery dameon for
>-	this many seconds, the main dameon will log a message that the recovery
>-	daemon is potentially hung.
>+	How frequently in seconds should the recovery daemon perform
>+	the consistency checks that determine if we need to perform a
>+	recovery or not.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>RecdFailCount</title>
>-      <para>Default: 10</para>
>+      <title>RecoverPDBBySeqNum</title>
>+      <para>Default: 1</para>
>+      <para>
>+	When set to zero, database recovery for persistent databases is
>+	record-by-record and recovery process simply collects the most
>+	recent version of every individual record.
>+      </para>
>+      <para>
>+	When set to non-zero, persistent databases will instead be
>+	recovered as a whole db and not by individual records. The
>+	node that contains the highest value stored in the record
>+	"__db_sequence_number__" is selected and the copy of that nodes
>+	database is used as the recovered database.
>+      </para>
>       <para>
>-	If the recovery daemon has failed to ping the main dameon for this many
>-	consecutive intervals, the main daemon will consider the recovery daemon
>-	as hung and will try to restart it to recover.
>+	By default, recovery of persistent databses is done using
>+	__db_sequence_number__ record.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>LogLatencyMs</title>
>-      <para>Default: 0</para>
>+      <title>RecoverTimeout</title>
>+      <para>Default: 20</para>
>       <para>
>-	When set to non-zero, this will make the main daemon log any operation that
>-	took longer than this value, in 'ms', to complete.
>-	These include "how long time a lockwait child process needed", 
>-	"how long time to write to a persistent database" but also
>-	"how long did it take to get a response to a CALL from a remote node".
>+	This is the default setting for timeouts for controls when sent
>+	from the recovery daemon. We allow longer control timeouts from
>+	the recovery daemon than from normal use since the recovery
>+	dameon often use controls that can take a lot longer than normal
>+	controls.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>RecLockLatencyMs</title>
>-      <para>Default: 1000</para>
>+      <title>RecoveryBanPeriod</title>
>+      <para>Default: 300</para>
>       <para>
>-	When using a reclock file for split brain prevention, if set to non-zero
>-	this tunable will make the recovery dameon log a message if the fcntl()
>-	call to lock/testlock the recovery file takes longer than this number of 
>-	ms.
>+	If a node becomes banned causing repetitive recovery failures. The
>+	node will eventually become banned from the cluster.  This
>+	controls how long the culprit node will be banned from the
>+	cluster before it is allowed to try to join the cluster again.
>+	Don't set to small. A node gets banned for a reason and it is
>+	usually due to real problems with the node.
>       </para>
>     </refsect2>
> 
>@@ -448,21 +467,13 @@
>     </refsect2>
> 
>     <refsect2>
>-      <title>VacuumInterval</title>
>-      <para>Default: 10</para>
>-      <para>
>-        Periodic interval in seconds when vacuuming is triggered for
>-        volatile databases.
>-      </para>
>-    </refsect2>
>-
>-    <refsect2>
>-      <title>VacuumMaxRunTime</title>
>+      <title>RecoveryGracePeriod</title>
>       <para>Default: 120</para>
>       <para>
>-        The maximum time in seconds for which the vacuuming process is
>-        allowed to run.  If vacuuming process takes longer than this
>-        value, then the vacuuming process is terminated.
>+       During recoveries, if a node has not caused recovery failures
>+       during the last grace period, any records of transgressions that
>+       the node has caused recovery failures will be forgiven. This
>+       resets the ban-counter back to zero for that node.
>       </para>
>     </refsect2>
> 
>@@ -470,76 +481,62 @@
>       <title>RepackLimit</title>
>       <para>Default: 10000</para>
>       <para>
>-        During vacuuming, if the number of freelist records are more
>-        than <varname>RepackLimit</varname>, then databases are
>-        repacked to get rid of the freelist records to avoid
>-        fragmentation.
>+        During vacuuming, if the number of freelist records are more than
>+        <varname>RepackLimit</varname>, then databases are repacked to
>+        get rid of the freelist records to avoid fragmentation.
>       </para>
>       <para>
>-        Databases are repacked only if both
>-        <varname>RepackLimit</varname> and
>-        <varname>VacuumLimit</varname> are exceeded.
>+        Databases are repacked only if both <varname>RepackLimit</varname>
>+        and <varname>VacuumLimit</varname> are exceeded.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>VacuumLimit</title>
>-      <para>Default: 5000</para>
>-      <para>
>-        During vacuuming, if the number of deleted records are more
>-        than <varname>VacuumLimit</varname>, then databases are
>-        repacked to avoid fragmentation.
>-      </para>
>+      <title>RerecoveryTimeout</title>
>+      <para>Default: 10</para>
>       <para>
>-        Databases are repacked only if both
>-        <varname>RepackLimit</varname> and
>-        <varname>VacuumLimit</varname> are exceeded.
>+	Once a recovery has completed, no additional recoveries are
>+	permitted until this timeout has expired.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>VacuumFastPathCount</title>
>-      <para>Default: 60</para>
>+      <title>Samba3AvoidDeadlocks</title>
>+      <para>Default: 0</para>
>+      <para>
>+        Enable code that prevents deadlocks with Samba (only for Samba
>+        3.x).
>+      </para>
>       <para>
>-        When a record is deleted, it is marked for deletion during
>-        vacuuming.  Vacuuming process usually processes this list to purge
>-        the records from the database.  If the number of records marked
>-        for deletion are more than VacuumFastPathCount, then vacuuming
>-	process will scan the complete database for empty records instead
>-	of using the list of records marked for deletion.
>+	This should be set to 1 when using Samba version 3.x to enable
>+	special code in CTDB to avoid deadlock with Samba version 3.x.
>+	This code is not required for Samba version 4.x and must not be
>+	enabled for Samba 4.x.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>DeferredAttachTO</title>
>-      <para>Default: 120</para>
>+      <title>SeqnumInterval</title>
>+      <para>Default: 1000</para>
>       <para>
>-	When databases are frozen we do not allow clients to attach to the
>-	databases. Instead of returning an error immediately to the application
>-	the attach request from the client is deferred until the database
>-	becomes available again at which stage we respond to the client.
>+	Some databases have seqnum tracking enabled, so that samba will
>+	be able to detect asynchronously when there has been updates
>+	to the database.  Everytime a database is updated its sequence
>+	number is increased.
>       </para>
>       <para>
>-	This timeout controls how long we will defer the request from the client
>-	before timing it out and returning an error to the client.
>+	This tunable is used to specify in 'ms' how frequently ctdb
>+	will send out updates to remote nodes to inform them that the
>+	sequence number is increased.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>HopcountMakeSticky</title>
>-      <para>Default: 50</para>
>-      <para>
>-	If the database is set to 'STICKY' mode, using the 'ctdb setdbsticky' 
>-	command, any record that is seen as very hot and migrating so fast that
>-	hopcount surpasses 50 is set to become a STICKY record for StickyDuration
>-	seconds. This means that after each migration the record will be kept on
>-	the node and prevented from being migrated off the node.
>-      </para>
>+      <title>StatHistoryInterval</title>
>+      <para>Default: 1</para>
>       <para>
>-	This setting allows one to try to identify such records and stop them from
>-	migrating across the cluster so fast. This will improve performance for
>-	certain workloads, such as locking.tdb if many clients are opening/closing
>-	the same file concurrently.
>+	Granularity of the statistics collected in the statistics
>+	history. This is reported by 'ctdb stats' command.
>       </para>
>     </refsect2>
> 
>@@ -547,9 +544,9 @@
>       <title>StickyDuration</title>
>       <para>Default: 600</para>
>       <para>
>-	Once a record has been found to be fetch-lock hot and has been flagged to
>-	become STICKY, this is for how long, in seconds, the record will be 
>-	flagged as a STICKY record.
>+	Once a record has been found to be fetch-lock hot and has been
>+	flagged to become STICKY, this is for how long, in seconds,
>+	the record will be flagged as a STICKY record.
>       </para>
>     </refsect2>
> 
>@@ -557,88 +554,97 @@
>       <title>StickyPindown</title>
>       <para>Default: 200</para>
>       <para>
>-	Once a STICKY record has been migrated onto a node, it will be pinned down
>-	on that node for this number of ms. Any request from other nodes to migrate
>-	the record off the node will be deferred until the pindown timer expires.
>+	Once a STICKY record has been migrated onto a node, it will be
>+	pinned down on that node for this number of ms. Any request from
>+	other nodes to migrate the record off the node will be deferred
>+	until the pindown timer expires.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>StatHistoryInterval</title>
>-      <para>Default: 1</para>
>+      <title>TakeoverTimeout</title>
>+      <para>Default: 9</para>
>       <para>
>-	Granularity of the statistics collected in the statistics history.
>+	This is how many seconds we allow controls to take for IP
>+	failover events.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>AllowClientDBAttach</title>
>-      <para>Default: 1</para>
>+      <title>TickleUpdateInterval</title>
>+      <para>Default: 20</para>
>       <para>
>-	When set to 0, clients are not allowed to attach to any databases.
>-	This can be used to temporarily block any new processes from attaching
>-	to and accessing the databases.
>+	How often will ctdb record and store the "tickle" information
>+	used to kickstart stalled tcp connections after a recovery.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>RecoverPDBBySeqNum</title>
>-      <para>Default: 1</para>
>+      <title>TraverseTimeout</title>
>+      <para>Default: 20</para>
>       <para>
>-	When set to zero, database recovery for persistent databases
>-	is record-by-record and recovery process simply collects the
>-	most recent version of every individual record.
>+	This setting controls how long we allow a traverse process to run.
>+	After this timeout triggers, the main ctdb daemon will abort
>+	the traverse if it has not yet finished.
>       </para>
>+    </refsect2>
>+
>+    <refsect2>
>+      <title>VacuumFastPathCount</title>
>+      <para>Default: 60</para>
>       <para>
>-	When set to non-zero, persistent databases will instead be
>-	recovered as a whole db and not by individual records. The
>-	node that contains the highest value stored in the record
>-	"__db_sequence_number__" is selected and the copy of that
>-	nodes database is used as the recovered database.
>+	When a record is deleted, it is marked for deletion during
>+	vacuuming.  Vacuuming process usually processes this list to
>+	purge the records from the database.  If the number of records
>+	marked for deletion are more than VacuumFastPathCount, then
>+	vacuuming process will scan the complete database for empty
>+	records instead of using the list of records marked for deletion.
>       </para>
>+    </refsect2>
>+
>+    <refsect2>
>+      <title>VacuumInterval</title>
>+      <para>Default: 10</para>
>       <para>
>-	By default, recovery of persistent databses is done using
>-	__db_sequence_number__ record.
>+        Periodic interval in seconds when vacuuming is triggered for
>+        volatile databases.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>FetchCollapse</title>
>-      <para>Default: 1</para>
>+      <title>VacuumLimit</title>
>+      <para>Default: 5000</para>
>       <para>
>-	When many clients across many nodes try to access the same record at the
>-	same time this can lead to a fetch storm where the record becomes very
>-	active and bounces between nodes very fast. This leads to high CPU
>-	utilization of the ctdbd daemon, trying to bounce that record around
>-	very fast, and poor performance.
>+        During vacuuming, if the number of deleted records are more than
>+        <varname>VacuumLimit</varname>, then databases are repacked to
>+        avoid fragmentation.
>       </para>
>       <para>
>-	This parameter is used to activate a fetch-collapse. A fetch-collapse
>-	is when we track which records we have requests in flight so that we only
>-	keep one request in flight from a certain node, even if multiple smbd
>-	processes are attemtping to fetch the record at the same time. This 
>-	can improve performance and reduce CPU utilization for certain
>-	workloads.
>+        Databases are repacked only if both <varname>RepackLimit</varname>
>+        and <varname>VacuumLimit</varname> are exceeded.
>       </para>
>+    </refsect2>
>+
>+    <refsect2>
>+      <title>VacuumMaxRunTime</title>
>+      <para>Default: 120</para>
>       <para>
>-	This timeout controls if we should collapse multiple fetch operations
>-	of the same record into a single request and defer all duplicates or not.
>+        The maximum time in seconds for which the vacuuming process is
>+        allowed to run.  If vacuuming process takes longer than this
>+        value, then the vacuuming process is terminated.
>       </para>
>     </refsect2>
> 
>     <refsect2>
>-      <title>Samba3AvoidDeadlocks</title>
>+      <title>VerboseMemoryNames</title>
>       <para>Default: 0</para>
>       <para>
>-	Enable code that prevents deadlocks with Samba (only for Samba 3.x).
>-      </para>
>-      <para>
>-	This should be set to 1 when using Samba version 3.x to enable special
>-	code in CTDB to avoid deadlock with Samba version 3.x.  This code
>-	is not required for Samba version 4.x and must not be enabled for
>-	Samba 4.x.
>+	This feature consumes additional memory. when used the talloc
>+	library will create more verbose names for all talloc allocated
>+	objects.
>       </para>
>     </refsect2>
>+
>   </refsect1>
> 
>   <refsect1>
>-- 
>2.5.5
>
>
>From ce4e69c6aa812e68906f6ea2d5e4cd8d7181e2b9 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Mon, 7 Mar 2016 19:26:43 +1100
>Subject: [PATCH 05/29] ctdb-doc: Update tunables documentation
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit e2539088e023ad88d314467a6a42b52c0decba77)
>---
> ctdb/doc/ctdb-tunables.7.xml | 247 +++++++++++++++++++++----------------------
> 1 file changed, 123 insertions(+), 124 deletions(-)
>
>diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml
>index a6ef435..2c544fc 100644
>--- a/ctdb/doc/ctdb-tunables.7.xml
>+++ b/ctdb/doc/ctdb-tunables.7.xml
>@@ -39,7 +39,8 @@
>       <para>
> 	When set to 0, clients are not allowed to attach to any databases.
> 	This can be used to temporarily block any new processes from
>-	attaching to and accessing the databases.
>+	attaching to and accessing the databases.  This is mainly used
>+	for detaching a volatile database using 'ctdb detach'.
>       </para>
>     </refsect2>
> 
>@@ -65,8 +66,8 @@
>       <title>DatabaseMaxDead</title>
>       <para>Default: 5</para>
>       <para>
>-	How many dead records per hashchain in the TDB database do we
>-	allow before the freelist needs to be processed.
>+	Maximum number of dead records per hash chain for the tdb databses
>+	managed by ctdb.
>       </para>
>     </refsect2>
> 
>@@ -74,10 +75,9 @@
>       <title>DBRecordCountWarn</title>
>       <para>Default: 100000</para>
>       <para>
>-	When set to non-zero, ctdb will log a warning when we try
>-	to recover a database with more than this many records. This
>-	will produce a warning if a database grows uncontrollably with
>-	orphaned records.
>+	When set to non-zero, ctdb will log a warning during recovery if
>+	a database has more than this many records. This will produce a
>+	warning if a database grows uncontrollably with orphaned records.
>       </para>
>     </refsect2>
> 
>@@ -85,10 +85,9 @@
>       <title>DBRecordSizeWarn</title>
>       <para>Default: 10000000</para>
>       <para>
>-	When set to non-zero, ctdb will log a warning when we try to
>-	recover a database where a single record is bigger than this. This
>-	will produce a warning if a database record grows uncontrollably
>-	with orphaned sub-records.
>+	When set to non-zero, ctdb will log a warning during recovery
>+	if a single record is bigger than this size. This will produce
>+	a warning if a database record grows uncontrollably.
>       </para>
>     </refsect2>
> 
>@@ -96,8 +95,8 @@
>       <title>DBSizeWarn</title>
>       <para>Default: 1000000000</para>
>       <para>
>-	When set to non-zero, ctdb will log a warning when we try to
>-	recover a database bigger than this. This will produce a warning
>+	When set to non-zero, ctdb will log a warning during recovery if
>+	a database size is bigger than this. This will produce a warning
> 	if a database grows uncontrollably.
>       </para>
>     </refsect2>
>@@ -108,7 +107,7 @@
>       <para>
> 	When databases are frozen we do not allow clients to attach to
> 	the databases. Instead of returning an error immediately to the
>-	application the attach request from the client is deferred until
>+	client, the attach request from the client is deferred until
> 	the database becomes available again at which stage we respond
> 	to the client.
>       </para>
>@@ -122,10 +121,10 @@
>       <title>DeterministicIPs</title>
>       <para>Default: 0</para>
>       <para>
>-	When enabled, this tunable makes ctdb try to keep public IP
>-	addresses locked to specific nodes as far as possible. This makes
>-	it easier for debugging since you can know that as long as all
>-	nodes are healthy public IP X will always be hosted by node Y.
>+	When set to 1, ctdb will try to keep public IP addresses locked
>+	to specific nodes as far as possible. This makes it easier
>+	for debugging since you can know that as long as all nodes are
>+	healthy public IP X will always be hosted by node Y.
>       </para>
>       <para>
> 	The cost of using deterministic IP address assignment is that it
>@@ -140,15 +139,15 @@
>       <title>DisableIPFailover</title>
>       <para>Default: 0</para>
>       <para>
>-	When enabled, ctdb will not perform failover or failback. Even
>-	if a node fails while holding public IPs, ctdb will not recover
>-	the IPs or assign them to another node.
>+	When set to non-zero, ctdb will not perform failover or
>+	failback. Even if a node fails while holding public IPs, ctdb
>+	will not recover the IPs or assign them to another node.
>       </para>
>       <para>
>-	When you enable this tunable, CTDB will no longer attempt
>+	When this tunable is enabled, ctdb will no longer attempt
> 	to recover the cluster by failing IP addresses over to other
> 	nodes. This leads to a service outage until the administrator
>-	has manually performed failover to replacement nodes using the
>+	has manually performed IP failover to replacement nodes using the
> 	'ctdb moveip' command.
>       </para>
>     </refsect2>
>@@ -157,9 +156,10 @@
>       <title>ElectionTimeout</title>
>       <para>Default: 3</para>
>       <para>
>-	When electing a new recovery master, this is how many seconds
>-	we allow the election to take before we either deem the election
>-	finished or we fail the election and start a new one.
>+	The number of seconds to wait for the election of recovery
>+	master to complete. If the election is not completed during this
>+	interval, then that round of election fails and ctdb starts a
>+	new election.
>       </para>
>     </refsect2>
> 
>@@ -167,7 +167,10 @@
>       <title>EnableBans</title>
>       <para>Default: 1</para>
>       <para>
>-	When set to 0, this disables BANNING completely in the cluster
>+        This parameter allows ctdb to ban a node if the node is misbehaving.
>+      </para>
>+      <para>
>+	When set to 0, this disables banning completely in the cluster
> 	and thus nodes can not get banned, even it they break. Don't
> 	set to 0 unless you know what you are doing.  You should set
> 	this to the same value on all nodes to avoid unexpected behaviour.
>@@ -194,24 +197,19 @@
>       <title>FetchCollapse</title>
>       <para>Default: 1</para>
>       <para>
>+       This parameter is used to avoid multiple migration requests for
>+       the same record from a single node. All the record requests for
>+       the same record are queued up and processed when the record is
>+       migrated to the current node.
>+      </para>
>+      <para>
> 	When many clients across many nodes try to access the same record
> 	at the same time this can lead to a fetch storm where the record
> 	becomes very active and bounces between nodes very fast. This
> 	leads to high CPU utilization of the ctdbd daemon, trying to
> 	bounce that record around very fast, and poor performance.
>-      </para>
>-      <para>
>-	This parameter is used to activate a fetch-collapse. A
>-	fetch-collapse is when we track which records we have requests in
>-	flight so that we only keep one request in flight from a certain
>-	node, even if multiple smbd processes are attemtping to fetch
>-	the record at the same time. This can improve performance and
>-	reduce CPU utilization for certain workloads.
>-      </para>
>-      <para>
>-	This timeout controls if we should collapse multiple fetch
>-	operations of the same record into a single request and defer
>-	all duplicates or not.
>+	This can improve performance and reduce CPU utilization for
>+	certain workloads.
>       </para>
>     </refsect2>
> 
>@@ -219,18 +217,18 @@
>       <title>HopcountMakeSticky</title>
>       <para>Default: 50</para>
>       <para>
>-	If the database is set to 'STICKY' mode, using the 'ctdb
>-	setdbsticky' command, any record that is seen as very hot and
>-	migrating so fast that hopcount surpasses 50 is set to become a
>-	STICKY record for StickyDuration seconds. This means that after
>-	each migration the record will be kept on the node and prevented
>-	from being migrated off the node.
>+	For database(s) marked STICKY (using 'ctdb setdbsticky'),
>+	any record that is migrating so fast that hopcount
>+	exceeds this limit is marked as STICKY record for
>+	<varname>StickyDuration</varname> seconds. This means that
>+	after each migration the sticky record will be kept on the node
>+	<varname>StickyPindown</varname>milliseconds and prevented from
>+	being migrated off the node.
>        </para>
>        <para>
>-	This setting allows one to try to identify such records and
>-	stop them from migrating across the cluster so fast. This will
>-	improve performance for certain workloads, such as locking.tdb
>-	if many clients are opening/closing the same file concurrently.
>+	This will improve performance for certain workloads, such as
>+	locking.tdb if many clients are opening/closing the same file
>+	concurrently.
>       </para>
>     </refsect2>
> 
>@@ -251,12 +249,13 @@
> 	a node wait until marking the peer as DISCONNECTED.
>        </para>
>        <para>
>-	If a node has hung, it can thus take
>-	KeepaliveInterval*(KeepaliveLimit+1) seconds before we determine
>-	that the node is DISCONNECTED and that we require a recovery. This
>-	limitshould not be set too high since we want a hung node to be
>-	detectec, and expunged from the cluster well before common CIFS
>-	timeouts (45-90 seconds) kick in.
>+	If a node has hung, it can take
>+	<varname>KeepaliveInterval</varname> *
>+	(<varname>KeepaliveLimit</varname> + 1) seconds before
>+	ctdb determines that the node is DISCONNECTED and performs
>+	a recovery. This limit should not be set too high to enable
>+	early detection and avoid any application timeouts (e.g. SMB1)
>+	to kick in before the fail over is completed.
>       </para>
>     </refsect2>
> 
>@@ -264,8 +263,7 @@
>       <title>LCP2PublicIPs</title>
>       <para>Default: 1</para>
>       <para>
>-	When enabled this switches ctdb to use the LCP2 ip allocation
>-	algorithm.
>+	When set to 1, ctdb uses the LCP2 ip allocation algorithm.
>       </para>
>     </refsect2>
> 
>@@ -273,11 +271,11 @@
>       <title>LogLatencyMs</title>
>       <para>Default: 0</para>
>       <para>
>-	When set to non-zero, this will make the main daemon log any
>-	operation that took longer than this value, in 'ms', to complete.
>-	These include "how long time a lockwait child process needed",
>-	"how long time to write to a persistent database" but also "how
>-	long did it take to get a response to a CALL from a remote node".
>+	When set to non-zero, ctdb will log if certains operations
>+	take longer than this value, in milliseconds, to complete.
>+	These operations include "process a record request from client",
>+	"take a record or database lock", "update a persistent database
>+	record" and "vaccum a database".
>       </para>
>     </refsect2>
> 
>@@ -304,7 +302,7 @@
>       <title>MonitorInterval</title>
>       <para>Default: 15</para>
>       <para>
>-	How often should ctdb run the event scripts in seconds to check
>+	How often should ctdb run the 'monitor' event in seconds to check
> 	for a node's health.
>       </para>
>     </refsect2>
>@@ -313,7 +311,7 @@
>       <title>MonitorTimeoutCount</title>
>       <para>Default: 20</para>
>       <para>
>-	How many monitor events in a row need to timeout before a node
>+	How many 'monitor' events in a row need to timeout before a node
> 	is flagged as UNHEALTHY.  This setting is useful if scripts can
> 	not be written so that they do not hang for benign reasons.
>       </para>
>@@ -324,9 +322,9 @@
>       <para>Default: 0</para>
>       <para>
> 	When set to 1, ctdb will not perform failback of IP addresses
>-	when a node becomes healthy. Ctdb WILL perform failover of public
>-	IP addresses when a node becomes UNHEALTHY, but when the node
>-	becomes HEALTHY again, ctdb will not fail the addresses back.
>+	when a node becomes healthy. When a node becomes UNHEALTHY,
>+	ctdb WILL perform failover of public IP addresses, but when the
>+	node becomes HEALTHY again, ctdb will not fail the addresses back.
>       </para>
>       <para>
> 	Use with caution! Normally when a node becomes available to the
>@@ -336,7 +334,7 @@
> 	approximately the same number of public addresses it hosts.
>       </para>
>       <para>
>-	When you enable this tunable, CTDB will no longer attempt to
>+	When you enable this tunable, ctdb will no longer attempt to
> 	rebalance the cluster by failing IP addresses back to the new
> 	nodes. An unbalanced cluster will therefore remain unbalanced
> 	until there is manual intervention from the administrator. When
>@@ -349,11 +347,11 @@
>       <title>NoIPHostOnAllDisabled</title>
>       <para>Default: 0</para>
>       <para>
>-	If no nodes are healthy then by default ctdb will happily host
>+	If no nodes are HEALTHY then by default ctdb will happily host
> 	public IPs on disabled (unhealthy or administratively disabled)
> 	nodes.	This can cause problems, for example if the underlying
> 	cluster filesystem is not mounted.  When set to 1 on a node and
>-	that node is disabled it, any IPs hosted by this node will be
>+	that node is disabled, any IPs hosted by this node will be
> 	released and the node will not takeover any IPs until it is no
> 	longer disabled.
>       </para>
>@@ -386,7 +384,9 @@
>       <para>
> 	If the main dameon has not heard a "ping" from the recovery dameon
> 	for this many seconds, the main dameon will log a message that
>-	the recovery daemon is potentially hung.
>+	the recovery daemon is potentially hung.  This also increments a
>+	counter which is checked against <varname>RecdFailCount</varname>
>+	for detection of hung recovery daemon.
>       </para>
>     </refsect2>
> 
>@@ -397,7 +397,7 @@
> 	When using a reclock file for split brain prevention, if set
> 	to non-zero this tunable will make the recovery dameon log a
> 	message if the fcntl() call to lock/testlock the recovery file
>-	takes longer than this number of ms.
>+	takes longer than this number of milliseconds.
>       </para>
>     </refsect2>
> 
>@@ -405,9 +405,8 @@
>       <title>RecoverInterval</title>
>       <para>Default: 1</para>
>       <para>
>-	How frequently in seconds should the recovery daemon perform
>-	the consistency checks that determine if we need to perform a
>-	recovery or not.
>+	How frequently in seconds should the recovery daemon perform the
>+	consistency checks to determine if it should perform a recovery.
>       </para>
>     </refsect2>
> 
>@@ -434,7 +433,7 @@
> 
>     <refsect2>
>       <title>RecoverTimeout</title>
>-      <para>Default: 20</para>
>+      <para>Default: 120</para>
>       <para>
> 	This is the default setting for timeouts for controls when sent
> 	from the recovery daemon. We allow longer control timeouts from
>@@ -448,12 +447,14 @@
>       <title>RecoveryBanPeriod</title>
>       <para>Default: 300</para>
>       <para>
>-	If a node becomes banned causing repetitive recovery failures. The
>-	node will eventually become banned from the cluster.  This
>-	controls how long the culprit node will be banned from the
>-	cluster before it is allowed to try to join the cluster again.
>-	Don't set to small. A node gets banned for a reason and it is
>-	usually due to real problems with the node.
>+       The duration in seconds for which a node is banned if the node
>+       fails during recovery.  After this time has elapsed the node will
>+       automatically get unbanned and will attempt to rejoin the cluster.
>+      </para>
>+      <para>
>+       A node usually gets banned due to real problems with the node.
>+       Don't set this value too small.  Otherwise, a problematic node
>+       will try to re-join cluster too soon causing unnecessary recoveries.
>       </para>
>     </refsect2>
> 
>@@ -461,8 +462,9 @@
>       <title>RecoveryDropAllIPs</title>
>       <para>Default: 120</para>
>       <para>
>-	If we have been stuck in recovery, or stopped, or banned, mode for
>-	this many seconds we will force drop all held public addresses.
>+	If a node is stuck in recovery, or stopped, or banned, for this
>+	many seconds, then ctdb will release all public addresses on
>+	that node.
>       </para>
>     </refsect2>
> 
>@@ -471,9 +473,9 @@
>       <para>Default: 120</para>
>       <para>
>        During recoveries, if a node has not caused recovery failures
>-       during the last grace period, any records of transgressions that
>-       the node has caused recovery failures will be forgiven. This
>-       resets the ban-counter back to zero for that node.
>+       during the last grace period in seconds, any records of
>+       transgressions that the node has caused recovery failures will be
>+       forgiven. This resets the ban-counter back to zero for that node.
>       </para>
>     </refsect2>
> 
>@@ -482,8 +484,8 @@
>       <para>Default: 10000</para>
>       <para>
>         During vacuuming, if the number of freelist records are more than
>-        <varname>RepackLimit</varname>, then databases are repacked to
>-        get rid of the freelist records to avoid fragmentation.
>+        <varname>RepackLimit</varname>, then the database is repacked
>+        to get rid of the freelist records to avoid fragmentation.
>       </para>
>       <para>
>         Databases are repacked only if both <varname>RepackLimit</varname>
>@@ -496,7 +498,7 @@
>       <para>Default: 10</para>
>       <para>
> 	Once a recovery has completed, no additional recoveries are
>-	permitted until this timeout has expired.
>+	permitted until this timeout in seconds has expired.
>       </para>
>     </refsect2>
> 
>@@ -504,14 +506,13 @@
>       <title>Samba3AvoidDeadlocks</title>
>       <para>Default: 0</para>
>       <para>
>-        Enable code that prevents deadlocks with Samba (only for Samba
>-        3.x).
>-      </para>
>-      <para>
>-	This should be set to 1 when using Samba version 3.x to enable
>-	special code in CTDB to avoid deadlock with Samba version 3.x.
>-	This code is not required for Samba version 4.x and must not be
>-	enabled for Samba 4.x.
>+        If set to non-zero, enable code that prevents deadlocks with Samba
>+        (only for Samba 3.x).
>+      </para> <para>
>+	This should be set to 1 only when using Samba version 3.x
>+	to enable special code in ctdb to avoid deadlock with Samba
>+	version 3.x.  This code is not required for Samba version 4.x
>+	and must not be enabled for Samba 4.x.
>       </para>
>     </refsect2>
> 
>@@ -525,9 +526,9 @@
> 	number is increased.
>       </para>
>       <para>
>-	This tunable is used to specify in 'ms' how frequently ctdb
>-	will send out updates to remote nodes to inform them that the
>-	sequence number is increased.
>+	This tunable is used to specify in milliseconds how frequently
>+	ctdb will send out updates to remote nodes to inform them that
>+	the sequence number is increased.
>       </para>
>     </refsect2>
> 
>@@ -544,9 +545,8 @@
>       <title>StickyDuration</title>
>       <para>Default: 600</para>
>       <para>
>-	Once a record has been found to be fetch-lock hot and has been
>-	flagged to become STICKY, this is for how long, in seconds,
>-	the record will be flagged as a STICKY record.
>+	Once a record has been marked STICKY, this is the duration in
>+	seconds, the record will be flagged as a STICKY record.
>       </para>
>     </refsect2>
> 
>@@ -555,9 +555,9 @@
>       <para>Default: 200</para>
>       <para>
> 	Once a STICKY record has been migrated onto a node, it will be
>-	pinned down on that node for this number of ms. Any request from
>-	other nodes to migrate the record off the node will be deferred
>-	until the pindown timer expires.
>+	pinned down on that node for this number of milliseconds. Any
>+	request from other nodes to migrate the record off the node will
>+	be deferred.
>       </para>
>     </refsect2>
> 
>@@ -565,8 +565,8 @@
>       <title>TakeoverTimeout</title>
>       <para>Default: 9</para>
>       <para>
>-	This is how many seconds we allow controls to take for IP
>-	failover events.
>+	This is the duration in seconds in which ctdb tries to complete IP
>+	failover.
>       </para>
>     </refsect2>
> 
>@@ -574,8 +574,8 @@
>       <title>TickleUpdateInterval</title>
>       <para>Default: 20</para>
>       <para>
>-	How often will ctdb record and store the "tickle" information
>-	used to kickstart stalled tcp connections after a recovery.
>+	Every <varname>TickleUpdateInterval</varname> seconds, ctdb
>+	synchronizes the client connection information across nodes.
>       </para>
>     </refsect2>
> 
>@@ -583,9 +583,9 @@
>       <title>TraverseTimeout</title>
>       <para>Default: 20</para>
>       <para>
>-	This setting controls how long we allow a traverse process to run.
>-	After this timeout triggers, the main ctdb daemon will abort
>-	the traverse if it has not yet finished.
>+	This is the duration in seconds for which a database traverse
>+	is allowed to run.  If the traverse does not complete during
>+	this interval, ctdb will abort the traverse.
>       </para>
>     </refsect2>
> 
>@@ -593,12 +593,11 @@
>       <title>VacuumFastPathCount</title>
>       <para>Default: 60</para>
>       <para>
>-	When a record is deleted, it is marked for deletion during
>-	vacuuming.  Vacuuming process usually processes this list to
>-	purge the records from the database.  If the number of records
>-	marked for deletion are more than VacuumFastPathCount, then
>-	vacuuming process will scan the complete database for empty
>-	records instead of using the list of records marked for deletion.
>+       During a vacuuming run, ctdb usually processes only the records
>+       marked for deletion also called the fast path vacuuming. After
>+       finishing <varname>VacuumFastPathCount</varname> number of fast
>+       path vacuuming runs, ctdb will trigger a scan of complete database
>+       for any empty records that need to be deleted.
>       </para>
>     </refsect2>
> 
>@@ -639,9 +638,9 @@
>       <title>VerboseMemoryNames</title>
>       <para>Default: 0</para>
>       <para>
>-	This feature consumes additional memory. when used the talloc
>-	library will create more verbose names for all talloc allocated
>-	objects.
>+	When set to non-zero, ctdb assigns verbose names for some of
>+	the talloc allocated memory objects.  These names are visible
>+	in the talloc memory report generated by 'ctdb dumpmemory'.
>       </para>
>     </refsect2>
> 
>-- 
>2.5.5
>
>
>From 911a9ea454004487f888a96bc3de8d9ccfdbf541 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 8 Mar 2016 14:37:41 +1100
>Subject: [PATCH 06/29] ctdb-doc: Add documentation for missing tunables
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 3680aba2126d967a5518ab2d35d9e6d3f338e75e)
>---
> ctdb/doc/ctdb-tunables.7.xml | 64 ++++++++++++++++++++++++++++++++++----------
> 1 file changed, 50 insertions(+), 14 deletions(-)
>
>diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml
>index 2c544fc..591dcf5 100644
>--- a/ctdb/doc/ctdb-tunables.7.xml
>+++ b/ctdb/doc/ctdb-tunables.7.xml
>@@ -45,6 +45,16 @@
>     </refsect2>
> 
>     <refsect2>
>+      <title>AllowUnhealthyDBRead</title>
>+      <para>Default: 0</para>
>+      <para>
>+	When set to 1, ctdb allows database traverses to read unhealthy
>+	databases.  By default, ctdb does not allow reading records from
>+	unhealthy databases.
>+      </para>
>+    </refsect2>
>+
>+    <refsect2>
>       <title>ControlTimeout</title>
>       <para>Default: 60</para>
>       <para>
>@@ -268,6 +278,17 @@
>     </refsect2>
> 
>     <refsect2>
>+      <title>LockProcessesPerDB</title>
>+      <para>Default: 200</para>
>+      <para>
>+	This is the maximum number of lock helper processes ctdb will
>+	create for obtaining record locks.  When ctdb cannot get a record
>+	lock without blocking, it creates a helper process that waits
>+	for the lock to be obtained.
>+      </para>
>+    </refsect2>
>+
>+    <refsect2>
>       <title>LogLatencyMs</title>
>       <para>Default: 0</para>
>       <para>
>@@ -280,21 +301,12 @@
>     </refsect2>
> 
>     <refsect2>
>-      <title>MaxRedirectCount</title>
>-      <para>Default: 3</para>
>+      <title>MaxQueueDropMsg</title>
>+      <para>Default: 1000000</para>
>       <para>
>-	If we are not the DMASTER and need to fetch a record across the
>-	network we first send the request to the LMASTER after which the
>-	record is passed onto the current DMASTER. If the DMASTER changes
>-	before the request has reached that node, the request will be
>-	passed onto the "next" DMASTER. For very hot records that migrate
>-	rapidly across the cluster this can cause a request to "chase"
>-	the record for many hops before it catches up with the record.
>-      </para>
>-      <para>
>-	When chasing a record, this is how many hops we will chase
>-	the record for before going back to the LMASTER to ask for
>-	new guidance.
>+	This is the maximum number of messages to be queued up for
>+	a client before ctdb will treat the client as hung and will
>+	terminate the client connection.
>       </para>
>     </refsect2>
> 
>@@ -369,6 +381,19 @@
>     </refsect2>
> 
>     <refsect2>
>+      <title>PullDBPreallocation</title>
>+      <para>Default: 10*1024*1024</para>
>+      <para>
>+	This is the size of a record buffer to pre-allocate for sending
>+	reply to PULLDB control. Usually record buffer starts with size
>+	of the first record and gets reallocated every time a new record
>+	is added to the record buffer. For a large number of records,
>+	this can be very inefficient to grow the record buffer one record
>+	at a time.
>+      </para>
>+    </refsect2>
>+
>+    <refsect2>
>       <title>RecdFailCount</title>
>       <para>Default: 10</para>
>       <para>
>@@ -571,6 +596,17 @@
>     </refsect2>
> 
>     <refsect2>
>+      <title>TDBMutexEnabled</title>
>+      <para>Default: 0</para>
>+      <para>
>+	This paramter enables TDB_MUTEX_LOCKING feature on volatile
>+	databases if the robust mutexes are supported. This optimizes the
>+	record locking using robust mutexes and is much more efficient
>+	that using posix locks.
>+      </para>
>+    </refsect2>
>+
>+    <refsect2>
>       <title>TickleUpdateInterval</title>
>       <para>Default: 20</para>
>       <para>
>-- 
>2.5.5
>
>
>From ae6922ff2f1dad412dc73658509d64a5e752a78b Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 8 Mar 2016 14:51:59 +1100
>Subject: [PATCH 07/29] ctdb-recovery-helper: Get tunables first, so control
> timeout can be set
>
>During the recovery process, the timeout value for sending all controls
>is decided by RecoverTimeout tunable.  So in the recovery process,
>first get the tunables, so the control timeout gets set correctly.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 700f39372ad9c893f8eb8884cd24999dc7026e60)
>---
> ctdb/server/ctdb_recovery_helper.c | 97 ++++++++++++++++++++------------------
> 1 file changed, 51 insertions(+), 46 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index f0fa12d..607faa6 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -34,7 +34,9 @@
> #include "protocol/protocol_api.h"
> #include "client/client.h"
> 
>-#define TIMEOUT()	timeval_current_ofs(10, 0)
>+static int recover_timeout = 120;
>+
>+#define TIMEOUT()	timeval_current_ofs(recover_timeout, 0)
> 
> static void LOG(const char *fmt, ...)
> {
>@@ -1215,10 +1217,10 @@ static bool db_recovery_recv(struct tevent_req *req, int *count)
> /*
>  * Run the parallel database recovery
>  *
>+ * - Get tunables
>  * - Get nodemap
>  * - Get vnnmap
>  * - Get capabilities from all nodes
>- * - Get tunables from all nodes
>  * - Get dbmap
>  * - Set RECOVERY_ACTIVE
>  * - Send START_RECOVERY
>@@ -1242,10 +1244,10 @@ struct recovery_state {
> 	struct ctdb_dbid_map *dbmap;
> };
> 
>+static void recovery_tunables_done(struct tevent_req *subreq);
> static void recovery_nodemap_done(struct tevent_req *subreq);
> static void recovery_vnnmap_done(struct tevent_req *subreq);
> static void recovery_capabilities_done(struct tevent_req *subreq);
>-static void recovery_tunables_done(struct tevent_req *subreq);
> static void recovery_dbmap_done(struct tevent_req *subreq);
> static void recovery_active_done(struct tevent_req *subreq);
> static void recovery_start_recovery_done(struct tevent_req *subreq);
>@@ -1273,17 +1275,59 @@ static struct tevent_req *recovery_send(TALLOC_CTX *mem_ctx,
> 	state->generation = generation;
> 	state->destnode = ctdb_client_pnn(client);
> 
>-	ctdb_req_control_get_nodemap(&request);
>-	subreq = ctdb_client_control_send(mem_ctx, ev, client, state->destnode,
>-					  TIMEOUT(), &request);
>+	ctdb_req_control_get_all_tunables(&request);
>+	subreq = ctdb_client_control_send(state, state->ev, state->client,
>+					  state->destnode, TIMEOUT(),
>+					  &request);
> 	if (tevent_req_nomem(subreq, req)) {
> 		return tevent_req_post(req, ev);
> 	}
>-	tevent_req_set_callback(subreq, recovery_nodemap_done, req);
>+	tevent_req_set_callback(subreq, recovery_tunables_done, req);
> 
> 	return req;
> }
> 
>+static void recovery_tunables_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct recovery_state *state = tevent_req_data(
>+		req, struct recovery_state);
>+	struct ctdb_reply_control *reply;
>+	struct ctdb_req_control request;
>+	int ret;
>+	bool status;
>+
>+	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret);
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	ret = ctdb_reply_control_get_all_tunables(reply, state,
>+						  &state->tun_list);
>+	if (ret != 0) {
>+		LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret);
>+		tevent_req_error(req, EPROTO);
>+		return;
>+	}
>+
>+	talloc_free(reply);
>+
>+	recover_timeout = state->tun_list->recover_timeout;
>+
>+	ctdb_req_control_get_nodemap(&request);
>+	subreq = ctdb_client_control_send(state, state->ev, state->client,
>+					  state->destnode, TIMEOUT(),
>+					  &request);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return;
>+	}
>+	tevent_req_set_callback(subreq, recovery_nodemap_done, req);
>+}
>+
> static void recovery_nodemap_done(struct tevent_req *subreq)
> {
> 	struct tevent_req *req = tevent_req_callback_data(
>@@ -1420,45 +1464,6 @@ static void recovery_capabilities_done(struct tevent_req *subreq)
> 
> 	talloc_free(reply);
> 
>-	ctdb_req_control_get_all_tunables(&request);
>-	subreq = ctdb_client_control_send(state, state->ev, state->client,
>-					  state->destnode, TIMEOUT(),
>-					  &request);
>-	if (tevent_req_nomem(subreq, req)) {
>-		return;
>-	}
>-	tevent_req_set_callback(subreq, recovery_tunables_done, req);
>-}
>-
>-static void recovery_tunables_done(struct tevent_req *subreq)
>-{
>-	struct tevent_req *req = tevent_req_callback_data(
>-		subreq, struct tevent_req);
>-	struct recovery_state *state = tevent_req_data(
>-		req, struct recovery_state);
>-	struct ctdb_reply_control *reply;
>-	struct ctdb_req_control request;
>-	int ret;
>-	bool status;
>-
>-	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>-	TALLOC_FREE(subreq);
>-	if (! status) {
>-		LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret);
>-		tevent_req_error(req, ret);
>-		return;
>-	}
>-
>-	ret = ctdb_reply_control_get_all_tunables(reply, state,
>-						  &state->tun_list);
>-	if (ret != 0) {
>-		LOG("control GET_ALL_TUNABLES failed, ret=%d\n", ret);
>-		tevent_req_error(req, EPROTO);
>-		return;
>-	}
>-
>-	talloc_free(reply);
>-
> 	ctdb_req_control_get_dbmap(&request);
> 	subreq = ctdb_client_control_send(state, state->ev, state->client,
> 					  state->destnode, TIMEOUT(),
>-- 
>2.5.5
>
>
>From f29700a559c94329cb97c082707babbca4377c9f Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Fri, 26 Feb 2016 14:12:19 +1100
>Subject: [PATCH 08/29] ctdb-client: Add client API for sending message to
> multiple nodes
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 67799c73af486962b897d1841a8a4234fdee824b)
>---
> ctdb/client/client.h         |  10 ++++
> ctdb/client/client_message.c | 132 +++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 142 insertions(+)
>
>diff --git a/ctdb/client/client.h b/ctdb/client/client.h
>index 48b69b6..66be313 100644
>--- a/ctdb/client/client.h
>+++ b/ctdb/client/client.h
>@@ -72,6 +72,16 @@ struct tevent_req *ctdb_client_message_send(TALLOC_CTX *mem_ctx,
> 
> bool ctdb_client_message_recv(struct tevent_req *req, int *perr);
> 
>+struct tevent_req *ctdb_client_message_multi_send(
>+				TALLOC_CTX *mem_ctx,
>+				struct tevent_context *ev,
>+				struct ctdb_client_context *client,
>+				uint32_t *pnn_list, int count,
>+				struct ctdb_req_message *message);
>+
>+bool ctdb_client_message_multi_recv(struct tevent_req *req, int *perr,
>+				    TALLOC_CTX *mem_ctx, int **perr_list);
>+
> int ctdb_client_message(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
> 			struct ctdb_client_context *client,
> 			uint32_t destnode, struct ctdb_req_message *message);
>diff --git a/ctdb/client/client_message.c b/ctdb/client/client_message.c
>index c316f42..8c75548 100644
>--- a/ctdb/client/client_message.c
>+++ b/ctdb/client/client_message.c
>@@ -157,6 +157,138 @@ void ctdb_client_req_message(struct ctdb_client_context *client,
> }
> 
> /*
>+ * Handle multiple nodes
>+ */
>+
>+struct ctdb_client_message_multi_state {
>+	uint32_t *pnn_list;
>+	int count;
>+	int done;
>+	int err;
>+	int *err_list;
>+};
>+
>+struct message_index_state {
>+	struct tevent_req *req;
>+	int index;
>+};
>+
>+static void ctdb_client_message_multi_done(struct tevent_req *subreq);
>+
>+struct tevent_req *ctdb_client_message_multi_send(
>+				TALLOC_CTX *mem_ctx,
>+				struct tevent_context *ev,
>+				struct ctdb_client_context *client,
>+				uint32_t *pnn_list, int count,
>+				struct ctdb_req_message *message)
>+{
>+	struct tevent_req *req, *subreq;
>+	struct ctdb_client_message_multi_state *state;
>+	int i;
>+
>+	if (pnn_list == NULL || count == 0) {
>+		return NULL;
>+	}
>+
>+	req = tevent_req_create(mem_ctx, &state,
>+				struct ctdb_client_message_multi_state);
>+	if (req == NULL) {
>+		return NULL;
>+	}
>+
>+	state->pnn_list = pnn_list;
>+	state->count = count;
>+	state->done = 0;
>+	state->err = 0;
>+	state->err_list = talloc_zero_array(state, int, count);
>+	if (tevent_req_nomem(state->err_list, req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+
>+	for (i=0; i<count; i++) {
>+		struct message_index_state *substate;
>+
>+		subreq = ctdb_client_message_send(state, ev, client,
>+						  pnn_list[i], message);
>+		if (tevent_req_nomem(subreq, req)) {
>+			return tevent_req_post(req, ev);
>+		}
>+
>+		substate = talloc(subreq, struct message_index_state);
>+		if (tevent_req_nomem(substate, req)) {
>+			return tevent_req_post(req, ev);
>+		}
>+
>+		substate->req = req;
>+		substate->index = i;
>+
>+		tevent_req_set_callback(subreq, ctdb_client_message_multi_done,
>+					substate);
>+	}
>+
>+	return req;
>+}
>+
>+static void ctdb_client_message_multi_done(struct tevent_req *subreq)
>+{
>+	struct message_index_state *substate = tevent_req_callback_data(
>+		subreq, struct message_index_state);
>+	struct tevent_req *req = substate->req;
>+	int idx = substate->index;
>+	struct ctdb_client_message_multi_state *state = tevent_req_data(
>+		req, struct ctdb_client_message_multi_state);
>+	bool status;
>+	int ret;
>+
>+	status = ctdb_client_message_recv(subreq, &ret);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		if (state->err == 0) {
>+			state->err = ret;
>+			state->err_list[idx] = state->err;
>+		}
>+	}
>+
>+	state->done += 1;
>+
>+	if (state->done == state->count) {
>+		tevent_req_done(req);
>+	}
>+}
>+
>+bool ctdb_client_message_multi_recv(struct tevent_req *req, int *perr,
>+				    TALLOC_CTX *mem_ctx, int **perr_list)
>+{
>+	struct ctdb_client_message_multi_state *state = tevent_req_data(
>+		req, struct ctdb_client_message_multi_state);
>+	int err;
>+
>+	if (tevent_req_is_unix_error(req, &err)) {
>+		if (perr != NULL) {
>+			*perr = err;
>+		}
>+		if (perr_list != NULL) {
>+			*perr_list = talloc_steal(mem_ctx, state->err_list);
>+		}
>+		return false;
>+	}
>+
>+	if (perr != NULL) {
>+		*perr = state->err;
>+	}
>+
>+	if (perr_list != NULL) {
>+		*perr_list = talloc_steal(mem_ctx, state->err_list);
>+	}
>+
>+	if (state->err != 0) {
>+		return false;
>+	}
>+
>+	return true;
>+}
>+
>+/*
>  * sync version of message send
>  */
> 
>-- 
>2.5.5
>
>
>From 1391f8e356a7663814bf94b2fb8109d05db15890 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 8 Mar 2016 17:29:31 +1100
>Subject: [PATCH 09/29] ctdb-tunables: Add new tunable RecBufferSizeLimit
>
>This will be used to limit the size of record buffer sent in newer
>controls for recovery and existing controls for vacuuming.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit c41808e6d292ec4cd861af545478e7dfb5c448e8)
>---
> ctdb/doc/ctdb-tunables.7.xml | 10 ++++++++++
> ctdb/protocol/protocol.h     |  1 +
> ctdb/server/ctdb_tunables.c  |  1 +
> 3 files changed, 12 insertions(+)
>
>diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml
>index 591dcf5..d2d2616 100644
>--- a/ctdb/doc/ctdb-tunables.7.xml
>+++ b/ctdb/doc/ctdb-tunables.7.xml
>@@ -394,6 +394,16 @@
>     </refsect2>
> 
>     <refsect2>
>+      <title>RecBufferSizeLimit</title>
>+      <para>Default: 1000000</para>
>+      <para>
>+        This is the limit on the size of the record buffer to be sent
>+        in various controls.  This limit is used by new controls used
>+        for recovery and controls used in vacuuming.
>+      </para>
>+    </refsect2>
>+
>+    <refsect2>
>       <title>RecdFailCount</title>
>       <para>Default: 10</para>
>       <para>
>diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h
>index a2a0c45..28d4c41 100644
>--- a/ctdb/protocol/protocol.h
>+++ b/ctdb/protocol/protocol.h
>@@ -619,6 +619,7 @@ struct ctdb_tunable_list {
> 	uint32_t samba3_hack;
> 	uint32_t mutex_enabled;
> 	uint32_t lock_processes_per_db;
>+	uint32_t rec_buffer_size_limit;
> };
> 
> struct ctdb_tickle_list {
>diff --git a/ctdb/server/ctdb_tunables.c b/ctdb/server/ctdb_tunables.c
>index ba184e7..50c5c77 100644
>--- a/ctdb/server/ctdb_tunables.c
>+++ b/ctdb/server/ctdb_tunables.c
>@@ -93,6 +93,7 @@ static const struct {
> 	{ "Samba3AvoidDeadlocks", 0, offsetof(struct ctdb_tunable_list, samba3_hack), false },
> 	{ "TDBMutexEnabled", 0, offsetof(struct ctdb_tunable_list, mutex_enabled), false },
> 	{ "LockProcessesPerDB", 200, offsetof(struct ctdb_tunable_list, lock_processes_per_db), false },
>+	{ "RecBufferSizeLimit", 1000000, offsetof(struct ctdb_tunable_list, rec_buffer_size_limit), false },
> };
> 
> /*
>-- 
>2.5.5
>
>
>From 8c6892f0a23893107874f9a4a470ce8fcee4ae72 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Fri, 19 Feb 2016 10:45:19 +1100
>Subject: [PATCH 10/29] ctdb-protocol: Add new data type ctdb_pulldb_ext for
> new control
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit fe69b72569494ef09dc8fa8673af02c0bfc18434)
>---
> ctdb/protocol/protocol.h         |  6 ++++++
> ctdb/protocol/protocol_private.h |  5 +++++
> ctdb/protocol/protocol_types.c   | 28 ++++++++++++++++++++++++++++
> 3 files changed, 39 insertions(+)
>
>diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h
>index 28d4c41..a44e768 100644
>--- a/ctdb/protocol/protocol.h
>+++ b/ctdb/protocol/protocol.h
>@@ -457,6 +457,12 @@ struct ctdb_pulldb {
> 	uint32_t lmaster;
> };
> 
>+struct ctdb_pulldb_ext {
>+	uint32_t db_id;
>+	uint32_t lmaster;
>+	uint64_t srvid;
>+};
>+
> #define CTDB_RECOVERY_NORMAL		0
> #define CTDB_RECOVERY_ACTIVE		1
> 
>diff --git a/ctdb/protocol/protocol_private.h b/ctdb/protocol/protocol_private.h
>index 65193d9..a67d622 100644
>--- a/ctdb/protocol/protocol_private.h
>+++ b/ctdb/protocol/protocol_private.h
>@@ -91,6 +91,11 @@ void ctdb_pulldb_push(struct ctdb_pulldb *pulldb, uint8_t *buf);
> int ctdb_pulldb_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx,
> 		     struct ctdb_pulldb **out);
> 
>+size_t ctdb_pulldb_ext_len(struct ctdb_pulldb_ext *pulldb);
>+void ctdb_pulldb_ext_push(struct ctdb_pulldb_ext *pulldb, uint8_t *buf);
>+int ctdb_pulldb_ext_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx,
>+			 struct ctdb_pulldb_ext **out);
>+
> size_t ctdb_traverse_start_len(struct ctdb_traverse_start *traverse);
> void ctdb_traverse_start_push(struct ctdb_traverse_start *traverse,
> 			      uint8_t *buf);
>diff --git a/ctdb/protocol/protocol_types.c b/ctdb/protocol/protocol_types.c
>index e36d2e8..a026333 100644
>--- a/ctdb/protocol/protocol_types.c
>+++ b/ctdb/protocol/protocol_types.c
>@@ -473,6 +473,34 @@ int ctdb_pulldb_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx,
> 	return 0;
> }
> 
>+size_t ctdb_pulldb_ext_len(struct ctdb_pulldb_ext *pulldb)
>+{
>+	return sizeof(struct ctdb_pulldb_ext);
>+}
>+
>+void ctdb_pulldb_ext_push(struct ctdb_pulldb_ext *pulldb, uint8_t *buf)
>+{
>+	memcpy(buf, pulldb, sizeof(struct ctdb_pulldb_ext));
>+}
>+
>+int ctdb_pulldb_ext_pull(uint8_t *buf, size_t buflen, TALLOC_CTX *mem_ctx,
>+			 struct ctdb_pulldb_ext **out)
>+{
>+	struct ctdb_pulldb_ext *pulldb;
>+
>+	if (buflen < sizeof(struct ctdb_pulldb_ext)) {
>+		return EMSGSIZE;
>+	}
>+
>+	pulldb = talloc_memdup(mem_ctx, buf, sizeof(struct ctdb_pulldb_ext));
>+	if (pulldb == NULL) {
>+		return ENOMEM;
>+	}
>+
>+	*out = pulldb;
>+	return 0;
>+}
>+
> size_t ctdb_ltdb_header_len(struct ctdb_ltdb_header *header)
> {
> 	return sizeof(struct ctdb_ltdb_header);
>-- 
>2.5.5
>
>
>From 25f1f205c76964e2a51487c1d8128972f606f983 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Fri, 19 Feb 2016 10:54:15 +1100
>Subject: [PATCH 11/29] ctdb-protocol: Add new controls DB_PULL and
> DB_PUSH_START/DB_PUSH_CONFIRM
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 0fd156ae8478f2da35a68c6bd3f90f50a06aecb8)
>---
> ctdb/protocol/protocol.h              |  5 +++
> ctdb/protocol/protocol_api.h          | 14 +++++++
> ctdb/protocol/protocol_client.c       | 70 +++++++++++++++++++++++++++++++++++
> ctdb/protocol/protocol_control.c      | 68 ++++++++++++++++++++++++++++++++++
> ctdb/tests/src/protocol_client_test.c | 48 ++++++++++++++++++++++++
> ctdb/tests/src/protocol_types_test.c  | 18 +++++++++
> 6 files changed, 223 insertions(+)
>
>diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h
>index a44e768..16d1801 100644
>--- a/ctdb/protocol/protocol.h
>+++ b/ctdb/protocol/protocol.h
>@@ -355,6 +355,9 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS          = 0,
> 		    CTDB_CONTROL_DB_TRANSACTION_START    = 143,
> 		    CTDB_CONTROL_DB_TRANSACTION_COMMIT   = 144,
> 		    CTDB_CONTROL_DB_TRANSACTION_CANCEL	 = 145,
>+		    CTDB_CONTROL_DB_PULL                 = 146,
>+		    CTDB_CONTROL_DB_PUSH_START           = 147,
>+		    CTDB_CONTROL_DB_PUSH_CONFIRM         = 148,
> };
> 
> #define CTDB_MONITORING_ACTIVE		0
>@@ -862,6 +865,7 @@ struct ctdb_req_control_data {
> 		struct ctdb_vnn_map *vnnmap;
> 		uint32_t loglevel;
> 		struct ctdb_pulldb *pulldb;
>+		struct ctdb_pulldb_ext *pulldb_ext;
> 		struct ctdb_rec_buffer *recbuf;
> 		uint32_t recmode;
> 		const char *db_name;
>@@ -930,6 +934,7 @@ struct ctdb_reply_control_data {
> 		struct ctdb_uint8_array *u8_array;
> 		struct ctdb_db_statistics *dbstats;
> 		enum ctdb_runstate runstate;
>+		uint32_t num_records;
> 	} data;
> };
> 
>diff --git a/ctdb/protocol/protocol_api.h b/ctdb/protocol/protocol_api.h
>index 6fb9cec..ed7bdcb 100644
>--- a/ctdb/protocol/protocol_api.h
>+++ b/ctdb/protocol/protocol_api.h
>@@ -634,6 +634,20 @@ void ctdb_req_control_db_transaction_cancel(struct ctdb_req_control *request,
> 					    uint32_t db_id);
> int ctdb_reply_control_db_transaction_cancel(struct ctdb_reply_control *reply);
> 
>+void ctdb_req_control_db_pull(struct ctdb_req_control *request,
>+			      struct ctdb_pulldb_ext *pulldb_ext);
>+int ctdb_reply_control_db_pull(struct ctdb_reply_control *reply,
>+			       uint32_t *num_records);
>+
>+void ctdb_req_control_db_push_start(struct ctdb_req_control *request,
>+				    struct ctdb_pulldb_ext *pulldb_ext);
>+int ctdb_reply_control_db_push_start(struct ctdb_reply_control *reply);
>+
>+void ctdb_req_control_db_push_confirm(struct ctdb_req_control *request,
>+				      uint32_t db_id);
>+int ctdb_reply_control_db_push_confirm(struct ctdb_reply_control *reply,
>+				       uint32_t *num_records);
>+
> /* From protocol/protocol_message.c */
> 
> int ctdb_req_message_push(struct ctdb_req_header *h,
>diff --git a/ctdb/protocol/protocol_client.c b/ctdb/protocol/protocol_client.c
>index a4c19d9..09cd7e9 100644
>--- a/ctdb/protocol/protocol_client.c
>+++ b/ctdb/protocol/protocol_client.c
>@@ -2468,3 +2468,73 @@ int ctdb_reply_control_db_transaction_cancel(struct ctdb_reply_control *reply)
> {
> 	return ctdb_reply_control_generic(reply);
> }
>+
>+/* CTDB_CONTROL_DB_PULL */
>+
>+void ctdb_req_control_db_pull(struct ctdb_req_control *request,
>+			      struct ctdb_pulldb_ext *pulldb_ext)
>+{
>+	request->opcode = CTDB_CONTROL_DB_PULL;
>+	request->pad = 0;
>+	request->srvid = 0;
>+	request->client_id = 0;
>+	request->flags = 0;
>+
>+	request->rdata.opcode = CTDB_CONTROL_DB_PULL;
>+	request->rdata.data.pulldb_ext = pulldb_ext;
>+}
>+
>+int ctdb_reply_control_db_pull(struct ctdb_reply_control *reply,
>+			       uint32_t *num_records)
>+{
>+	if (reply->status == 0 &&
>+	    reply->rdata.opcode == CTDB_CONTROL_DB_PULL) {
>+		*num_records = reply->rdata.data.num_records;
>+	}
>+	return reply->status;
>+}
>+
>+/* CTDB_CONTROL_DB_PUSH_START */
>+
>+void ctdb_req_control_db_push_start(struct ctdb_req_control *request,
>+				    struct ctdb_pulldb_ext *pulldb_ext)
>+{
>+	request->opcode = CTDB_CONTROL_DB_PUSH_START;
>+	request->pad = 0;
>+	request->srvid = 0;
>+	request->client_id = 0;
>+	request->flags = 0;
>+
>+	request->rdata.opcode = CTDB_CONTROL_DB_PUSH_START;
>+	request->rdata.data.pulldb_ext = pulldb_ext;
>+}
>+
>+int ctdb_reply_control_db_push_start(struct ctdb_reply_control *reply)
>+{
>+	return ctdb_reply_control_generic(reply);
>+}
>+
>+/* CTDB_CONTROL_DB_PUSH_CONFIRM */
>+
>+void ctdb_req_control_db_push_confirm(struct ctdb_req_control *request,
>+				      uint32_t db_id)
>+{
>+	request->opcode = CTDB_CONTROL_DB_PUSH_CONFIRM;
>+	request->pad = 0;
>+	request->srvid = 0;
>+	request->client_id = 0;
>+	request->flags = 0;
>+
>+	request->rdata.opcode = CTDB_CONTROL_DB_PUSH_CONFIRM;
>+	request->rdata.data.db_id = db_id;
>+}
>+
>+int ctdb_reply_control_db_push_confirm(struct ctdb_reply_control *reply,
>+				       uint32_t *num_records)
>+{
>+	if (reply->status == 0 &&
>+	    reply->rdata.opcode == CTDB_CONTROL_DB_PUSH_CONFIRM) {
>+		*num_records = reply->rdata.data.num_records;
>+	}
>+	return reply->status;
>+}
>diff --git a/ctdb/protocol/protocol_control.c b/ctdb/protocol/protocol_control.c
>index 1c8364e..e7487da 100644
>--- a/ctdb/protocol/protocol_control.c
>+++ b/ctdb/protocol/protocol_control.c
>@@ -481,6 +481,18 @@ static size_t ctdb_req_control_data_len(struct ctdb_req_control_data *cd)
> 	case CTDB_CONTROL_DB_TRANSACTION_CANCEL:
> 		len = ctdb_uint32_len(cd->data.db_id);
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		len = ctdb_pulldb_ext_len(cd->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_START:
>+		len = ctdb_pulldb_ext_len(cd->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		len = ctdb_uint32_len(cd->data.db_id);
>+		break;
> 	}
> 
> 	return len;
>@@ -791,6 +803,18 @@ static void ctdb_req_control_data_push(struct ctdb_req_control_data *cd,
> 	case CTDB_CONTROL_DB_TRANSACTION_CANCEL:
> 		ctdb_uint32_push(cd->data.db_id, buf);
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		ctdb_pulldb_ext_push(cd->data.pulldb_ext, buf);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_START:
>+		ctdb_pulldb_ext_push(cd->data.pulldb_ext, buf);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		ctdb_uint32_push(cd->data.db_id, buf);
>+		break;
> 	}
> }
> 
>@@ -1177,6 +1201,21 @@ static int ctdb_req_control_data_pull(uint8_t *buf, size_t buflen,
> 		ret = ctdb_uint32_pull(buf, buflen, mem_ctx,
> 					&cd->data.db_id);
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		ret = ctdb_pulldb_ext_pull(buf, buflen, mem_ctx,
>+					   &cd->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_START:
>+		ret = ctdb_pulldb_ext_pull(buf, buflen, mem_ctx,
>+					   &cd->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		ret = ctdb_uint32_pull(buf, buflen, mem_ctx,
>+				       &cd->data.db_id);
>+		break;
> 	}
> 
> 	return ret;
>@@ -1574,6 +1613,17 @@ static size_t ctdb_reply_control_data_len(struct ctdb_reply_control_data *cd)
> 
> 	case CTDB_CONTROL_DB_TRANSACTION_CANCEL:
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		len = ctdb_uint32_len(cd->data.num_records);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_START:
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		len = ctdb_uint32_len(cd->data.num_records);
>+		break;
> 	}
> 
> 	return len;
>@@ -1726,6 +1776,14 @@ static void ctdb_reply_control_data_push(struct ctdb_reply_control_data *cd,
> 	case CTDB_CONTROL_GET_NODES_FILE:
> 		ctdb_node_map_push(cd->data.nodemap, buf);
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		ctdb_uint32_push(cd->data.num_records, buf);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		ctdb_uint32_push(cd->data.num_records, buf);
>+		break;
> 	}
> }
> 
>@@ -1913,6 +1971,16 @@ static int ctdb_reply_control_data_pull(uint8_t *buf, size_t buflen,
> 		ret = ctdb_node_map_pull(buf, buflen, mem_ctx,
> 					 &cd->data.nodemap);
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		ret = ctdb_uint32_pull(buf, buflen, mem_ctx,
>+				       &cd->data.num_records);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		ret = ctdb_uint32_pull(buf, buflen, mem_ctx,
>+				       &cd->data.num_records);
>+		break;
> 	}
> 
> 	return ret;
>diff --git a/ctdb/tests/src/protocol_client_test.c b/ctdb/tests/src/protocol_client_test.c
>index f5a3c35..64923e9 100644
>--- a/ctdb/tests/src/protocol_client_test.c
>+++ b/ctdb/tests/src/protocol_client_test.c
>@@ -656,6 +656,23 @@ static void fill_ctdb_req_control_data(TALLOC_CTX *mem_ctx,
> 	case CTDB_CONTROL_DB_TRANSACTION_CANCEL:
> 		cd->data.db_id = rand32();
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		cd->data.pulldb_ext = talloc(mem_ctx, struct ctdb_pulldb_ext);
>+		assert(cd->data.pulldb_ext != NULL);
>+		fill_ctdb_pulldb_ext(mem_ctx, cd->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_START:
>+		cd->data.pulldb_ext = talloc(mem_ctx, struct ctdb_pulldb_ext);
>+		assert(cd->data.pulldb_ext != NULL);
>+		fill_ctdb_pulldb_ext(mem_ctx, cd->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		cd->data.db_id = rand32();
>+		break;
>+
> 	}
> }
> 
>@@ -1103,6 +1120,21 @@ static void verify_ctdb_req_control_data(struct ctdb_req_control_data *cd,
> 	case CTDB_CONTROL_DB_TRANSACTION_CANCEL:
> 		assert(cd->data.db_id == cd2->data.db_id);
> 		break;
>+
>+	case CTDB_CONTROL_DB_PULL:
>+		verify_ctdb_pulldb_ext(cd->data.pulldb_ext,
>+				       cd2->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_START:
>+		verify_ctdb_pulldb_ext(cd->data.pulldb_ext,
>+				       cd2->data.pulldb_ext);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		assert(cd->data.db_id == cd2->data.db_id);
>+		break;
>+
> 	}
> }
> 
>@@ -1559,6 +1591,14 @@ static void fill_ctdb_reply_control_data(TALLOC_CTX *mem_ctx,
> 		fill_ctdb_node_map(mem_ctx, cd->data.nodemap);
> 		break;
> 
>+	case CTDB_CONTROL_DB_PULL:
>+		cd->data.num_records = rand32();
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		cd->data.num_records = rand32();
>+		break;
>+
> 	}
> }
> 
>@@ -1942,6 +1982,14 @@ static void verify_ctdb_reply_control_data(struct ctdb_reply_control_data *cd,
> 		verify_ctdb_node_map(cd->data.nodemap, cd2->data.nodemap);
> 		break;
> 
>+	case CTDB_CONTROL_DB_PULL:
>+		assert(cd->data.num_records == cd2->data.num_records);
>+		break;
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		assert(cd->data.num_records == cd2->data.num_records);
>+		break;
>+
> 	}
> }
> 
>diff --git a/ctdb/tests/src/protocol_types_test.c b/ctdb/tests/src/protocol_types_test.c
>index bab104c..55ee743 100644
>--- a/ctdb/tests/src/protocol_types_test.c
>+++ b/ctdb/tests/src/protocol_types_test.c
>@@ -182,6 +182,22 @@ static void verify_ctdb_pulldb(struct ctdb_pulldb *p1, struct ctdb_pulldb *p2)
> 	assert(p1->lmaster == p2->lmaster);
> }
> 
>+static void fill_ctdb_pulldb_ext(TALLOC_CTX *mem_ctx,
>+				 struct ctdb_pulldb_ext *p)
>+{
>+	p->db_id = rand32();
>+	p->lmaster = rand32();
>+	p->srvid = rand64();
>+}
>+
>+static void verify_ctdb_pulldb_ext(struct ctdb_pulldb_ext *p1,
>+				   struct ctdb_pulldb_ext *p2)
>+{
>+	assert(p1->db_id == p2->db_id);
>+	assert(p1->lmaster == p2->lmaster);
>+	assert(p1->srvid == p2->srvid);
>+}
>+
> static void fill_ctdb_ltdb_header(TALLOC_CTX *mem_ctx,
> 				  struct ctdb_ltdb_header *p)
> {
>@@ -1177,6 +1193,7 @@ DEFINE_TEST(struct ctdb_statistics, ctdb_statistics);
> DEFINE_TEST(struct ctdb_vnn_map, ctdb_vnn_map);
> DEFINE_TEST(struct ctdb_dbid_map, ctdb_dbid_map);
> DEFINE_TEST(struct ctdb_pulldb, ctdb_pulldb);
>+DEFINE_TEST(struct ctdb_pulldb_ext, ctdb_pulldb_ext);
> DEFINE_TEST(struct ctdb_rec_data, ctdb_rec_data);
> DEFINE_TEST(struct ctdb_rec_buffer, ctdb_rec_buffer);
> DEFINE_TEST(struct ctdb_traverse_start, ctdb_traverse_start);
>@@ -1240,6 +1257,7 @@ int main(int argc, char *argv[])
> 	TEST_FUNC(ctdb_vnn_map)();
> 	TEST_FUNC(ctdb_dbid_map)();
> 	TEST_FUNC(ctdb_pulldb)();
>+	TEST_FUNC(ctdb_pulldb_ext)();
> 	TEST_FUNC(ctdb_rec_data)();
> 	TEST_FUNC(ctdb_rec_buffer)();
> 	TEST_FUNC(ctdb_traverse_start)();
>-- 
>2.5.5
>
>
>From 66191228517ab77a890b825531b05d6d350ce8e1 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Fri, 19 Feb 2016 17:32:09 +1100
>Subject: [PATCH 12/29] ctdb-daemon: Implement new controls DB_PULL and
> DB_PUSH_START/DB_PUSH_CONFIRM
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 95a15cde45c47e36d87d54464bcd769ee96e43c2)
>---
> ctdb/include/ctdb_private.h |  11 ++
> ctdb/server/ctdb_control.c  |  12 ++
> ctdb/server/ctdb_recover.c  | 340 ++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 363 insertions(+)
>
>diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
>index b7c3e5d..04574fe 100644
>--- a/ctdb/include/ctdb_private.h
>+++ b/ctdb/include/ctdb_private.h
>@@ -437,6 +437,9 @@ struct ctdb_db_context {
> 	bool freeze_transaction_started;
> 	uint32_t freeze_transaction_id;
> 	uint32_t generation;
>+
>+	bool push_started;
>+	void *push_state;
> };
> 
> 
>@@ -873,6 +876,14 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata,
> 			     TDB_DATA *outdata);
> int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata);
> 
>+int32_t ctdb_control_db_pull(struct ctdb_context *ctdb,
>+			     struct ctdb_req_control_old *c,
>+			     TDB_DATA indata, TDB_DATA *outdata);
>+int32_t ctdb_control_db_push_start(struct ctdb_context *ctdb,
>+				   TDB_DATA indata);
>+int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb,
>+				     TDB_DATA indata, TDB_DATA *outdata);
>+
> int ctdb_deferred_drop_all_ips(struct ctdb_context *ctdb);
> 
> int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
>diff --git a/ctdb/server/ctdb_control.c b/ctdb/server/ctdb_control.c
>index e6f8a0d..d840174 100644
>--- a/ctdb/server/ctdb_control.c
>+++ b/ctdb/server/ctdb_control.c
>@@ -715,6 +715,18 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
> 		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
> 		return ctdb_control_db_transaction_cancel(ctdb, indata);
> 
>+	case CTDB_CONTROL_DB_PULL:
>+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_pulldb_ext));
>+		return ctdb_control_db_pull(ctdb, c, indata, outdata);
>+
>+	case CTDB_CONTROL_DB_PUSH_START:
>+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_pulldb_ext));
>+		return ctdb_control_db_push_start(ctdb, indata);
>+
>+	case CTDB_CONTROL_DB_PUSH_CONFIRM:
>+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
>+		return ctdb_control_db_push_confirm(ctdb, indata, outdata);
>+
> 	default:
> 		DEBUG(DEBUG_CRIT,(__location__ " Unknown CTDB control opcode %u\n", opcode));
> 		return -1;
>diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c
>index 127ff6b..49f568a 100644
>--- a/ctdb/server/ctdb_recover.c
>+++ b/ctdb/server/ctdb_recover.c
>@@ -313,6 +313,133 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT
> 	return 0;
> }
> 
>+struct db_pull_state {
>+	struct ctdb_context *ctdb;
>+	struct ctdb_db_context *ctdb_db;
>+	struct ctdb_marshall_buffer *recs;
>+	uint32_t pnn;
>+	uint64_t srvid;
>+	uint32_t num_records;
>+};
>+
>+static int traverse_db_pull(struct tdb_context *tdb, TDB_DATA key,
>+			    TDB_DATA data, void *private_data)
>+{
>+	struct db_pull_state *state = (struct db_pull_state *)private_data;
>+	struct ctdb_marshall_buffer *recs;
>+
>+	recs = ctdb_marshall_add(state->ctdb, state->recs,
>+				 state->ctdb_db->db_id, 0, key, NULL, data);
>+	if (recs == NULL) {
>+		TALLOC_FREE(state->recs);
>+		return -1;
>+	}
>+	state->recs = recs;
>+
>+	if (talloc_get_size(state->recs) >=
>+			state->ctdb->tunable.rec_buffer_size_limit) {
>+		TDB_DATA buffer;
>+		int ret;
>+
>+		buffer = ctdb_marshall_finish(state->recs);
>+		ret = ctdb_daemon_send_message(state->ctdb, state->pnn,
>+					       state->srvid, buffer);
>+		if (ret != 0) {
>+			TALLOC_FREE(state->recs);
>+			return -1;
>+		}
>+
>+		state->num_records += state->recs->count;
>+		TALLOC_FREE(state->recs);
>+	}
>+
>+	return 0;
>+}
>+
>+int32_t ctdb_control_db_pull(struct ctdb_context *ctdb,
>+			     struct ctdb_req_control_old *c,
>+			     TDB_DATA indata, TDB_DATA *outdata)
>+{
>+	struct ctdb_pulldb_ext *pulldb_ext;
>+	struct ctdb_db_context *ctdb_db;
>+	struct db_pull_state state;
>+	int ret;
>+
>+	pulldb_ext = (struct ctdb_pulldb_ext *)indata.dptr;
>+
>+	ctdb_db = find_ctdb_db(ctdb, pulldb_ext->db_id);
>+	if (ctdb_db == NULL) {
>+		DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n",
>+				 pulldb_ext->db_id));
>+		return -1;
>+	}
>+
>+	if (!ctdb_db_frozen(ctdb_db)) {
>+		DEBUG(DEBUG_ERR,
>+		      ("rejecting ctdb_control_pull_db when not frozen\n"));
>+		return -1;
>+	}
>+
>+	if (ctdb_db->unhealthy_reason) {
>+		/* this is just a warning, as the tdb should be empty anyway */
>+		DEBUG(DEBUG_WARNING,
>+		      ("db(%s) unhealty in ctdb_control_db_pull: %s\n",
>+		       ctdb_db->db_name, ctdb_db->unhealthy_reason));
>+	}
>+
>+	state.ctdb = ctdb;
>+	state.ctdb_db = ctdb_db;
>+	state.recs = NULL;
>+	state.pnn = c->hdr.srcnode;
>+	state.srvid = pulldb_ext->srvid;
>+	state.num_records = 0;
>+
>+	if (ctdb_lockdb_mark(ctdb_db) != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      (__location__ " Failed to get lock on entire db - failing\n"));
>+		return -1;
>+	}
>+
>+	ret = tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_db_pull, &state);
>+	if (ret == -1) {
>+		DEBUG(DEBUG_ERR,
>+		      (__location__ " Failed to get traverse db '%s'\n",
>+		       ctdb_db->db_name));
>+		ctdb_lockdb_unmark(ctdb_db);
>+		return -1;
>+	}
>+
>+	/* Last few records */
>+	if (state.recs != NULL) {
>+		TDB_DATA buffer;
>+
>+		buffer = ctdb_marshall_finish(state.recs);
>+		ret = ctdb_daemon_send_message(state.ctdb, state.pnn,
>+					       state.srvid, buffer);
>+		if (ret != 0) {
>+			TALLOC_FREE(state.recs);
>+			ctdb_lockdb_unmark(ctdb_db);
>+			return -1;
>+		}
>+
>+		state.num_records += state.recs->count;
>+		TALLOC_FREE(state.recs);
>+	}
>+
>+	ctdb_lockdb_unmark(ctdb_db);
>+
>+	outdata->dptr = talloc_size(outdata, sizeof(uint32_t));
>+	if (outdata->dptr == NULL) {
>+		DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n"));
>+		return -1;
>+	}
>+
>+	memcpy(outdata->dptr, (uint8_t *)&state.num_records, sizeof(uint32_t));
>+	outdata->dsize = sizeof(uint32_t);
>+
>+	return 0;
>+}
>+
> /*
>   push a bunch of records into a ltdb, filtering by rsn
>  */
>@@ -407,6 +534,219 @@ failed:
> 	return -1;
> }
> 
>+struct db_push_state {
>+	struct ctdb_context *ctdb;
>+	struct ctdb_db_context *ctdb_db;
>+	uint64_t srvid;
>+	uint32_t num_records;
>+	bool failed;
>+};
>+
>+static void db_push_msg_handler(uint64_t srvid, TDB_DATA indata,
>+				void *private_data)
>+{
>+	struct db_push_state *state = talloc_get_type(
>+		private_data, struct db_push_state);
>+	struct ctdb_marshall_buffer *recs;
>+	struct ctdb_rec_data_old *rec;
>+	int i, ret;
>+
>+	if (state->failed) {
>+		return;
>+	}
>+
>+	recs = (struct ctdb_marshall_buffer *)indata.dptr;
>+	rec = (struct ctdb_rec_data_old *)&recs->data[0];
>+
>+	DEBUG(DEBUG_INFO, ("starting push of %u records for dbid 0x%x\n",
>+			   recs->count, recs->db_id));
>+
>+	for (i=0; i<recs->count; i++) {
>+		TDB_DATA key, data;
>+		struct ctdb_ltdb_header *hdr;
>+
>+		key.dptr = &rec->data[0];
>+		key.dsize = rec->keylen;
>+		data.dptr = &rec->data[key.dsize];
>+		data.dsize = rec->datalen;
>+
>+		if (data.dsize < sizeof(struct ctdb_ltdb_header)) {
>+			DEBUG(DEBUG_CRIT,(__location__ " bad ltdb record\n"));
>+			goto failed;
>+		}
>+
>+		hdr = (struct ctdb_ltdb_header *)data.dptr;
>+		/* Strip off any read only record flags.
>+		 * All readonly records are revoked implicitely by a recovery.
>+		 */
>+		hdr->flags &= ~CTDB_REC_RO_FLAGS;
>+
>+		data.dptr += sizeof(*hdr);
>+		data.dsize -= sizeof(*hdr);
>+
>+		ret = ctdb_ltdb_store(state->ctdb_db, key, hdr, data);
>+		if (ret != 0) {
>+			DEBUG(DEBUG_ERR,
>+			      (__location__ " Unable to store record\n"));
>+			goto failed;
>+		}
>+
>+		rec = (struct ctdb_rec_data_old *)(rec->length + (uint8_t *)rec);
>+	}
>+
>+	DEBUG(DEBUG_DEBUG, ("finished push of %u records for dbid 0x%x\n",
>+			    recs->count, recs->db_id));
>+
>+	state->num_records += recs->count;
>+	return;
>+
>+failed:
>+	state->failed = true;
>+}
>+
>+int32_t ctdb_control_db_push_start(struct ctdb_context *ctdb, TDB_DATA indata)
>+{
>+	struct ctdb_pulldb_ext *pulldb_ext;
>+	struct ctdb_db_context *ctdb_db;
>+	struct db_push_state *state;
>+	int ret;
>+
>+	pulldb_ext = (struct ctdb_pulldb_ext *)indata.dptr;
>+
>+	ctdb_db = find_ctdb_db(ctdb, pulldb_ext->db_id);
>+	if (ctdb_db == NULL) {
>+		DEBUG(DEBUG_ERR,
>+		      (__location__ " Unknown db 0x%08x\n", pulldb_ext->db_id));
>+		return -1;
>+	}
>+
>+	if (!ctdb_db_frozen(ctdb_db)) {
>+		DEBUG(DEBUG_ERR,
>+		      ("rejecting ctdb_control_db_push_start when not frozen\n"));
>+		return -1;
>+	}
>+
>+	if (ctdb_db->push_started) {
>+		DEBUG(DEBUG_WARNING,
>+		      (__location__ " DB push already started for %s\n",
>+		       ctdb_db->db_name));
>+
>+		/* De-register old state */
>+		state = (struct db_push_state *)ctdb_db->push_state;
>+		if (state != NULL) {
>+			srvid_deregister(ctdb->srv, state->srvid, state);
>+			talloc_free(state);
>+			ctdb_db->push_state = NULL;
>+		}
>+	}
>+
>+	state = talloc_zero(ctdb_db, struct db_push_state);
>+	if (state == NULL) {
>+		DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n"));
>+		return -1;
>+	}
>+
>+	state->ctdb = ctdb;
>+	state->ctdb_db = ctdb_db;
>+	state->srvid = pulldb_ext->srvid;
>+	state->failed = false;
>+
>+	ret = srvid_register(ctdb->srv, state, state->srvid,
>+			     db_push_msg_handler, state);
>+	if (ret != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      (__location__ " Failed to register srvid for db push\n"));
>+		talloc_free(state);
>+		return -1;
>+	}
>+
>+	if (ctdb_lockdb_mark(ctdb_db) != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      (__location__ " Failed to get lock on entire db - failing\n"));
>+		srvid_deregister(ctdb->srv, state->srvid, state);
>+		talloc_free(state);
>+		return -1;
>+	}
>+
>+	ctdb_db->push_started = true;
>+	ctdb_db->push_state = state;
>+
>+	return 0;
>+}
>+
>+int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb,
>+				     TDB_DATA indata, TDB_DATA *outdata)
>+{
>+	uint32_t db_id;
>+	struct ctdb_db_context *ctdb_db;
>+	struct db_push_state *state;
>+
>+	db_id = *(uint32_t *)indata.dptr;
>+
>+	ctdb_db = find_ctdb_db(ctdb, db_id);
>+	if (ctdb_db == NULL) {
>+		DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", db_id));
>+		return -1;
>+	}
>+
>+	if (!ctdb_db_frozen(ctdb_db)) {
>+		DEBUG(DEBUG_ERR,
>+		      ("rejecting ctdb_control_db_push_confirm when not frozen\n"));
>+		return -1;
>+	}
>+
>+	if (!ctdb_db->push_started) {
>+		DEBUG(DEBUG_ERR, (__location__ " DB push not started\n"));
>+		return -1;
>+	}
>+
>+	if (ctdb_db->readonly) {
>+		DEBUG(DEBUG_ERR,
>+		      ("Clearing the tracking database for dbid 0x%x\n",
>+		       ctdb_db->db_id));
>+		if (tdb_wipe_all(ctdb_db->rottdb) != 0) {
>+			DEBUG(DEBUG_ERR,
>+			      ("Failed to wipe tracking database for 0x%x."
>+			       " Dropping read-only delegation support\n",
>+			       ctdb_db->db_id));
>+			ctdb_db->readonly = false;
>+			tdb_close(ctdb_db->rottdb);
>+			ctdb_db->rottdb = NULL;
>+			ctdb_db->readonly = false;
>+		}
>+
>+		while (ctdb_db->revokechild_active != NULL) {
>+			talloc_free(ctdb_db->revokechild_active);
>+		}
>+	}
>+
>+	ctdb_lockdb_unmark(ctdb_db);
>+
>+	state = (struct db_push_state *)ctdb_db->push_state;
>+	if (state == NULL) {
>+		DEBUG(DEBUG_ERR, (__location__ " Missing push db state\n"));
>+		return -1;
>+	}
>+
>+	srvid_deregister(ctdb->srv, state->srvid, state);
>+
>+	outdata->dptr = talloc_size(outdata, sizeof(uint32_t));
>+	if (outdata->dptr == NULL) {
>+		DEBUG(DEBUG_ERR, (__location__ " Memory allocation error\n"));
>+		talloc_free(state);
>+		ctdb_db->push_state = NULL;
>+		return -1;
>+	}
>+
>+	memcpy(outdata->dptr, (uint8_t *)&state->num_records, sizeof(uint32_t));
>+	outdata->dsize = sizeof(uint32_t);
>+
>+	talloc_free(state);
>+	ctdb_db->push_state = NULL;
>+
>+	return 0;
>+}
>+
> struct ctdb_set_recmode_state {
> 	struct ctdb_context *ctdb;
> 	struct ctdb_req_control_old *c;
>-- 
>2.5.5
>
>
>From cab299abb4029cfe7bf6aac741751e3404ea63d7 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Fri, 19 Feb 2016 11:14:40 +1100
>Subject: [PATCH 13/29] ctdb-client: Add client API functions for new controls
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 338e0dccd903b63bf15ae9af38c47aad01c110d0)
>---
> ctdb/client/client.h              | 15 +++++++
> ctdb/client/client_control_sync.c | 86 +++++++++++++++++++++++++++++++++++++++
> 2 files changed, 101 insertions(+)
>
>diff --git a/ctdb/client/client.h b/ctdb/client/client.h
>index 66be313..eebb7a6 100644
>--- a/ctdb/client/client.h
>+++ b/ctdb/client/client.h
>@@ -737,6 +737,21 @@ int ctdb_ctrl_db_transaction_cancel(TALLOC_CTX *mem_ctx,
> 				    int destnode, struct timeval timeout,
> 				    uint32_t db_id);
> 
>+int ctdb_ctrl_db_pull(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
>+		      struct ctdb_client_context *client,
>+		      int destnode, struct timeval timeout,
>+		      struct ctdb_pulldb_ext *pulldb, uint32_t *num_records);
>+
>+int ctdb_ctrl_db_push_start(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
>+			    struct ctdb_client_context *client,
>+			    int destnode, struct timeval timeout,
>+			    struct ctdb_pulldb_ext *pulldb);
>+
>+int ctdb_ctrl_db_push_confirm(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
>+			      struct ctdb_client_context *client,
>+			      int destnode, struct timeval timeout,
>+			      uint32_t db_id, uint32_t *num_records);
>+
> /* from client/client_db.c */
> 
> struct tevent_req *ctdb_attach_send(TALLOC_CTX *mem_ctx,
>diff --git a/ctdb/client/client_control_sync.c b/ctdb/client/client_control_sync.c
>index de52b47..753007e 100644
>--- a/ctdb/client/client_control_sync.c
>+++ b/ctdb/client/client_control_sync.c
>@@ -3117,3 +3117,89 @@ int ctdb_ctrl_db_transaction_cancel(TALLOC_CTX *mem_ctx,
> 
> 	return 0;
> }
>+
>+int ctdb_ctrl_db_pull(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
>+		      struct ctdb_client_context *client,
>+		      int destnode, struct timeval timeout,
>+		      struct ctdb_pulldb_ext *pulldb, uint32_t *num_records)
>+{
>+	struct ctdb_req_control request;
>+	struct ctdb_reply_control *reply;
>+	int ret;
>+
>+	ctdb_req_control_db_pull(&request, pulldb);
>+	ret = ctdb_client_control(mem_ctx, ev, client, destnode, timeout,
>+				  &request, &reply);
>+	if (ret != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      ("Control DB_PULL failed to node %u, ret=%d\n",
>+		       destnode, ret));
>+		return ret;
>+	}
>+
>+	ret = ctdb_reply_control_db_pull(reply, num_records);
>+	if (ret != 0) {
>+		DEBUG(DEBUG_ERR, ("Control DB_PULL failed, ret=%d\n", ret));
>+		return ret;
>+	}
>+
>+	return 0;
>+}
>+
>+int ctdb_ctrl_db_push_start(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
>+			    struct ctdb_client_context *client,
>+			    int destnode, struct timeval timeout,
>+			    struct ctdb_pulldb_ext *pulldb)
>+{
>+	struct ctdb_req_control request;
>+	struct ctdb_reply_control *reply;
>+	int ret;
>+
>+	ctdb_req_control_db_push_start(&request, pulldb);
>+	ret = ctdb_client_control(mem_ctx, ev, client, destnode, timeout,
>+				  &request, &reply);
>+	if (ret != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      ("Control DB_PUSH failed to node %u, ret=%d\n",
>+		       destnode, ret));
>+		return ret;
>+	}
>+
>+	ret = ctdb_reply_control_db_push_start(reply);
>+	if (ret != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      ("Control DB_PUSH failed, ret=%d\n", ret));
>+		return ret;
>+	}
>+
>+	return 0;
>+}
>+
>+int ctdb_ctrl_db_push_confirm(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
>+			      struct ctdb_client_context *client,
>+			      int destnode, struct timeval timeout,
>+			      uint32_t db_id, uint32_t *num_records)
>+{
>+	struct ctdb_req_control request;
>+	struct ctdb_reply_control *reply;
>+	int ret;
>+
>+	ctdb_req_control_db_push_confirm(&request, db_id);
>+	ret = ctdb_client_control(mem_ctx, ev, client, destnode, timeout,
>+				  &request, &reply);
>+	if (ret != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      ("Control DB_PUSH failed to node %u, ret=%d\n",
>+		       destnode, ret));
>+		return ret;
>+	}
>+
>+	ret = ctdb_reply_control_db_push_confirm(reply, num_records);
>+	if (ret != 0) {
>+		DEBUG(DEBUG_ERR,
>+		      ("Control DB_PUSH failed, ret=%d\n", ret));
>+		return ret;
>+	}
>+
>+	return 0;
>+}
>-- 
>2.5.5
>
>
>From c1a2d1001a296e5d02ec864fe25f2417e4409bdb Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Fri, 26 Feb 2016 17:36:39 +1100
>Subject: [PATCH 14/29] ctdb-recovery-helper: Factor out generic recv function
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 5f43f92796f4f8130067e23555f94842bb803fcf)
>---
> ctdb/server/ctdb_recovery_helper.c | 53 +++++++++++++-------------------------
> 1 file changed, 18 insertions(+), 35 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 607faa6..4338818 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -65,6 +65,20 @@ static ssize_t sys_write(int fd, const void *buf, size_t count)
>         return ret;
> }
> 
>+static bool generic_recv(struct tevent_req *req, int *perr)
>+{
>+	int err;
>+
>+	if (tevent_req_is_unix_error(req, &err)) {
>+		if (perr != NULL) {
>+			*perr = err;
>+		}
>+		return false;
>+	}
>+
>+	return true;
>+}
>+
> /*
>  * Recovery database functions
>  */
>@@ -443,16 +457,7 @@ static void collect_highseqnum_db_pulldb_done(struct tevent_req *subreq)
> 
> static bool collect_highseqnum_db_recv(struct tevent_req *req, int *perr)
> {
>-	int err;
>-
>-	if (tevent_req_is_unix_error(req, &err)) {
>-		if (perr != NULL) {
>-			*perr = err;
>-		}
>-		return false;
>-	}
>-
>-	return true;
>+	return generic_recv(req, perr);
> }
> 
> /*
>@@ -570,16 +575,7 @@ static void collect_all_db_pulldb_done(struct tevent_req *subreq)
> 
> static bool collect_all_db_recv(struct tevent_req *req, int *perr)
> {
>-	int err;
>-
>-	if (tevent_req_is_unix_error(req, &err)) {
>-		if (perr != NULL) {
>-			*perr = err;
>-		}
>-		return false;
>-	}
>-
>-	return true;
>+	return generic_recv(req, perr);
> }
> 
> 
>@@ -1050,13 +1046,7 @@ static void recover_db_thaw_done(struct tevent_req *subreq)
> 
> static bool recover_db_recv(struct tevent_req *req)
> {
>-	int err;
>-
>-	if (tevent_req_is_unix_error(req, &err)) {
>-		return false;
>-	}
>-
>-	return true;
>+	return generic_recv(req, NULL);
> }
> 
> 
>@@ -1808,14 +1798,7 @@ static void recovery_end_recovery_done(struct tevent_req *subreq)
> 
> static void recovery_recv(struct tevent_req *req, int *perr)
> {
>-	int err;
>-
>-	if (tevent_req_is_unix_error(req, &err)) {
>-		if (perr != NULL) {
>-			*perr = err;
>-		}
>-		return;
>-	}
>+	generic_recv(req, perr);
> }
> 
> static void usage(const char *progname)
>-- 
>2.5.5
>
>
>From 5c8d980997d48c25edff07240cc6d5a552e46ebe Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Thu, 25 Feb 2016 11:04:51 +1100
>Subject: [PATCH 15/29] ctdb-recovery-helper: Pass capabilities to database
> recovery functions
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 5b926d882e3cf63d4ad3a9714fd0d440d6b5f3e5)
>---
> ctdb/server/ctdb_recovery_helper.c | 24 +++++++++++++++++-------
> 1 file changed, 17 insertions(+), 7 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 4338818..ef5303f 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -304,6 +304,7 @@ struct collect_highseqnum_db_state {
> 	struct ctdb_client_context *client;
> 	uint32_t *pnn_list;
> 	int count;
>+	uint32_t *caps;
> 	uint32_t db_id;
> 	struct recdb_context *recdb;
> 	uint32_t max_pnn;
>@@ -316,7 +317,7 @@ static struct tevent_req *collect_highseqnum_db_send(
> 			TALLOC_CTX *mem_ctx,
> 			struct tevent_context *ev,
> 			struct ctdb_client_context *client,
>-			uint32_t *pnn_list, int count,
>+			uint32_t *pnn_list, int count, uint32_t *caps,
> 			uint32_t db_id, struct recdb_context *recdb)
> {
> 	struct tevent_req *req, *subreq;
>@@ -333,6 +334,7 @@ static struct tevent_req *collect_highseqnum_db_send(
> 	state->client = client;
> 	state->pnn_list = pnn_list;
> 	state->count = count;
>+	state->caps = caps;
> 	state->db_id = db_id;
> 	state->recdb = recdb;
> 
>@@ -469,6 +471,7 @@ struct collect_all_db_state {
> 	struct ctdb_client_context *client;
> 	uint32_t *pnn_list;
> 	int count;
>+	uint32_t *caps;
> 	uint32_t db_id;
> 	struct recdb_context *recdb;
> 	struct ctdb_pulldb pulldb;
>@@ -481,7 +484,7 @@ static struct tevent_req *collect_all_db_send(
> 			TALLOC_CTX *mem_ctx,
> 			struct tevent_context *ev,
> 			struct ctdb_client_context *client,
>-			uint32_t *pnn_list, int count,
>+			uint32_t *pnn_list, int count, uint32_t *caps,
> 			uint32_t db_id, struct recdb_context *recdb)
> {
> 	struct tevent_req *req, *subreq;
>@@ -598,6 +601,7 @@ struct recover_db_state {
> 	struct ctdb_tunable_list *tun_list;
> 	uint32_t *pnn_list;
> 	int count;
>+	uint32_t *caps;
> 	uint32_t db_id;
> 	bool persistent;
> 
>@@ -625,6 +629,7 @@ static struct tevent_req *recover_db_send(TALLOC_CTX *mem_ctx,
> 					  struct ctdb_client_context *client,
> 					  struct ctdb_tunable_list *tun_list,
> 					  uint32_t *pnn_list, int count,
>+					  uint32_t *caps,
> 					  uint32_t generation,
> 					  uint32_t db_id, bool persistent)
> {
>@@ -642,6 +647,7 @@ static struct tevent_req *recover_db_send(TALLOC_CTX *mem_ctx,
> 	state->tun_list = tun_list;
> 	state->pnn_list = pnn_list;
> 	state->count = count;
>+	state->caps = caps;
> 	state->db_id = db_id;
> 	state->persistent = persistent;
> 
>@@ -826,12 +832,12 @@ static void recover_db_transaction_started(struct tevent_req *subreq)
> 	if (state->persistent && state->tun_list->recover_pdb_by_seqnum != 0) {
> 		subreq = collect_highseqnum_db_send(
> 				state, state->ev, state->client,
>-				state->pnn_list, state->count,
>+				state->pnn_list, state->count, state->caps,
> 				state->db_id, state->recdb);
> 	} else {
> 		subreq = collect_all_db_send(
> 				state, state->ev, state->client,
>-				state->pnn_list, state->count,
>+				state->pnn_list, state->count, state->caps,
> 				state->db_id, state->recdb);
> 	}
> 	if (tevent_req_nomem(subreq, req)) {
>@@ -1070,6 +1076,7 @@ struct db_recovery_one_state {
> 	struct ctdb_tunable_list *tun_list;
> 	uint32_t *pnn_list;
> 	int count;
>+	uint32_t *caps;
> 	uint32_t generation;
> 	uint32_t db_id;
> 	bool persistent;
>@@ -1084,6 +1091,7 @@ static struct tevent_req *db_recovery_send(TALLOC_CTX *mem_ctx,
> 					   struct ctdb_dbid_map *dbmap,
> 					   struct ctdb_tunable_list *tun_list,
> 					   uint32_t *pnn_list, int count,
>+					   uint32_t *caps,
> 					   uint32_t generation)
> {
> 	struct tevent_req *req, *subreq;
>@@ -1119,14 +1127,15 @@ static struct tevent_req *db_recovery_send(TALLOC_CTX *mem_ctx,
> 		substate->tun_list = tun_list;
> 		substate->pnn_list = pnn_list;
> 		substate->count = count;
>+		substate->caps = caps;
> 		substate->generation = generation;
> 		substate->db_id = dbmap->dbs[i].db_id;
> 		substate->persistent = dbmap->dbs[i].flags &
> 				       CTDB_DB_FLAGS_PERSISTENT;
> 
> 		subreq = recover_db_send(state, ev, client, tun_list,
>-					 pnn_list, count, generation,
>-					 substate->db_id,
>+					 pnn_list, count, caps,
>+					 generation, substate->db_id,
> 					 substate->persistent);
> 		if (tevent_req_nomem(subreq, req)) {
> 			return tevent_req_post(req, ev);
>@@ -1161,6 +1170,7 @@ static void db_recovery_one_done(struct tevent_req *subreq)
> 		subreq = recover_db_send(state, state->ev, substate->client,
> 					 substate->tun_list,
> 					 substate->pnn_list, substate->count,
>+					 substate->caps,
> 					 substate->generation, substate->db_id,
> 					 substate->persistent);
> 		if (tevent_req_nomem(subreq, req)) {
>@@ -1678,7 +1688,7 @@ static void recovery_vnnmap_update_done(struct tevent_req *subreq)
> 	subreq = db_recovery_send(state, state->ev, state->client,
> 				  state->dbmap, state->tun_list,
> 				  state->pnn_list, state->count,
>-				  state->vnnmap->generation);
>+				  state->caps, state->vnnmap->generation);
> 	if (tevent_req_nomem(subreq, req)) {
> 		return;
> 	}
>-- 
>2.5.5
>
>
>From 097623158c286f4255f3f4575e032d7b509014f7 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Fri, 26 Feb 2016 15:42:53 +1100
>Subject: [PATCH 16/29] ctdb-recovery-helper: Rename pnn to dmaster in
> recdb_records()
>
>This variable is used to set the dmaster value for each record in
>recdb_traverse().
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 70011a1bfb24d9f4b2a042d353f907ef7c49bc1c)
>---
> ctdb/server/ctdb_recovery_helper.c | 9 +++++----
> 1 file changed, 5 insertions(+), 4 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index ef5303f..de9c3da 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -203,7 +203,7 @@ static bool recdb_add(struct recdb_context *recdb, int mypnn,
> 
> struct recdb_traverse_state {
> 	struct ctdb_rec_buffer *recbuf;
>-	uint32_t pnn;
>+	uint32_t dmaster;
> 	uint32_t reqid;
> 	bool persistent;
> 	bool failed;
>@@ -255,7 +255,7 @@ static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
> 	/* update the dmaster field to point to us */
> 	header = (struct ctdb_ltdb_header *)data.dptr;
> 	if (!state->persistent) {
>-		header->dmaster = state->pnn;
>+		header->dmaster = state->dmaster;
> 		header->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA;
> 	}
> 
>@@ -270,7 +270,8 @@ static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
> }
> 
> static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb,
>-					     TALLOC_CTX *mem_ctx, uint32_t pnn)
>+					     TALLOC_CTX *mem_ctx,
>+					     uint32_t dmaster)
> {
> 	struct recdb_traverse_state state;
> 	int ret;
>@@ -279,7 +280,7 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb,
> 	if (state.recbuf == NULL) {
> 		return NULL;
> 	}
>-	state.pnn = pnn;
>+	state.dmaster = dmaster;
> 	state.reqid = 0;
> 	state.persistent = recdb->persistent;
> 	state.failed = false;
>-- 
>2.5.5
>
>
>From 6e208aca8683b5157fc99053285143e2ecf39277 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Mon, 29 Feb 2016 13:53:52 +1100
>Subject: [PATCH 17/29] ctdb-recovery-helper: Create accessors for recdb
> structure fields
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit a80ff09ed3da49c544fb121ae88e3ccb351fc4e7)
>---
> ctdb/server/ctdb_recovery_helper.c | 32 ++++++++++++++++++++++++++------
> 1 file changed, 26 insertions(+), 6 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index de9c3da..d7a2b5b 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -135,11 +135,31 @@ static struct recdb_context *recdb_create(TALLOC_CTX *mem_ctx, uint32_t db_id,
> 	return recdb;
> }
> 
>+static uint32_t recdb_id(struct recdb_context *recdb)
>+{
>+	return recdb->db_id;
>+}
>+
> static const char *recdb_name(struct recdb_context *recdb)
> {
> 	return recdb->db_name;
> }
> 
>+static const char *recdb_path(struct recdb_context *recdb)
>+{
>+	return recdb->db_path;
>+}
>+
>+static struct tdb_context *recdb_tdb(struct recdb_context *recdb)
>+{
>+	return recdb->db->tdb;
>+}
>+
>+static bool recdb_persistent(struct recdb_context *recdb)
>+{
>+	return recdb->persistent;
>+}
>+
> struct recdb_add_traverse_state {
> 	struct recdb_context *recdb;
> 	int mypnn;
>@@ -163,7 +183,7 @@ static int recdb_add_traverse(uint32_t reqid, struct ctdb_ltdb_header *header,
> 	hdr = (struct ctdb_ltdb_header *)data.dptr;
> 
> 	/* fetch the existing record, if any */
>-	prev_data = tdb_fetch(state->recdb->db->tdb, key);
>+	prev_data = tdb_fetch(recdb_tdb(state->recdb), key);
> 
> 	if (prev_data.dptr != NULL) {
> 		struct ctdb_ltdb_header prev_hdr;
>@@ -177,7 +197,7 @@ static int recdb_add_traverse(uint32_t reqid, struct ctdb_ltdb_header *header,
> 		}
> 	}
> 
>-	ret = tdb_store(state->recdb->db->tdb, key, data, TDB_REPLACE);
>+	ret = tdb_store(recdb_tdb(state->recdb), key, data, TDB_REPLACE);
> 	if (ret != 0) {
> 		return -1;
> 	}
>@@ -276,19 +296,19 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb,
> 	struct recdb_traverse_state state;
> 	int ret;
> 
>-	state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb->db_id);
>+	state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb_id(recdb));
> 	if (state.recbuf == NULL) {
> 		return NULL;
> 	}
> 	state.dmaster = dmaster;
> 	state.reqid = 0;
>-	state.persistent = recdb->persistent;
>+	state.persistent = recdb_persistent(recdb);
> 	state.failed = false;
> 
>-	ret = tdb_traverse_read(recdb->db->tdb, recdb_traverse, &state);
>+	ret = tdb_traverse_read(recdb_tdb(recdb), recdb_traverse, &state);
> 	if (ret == -1 || state.failed) {
> 		LOG("Failed to marshall recovery records for %s\n",
>-		    recdb->db_name);
>+		    recdb_name(recdb));
> 		TALLOC_FREE(state.recbuf);
> 		return NULL;
> 	}
>-- 
>2.5.5
>
>
>From e332108f0a4f6c523121a473d303a3bee4916292 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 8 Mar 2016 17:20:30 +1100
>Subject: [PATCH 18/29] ctdb-protocol: Add file IO functions for
> ctdb_rec_buffer
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 76f653f0bc016dfa98a4d8688469be4ac450a17a)
>---
> ctdb/protocol/protocol_api.h         |  4 +++
> ctdb/protocol/protocol_types.c       | 62 ++++++++++++++++++++++++++++++++++++
> ctdb/tests/src/protocol_types_test.c | 49 ++++++++++++++++++++++++++++
> 3 files changed, 115 insertions(+)
>
>diff --git a/ctdb/protocol/protocol_api.h b/ctdb/protocol/protocol_api.h
>index ed7bdcb..4cd8b11 100644
>--- a/ctdb/protocol/protocol_api.h
>+++ b/ctdb/protocol/protocol_api.h
>@@ -52,6 +52,10 @@ int ctdb_rec_buffer_traverse(struct ctdb_rec_buffer *recbuf,
> 			     ctdb_rec_parser_func_t func,
> 			     void *private_data);
> 
>+int ctdb_rec_buffer_write(struct ctdb_rec_buffer *recbuf, int fd);
>+int ctdb_rec_buffer_read(int fd, TALLOC_CTX *mem_ctx,
>+			 struct ctdb_rec_buffer **out);
>+
> size_t ctdb_server_id_len(struct ctdb_server_id *sid);
> void ctdb_server_id_push(struct ctdb_server_id *sid, uint8_t *buf);
> int ctdb_server_id_pull(uint8_t *buf, size_t buflen,
>diff --git a/ctdb/protocol/protocol_types.c b/ctdb/protocol/protocol_types.c
>index a026333..cd04aca 100644
>--- a/ctdb/protocol/protocol_types.c
>+++ b/ctdb/protocol/protocol_types.c
>@@ -800,6 +800,68 @@ int ctdb_rec_buffer_traverse(struct ctdb_rec_buffer *recbuf,
> 	return ret;
> }
> 
>+int ctdb_rec_buffer_write(struct ctdb_rec_buffer *recbuf, int fd)
>+{
>+	ssize_t n;
>+
>+	n = write(fd, &recbuf->db_id, sizeof(uint32_t));
>+	if (n == -1 || n != sizeof(uint32_t)) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+	n = write(fd, &recbuf->count, sizeof(uint32_t));
>+	if (n == -1 || n != sizeof(uint32_t)) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+	n = write(fd, &recbuf->buflen, sizeof(size_t));
>+	if (n == -1 || n != sizeof(size_t)) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+	n = write(fd, recbuf->buf, recbuf->buflen);
>+	if (n == -1 || n != recbuf->buflen) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+
>+	return 0;
>+}
>+
>+int ctdb_rec_buffer_read(int fd, TALLOC_CTX *mem_ctx,
>+			 struct ctdb_rec_buffer **out)
>+{
>+	struct ctdb_rec_buffer *recbuf;
>+	ssize_t n;
>+
>+	recbuf = talloc(mem_ctx, struct ctdb_rec_buffer);
>+	if (recbuf == NULL) {
>+		return ENOMEM;
>+	}
>+
>+	n = read(fd, &recbuf->db_id, sizeof(uint32_t));
>+	if (n == -1 || n != sizeof(uint32_t)) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+	n = read(fd, &recbuf->count, sizeof(uint32_t));
>+	if (n == -1 || n != sizeof(uint32_t)) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+	n = read(fd, &recbuf->buflen, sizeof(size_t));
>+	if (n == -1 || n != sizeof(size_t)) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+
>+	recbuf->buf = talloc_size(recbuf, recbuf->buflen);
>+	if (recbuf->buf == NULL) {
>+		return ENOMEM;
>+	}
>+
>+	n = read(fd, recbuf->buf, recbuf->buflen);
>+	if (n == -1 || n != recbuf->buflen) {
>+		return (errno != 0 ? errno : EIO);
>+	}
>+
>+	*out = recbuf;
>+	return 0;
>+}
>+
> size_t ctdb_traverse_start_len(struct ctdb_traverse_start *traverse)
> {
> 	return sizeof(struct ctdb_traverse_start);
>diff --git a/ctdb/tests/src/protocol_types_test.c b/ctdb/tests/src/protocol_types_test.c
>index 55ee743..e39a2c9 100644
>--- a/ctdb/tests/src/protocol_types_test.c
>+++ b/ctdb/tests/src/protocol_types_test.c
>@@ -19,6 +19,7 @@
> 
> #include "replace.h"
> #include "system/network.h"
>+#include "system/filesys.h"
> 
> #include <assert.h>
> 
>@@ -1235,6 +1236,52 @@ DEFINE_TEST(struct ctdb_srvid_message, ctdb_srvid_message);
> DEFINE_TEST(struct ctdb_disable_message, ctdb_disable_message);
> DEFINE_TEST(struct ctdb_g_lock_list, ctdb_g_lock_list);
> 
>+static void test_ctdb_rec_buffer_read_write(void)
>+{
>+	TALLOC_CTX *mem_ctx = talloc_new(NULL);
>+	struct ctdb_rec_buffer *p1, **p2;
>+	const char *filename = "ctdb_rec_buffer_test.dat";
>+	int count = 100;
>+	int fd, i, ret;
>+	off_t offset;
>+
>+	p1 = talloc_array(mem_ctx, struct ctdb_rec_buffer, count);
>+	assert(p1 != NULL);
>+	for (i=0; i<count; i++) {
>+		fill_ctdb_rec_buffer(mem_ctx, &p1[i]);
>+	}
>+
>+	fd = open(filename, O_RDWR|O_CREAT, 0600);
>+	assert(fd != -1);
>+	unlink(filename);
>+
>+	for (i=0; i<count; i++) {
>+		ret = ctdb_rec_buffer_write(&p1[i], fd);
>+		assert(ret == 0);
>+	}
>+
>+	offset = lseek(fd, 0, SEEK_CUR);
>+	assert(offset != -1);
>+	offset = lseek(fd, -offset, SEEK_CUR);
>+	assert(offset == 0);
>+
>+	p2 = talloc_array(mem_ctx, struct ctdb_rec_buffer *, count);
>+	assert(p2 != NULL);
>+
>+	for (i=0; i<count; i++) {
>+		ret = ctdb_rec_buffer_read(fd, mem_ctx, &p2[i]);
>+		assert(ret == 0);
>+	}
>+
>+	close(fd);
>+
>+	for (i=0; i<count; i++) {
>+		verify_ctdb_rec_buffer(&p1[i], p2[i]);
>+	}
>+
>+	talloc_free(mem_ctx);
>+}
>+
> int main(int argc, char *argv[])
> {
> 	if (argc == 2) {
>@@ -1299,6 +1346,8 @@ int main(int argc, char *argv[])
> 	TEST_FUNC(ctdb_disable_message)();
> 	TEST_FUNC(ctdb_g_lock_list)();
> 
>+	test_ctdb_rec_buffer_read_write();
>+
> 	return 0;
> }
> 
>-- 
>2.5.5
>
>
>From 8b191b63b0c5422a45e693391b79d070e9ec8cad Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 8 Mar 2016 17:48:46 +1100
>Subject: [PATCH 19/29] ctdb-recovery-helper: Re-factor function to retain
> records from recdb
>
>Also, rename traverse function and traverse state for recdb_records
>consistently.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit 9058fe06df7f24e9a1b8fd5792a537a1fa8f5a60)
>---
> ctdb/server/ctdb_recovery_helper.c | 56 ++++++++++++++++++++++++--------------
> 1 file changed, 36 insertions(+), 20 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index d7a2b5b..05900ef 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -221,19 +221,11 @@ static bool recdb_add(struct recdb_context *recdb, int mypnn,
> 	return true;
> }
> 
>-struct recdb_traverse_state {
>-	struct ctdb_rec_buffer *recbuf;
>-	uint32_t dmaster;
>-	uint32_t reqid;
>-	bool persistent;
>-	bool failed;
>-};
>-
>-static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
>-			  void *private_data)
>+/* This function decides which records from recdb are retained */
>+static int recbuf_filter_add(struct ctdb_rec_buffer *recbuf, bool persistent,
>+			     uint32_t reqid, uint32_t dmaster,
>+			     TDB_DATA key, TDB_DATA data)
> {
>-	struct recdb_traverse_state *state =
>-		(struct recdb_traverse_state *)private_data;
> 	struct ctdb_ltdb_header *header;
> 	int ret;
> 
>@@ -267,20 +259,43 @@ static int recdb_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
> 	 * On databases like Samba's registry, this can damage the higher-level
> 	 * data structures built from the various tdb-level records.
> 	 */
>-	if (!state->persistent &&
>-	    data.dsize <= sizeof(struct ctdb_ltdb_header)) {
>+	if (!persistent && data.dsize <= sizeof(struct ctdb_ltdb_header)) {
> 		return 0;
> 	}
> 
> 	/* update the dmaster field to point to us */
> 	header = (struct ctdb_ltdb_header *)data.dptr;
>-	if (!state->persistent) {
>-		header->dmaster = state->dmaster;
>+	if (!persistent) {
>+		header->dmaster = dmaster;
> 		header->flags |= CTDB_REC_FLAG_MIGRATED_WITH_DATA;
> 	}
> 
>-	ret = ctdb_rec_buffer_add(state->recbuf, state->recbuf, state->reqid,
>-				  NULL, key, data);
>+	ret = ctdb_rec_buffer_add(recbuf, recbuf, reqid, NULL, key, data);
>+	if (ret != 0) {
>+		return ret;
>+	}
>+
>+	return 0;
>+}
>+
>+struct recdb_records_traverse_state {
>+	struct ctdb_rec_buffer *recbuf;
>+	uint32_t dmaster;
>+	uint32_t reqid;
>+	bool persistent;
>+	bool failed;
>+};
>+
>+static int recdb_records_traverse(struct tdb_context *tdb,
>+				  TDB_DATA key, TDB_DATA data,
>+				  void *private_data)
>+{
>+	struct recdb_records_traverse_state *state =
>+		(struct recdb_records_traverse_state *)private_data;
>+	int ret;
>+
>+	ret = recbuf_filter_add(state->recbuf, state->persistent,
>+				state->reqid, state->dmaster, key, data);
> 	if (ret != 0) {
> 		state->failed = true;
> 		return ret;
>@@ -293,7 +308,7 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb,
> 					     TALLOC_CTX *mem_ctx,
> 					     uint32_t dmaster)
> {
>-	struct recdb_traverse_state state;
>+	struct recdb_records_traverse_state state;
> 	int ret;
> 
> 	state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb_id(recdb));
>@@ -305,7 +320,8 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb,
> 	state.persistent = recdb_persistent(recdb);
> 	state.failed = false;
> 
>-	ret = tdb_traverse_read(recdb_tdb(recdb), recdb_traverse, &state);
>+	ret = tdb_traverse_read(recdb_tdb(recdb), recdb_records_traverse,
>+				&state);
> 	if (ret == -1 || state.failed) {
> 		LOG("Failed to marshall recovery records for %s\n",
> 		    recdb_name(recdb));
>-- 
>2.5.5
>
>
>From 70db289e2c2ee375f3887f73fb9ebdab6a55018e Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 8 Mar 2016 18:32:19 +1100
>Subject: [PATCH 20/29] ctdb-recovery-helper: Write recovery records to a
> recovery file
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit e1fdfdd1c13b0a81e4d2e8b87aeb6f6bafef1a92)
>---
> ctdb/server/ctdb_recovery_helper.c | 89 ++++++++++++++++++++++++++++++++++++++
> 1 file changed, 89 insertions(+)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 05900ef..78fb3f9 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -332,6 +332,95 @@ static struct ctdb_rec_buffer *recdb_records(struct recdb_context *recdb,
> 	return state.recbuf;
> }
> 
>+struct recdb_file_traverse_state {
>+	struct ctdb_rec_buffer *recbuf;
>+	struct recdb_context *recdb;
>+	TALLOC_CTX *mem_ctx;
>+	uint32_t dmaster;
>+	uint32_t reqid;
>+	bool persistent;
>+	bool failed;
>+	int fd;
>+	int max_size;
>+	int num_buffers;
>+};
>+
>+static int recdb_file_traverse(struct tdb_context *tdb,
>+			       TDB_DATA key, TDB_DATA data,
>+			       void *private_data)
>+{
>+	struct recdb_file_traverse_state *state =
>+		(struct recdb_file_traverse_state *)private_data;
>+	int ret;
>+
>+	ret = recbuf_filter_add(state->recbuf, state->persistent,
>+				state->reqid, state->dmaster, key, data);
>+	if (ret != 0) {
>+		state->failed = true;
>+		return ret;
>+	}
>+
>+	if (ctdb_rec_buffer_len(state->recbuf) > state->max_size) {
>+		ret = ctdb_rec_buffer_write(state->recbuf, state->fd);
>+		if (ret != 0) {
>+			LOG("Failed to collect recovery records for %s\n",
>+			    recdb_name(state->recdb));
>+			state->failed = true;
>+			return ret;
>+		}
>+
>+		state->num_buffers += 1;
>+
>+		TALLOC_FREE(state->recbuf);
>+		state->recbuf = ctdb_rec_buffer_init(state->mem_ctx,
>+						     recdb_id(state->recdb));
>+		if (state->recbuf == NULL) {
>+			state->failed = true;
>+			return ENOMEM;
>+		}
>+	}
>+
>+	return 0;
>+}
>+
>+static int recdb_file(struct recdb_context *recdb, TALLOC_CTX *mem_ctx,
>+		      uint32_t dmaster, int fd, int max_size)
>+{
>+	struct recdb_file_traverse_state state;
>+	int ret;
>+
>+	state.recbuf = ctdb_rec_buffer_init(mem_ctx, recdb_id(recdb));
>+	if (state.recbuf == NULL) {
>+		return -1;
>+	}
>+	state.recdb = recdb;
>+	state.mem_ctx = mem_ctx;
>+	state.dmaster = dmaster;
>+	state.reqid = 0;
>+	state.persistent = recdb_persistent(recdb);
>+	state.failed = false;
>+	state.fd = fd;
>+	state.max_size = max_size;
>+	state.num_buffers = 0;
>+
>+	ret = tdb_traverse_read(recdb_tdb(recdb), recdb_file_traverse, &state);
>+	if (ret == -1 || state.failed) {
>+		TALLOC_FREE(state.recbuf);
>+		return -1;
>+	}
>+
>+	ret = ctdb_rec_buffer_write(state.recbuf, fd);
>+	if (ret != 0) {
>+		LOG("Failed to collect recovery records for %s\n",
>+		    recdb_name(recdb));
>+		TALLOC_FREE(state.recbuf);
>+		return -1;
>+	}
>+	state.num_buffers += 1;
>+
>+	return state.num_buffers;
>+}
>+
> /*
>  * Collect databases using highest sequence number
>  */
>-- 
>2.5.5
>
>
>From 0b40262231315376971455143b5342d84932bad7 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 23 Feb 2016 18:03:10 +1100
>Subject: [PATCH 21/29] ctdb-protocol: Add srvid for messages during recovery
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit c5776f0529dd23979dd3ef36ffe7d7b8eb444d0d)
>---
> ctdb/protocol/protocol.h | 3 +++
> 1 file changed, 3 insertions(+)
>
>diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h
>index 16d1801..796b7e5 100644
>--- a/ctdb/protocol/protocol.h
>+++ b/ctdb/protocol/protocol.h
>@@ -121,6 +121,9 @@ struct ctdb_call {
> /* SRVID prefix used by CTDB */
> #define CTDB_SRVID_PREFIX	0xF000000000000000LL
> 
>+/* SRVID prefix used during recovery for pulling and pushing databases */
>+#define CTDB_SRVID_RECOVERY	0xF001000000000000LL
>+
> /* SRVID to inform of election data */
> #define CTDB_SRVID_ELECTION	0xF100000000000000LL
> 
>-- 
>2.5.5
>
>
>From 99a4dc5633b163302e9502d35a7c9d9303d61c1f Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Tue, 23 Feb 2016 14:52:51 +1100
>Subject: [PATCH 22/29] ctdb-protocol: Add new capability
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit bb6541b386c7dfcc6dcbbd4cea39dc1ff80fe578)
>---
> ctdb/protocol/protocol.h | 4 +++-
> 1 file changed, 3 insertions(+), 1 deletion(-)
>
>diff --git a/ctdb/protocol/protocol.h b/ctdb/protocol/protocol.h
>index 796b7e5..798c928 100644
>--- a/ctdb/protocol/protocol.h
>+++ b/ctdb/protocol/protocol.h
>@@ -717,8 +717,10 @@ struct ctdb_public_ip_list {
>  * Node features
>  */
> #define CTDB_CAP_PARALLEL_RECOVERY	0x00010000
>+#define CTDB_CAP_FRAGMENTED_CONTROLS	0x00020000
> 
>-#define CTDB_CAP_FEATURES		(CTDB_CAP_PARALLEL_RECOVERY)
>+#define CTDB_CAP_FEATURES		(CTDB_CAP_PARALLEL_RECOVERY | \
>+					 CTDB_CAP_FRAGMENTED_CONTROLS)
> 
> #define CTDB_CAP_DEFAULT		(CTDB_CAP_RECMASTER | \
> 					 CTDB_CAP_LMASTER   | \
>-- 
>2.5.5
>
>
>From e4269bf89b75eea37691c82e67607ffe1a6c57bb Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Wed, 24 Feb 2016 18:10:49 +1100
>Subject: [PATCH 23/29] ctdb-recovery-helper: Introduce pull database
> abstraction
>
>This abstraction depending on the capability of the remote node either
>uses older PULL_DB control or newer DB_PULL control.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit b96a4759b397d873d56ccdd0c0b26e770cc10b89)
>---
> ctdb/server/ctdb_recovery_helper.c | 349 ++++++++++++++++++++++++++++++-------
> 1 file changed, 282 insertions(+), 67 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 78fb3f9..32fadc0 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -79,6 +79,14 @@ static bool generic_recv(struct tevent_req *req, int *perr)
> 	return true;
> }
> 
>+static uint64_t rec_srvid = CTDB_SRVID_RECOVERY;
>+
>+static uint64_t srvid_next(void)
>+{
>+	rec_srvid += 1;
>+	return rec_srvid;
>+}
>+
> /*
>  * Recovery database functions
>  */
>@@ -422,6 +430,265 @@ static int recdb_file(struct recdb_context *recdb, TALLOC_CTX *mem_ctx,
> }
> 
> /*
>+ * Pull database from a single node
>+ */
>+
>+struct pull_database_state {
>+	struct tevent_context *ev;
>+	struct ctdb_client_context *client;
>+	struct recdb_context *recdb;
>+	uint32_t pnn;
>+	uint64_t srvid;
>+	int num_records;
>+};
>+
>+static void pull_database_handler(uint64_t srvid, TDB_DATA data,
>+				  void *private_data);
>+static void pull_database_register_done(struct tevent_req *subreq);
>+static void pull_database_old_done(struct tevent_req *subreq);
>+static void pull_database_unregister_done(struct tevent_req *subreq);
>+static void pull_database_new_done(struct tevent_req *subreq);
>+
>+static struct tevent_req *pull_database_send(
>+			TALLOC_CTX *mem_ctx,
>+			struct tevent_context *ev,
>+			struct ctdb_client_context *client,
>+			uint32_t pnn, uint32_t caps,
>+			struct recdb_context *recdb)
>+{
>+	struct tevent_req *req, *subreq;
>+	struct pull_database_state *state;
>+	struct ctdb_req_control request;
>+
>+	req = tevent_req_create(mem_ctx, &state, struct pull_database_state);
>+	if (req == NULL) {
>+		return NULL;
>+	}
>+
>+	state->ev = ev;
>+	state->client = client;
>+	state->recdb = recdb;
>+	state->pnn = pnn;
>+	state->srvid = srvid_next();
>+
>+	if (caps & CTDB_CAP_FRAGMENTED_CONTROLS) {
>+		subreq = ctdb_client_set_message_handler_send(
>+					state, state->ev, state->client,
>+					state->srvid, pull_database_handler,
>+					req);
>+		if (tevent_req_nomem(subreq, req)) {
>+			return tevent_req_post(req, ev);
>+		}
>+
>+		tevent_req_set_callback(subreq, pull_database_register_done,
>+					req);
>+
>+	} else {
>+		struct ctdb_pulldb pulldb;
>+
>+		pulldb.db_id = recdb_id(recdb);
>+		pulldb.lmaster = CTDB_LMASTER_ANY;
>+
>+		ctdb_req_control_pull_db(&request, &pulldb);
>+		subreq = ctdb_client_control_send(state, state->ev,
>+						  state->client,
>+						  pnn, TIMEOUT(),
>+						  &request);
>+		if (tevent_req_nomem(subreq, req)) {
>+			return tevent_req_post(req, ev);
>+		}
>+		tevent_req_set_callback(subreq, pull_database_old_done, req);
>+	}
>+
>+	return req;
>+}
>+
>+static void pull_database_handler(uint64_t srvid, TDB_DATA data,
>+				  void *private_data)
>+{
>+	struct tevent_req *req = talloc_get_type_abort(
>+		private_data, struct tevent_req);
>+	struct pull_database_state *state = tevent_req_data(
>+		req, struct pull_database_state);
>+	struct ctdb_rec_buffer *recbuf;
>+	int ret;
>+	bool status;
>+
>+	if (srvid != state->srvid) {
>+		return;
>+	}
>+
>+	ret = ctdb_rec_buffer_pull(data.dptr, data.dsize, state, &recbuf);
>+	if (ret != 0) {
>+		LOG("Invalid data received for DB_PULL messages\n");
>+		return;
>+	}
>+
>+	if (recbuf->db_id != recdb_id(state->recdb)) {
>+		talloc_free(recbuf);
>+		LOG("Invalid dbid:%08x for DB_PULL messages for %s\n",
>+		    recbuf->db_id, recdb_name(state->recdb));
>+		return;
>+	}
>+
>+	status = recdb_add(state->recdb, ctdb_client_pnn(state->client),
>+			   recbuf);
>+	if (! status) {
>+		talloc_free(recbuf);
>+		LOG("Failed to add records to recdb for %s\n",
>+		    recdb_name(state->recdb));
>+		return;
>+	}
>+
>+	state->num_records += recbuf->count;
>+	talloc_free(recbuf);
>+}
>+
>+static void pull_database_register_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct pull_database_state *state = tevent_req_data(
>+		req, struct pull_database_state);
>+	struct ctdb_req_control request;
>+	struct ctdb_pulldb_ext pulldb_ext;
>+	int ret;
>+	bool status;
>+
>+	status = ctdb_client_set_message_handler_recv(subreq, &ret);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		LOG("failed to set message handler for DB_PULL for %s\n",
>+		    recdb_name(state->recdb));
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	pulldb_ext.db_id = recdb_id(state->recdb);
>+	pulldb_ext.lmaster = CTDB_LMASTER_ANY;
>+	pulldb_ext.srvid = state->srvid;
>+
>+	ctdb_req_control_db_pull(&request, &pulldb_ext);
>+	subreq = ctdb_client_control_send(state, state->ev, state->client,
>+					  state->pnn, TIMEOUT(), &request);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return;
>+	}
>+	tevent_req_set_callback(subreq, pull_database_new_done, req);
>+}
>+
>+static void pull_database_old_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct pull_database_state *state = tevent_req_data(
>+		req, struct pull_database_state);
>+	struct ctdb_reply_control *reply;
>+	struct ctdb_rec_buffer *recbuf;
>+	int ret;
>+	bool status;
>+
>+	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		LOG("control PULL_DB failed for %s on node %u, ret=%d\n",
>+		    recdb_name(state->recdb), state->pnn, ret);
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	ret = ctdb_reply_control_pull_db(reply, state, &recbuf);
>+	talloc_free(reply);
>+	if (ret != 0) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	status = recdb_add(state->recdb, ctdb_client_pnn(state->client),
>+			   recbuf);
>+	if (! status) {
>+		talloc_free(recbuf);
>+		tevent_req_error(req, EIO);
>+		return;
>+	}
>+
>+	state->num_records = recbuf->count;
>+	talloc_free(recbuf);
>+
>+	LOG("Pulled %d records for db %s from node %d\n",
>+	    state->num_records, recdb_name(state->recdb), state->pnn);
>+
>+	tevent_req_done(req);
>+}
>+
>+static void pull_database_new_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct pull_database_state *state = tevent_req_data(
>+		req, struct pull_database_state);
>+	struct ctdb_reply_control *reply;
>+	uint32_t num_records;
>+	int ret;
>+	bool status;
>+
>+	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		LOG("control DB_PULL failed for %s on node %u, ret=%d\n",
>+		    recdb_name(state->recdb), state->pnn, ret);
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	ret = ctdb_reply_control_db_pull(reply, &num_records);
>+	talloc_free(reply);
>+	if (num_records != state->num_records) {
>+		LOG("mismatch (%u != %u) in DB_PULL records for %s\n",
>+		    num_records, state->num_records, recdb_name(state->recdb));
>+		tevent_req_error(req, EIO);
>+		return;
>+	}
>+
>+	LOG("Pulled %d records for db %s from node %d\n",
>+	    state->num_records, recdb_name(state->recdb), state->pnn);
>+
>+	subreq = ctdb_client_remove_message_handler_send(
>+					state, state->ev, state->client,
>+					state->srvid, req);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return;
>+	}
>+	tevent_req_set_callback(subreq, pull_database_unregister_done, req);
>+}
>+
>+static void pull_database_unregister_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct pull_database_state *state = tevent_req_data(
>+		req, struct pull_database_state);
>+	int ret;
>+	bool status;
>+
>+	status = ctdb_client_remove_message_handler_recv(subreq, &ret);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		LOG("failed to remove message handler for DB_PULL for %s\n",
>+		    recdb_name(state->recdb));
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	tevent_req_done(req);
>+}
>+
>+static bool pull_database_recv(struct tevent_req *req, int *perr)
>+{
>+	return generic_recv(req, perr);
>+}
>+
>+/*
>  * Collect databases using highest sequence number
>  */
> 
>@@ -484,8 +751,6 @@ static void collect_highseqnum_db_seqnum_done(struct tevent_req *subreq)
> 	struct collect_highseqnum_db_state *state = tevent_req_data(
> 		req, struct collect_highseqnum_db_state);
> 	struct ctdb_reply_control **reply;
>-	struct ctdb_req_control request;
>-	struct ctdb_pulldb pulldb;
> 	int *err_list;
> 	bool status;
> 	int ret, i;
>@@ -532,12 +797,10 @@ static void collect_highseqnum_db_seqnum_done(struct tevent_req *subreq)
> 	LOG("Pull persistent db %s from node %d with seqnum 0x%"PRIx64"\n",
> 	    recdb_name(state->recdb), state->max_pnn, max_seqnum);
> 
>-	pulldb.db_id = state->db_id;
>-	pulldb.lmaster = CTDB_LMASTER_ANY;
>-
>-	ctdb_req_control_pull_db(&request, &pulldb);
>-	subreq = ctdb_client_control_send(state, state->ev, state->client,
>-					  state->max_pnn, TIMEOUT(), &request);
>+	subreq = pull_database_send(state, state->ev, state->client,
>+				    state->max_pnn,
>+				    state->caps[state->max_pnn],
>+				    state->recdb);
> 	if (tevent_req_nomem(subreq, req)) {
> 		return;
> 	}
>@@ -549,37 +812,16 @@ static void collect_highseqnum_db_pulldb_done(struct tevent_req *subreq)
> {
> 	struct tevent_req *req = tevent_req_callback_data(
> 		subreq, struct tevent_req);
>-	struct collect_highseqnum_db_state *state = tevent_req_data(
>-		req, struct collect_highseqnum_db_state);
>-	struct ctdb_reply_control *reply;
>-	struct ctdb_rec_buffer *recbuf;
> 	int ret;
> 	bool status;
> 
>-	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>+	status = pull_database_recv(subreq, &ret);
> 	TALLOC_FREE(subreq);
> 	if (! status) {
>-		LOG("control PULL_DB failed for %s on node %u, ret=%d\n",
>-		    recdb_name(state->recdb), state->max_pnn, ret);
> 		tevent_req_error(req, ret);
> 		return;
> 	}
> 
>-	ret = ctdb_reply_control_pull_db(reply, state, &recbuf);
>-	if (ret != 0) {
>-		tevent_req_error(req, EPROTO);
>-		return;
>-	}
>-
>-	talloc_free(reply);
>-
>-	ret = recdb_add(state->recdb, ctdb_client_pnn(state->client), recbuf);
>-	talloc_free(recbuf);
>-	if (! ret) {
>-		tevent_req_error(req, EIO);
>-		return;
>-	}
>-
> 	tevent_req_done(req);
> }
> 
>@@ -615,7 +857,7 @@ static struct tevent_req *collect_all_db_send(
> {
> 	struct tevent_req *req, *subreq;
> 	struct collect_all_db_state *state;
>-	struct ctdb_req_control request;
>+	uint32_t pnn;
> 
> 	req = tevent_req_create(mem_ctx, &state,
> 				struct collect_all_db_state);
>@@ -627,18 +869,14 @@ static struct tevent_req *collect_all_db_send(
> 	state->client = client;
> 	state->pnn_list = pnn_list;
> 	state->count = count;
>+	state->caps = caps;
> 	state->db_id = db_id;
> 	state->recdb = recdb;
>-
>-	state->pulldb.db_id = db_id;
>-	state->pulldb.lmaster = CTDB_LMASTER_ANY;
>-
> 	state->index = 0;
> 
>-	ctdb_req_control_pull_db(&request, &state->pulldb);
>-	subreq = ctdb_client_control_send(state, ev, client,
>-					  state->pnn_list[state->index],
>-					  TIMEOUT(), &request);
>+	pnn = state->pnn_list[state->index];
>+
>+	subreq = pull_database_send(state, ev, client, pnn, caps[pnn], recdb);
> 	if (tevent_req_nomem(subreq, req)) {
> 		return tevent_req_post(req, ev);
> 	}
>@@ -653,49 +891,26 @@ static void collect_all_db_pulldb_done(struct tevent_req *subreq)
> 		subreq, struct tevent_req);
> 	struct collect_all_db_state *state = tevent_req_data(
> 		req, struct collect_all_db_state);
>-	struct ctdb_reply_control *reply;
>-	struct ctdb_req_control request;
>-	struct ctdb_rec_buffer *recbuf;
>+	uint32_t pnn;
> 	int ret;
> 	bool status;
> 
>-	status = ctdb_client_control_recv(subreq, &ret, state, &reply);
>+	status = pull_database_recv(subreq, &ret);
> 	TALLOC_FREE(subreq);
> 	if (! status) {
>-		LOG("control PULL_DB failed for %s from node %u, ret=%d\n",
>-		    recdb_name(state->recdb), state->pnn_list[state->index],
>-		    ret);
> 		tevent_req_error(req, ret);
> 		return;
> 	}
> 
>-	ret = ctdb_reply_control_pull_db(reply, state, &recbuf);
>-	if (ret != 0) {
>-		LOG("control PULL_DB failed for %s, ret=%d\n",
>-		    recdb_name(state->recdb), ret);
>-		tevent_req_error(req, EPROTO);
>-		return;
>-	}
>-
>-	talloc_free(reply);
>-
>-	status = recdb_add(state->recdb, ctdb_client_pnn(state->client), recbuf);
>-	talloc_free(recbuf);
>-	if (! status) {
>-		tevent_req_error(req, EIO);
>-		return;
>-	}
>-
> 	state->index += 1;
> 	if (state->index == state->count) {
> 		tevent_req_done(req);
> 		return;
> 	}
> 
>-	ctdb_req_control_pull_db(&request, &state->pulldb);
>-	subreq = ctdb_client_control_send(state, state->ev, state->client,
>-					  state->pnn_list[state->index],
>-					  TIMEOUT(), &request);
>+	pnn = state->pnn_list[state->index];
>+	subreq = pull_database_send(state, state->ev, state->client,
>+				    pnn, state->caps[pnn], state->recdb);
> 	if (tevent_req_nomem(subreq, req)) {
> 		return;
> 	}
>-- 
>2.5.5
>
>
>From 19ed1f6743af77d69f1740d1b6a474c93c7bfb4c Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Thu, 25 Feb 2016 18:07:11 +1100
>Subject: [PATCH 24/29] ctdb-recovery-helper: Introduce push database
> abstraction
>
>This abstraction uses capabilities of the remote nodes to either send
>older PUSH_DB controls or newer DB_PUSH_START and DB_PUSH_CONFIRM
>controls.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit ffea827bae2a8054ad488ae82eedb021cdfb71c4)
>---
> ctdb/server/ctdb_recovery_helper.c | 556 ++++++++++++++++++++++++++++++++++---
> 1 file changed, 524 insertions(+), 32 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 32fadc0..9991d31 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -426,6 +426,9 @@ static int recdb_file(struct recdb_context *recdb, TALLOC_CTX *mem_ctx,
> 	}
> 	state.num_buffers += 1;
> 
>+	LOG("Wrote %d buffers of recovery records for %s\n",
>+	    state.num_buffers, recdb_name(recdb));
>+
> 	return state.num_buffers;
> }
> 
>@@ -689,6 +692,521 @@ static bool pull_database_recv(struct tevent_req *req, int *perr)
> }
> 
> /*
>+ * Push database to specified nodes (old style)
>+ */
>+
>+struct push_database_old_state {
>+	struct tevent_context *ev;
>+	struct ctdb_client_context *client;
>+	struct recdb_context *recdb;
>+	uint32_t *pnn_list;
>+	int count;
>+	struct ctdb_rec_buffer *recbuf;
>+	int index;
>+};
>+
>+static void push_database_old_push_done(struct tevent_req *subreq);
>+
>+static struct tevent_req *push_database_old_send(
>+			TALLOC_CTX *mem_ctx,
>+			struct tevent_context *ev,
>+			struct ctdb_client_context *client,
>+			uint32_t *pnn_list, int count,
>+			struct recdb_context *recdb)
>+{
>+	struct tevent_req *req, *subreq;
>+	struct push_database_old_state *state;
>+	struct ctdb_req_control request;
>+	uint32_t pnn;
>+
>+	req = tevent_req_create(mem_ctx, &state,
>+				struct push_database_old_state);
>+	if (req == NULL) {
>+		return NULL;
>+	}
>+
>+	state->ev = ev;
>+	state->client = client;
>+	state->recdb = recdb;
>+	state->pnn_list = pnn_list;
>+	state->count = count;
>+	state->index = 0;
>+
>+	state->recbuf = recdb_records(recdb, state,
>+				      ctdb_client_pnn(client));
>+	if (tevent_req_nomem(state->recbuf, req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+
>+	pnn = state->pnn_list[state->index];
>+
>+	ctdb_req_control_push_db(&request, state->recbuf);
>+	subreq = ctdb_client_control_send(state, ev, client, pnn,
>+					  TIMEOUT(), &request);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+	tevent_req_set_callback(subreq, push_database_old_push_done, req);
>+
>+	return req;
>+}
>+
>+static void push_database_old_push_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct push_database_old_state *state = tevent_req_data(
>+		req, struct push_database_old_state);
>+	struct ctdb_req_control request;
>+	uint32_t pnn;
>+	int ret;
>+	bool status;
>+
>+	status = ctdb_client_control_recv(subreq, &ret, NULL, NULL);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		LOG("control PUSH_DB failed for db %s on node %u, ret=%d\n",
>+		    recdb_name(state->recdb), state->pnn_list[state->index],
>+		    ret);
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	state->index += 1;
>+	if (state->index == state->count) {
>+		TALLOC_FREE(state->recbuf);
>+		tevent_req_done(req);
>+		return;
>+	}
>+
>+	pnn = state->pnn_list[state->index];
>+
>+	ctdb_req_control_push_db(&request, state->recbuf);
>+	subreq = ctdb_client_control_send(state, state->ev, state->client,
>+					  pnn, TIMEOUT(), &request);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return;
>+	}
>+	tevent_req_set_callback(subreq, push_database_old_push_done, req);
>+}
>+
>+static bool push_database_old_recv(struct tevent_req *req, int *perr)
>+{
>+	return generic_recv(req, perr);
>+}
>+
>+/*
>+ * Push database to specified nodes (new style)
>+ */
>+
>+struct push_database_new_state {
>+	struct tevent_context *ev;
>+	struct ctdb_client_context *client;
>+	struct recdb_context *recdb;
>+	uint32_t *pnn_list;
>+	int count;
>+	uint64_t srvid;
>+	uint32_t dmaster;
>+	int fd;
>+	int num_buffers;
>+	int num_buffers_sent;
>+	int num_records;
>+};
>+
>+static void push_database_new_started(struct tevent_req *subreq);
>+static void push_database_new_send_msg(struct tevent_req *req);
>+static void push_database_new_send_done(struct tevent_req *subreq);
>+static void push_database_new_confirmed(struct tevent_req *subreq);
>+
>+static struct tevent_req *push_database_new_send(
>+			TALLOC_CTX *mem_ctx,
>+			struct tevent_context *ev,
>+			struct ctdb_client_context *client,
>+			uint32_t *pnn_list, int count,
>+			struct recdb_context *recdb,
>+			int max_size)
>+{
>+	struct tevent_req *req, *subreq;
>+	struct push_database_new_state *state;
>+	struct ctdb_req_control request;
>+	struct ctdb_pulldb_ext pulldb_ext;
>+	char *filename;
>+	off_t offset;
>+
>+	req = tevent_req_create(mem_ctx, &state,
>+				struct push_database_new_state);
>+	if (req == NULL) {
>+		return NULL;
>+	}
>+
>+	state->ev = ev;
>+	state->client = client;
>+	state->recdb = recdb;
>+	state->pnn_list = pnn_list;
>+	state->count = count;
>+
>+	state->srvid = srvid_next();
>+	state->dmaster = ctdb_client_pnn(client);
>+	state->num_buffers_sent = 0;
>+	state->num_records = 0;
>+
>+	filename = talloc_asprintf(state, "%s.dat", recdb_path(recdb));
>+	if (tevent_req_nomem(filename, req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+
>+	state->fd = open(filename, O_RDWR|O_CREAT, 0644);
>+	if (state->fd == -1) {
>+		tevent_req_error(req, errno);
>+		return tevent_req_post(req, ev);
>+	}
>+	unlink(filename);
>+	talloc_free(filename);
>+
>+	state->num_buffers = recdb_file(recdb, state, state->dmaster,
>+					state->fd, max_size);
>+	if (state->num_buffers == -1) {
>+		tevent_req_error(req, ENOMEM);
>+		return tevent_req_post(req, ev);
>+	}
>+
>+	offset = lseek(state->fd, 0, SEEK_SET);
>+	if (offset != 0) {
>+		tevent_req_error(req, EIO);
>+		return tevent_req_post(req, ev);
>+	}
>+
>+	pulldb_ext.db_id = recdb_id(recdb);
>+	pulldb_ext.srvid = state->srvid;
>+
>+	ctdb_req_control_db_push_start(&request, &pulldb_ext);
>+	subreq = ctdb_client_control_multi_send(state, ev, client,
>+						pnn_list, count,
>+						TIMEOUT(), &request);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+	tevent_req_set_callback(subreq, push_database_new_started, req);
>+
>+	return req;
>+}
>+
>+static void push_database_new_started(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct push_database_new_state *state = tevent_req_data(
>+		req, struct push_database_new_state);
>+	int *err_list;
>+	int ret;
>+	bool status;
>+
>+	status = ctdb_client_control_multi_recv(subreq, &ret, state,
>+						&err_list, NULL);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		int ret2;
>+		uint32_t pnn;
>+
>+		ret2 = ctdb_client_control_multi_error(state->pnn_list,
>+						       state->count,
>+						       err_list, &pnn);
>+		if (ret2 != 0) {
>+			LOG("control DB_PUSH_START failed for db %s "
>+			    "on node %u, ret=%d\n",
>+			    recdb_name(state->recdb), pnn, ret2);
>+		} else {
>+			LOG("control DB_PUSH_START failed for db %s, ret=%d\n",
>+			    recdb_name(state->recdb), ret);
>+		}
>+		talloc_free(err_list);
>+
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	push_database_new_send_msg(req);
>+}
>+
>+static void push_database_new_send_msg(struct tevent_req *req)
>+{
>+	struct push_database_new_state *state = tevent_req_data(
>+		req, struct push_database_new_state);
>+	struct tevent_req *subreq;
>+	struct ctdb_rec_buffer *recbuf;
>+	struct ctdb_req_message message;
>+	TDB_DATA data;
>+	int ret;
>+
>+	if (state->num_buffers_sent == state->num_buffers) {
>+		struct ctdb_req_control request;
>+
>+		ctdb_req_control_db_push_confirm(&request,
>+						 recdb_id(state->recdb));
>+		subreq = ctdb_client_control_multi_send(state, state->ev,
>+							state->client,
>+							state->pnn_list,
>+							state->count,
>+							TIMEOUT(), &request);
>+		if (tevent_req_nomem(subreq, req)) {
>+			return;
>+		}
>+		tevent_req_set_callback(subreq, push_database_new_confirmed,
>+					req);
>+		return;
>+	}
>+
>+	ret = ctdb_rec_buffer_read(state->fd, state, &recbuf);
>+	if (ret != 0) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	data.dsize = ctdb_rec_buffer_len(recbuf);
>+	data.dptr = talloc_size(state, data.dsize);
>+	if (tevent_req_nomem(data.dptr, req)) {
>+		return;
>+	}
>+
>+	ctdb_rec_buffer_push(recbuf, data.dptr);
>+
>+	message.srvid = state->srvid;
>+	message.data.data = data;
>+
>+	LOG("Pushing buffer %d with %d records for %s\n",
>+	    state->num_buffers_sent, recbuf->count, recdb_name(state->recdb));
>+
>+	subreq = ctdb_client_message_multi_send(state, state->ev,
>+						state->client,
>+						state->pnn_list, state->count,
>+						&message);
>+	if (tevent_req_nomem(subreq, req)) {
>+		return;
>+	}
>+	tevent_req_set_callback(subreq, push_database_new_send_done, req);
>+
>+	state->num_records += recbuf->count;
>+
>+	talloc_free(data.dptr);
>+	talloc_free(recbuf);
>+}
>+
>+static void push_database_new_send_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct push_database_new_state *state = tevent_req_data(
>+		req, struct push_database_new_state);
>+	bool status;
>+	int ret;
>+
>+	status = ctdb_client_message_multi_recv(subreq, &ret, NULL, NULL);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		LOG("Sending recovery records failed for %s\n",
>+		    recdb_name(state->recdb));
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	state->num_buffers_sent += 1;
>+
>+	push_database_new_send_msg(req);
>+}
>+
>+static void push_database_new_confirmed(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct push_database_new_state *state = tevent_req_data(
>+		req, struct push_database_new_state);
>+	struct ctdb_reply_control **reply;
>+	int *err_list;
>+	bool status;
>+	int ret, i;
>+	uint32_t num_records;
>+
>+	status = ctdb_client_control_multi_recv(subreq, &ret, state,
>+						&err_list, &reply);
>+	TALLOC_FREE(subreq);
>+	if (! status) {
>+		int ret2;
>+		uint32_t pnn;
>+
>+		ret2 = ctdb_client_control_multi_error(state->pnn_list,
>+						       state->count, err_list,
>+						       &pnn);
>+		if (ret2 != 0) {
>+			LOG("control DB_PUSH_CONFIRM failed for %s on node %u,"
>+			    " ret=%d\n", recdb_name(state->recdb), pnn, ret2);
>+		} else {
>+			LOG("control DB_PUSH_CONFIRM failed for %s, ret=%d\n",
>+			    recdb_name(state->recdb), ret);
>+		}
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	for (i=0; i<state->count; i++) {
>+		ret = ctdb_reply_control_db_push_confirm(reply[i],
>+							 &num_records);
>+		if (ret != 0) {
>+			tevent_req_error(req, EPROTO);
>+			return;
>+		}
>+
>+		if (num_records != state->num_records) {
>+			LOG("Node %u received %d of %d records for %s\n",
>+			    state->pnn_list[i], num_records,
>+			    state->num_records, recdb_name(state->recdb));
>+			tevent_req_error(req, EPROTO);
>+			return;
>+		}
>+	}
>+
>+	talloc_free(reply);
>+
>+	LOG("Pushed %d records for db %s\n",
>+	    state->num_records, recdb_name(state->recdb));
>+
>+	tevent_req_done(req);
>+}
>+
>+static bool push_database_new_recv(struct tevent_req *req, int *perr)
>+{
>+	return generic_recv(req, perr);
>+}
>+
>+/*
>+ * wrapper for push_database_old and push_database_new
>+ */
>+
>+struct push_database_state {
>+	bool old_done, new_done;
>+};
>+
>+static void push_database_old_done(struct tevent_req *subreq);
>+static void push_database_new_done(struct tevent_req *subreq);
>+
>+static struct tevent_req *push_database_send(
>+			TALLOC_CTX *mem_ctx,
>+			struct tevent_context *ev,
>+			struct ctdb_client_context *client,
>+			uint32_t *pnn_list, int count, uint32_t *caps,
>+			struct ctdb_tunable_list *tun_list,
>+			struct recdb_context *recdb)
>+{
>+	struct tevent_req *req, *subreq;
>+	struct push_database_state *state;
>+	uint32_t *old_list, *new_list;
>+	int old_count, new_count;
>+	int i;
>+
>+	req = tevent_req_create(mem_ctx, &state, struct push_database_state);
>+	if (req == NULL) {
>+		return NULL;
>+	}
>+
>+	state->old_done = false;
>+	state->new_done = false;
>+
>+	old_count = 0;
>+	new_count = 0;
>+	old_list = talloc_array(state, uint32_t, count);
>+	new_list = talloc_array(state, uint32_t, count);
>+	if (tevent_req_nomem(old_list, req) ||
>+	    tevent_req_nomem(new_list,req)) {
>+		return tevent_req_post(req, ev);
>+	}
>+
>+	for (i=0; i<count; i++) {
>+		uint32_t pnn = pnn_list[i];
>+
>+		if (caps[pnn] & CTDB_CAP_FRAGMENTED_CONTROLS) {
>+			new_list[new_count] = pnn;
>+			new_count += 1;
>+		} else {
>+			old_list[old_count] = pnn;
>+			old_count += 1;
>+		}
>+	}
>+
>+	if (old_count > 0) {
>+		subreq = push_database_old_send(state, ev, client,
>+						old_list, old_count, recdb);
>+		if (tevent_req_nomem(subreq, req)) {
>+			return tevent_req_post(req, ev);
>+		}
>+		tevent_req_set_callback(subreq, push_database_old_done, req);
>+	} else {
>+		state->old_done = true;
>+	}
>+
>+	if (new_count > 0) {
>+		subreq = push_database_new_send(state, ev, client,
>+						new_list, new_count, recdb,
>+						tun_list->rec_buffer_size_limit);
>+		if (tevent_req_nomem(subreq, req)) {
>+			return tevent_req_post(req, ev);
>+		}
>+		tevent_req_set_callback(subreq, push_database_new_done, req);
>+	} else {
>+		state->new_done = true;
>+	}
>+
>+	return req;
>+}
>+
>+static void push_database_old_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct push_database_state *state = tevent_req_data(
>+		req, struct push_database_state);
>+	bool status;
>+	int ret;
>+
>+	status = push_database_old_recv(subreq, &ret);
>+	if (! status) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	state->old_done = true;
>+
>+	if (state->old_done && state->new_done) {
>+		tevent_req_done(req);
>+	}
>+}
>+
>+static void push_database_new_done(struct tevent_req *subreq)
>+{
>+	struct tevent_req *req = tevent_req_callback_data(
>+		subreq, struct tevent_req);
>+	struct push_database_state *state = tevent_req_data(
>+		req, struct push_database_state);
>+	bool status;
>+	int ret;
>+
>+	status = push_database_new_recv(subreq, &ret);
>+	if (! status) {
>+		tevent_req_error(req, ret);
>+		return;
>+	}
>+
>+	state->new_done = true;
>+
>+	if (state->old_done && state->new_done) {
>+		tevent_req_done(req);
>+	}
>+}
>+
>+static bool push_database_recv(struct tevent_req *req, int *perr)
>+{
>+	return generic_recv(req, perr);
>+}
>+
>+/*
>  * Collect databases using highest sequence number
>  */
> 
>@@ -951,8 +1469,6 @@ struct recover_db_state {
> 
> 	const char *db_name, *db_path;
> 	struct recdb_context *recdb;
>-	struct ctdb_rec_buffer *recbuf;
>-
> };
> 
> static void recover_db_name_done(struct tevent_req *subreq);
>@@ -1225,7 +1741,6 @@ static void recover_db_wipedb_done(struct tevent_req *subreq)
> 		subreq, struct tevent_req);
> 	struct recover_db_state *state = tevent_req_data(
> 		req, struct recover_db_state);
>-	struct ctdb_req_control request;
> 	int *err_list;
> 	int ret;
> 	bool status;
>@@ -1251,18 +1766,10 @@ static void recover_db_wipedb_done(struct tevent_req *subreq)
> 		return;
> 	}
> 
>-	state->recbuf = recdb_records(state->recdb, state, state->destnode);
>-	if (tevent_req_nomem(state->recbuf, req)) {
>-		return;
>-	}
>-
>-	TALLOC_FREE(state->recdb);
>-
>-	ctdb_req_control_push_db(&request, state->recbuf);
>-	subreq = ctdb_client_control_multi_send(state, state->ev,
>-						state->client,
>-						state->pnn_list, state->count,
>-						TIMEOUT(), &request);
>+	subreq = push_database_send(state, state->ev, state->client,
>+				    state->pnn_list, state->count,
>+				    state->caps, state->tun_list,
>+				    state->recdb);
> 	if (tevent_req_nomem(subreq, req)) {
> 		return;
> 	}
>@@ -1276,32 +1783,17 @@ static void recover_db_pushdb_done(struct tevent_req *subreq)
> 	struct recover_db_state *state = tevent_req_data(
> 		req, struct recover_db_state);
> 	struct ctdb_req_control request;
>-	int *err_list;
> 	int ret;
> 	bool status;
> 
>-	status = ctdb_client_control_multi_recv(subreq, &ret, NULL, &err_list,
>-						NULL);
>+	status = push_database_recv(subreq, &ret);
> 	TALLOC_FREE(subreq);
> 	if (! status) {
>-		int ret2;
>-		uint32_t pnn;
>-
>-		ret2 = ctdb_client_control_multi_error(state->pnn_list,
>-						       state->count,
>-						       err_list, &pnn);
>-		if (ret2 != 0) {
>-			LOG("control PUSHDB failed for db %s on node %u,"
>-			    " ret=%d\n", state->db_name, pnn, ret2);
>-		} else {
>-			LOG("control PUSHDB failed for db %s, ret=%d\n",
>-			    state->db_name, ret);
>-		}
> 		tevent_req_error(req, ret);
> 		return;
> 	}
> 
>-	TALLOC_FREE(state->recbuf);
>+	TALLOC_FREE(state->recdb);
> 
> 	ctdb_req_control_db_transaction_commit(&request, &state->transdb);
> 	subreq = ctdb_client_control_multi_send(state, state->ev,
>-- 
>2.5.5
>
>
>From 0cc13ecde2d16f925d2a6b3a1826d6828c7f0f93 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Sat, 5 Mar 2016 15:04:48 +1100
>Subject: [PATCH 25/29] ctdb-tests: Add a test for recovery of large databases
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit a7b8ee87fe1b58ffb9da02ae8e3959a385e628d9)
>---
> ctdb/tests/simple/78_ctdb_large_db_recovery.sh | 107 +++++++++++++++++++++++++
> 1 file changed, 107 insertions(+)
> create mode 100755 ctdb/tests/simple/78_ctdb_large_db_recovery.sh
>
>diff --git a/ctdb/tests/simple/78_ctdb_large_db_recovery.sh b/ctdb/tests/simple/78_ctdb_large_db_recovery.sh
>new file mode 100755
>index 0000000..51fbd25
>--- /dev/null
>+++ b/ctdb/tests/simple/78_ctdb_large_db_recovery.sh
>@@ -0,0 +1,107 @@
>+#!/bin/bash
>+
>+test_info()
>+{
>+    cat <<EOF
>+Older style of recovery using PULL_DB and PUSH_DB controls tries to
>+construct a single large marshall buffer for all the records in the
>+database.  However, this approach is problematic as talloc restricts the
>+maximum size of buffer to 256M.  Also, trying to construct and send large
>+buffers is inefficient and can cause CTDB daemon to be tied up for long
>+periods of time.
>+
>+Instead new style recovery is introduced using DB_PULL and
>+DB_PUSH_START/DB_PUSH_CONFIRM controls.  This sends the records in
>+batches of ~RecBufferSizeLimit in size at a time.
>+
>+Expected results:
>+
>+* The recovery should complete successfully
>+
>+EOF
>+}
>+
>+. "${TEST_SCRIPTS_DIR}/integration.bash"
>+
>+ctdb_test_init "$@"
>+
>+set -e
>+
>+cluster_is_healthy
>+
>+# Reset configuration
>+ctdb_restart_when_done
>+
>+#
>+# Main test
>+#
>+TEST1DB="large_persistent_db.tdb"
>+TEST2DB="large_volatile_db.tdb"
>+RECDATA=$(onnode 0 mktemp)
>+
>+# Create a persistent database to test
>+echo "create persistent test database $TEST1DB"
>+try_command_on_node 0 $CTDB attach $TEST1DB persistent
>+
>+# Wipe Test database
>+echo "wipe test database $TEST1DB"
>+try_command_on_node 0 $CTDB wipedb $TEST1DB
>+
>+# Create dummy record data
>+echo "creating dummy record data"
>+onnode 0 dd if=/dev/urandom of=$RECDATA bs=10K count=1
>+
>+# Add 345 records
>+echo "Adding 345 records"
>+for i in $(seq 1 345) ; do
>+    try_command_on_node 0 $CTDB pstore $TEST1DB record$i $RECDATA || exit 1
>+done
>+
>+# Create a volatile database to test
>+echo "create volatile test database $TEST2DB"
>+try_command_on_node 0 $CTDB attach $TEST2DB
>+
>+# Wipe Test database
>+echo "wipe test database $TEST2DB"
>+try_command_on_node 0 $CTDB wipedb $TEST2DB
>+
>+# Create dummy record data
>+v1="1234567890"
>+v2="$v1$v1$v1$v1$v1$v1$v1$v1$v1$v1"
>+v3="$v2$v2$v2$v2$v2$v2$v2$v2$v2$v2"
>+
>+# Add 1234 records
>+echo "Adding 1234 records"
>+for i in $(seq 1 1234) ; do
>+    try_command_on_node 0 $CTDB writekey $TEST2DB record$i $v3 || exit 1
>+done
>+
>+echo "Find out which node is recmaster"
>+try_command_on_node 0 $CTDB recmaster
>+recmaster="$out"
>+
>+# Set RecBufferSizeLimit to 10000
>+try_command_on_node $recmaster $CTDB setvar RecBufferSizeLimit 10000
>+
>+# Do a recovery
>+echo "force recovery"
>+try_command_on_node 0 $CTDB recover
>+
>+wait_until_node_has_status 0 recovered 30
>+
>+# check that there are correct number of records
>+num_records=$(db_ctdb_cattdb_count_records 0 $TEST1DB)
>+if [ $num_records = "345" ] ; then
>+    echo "OK: persistent database recovered correctly"
>+else
>+    echo "BAD: persistent database ended up with $num_records of 345 records"
>+    exit 1
>+fi
>+
>+num_records=$(db_ctdb_cattdb_count_records 0 $TEST2DB)
>+if [ $num_records = "1234" ] ; then
>+    echo "OK: volatile database recovered correctly"
>+else
>+    echo "BAD: volatile database ended up with $num_records of 1234 records"
>+    exit 1
>+fi
>-- 
>2.5.5
>
>
>From 7aaf61ca902c9e1c0276c258156f46b28f209080 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Thu, 17 Mar 2016 14:13:02 +1100
>Subject: [PATCH 26/29] ctdb-recovery-helper: Improve log message
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit e5a714a3c29fd8fdebe0203e68c7ce19f54af2d3)
>---
> ctdb/server/ctdb_recovery_helper.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 9991d31..c741c2c 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -2541,7 +2541,7 @@ static void recovery_db_recovery_done(struct tevent_req *subreq)
> 	status = db_recovery_recv(subreq, &count);
> 	TALLOC_FREE(subreq);
> 
>-	LOG("%d databases recovered\n", count);
>+	LOG("%d of %d databases recovered\n", count, state->dbmap->num);
> 
> 	if (! status) {
> 		tevent_req_error(req, EIO);
>-- 
>2.5.5
>
>
>From df0b2a82f4badb77c1b93ad6df6443354d3b6284 Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Thu, 17 Mar 2016 14:22:17 +1100
>Subject: [PATCH 27/29] ctdb-recovery-helper: Introduce new #define variable
>
>... instead of hardcoding number of retries.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>(cherry picked from commit ad7a407a13b87ec13d94a808111d2583bfd1d217)
>---
> ctdb/server/ctdb_recovery_helper.c | 4 +++-
> 1 file changed, 3 insertions(+), 1 deletion(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index c741c2c..5911644 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -36,6 +36,8 @@
> 
> static int recover_timeout = 120;
> 
>+#define NUM_RETRIES	5
>+
> #define TIMEOUT()	timeval_current_ofs(recover_timeout, 0)
> 
> static void LOG(const char *fmt, ...)
>@@ -1999,7 +2001,7 @@ static void db_recovery_one_done(struct tevent_req *subreq)
> 	}
> 
> 	substate->num_fails += 1;
>-	if (substate->num_fails < 5) {
>+	if (substate->num_fails < NUM_RETRIES) {
> 		subreq = recover_db_send(state, state->ev, substate->client,
> 					 substate->tun_list,
> 					 substate->pnn_list, substate->count,
>-- 
>2.5.5
>
>
>From aee8459b09215abc26f10aeb4187fee92374bd2f Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Thu, 2 Jun 2016 18:27:29 +1000
>Subject: [PATCH 28/29] ctdb-recovery: Update timeout and number of retries
> during recovery
>
>The timeout RecoverTimeout (default 120) is used for control messages
>sent during the recovery.  If any of the nodes does not respond to any
>of the recovery control messages for RecoverTimeout seconds, then it
>will cause a failure of recovery of a database.  Recovery helper will
>retry the recovery for a database 5 times.
>
>In the worst case, if a database could not be recovered within 5 attempts,
>a total of 600 seconds would have passed.  During this time period other
>timeouts will be triggered causing unnecessary failures as follows:
>
>1. During the recovery, even though recoverd is processing events,
>   it does not send a ping message to ctdb daemon.  If a ping message is
>   not received for RecdPingTimeout (default 60) seconds, then ctdb will
>   count it as unresponsive recovery daemon.  If the recovery daemon
>   fails for RecdFailCount (default 10) times, then ctdb daemon will
>   restart recovery daemon.  So after 600 seconds, ctdb daemon will
>   restart recovery daemon.
>
>2. If ctdb daemon stays in recovery for RecoveryDropAllIPs (default 120),
>   then it will drop all the public addresses.  This will cause all
>   SMB client to be disconnected unnecessarily.  The released public
>   addresses will not be taken over till the recovery is complete.
>
>To avoid dropping of IPs and restarting recovery daemon during a delayed
>recovery, adjust RecoverTimeout to 30 seconds and limit number of
>retries for recovering a database to 3.  If we don't hear from a node
>for more than 25 seconds, then the node is considered disconnected.
>So 30 seconds is sufficient timeout for controls during recovery.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>
>Autobuild-User(master): Martin Schwenke <martins@samba.org>
>Autobuild-Date(master): Mon Jun  6 08:49:15 CEST 2016 on sn-devel-144
>
>(cherry picked from commit 93dcca2a5f7af9698c9ba1024dbce1d1a66d4efb)
>---
> ctdb/server/ctdb_recovery_helper.c | 4 ++--
> ctdb/server/ctdb_tunables.c        | 2 +-
> 2 files changed, 3 insertions(+), 3 deletions(-)
>
>diff --git a/ctdb/server/ctdb_recovery_helper.c b/ctdb/server/ctdb_recovery_helper.c
>index 5911644..d1ec1e7 100644
>--- a/ctdb/server/ctdb_recovery_helper.c
>+++ b/ctdb/server/ctdb_recovery_helper.c
>@@ -34,9 +34,9 @@
> #include "protocol/protocol_api.h"
> #include "client/client.h"
> 
>-static int recover_timeout = 120;
>+static int recover_timeout = 30;
> 
>-#define NUM_RETRIES	5
>+#define NUM_RETRIES	3
> 
> #define TIMEOUT()	timeval_current_ofs(recover_timeout, 0)
> 
>diff --git a/ctdb/server/ctdb_tunables.c b/ctdb/server/ctdb_tunables.c
>index 50c5c77..7cb1a24 100644
>--- a/ctdb/server/ctdb_tunables.c
>+++ b/ctdb/server/ctdb_tunables.c
>@@ -41,7 +41,7 @@ static const struct {
> 	{ "TraverseTimeout",     20, offsetof(struct ctdb_tunable_list, traverse_timeout), false },
> 	{ "KeepaliveInterval",    5,  offsetof(struct ctdb_tunable_list, keepalive_interval), false },
> 	{ "KeepaliveLimit",       5,  offsetof(struct ctdb_tunable_list, keepalive_limit), false },
>-	{ "RecoverTimeout",     120,  offsetof(struct ctdb_tunable_list, recover_timeout), false },
>+	{ "RecoverTimeout",      30,  offsetof(struct ctdb_tunable_list, recover_timeout), false },
> 	{ "RecoverInterval",      1,  offsetof(struct ctdb_tunable_list, recover_interval), false },
> 	{ "ElectionTimeout",      3,  offsetof(struct ctdb_tunable_list, election_timeout), false },
> 	{ "TakeoverTimeout",      9,  offsetof(struct ctdb_tunable_list, takeover_timeout), false },
>-- 
>2.5.5
>
>
>From ff6780d50a5f0b3028db0ebc80a9751cdc41e87d Mon Sep 17 00:00:00 2001
>From: Amitay Isaacs <amitay@gmail.com>
>Date: Wed, 8 Jun 2016 15:04:52 +1000
>Subject: [PATCH 29/29] ctdb-daemon: Reset push_started flag once
> DB_PUSH_CONFIRM is done
>
>Once DB_PUSH_START is processed as part of recovery, push_started
>flag tracks if there are multiple attempts to send DB_PUSH_START.
>In DB_PUSH_CONFIRM, once the record count is confirmed, all information
>related to DB_PUSH should be reset.  However, The push_started flag was
>not reset when the push_state was reset.
>
>Signed-off-by: Amitay Isaacs <amitay@gmail.com>
>Reviewed-by: Martin Schwenke <martin@meltin.net>
>
>Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
>Autobuild-Date(master): Wed Jun  8 14:31:52 CEST 2016 on sn-devel-144
>
>(cherry picked from commit c620bf5debd57a4a5d7f893a2b6383098ff7a919)
>---
> ctdb/server/ctdb_recover.c | 1 +
> 1 file changed, 1 insertion(+)
>
>diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c
>index 49f568a..47461f4 100644
>--- a/ctdb/server/ctdb_recover.c
>+++ b/ctdb/server/ctdb_recover.c
>@@ -742,6 +742,7 @@ int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb,
> 	outdata->dsize = sizeof(uint32_t);
> 
> 	talloc_free(state);
>+	ctdb_db->push_started = false;
> 	ctdb_db->push_state = NULL;
> 
> 	return 0;
>-- 
>2.5.5
>

Flags:

martins: review+

Actions: View

Attachments on bug 11940: 12169