From efe26156d0997d5c06c047aaac69cf85a816bf4e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:46:23 +0100 Subject: [PATCH 01/18] s4:scripting/python: always treat the highwatermark as opaque (bug #9508) Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 257ae5443631e645842cfcc9c1cedce6c41d5afa) --- source4/scripting/devel/getncchanges | 2 +- source4/scripting/devel/repl_cleartext_pwd.py | 2 +- source4/scripting/python/samba/drs_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source4/scripting/devel/getncchanges b/source4/scripting/devel/getncchanges index 736e3db..d401c82 100755 --- a/source4/scripting/devel/getncchanges +++ b/source4/scripting/devel/getncchanges @@ -177,4 +177,4 @@ if __name__ == "__main__": nb_iter += 1 if ctr.more_data == 0 or opts.nb_iter == nb_iter: break - req8.highwatermark.tmp_highest_usn = ctr.new_highwatermark.tmp_highest_usn + req8.highwatermark = ctr.new_highwatermark diff --git a/source4/scripting/devel/repl_cleartext_pwd.py b/source4/scripting/devel/repl_cleartext_pwd.py index 5eebc86..9637d88 100755 --- a/source4/scripting/devel/repl_cleartext_pwd.py +++ b/source4/scripting/devel/repl_cleartext_pwd.py @@ -406,4 +406,4 @@ if __name__ == "__main__": print "# up to usn[%d]" % (ctr.new_highwatermark.highest_usn) break print "# up to tmp_usn[%d]" % (ctr.new_highwatermark.highest_usn) - req8.highwatermark.tmp_highest_usn = ctr.new_highwatermark.tmp_highest_usn + req8.highwatermark = ctr.new_highwatermark diff --git a/source4/scripting/python/samba/drs_utils.py b/source4/scripting/python/samba/drs_utils.py index 481eec2..6e2cfea 100644 --- a/source4/scripting/python/samba/drs_utils.py +++ b/source4/scripting/python/samba/drs_utils.py @@ -252,4 +252,4 @@ class drs_Replicate(object): schema=schema, req_level=req_level, req=req) if ctr.more_data == 0: break - req.highwatermark.tmp_highest_usn = ctr.new_highwatermark.tmp_highest_usn + req.highwatermark = ctr.new_highwatermark -- 1.7.9.5 From 22949cea5acfaa0acf7a28e6ce8fb692bee8fa5d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:46:23 +0100 Subject: [PATCH 02/18] s4:dsdb/repl_meta_data: always treat the highwatermark as opaque (bug #9508) Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit a37f46a9a83a03157276485eb583649b36fb6ee1) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 3ac1e6a..868484d 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4652,7 +4652,6 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a ZERO_STRUCT(nrf); nrf.version = 1; nrf.ctr.ctr1 = *ar->objs->source_dsa; - nrf.ctr.ctr1.highwatermark.highest_usn = nrf.ctr.ctr1.highwatermark.tmp_highest_usn; /* * first see if we already have a repsFrom value for the current source dsa -- 1.7.9.5 From e061a9f6bf2182d0c3831267c90719c9f77d499d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 17:29:04 +0100 Subject: [PATCH 03/18] s4:dsdb/repl_meta_data: store the last results and timestamps in the repsFrom Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 634f8cf7c43bd60507d842d35cf46c0017e34dce) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 868484d..6a69f7d 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4652,6 +4652,9 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a ZERO_STRUCT(nrf); nrf.version = 1; nrf.ctr.ctr1 = *ar->objs->source_dsa; + nrf.ctr.ctr1.last_attempt = now; + nrf.ctr.ctr1.last_success = now; + nrf.ctr.ctr1.result_last_attempt = WERR_OK; /* * first see if we already have a repsFrom value for the current source dsa -- 1.7.9.5 From a1149f0d81df712b9fa9cf138d1bf58ac91ed406 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 20 Dec 2012 15:46:05 +0100 Subject: [PATCH 04/18] s4:dsdb/repl_meta_data: also update the last_sync_success in replUpToDateVector This matches Windows 2008R2 and Windows 2012. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit ad43bb6086a7dbf48b405d0372ae85d2244384d9) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 6a69f7d..b7d9248 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4563,12 +4563,8 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a found = true; - /* - * we update only the highest_usn and not the latest_sync_success time, - * because the last success stands for direct replication - */ if (ruv->cursors[i].highest_usn > nuv.ctr.ctr2.cursors[j].highest_usn) { - nuv.ctr.ctr2.cursors[j].highest_usn = ruv->cursors[i].highest_usn; + nuv.ctr.ctr2.cursors[j] = ruv->cursors[i]; } break; } -- 1.7.9.5 From 7dbb6d5c1b8cb9bfac625468fbc579b859ece37f Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:46:23 +0100 Subject: [PATCH 05/18] s4:dsdb/repl_meta_data: don't merge highwatermark and uptodatevector (bug #9508) We should not do any magic regarding the highwatermark we got from the source dsa. We need to treat it as opaque and not try to be smart and merge it into the uptodatevector. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 5ecbc892b5226d3d31da2c62ae5261a8d8a73072) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 39 +---------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index b7d9248..30b2a42 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4529,7 +4529,7 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a * * plus optional values from our old vector and the one from the source_dsa */ - nuv.ctr.ctr2.count = 1 + ouv.ctr.ctr2.count; + nuv.ctr.ctr2.count = ouv.ctr.ctr2.count; if (ruv) nuv.ctr.ctr2.count += ruv->count; nuv.ctr.ctr2.cursors = talloc_array(ar, struct drsuapi_DsReplicaCursor2, @@ -4577,43 +4577,6 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a } /* - * merge in the current highwatermark for the source_dsa - */ - found = false; - for (j=0; j < ni; j++) { - if (!GUID_equal(&ar->objs->source_dsa->source_dsa_invocation_id, - &nuv.ctr.ctr2.cursors[j].source_dsa_invocation_id)) { - continue; - } - - found = true; - - /* - * here we update the highest_usn and last_sync_success time - * because we're directly replicating from the source_dsa - * - * and use the tmp_highest_usn because this is what we have just applied - * to our ldb - */ - nuv.ctr.ctr2.cursors[j].highest_usn = ar->objs->source_dsa->highwatermark.tmp_highest_usn; - nuv.ctr.ctr2.cursors[j].last_sync_success = now; - break; - } - if (!found) { - /* - * here we update the highest_usn and last_sync_success time - * because we're directly replicating from the source_dsa - * - * and use the tmp_highest_usn because this is what we have just applied - * to our ldb - */ - nuv.ctr.ctr2.cursors[ni].source_dsa_invocation_id= ar->objs->source_dsa->source_dsa_invocation_id; - nuv.ctr.ctr2.cursors[ni].highest_usn = ar->objs->source_dsa->highwatermark.tmp_highest_usn; - nuv.ctr.ctr2.cursors[ni].last_sync_success = now; - ni++; - } - - /* * finally correct the size of the cursors array */ nuv.ctr.ctr2.count = ni; -- 1.7.9.5 From a7db24d5c1b33dd69b690d15188b08d6ff27de6b Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 12:47:43 +0100 Subject: [PATCH 06/18] s4:dsdb/common: use LDB_SEQ_HIGHEST_SEQ for our entry in the uptodatevector We should use the global highestCommittedUSN, not the per partition value. This matches a Windows 2008R2 and 2012 server. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 81fa179b155a62f2f652fbb1fc4978c9f6eb5462) --- source4/dsdb/common/util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/dsdb/common/util.c b/source4/dsdb/common/util.c index 4543003..894fd3c 100644 --- a/source4/dsdb/common/util.c +++ b/source4/dsdb/common/util.c @@ -3487,7 +3487,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m const struct ldb_val *ouv_value; unsigned int i; int ret; - uint64_t highest_usn; + uint64_t highest_usn = 0; const struct GUID *our_invocation_id; struct timeval now = timeval_current(); @@ -3530,7 +3530,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m return ldb_operr(samdb); } - ret = dsdb_load_partition_usn(samdb, dn, &highest_usn, NULL); + ret = ldb_sequence_number(samdb, LDB_SEQ_HIGHEST_SEQ, &highest_usn); if (ret != LDB_SUCCESS) { /* nothing to add - this can happen after a vampire */ TYPESAFE_QSORT(*cursors, *count, drsuapi_DsReplicaCursor2_compare); -- 1.7.9.5 From c23d005dda169bbdb705f73e42c2f7f71332caa2 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 17:33:13 +0100 Subject: [PATCH 07/18] s4:dsdb/common: use 01.01.1970 as last_sync_success for our entry in the uptodatevector This matches a Windows 2008R2 and 2012 server. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit e7a26d02413005294180a1d9cd4c90d4ac4d9733) --- source4/dsdb/common/util.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source4/dsdb/common/util.c b/source4/dsdb/common/util.c index 894fd3c..2b96bd4 100644 --- a/source4/dsdb/common/util.c +++ b/source4/dsdb/common/util.c @@ -3489,7 +3489,8 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m int ret; uint64_t highest_usn = 0; const struct GUID *our_invocation_id; - struct timeval now = timeval_current(); + static const struct timeval tv1970; + NTTIME nt1970 = timeval_to_nttime(&tv1970); ret = ldb_search(samdb, mem_ctx, &r, dn, LDB_SCOPE_BASE, attrs, NULL); if (ret != LDB_SUCCESS) { @@ -3540,7 +3541,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m for (i=0; i<*count; i++) { if (GUID_equal(our_invocation_id, &(*cursors)[i].source_dsa_invocation_id)) { (*cursors)[i].highest_usn = highest_usn; - (*cursors)[i].last_sync_success = timeval_to_nttime(&now); + (*cursors)[i].last_sync_success = nt1970; TYPESAFE_QSORT(*cursors, *count, drsuapi_DsReplicaCursor2_compare); return LDB_SUCCESS; } @@ -3553,7 +3554,7 @@ int dsdb_load_udv_v2(struct ldb_context *samdb, struct ldb_dn *dn, TALLOC_CTX *m (*cursors)[*count].source_dsa_invocation_id = *our_invocation_id; (*cursors)[*count].highest_usn = highest_usn; - (*cursors)[*count].last_sync_success = timeval_to_nttime(&now); + (*cursors)[*count].last_sync_success = nt1970; (*count)++; TYPESAFE_QSORT(*cursors, *count, drsuapi_DsReplicaCursor2_compare); -- 1.7.9.5 From f1c5121ff399a97ff7e9c06c0ee22abc2690998e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 19 Dec 2012 17:31:28 +0100 Subject: [PATCH 08/18] s4:dsdb/drepl: update the source_dsa_obj/invocation_id in repsFrom The highwatermark is relative to the source_dsa_invocation_id. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 2e9b06412b09163d4b851135ef509d73bb6d61fc) --- source4/dsdb/repl/drepl_out_helpers.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source4/dsdb/repl/drepl_out_helpers.c b/source4/dsdb/repl/drepl_out_helpers.c index 16825d4..57205a8 100644 --- a/source4/dsdb/repl/drepl_out_helpers.c +++ b/source4/dsdb/repl/drepl_out_helpers.c @@ -627,6 +627,8 @@ static void dreplsrv_op_pull_source_apply_changes_trigger(struct tevent_req *req first_object = ctr1->first_object; linked_attributes_count = 0; linked_attributes = NULL; + rf1.source_dsa_obj_guid = ctr1->source_dsa_guid; + rf1.source_dsa_invocation_id = ctr1->source_dsa_invocation_id; rf1.highwatermark = ctr1->new_highwatermark; uptodateness_vector = NULL; /* TODO: map it */ more_data = ctr1->more_data; @@ -637,6 +639,8 @@ static void dreplsrv_op_pull_source_apply_changes_trigger(struct tevent_req *req first_object = ctr6->first_object; linked_attributes_count = ctr6->linked_attributes_count; linked_attributes = ctr6->linked_attributes; + rf1.source_dsa_obj_guid = ctr6->source_dsa_guid; + rf1.source_dsa_invocation_id = ctr6->source_dsa_invocation_id; rf1.highwatermark = ctr6->new_highwatermark; uptodateness_vector = ctr6->uptodateness_vector; more_data = ctr6->more_data; -- 1.7.9.5 From ba201668c4b952f313715a95fa9dd4e40b9cbd85 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 14:08:56 +0100 Subject: [PATCH 09/18] s4:drsuapi: move struct drsuapi_getncchanges_state to the top of getncchanges.c Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 551bb2ccea6a1d82dbe0d4a21c19a8d8bd13ccbc) --- source4/rpc_server/drsuapi/getncchanges.c | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 09406d6..00988fc 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -37,6 +37,23 @@ #include "auth/session.h" #include "dsdb/common/util.h" +/* state of a partially completed getncchanges call */ +struct drsuapi_getncchanges_state { + struct GUID *guids; + uint32_t num_records; + uint32_t num_processed; + struct ldb_dn *ncRoot_dn; + bool is_schema_nc; + uint64_t min_usn; + uint64_t highest_usn; + struct ldb_dn *last_dn; + struct drsuapi_DsReplicaLinkedAttribute *la_list; + uint32_t la_count; + bool la_sorted; + uint32_t la_idx; + struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector; +}; + /* build a DsReplicaObjectIdentifier from a ldb msg */ @@ -1147,23 +1164,6 @@ static WERROR getncchanges_change_master(struct drsuapi_bind_state *b_state, return WERR_OK; } -/* state of a partially completed getncchanges call */ -struct drsuapi_getncchanges_state { - struct GUID *guids; - uint32_t num_records; - uint32_t num_processed; - struct ldb_dn *ncRoot_dn; - bool is_schema_nc; - uint64_t min_usn; - uint64_t highest_usn; - struct ldb_dn *last_dn; - struct drsuapi_DsReplicaLinkedAttribute *la_list; - uint32_t la_count; - bool la_sorted; - uint32_t la_idx; - struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector; -}; - /* see if this getncchanges request includes a request to reveal secret information */ -- 1.7.9.5 From 8e9f9da36bf6860c21aec898aef5cf2f5e656566 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 13:48:01 +0100 Subject: [PATCH 10/18] s4:drsuapi: remove unused 'highest_usn' from drsuapi_getncchanges_state Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 30be17bc5d6b3cf2ee0aef6663af78b153b2ab9a) --- source4/rpc_server/drsuapi/getncchanges.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 00988fc..b7b488c 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -45,7 +45,6 @@ struct drsuapi_getncchanges_state { struct ldb_dn *ncRoot_dn; bool is_schema_nc; uint64_t min_usn; - uint64_t highest_usn; struct ldb_dn *last_dn; struct drsuapi_DsReplicaLinkedAttribute *la_list; uint32_t la_count; @@ -1876,9 +1875,6 @@ allowed: if (uSN > r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn) { r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn = uSN; } - if (uSN > getnc_state->highest_usn) { - getnc_state->highest_usn = uSN; - } if (obj->meta_data_ctr == NULL) { DEBUG(8,(__location__ ": getncchanges skipping send of object %s\n", -- 1.7.9.5 From 534c0b1309c5d6a70d0b116c9b1c24dc784fc79d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 12:44:43 +0100 Subject: [PATCH 11/18] s4:drsuapi: avoid a ldb_dn_copy() and use talloc_move() instead Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 025c6d62f3c1b0f760aaacb7b3960135319031da) --- source4/rpc_server/drsuapi/getncchanges.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index b7b488c..7ed47f1 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -1889,11 +1889,11 @@ allowed: *currentObject = obj; currentObject = &obj->next_object; - talloc_free(getnc_state->last_dn); - getnc_state->last_dn = ldb_dn_copy(getnc_state, msg->dn); - DEBUG(8,(__location__ ": replicating object %s\n", ldb_dn_get_linearized(msg->dn))); + talloc_free(getnc_state->last_dn); + getnc_state->last_dn = talloc_move(getnc_state, &msg->dn); + talloc_free(msg_res); talloc_free(msg_dn); } -- 1.7.9.5 From d3c380b724305808ce6185c5b00a7febe09806dd Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 16:34:25 +0100 Subject: [PATCH 12/18] s4:drsuapi: always use the current uptodateness_vector Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 02de5b140cfe6ea31e0686e5f0ff726a22153020) --- source4/rpc_server/drsuapi/getncchanges.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 7ed47f1..b3ceb50 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -50,7 +50,6 @@ struct drsuapi_getncchanges_state { uint32_t la_count; bool la_sorted; uint32_t la_idx; - struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector; }; /* @@ -1730,14 +1729,6 @@ allowed: site_res_cmp_dn_usn_order); } - getnc_state->uptodateness_vector = talloc_steal(getnc_state, req10->uptodateness_vector); - if (getnc_state->uptodateness_vector) { - /* make sure its sorted */ - TYPESAFE_QSORT(getnc_state->uptodateness_vector->cursors, - getnc_state->uptodateness_vector->count, - drsuapi_DsReplicaCursor_compare); - } - for (i=0; i < getnc_state->num_records; i++) { getnc_state->guids[i] = changes[i].guid; if (GUID_all_zero(&getnc_state->guids[i])) { @@ -1751,6 +1742,13 @@ allowed: talloc_free(changes); } + if (req10->uptodateness_vector) { + /* make sure its sorted */ + TYPESAFE_QSORT(req10->uptodateness_vector->cursors, + req10->uptodateness_vector->count, + drsuapi_DsReplicaCursor_compare); + } + /* Prefix mapping */ schema = dsdb_get_schema(sam_ctx, mem_ctx); if (!schema) { @@ -1852,7 +1850,7 @@ allowed: schema, &session_key, getnc_state->min_usn, req10->replica_flags, req10->partial_attribute_set, - getnc_state->uptodateness_vector, + req10->uptodateness_vector, req10->extended_op, max_wait_reached); if (!W_ERROR_IS_OK(werr)) { @@ -1866,7 +1864,7 @@ allowed: msg, &getnc_state->la_list, &getnc_state->la_count, - getnc_state->uptodateness_vector); + req10->uptodateness_vector); if (!W_ERROR_IS_OK(werr)) { return werr; } -- 1.7.9.5 From 2dc13133abb6aa9094d3e0f9f270754b42a33a06 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 11:13:43 +0100 Subject: [PATCH 13/18] s4:drsuapi: add drsuapi_DsReplicaHighWaterMark_cmp() Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 7e511b58318cef1b325a8191685ee156a7fc0cb7) --- source4/rpc_server/drsuapi/getncchanges.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index b3ceb50..1f597c6 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -52,6 +52,26 @@ struct drsuapi_getncchanges_state { uint32_t la_idx; }; +static int drsuapi_DsReplicaHighWaterMark_cmp(const struct drsuapi_DsReplicaHighWaterMark *h1, + const struct drsuapi_DsReplicaHighWaterMark *h2) +{ + if (h1->highest_usn < h2->highest_usn) { + return -1; + } else if (h1->highest_usn > h2->highest_usn) { + return 1; + } else if (h1->tmp_highest_usn < h2->tmp_highest_usn) { + return -1; + } else if (h1->tmp_highest_usn > h2->tmp_highest_usn) { + return 1; + } else if (h1->reserved_usn < h2->reserved_usn) { + return -1; + } else if (h1->reserved_usn > h2->reserved_usn) { + return 1; + } + + return 0; +} + /* build a DsReplicaObjectIdentifier from a ldb msg */ -- 1.7.9.5 From 2384caabd34fbf3758b8edac0850fa14224a9b81 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 17 Dec 2012 11:30:26 +0100 Subject: [PATCH 14/18] s4:drsuapi: make sure we never return the same highwatermark twice in a replication cycle (bug #9508) If the highwatermark given by the client is not the one we expect, we need to start a new replication cycle. Otherwise the destination dsa skips objects and linked attribute values. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 91f7f2c04fd00e281b0755a331ca632a4905e3b5) --- source4/rpc_server/drsuapi/getncchanges.c | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 1f597c6..e06aeed 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -45,6 +45,7 @@ struct drsuapi_getncchanges_state { struct ldb_dn *ncRoot_dn; bool is_schema_nc; uint64_t min_usn; + struct drsuapi_DsReplicaHighWaterMark last_hwm; struct ldb_dn *last_dn; struct drsuapi_DsReplicaLinkedAttribute *la_list; uint32_t la_count; @@ -1620,6 +1621,20 @@ allowed: } } + if (getnc_state) { + ret = drsuapi_DsReplicaHighWaterMark_cmp(&getnc_state->last_hwm, + &req10->highwatermark); + if (ret != 0) { + DEBUG(0,(__location__ ": DsGetNCChanges 2nd replication " + "on DN %s %s highwatermark (last_dn %s)\n", + ldb_dn_get_linearized(getnc_state->ncRoot_dn), + (ret > 0) ? "older" : "newer", + ldb_dn_get_linearized(getnc_state->last_dn))); + talloc_free(getnc_state); + getnc_state = NULL; + } + } + if (getnc_state == NULL) { getnc_state = talloc_zero(b_state, struct drsuapi_getncchanges_state); if (getnc_state == NULL) { @@ -1892,6 +1907,7 @@ allowed: uSN = ldb_msg_find_attr_as_int(msg, "uSNChanged", -1); if (uSN > r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn) { r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn = uSN; + r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; } if (obj->meta_data_ctr == NULL) { @@ -1993,6 +2009,7 @@ allowed: r->out.ctr->ctr6.uptodateness_vector = talloc(mem_ctx, struct drsuapi_DsReplicaCursor2CtrEx); r->out.ctr->ctr6.new_highwatermark.highest_usn = r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn; + r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; werr = get_nc_changes_udv(sam_ctx, getnc_state->ncRoot_dn, r->out.ctr->ctr6.uptodateness_vector); @@ -2002,6 +2019,25 @@ allowed: talloc_free(getnc_state); b_state->getncchanges_state = NULL; + } else { + ret = drsuapi_DsReplicaHighWaterMark_cmp(&r->out.ctr->ctr6.old_highwatermark, + &r->out.ctr->ctr6.new_highwatermark); + if (ret == 0) { + /* + * We need to make sure that we never return the + * same highwatermark within the same replication + * cycle more than once. Otherwise we cannot detect + * when the client uses an unexptected highwatermark. + * + * This is a HACK which is needed because our + * object ordering is wrong and set tmp_highest_usn + * to a value that is higher than what we already + * sent to the client (destination dsa). + */ + r->out.ctr->ctr6.new_highwatermark.reserved_usn += 1; + } + + getnc_state->last_hwm = r->out.ctr->ctr6.new_highwatermark; } if (req10->extended_op != DRSUAPI_EXOP_NONE) { -- 1.7.9.5 From 523836905d39dcda640bcdb9663faeda404123cd Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 13:40:33 +0100 Subject: [PATCH 15/18] s4:drsuapi: check the source_dsa_invocation_id (bug #9508) The given highwatermark is only valid relative to the specified source_dsa_invocation_id. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 1f89d641d09ef983f6a5055bb75099dc0ce57aa8) --- source4/rpc_server/drsuapi/getncchanges.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index e06aeed..10965a3 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -1483,12 +1483,15 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_ time_t start = time(NULL); bool max_wait_reached = false; bool has_get_all_changes = false; + struct GUID invocation_id; DCESRV_PULL_HANDLE_WERR(h, r->in.bind_handle, DRSUAPI_BIND_HANDLE); b_state = h->data; sam_ctx = b_state->sam_ctx_system?b_state->sam_ctx_system:b_state->sam_ctx; + invocation_id = *(samdb_ntds_invocation_id(sam_ctx)); + *r->out.level_out = 6; /* TODO: linked attributes*/ r->out.ctr->ctr6.linked_attributes_count = 0; @@ -1606,6 +1609,18 @@ allowed: req10->uptodateness_vector = NULL; } + if (GUID_all_zero(&req10->source_dsa_invocation_id)) { + req10->source_dsa_invocation_id = invocation_id; + } + + if (!GUID_equal(&req10->source_dsa_invocation_id, &invocation_id)) { + /* + * The given highwatermark is only valid relative to the + * specified source_dsa_invocation_id. + */ + ZERO_STRUCT(req10->highwatermark); + } + getnc_state = b_state->getncchanges_state; /* see if a previous replication has been abandoned */ -- 1.7.9.5 From a6535bbbc5ba095516fb6a8eb7bf535077164481 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 14:59:20 +0100 Subject: [PATCH 16/18] s4:drsuapi: make sure we report the meta data from the cycle start (bug #9508) We should build the final highwatermark and uptodatevector of a replication cycle at the start of the cycle. Before we search for the currently missing objects. Otherwise we risk that some objects get lost. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 88833b089a90e8f685d15b508f2e4615afb3a16f) --- source4/rpc_server/drsuapi/getncchanges.c | 47 +++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 10965a3..511925a 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -45,8 +45,11 @@ struct drsuapi_getncchanges_state { struct ldb_dn *ncRoot_dn; bool is_schema_nc; uint64_t min_usn; + uint64_t max_usn; struct drsuapi_DsReplicaHighWaterMark last_hwm; struct ldb_dn *last_dn; + struct drsuapi_DsReplicaHighWaterMark final_hwm; + struct drsuapi_DsReplicaCursor2CtrEx *final_udv; struct drsuapi_DsReplicaLinkedAttribute *la_list; uint32_t la_count; bool la_sorted; @@ -1740,6 +1743,18 @@ allowed: extra_filter = lpcfg_parm_string(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "object filter"); getnc_state->min_usn = req10->highwatermark.highest_usn; + getnc_state->max_usn = getnc_state->min_usn; + + getnc_state->final_udv = talloc_zero(getnc_state, + struct drsuapi_DsReplicaCursor2CtrEx); + if (getnc_state->final_udv == NULL) { + return WERR_NOMEM; + } + werr = get_nc_changes_udv(sam_ctx, getnc_state->ncRoot_dn, + getnc_state->final_udv); + if (!W_ERROR_IS_OK(werr)) { + return werr; + } if (req10->extended_op == DRSUAPI_EXOP_NONE) { werr = getncchanges_collect_objects(b_state, mem_ctx, req10, @@ -1767,6 +1782,10 @@ allowed: changes[i].dn = search_res->msgs[i]->dn; changes[i].guid = samdb_result_guid(search_res->msgs[i], "objectGUID"); changes[i].usn = ldb_msg_find_attr_as_uint64(search_res->msgs[i], "uSNChanged", 0); + + if (changes[i].usn > getnc_state->max_usn) { + getnc_state->max_usn = changes[i].usn; + } } if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) { @@ -1788,6 +1807,10 @@ allowed: } } + getnc_state->final_hwm.tmp_highest_usn = getnc_state->max_usn; + getnc_state->final_hwm.reserved_usn = 0; + getnc_state->final_hwm.highest_usn = getnc_state->max_usn; + talloc_free(search_res); talloc_free(changes); } @@ -1920,6 +1943,18 @@ allowed: } uSN = ldb_msg_find_attr_as_int(msg, "uSNChanged", -1); + if (uSN > getnc_state->max_usn) { + /* + * Only report the max_usn we had at the start + * of the replication cycle. + * + * If this object has changed lately we better + * let the destination dsa refetch the change. + * This is better than the risk of loosing some + * objects or linked attributes. + */ + uSN = 0; + } if (uSN > r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn) { r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn = uSN; r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; @@ -2022,15 +2057,9 @@ allowed: if (!r->out.ctr->ctr6.more_data) { talloc_steal(mem_ctx, getnc_state->la_list); - r->out.ctr->ctr6.uptodateness_vector = talloc(mem_ctx, struct drsuapi_DsReplicaCursor2CtrEx); - r->out.ctr->ctr6.new_highwatermark.highest_usn = r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn; - r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0; - - werr = get_nc_changes_udv(sam_ctx, getnc_state->ncRoot_dn, - r->out.ctr->ctr6.uptodateness_vector); - if (!W_ERROR_IS_OK(werr)) { - return werr; - } + r->out.ctr->ctr6.new_highwatermark = getnc_state->final_hwm; + r->out.ctr->ctr6.uptodateness_vector = talloc_move(mem_ctx, + &getnc_state->final_udv); talloc_free(getnc_state); b_state->getncchanges_state = NULL; -- 1.7.9.5 From 8583c05fe3f82116e6c07719e211a47b484eba05 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 18 Dec 2012 15:16:28 +0100 Subject: [PATCH 17/18] s4:drsuapi: make use of LDB_TYPESAFE_QSORT() and pass getnc_state Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 16aef75c4f83c114206aa7637fedc9c2c2486877) --- source4/rpc_server/drsuapi/getncchanges.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 511925a..8ea17e5 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -686,8 +686,9 @@ struct drsuapi_changed_objects { /* sort the objects we send by tree order */ -static int site_res_cmp_parent_order(struct drsuapi_changed_objects *m1, - struct drsuapi_changed_objects *m2) +static int site_res_cmp_anc_order(struct drsuapi_changed_objects *m1, + struct drsuapi_changed_objects *m2, + struct drsuapi_getncchanges_state *getnc_state) { return ldb_dn_compare(m2->dn, m1->dn); } @@ -695,8 +696,9 @@ static int site_res_cmp_parent_order(struct drsuapi_changed_objects *m1, /* sort the objects we send first by uSNChanged */ -static int site_res_cmp_dn_usn_order(struct drsuapi_changed_objects *m1, - struct drsuapi_changed_objects *m2) +static int site_res_cmp_usn_order(struct drsuapi_changed_objects *m1, + struct drsuapi_changed_objects *m2, + struct drsuapi_getncchanges_state *getnc_state) { unsigned usnchanged1, usnchanged2; unsigned cn1, cn2; @@ -1789,13 +1791,15 @@ allowed: } if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) { - TYPESAFE_QSORT(changes, - getnc_state->num_records, - site_res_cmp_parent_order); + LDB_TYPESAFE_QSORT(changes, + getnc_state->num_records, + getnc_state, + site_res_cmp_anc_order); } else { - TYPESAFE_QSORT(changes, - getnc_state->num_records, - site_res_cmp_dn_usn_order); + LDB_TYPESAFE_QSORT(changes, + getnc_state->num_records, + getnc_state, + site_res_cmp_usn_order); } for (i=0; i < getnc_state->num_records; i++) { -- 1.7.9.5 From 8036be5359bfef2218ca3b10f9a71906a4901ce1 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sat, 15 Dec 2012 10:18:08 +0100 Subject: [PATCH 18/18] s4:drsuapi: try to behave more like windows for usn order (bug #9508) We don't behave completely like a Windows server, but it's much more identical than before. The partition head is always the first object followed by the rest sorted by uSNChanged. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Stefan Metzmacher Autobuild-Date(master): Tue Jan 1 21:09:42 CET 2013 on sn-devel-104 (cherry picked from commit f77bfed088b93f3ed0f00d0c172ad495c6c2b09b) --- source4/rpc_server/drsuapi/getncchanges.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 8ea17e5..c3fd000 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -700,20 +700,27 @@ static int site_res_cmp_usn_order(struct drsuapi_changed_objects *m1, struct drsuapi_changed_objects *m2, struct drsuapi_getncchanges_state *getnc_state) { - unsigned usnchanged1, usnchanged2; - unsigned cn1, cn2; + int ret; - cn1 = ldb_dn_get_comp_num(m1->dn); - cn2 = ldb_dn_get_comp_num(m2->dn); - if (cn1 != cn2) { - return cn1 > cn2 ? 1 : -1; + ret = ldb_dn_compare(getnc_state->ncRoot_dn, m1->dn); + if (ret == 0) { + return -1; } - usnchanged1 = m1->usn; - usnchanged2 = m2->usn; - if (usnchanged1 == usnchanged2) { - return 0; + + ret = ldb_dn_compare(getnc_state->ncRoot_dn, m2->dn); + if (ret == 0) { + return 1; } - return usnchanged1 > usnchanged2 ? 1 : -1; + + if (m1->usn == m2->usn) { + return ldb_dn_compare(m2->dn, m1->dn); + } + + if (m1->usn < m2->usn) { + return -1; + } + + return 1; } -- 1.7.9.5