From c2179fdbf916e7c8bbd4102c513515ec49add3a8 Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Wed, 23 Jan 2013 11:33:30 -0800 Subject: [PATCH 01/16] dsdb-repl: make message more clearer Reviewed-by: Andrew Bartlett (cherry picked from commit 1c0d3486a485cf01338dd5eff49ce847628d1b83) --- source4/dsdb/repl/replicated_objects.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index 829c440..44c5de8 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -150,8 +150,8 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, } talloc_free(tmp_ctx); - DEBUG(4,("Schema load pass %d: %d/%d of %d objects left to be converted.\n", - pass_no, failed_obj_count, converted_obj_count, object_count)); + DEBUG(4,("Schema load pass %d: converted %d, %d of %d objects left to be converted.\n", + pass_no, converted_obj_count, failed_obj_count, object_count)); pass_no++; /* check if we converted any objects in this pass */ -- 1.7.9.5 From 21577acf5bb35c0d05790df68ce8d5a5a9596d05 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Tue, 26 Mar 2013 11:51:38 +1100 Subject: [PATCH 02/16] dsdb-repl: Allow the name attribute (and name-based schema lookups) to be skipped in dsdb_repl_make_working_schema() This allows us to use a schema that may only be valid for attributeID based lookups, during the schema load. Andrew Bartlett Reviewed-by: Stefan Metzmacher (cherry picked from commit 25402e06bcdf98e346fdbbfa7e8740504329b42f) --- source4/dsdb/repl/replicated_objects.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index 44c5de8..29b494b 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -209,10 +209,6 @@ WERROR dsdb_convert_object_ex(struct ldb_context *ldb, NTTIME whenChanged = 0; time_t whenChanged_t; const char *whenChanged_s; - const char *rdn_name = NULL; - const struct ldb_val *rdn_value = NULL; - const struct dsdb_attribute *rdn_attr = NULL; - uint32_t rdn_attid; struct drsuapi_DsReplicaAttribute *name_a = NULL; struct drsuapi_DsReplicaMetaData *name_d = NULL; struct replPropertyMetaData1 *rdn_m = NULL; @@ -248,14 +244,6 @@ WERROR dsdb_convert_object_ex(struct ldb_context *ldb, msg->dn = ldb_dn_new(msg, ldb, in->object.identifier->dn); W_ERROR_HAVE_NO_MEMORY(msg->dn); - rdn_name = ldb_dn_get_rdn_name(msg->dn); - rdn_attr = dsdb_attribute_by_lDAPDisplayName(schema, rdn_name); - if (!rdn_attr) { - return WERR_FOOBAR; - } - rdn_attid = rdn_attr->attributeID_id; - rdn_value = ldb_dn_get_rdn_val(msg->dn); - msg->num_elements = in->object.attribute_ctr.num_attributes; msg->elements = talloc_array(msg, struct ldb_message_element, msg->num_elements + 1); /* +1 because of the RDN attribute */ @@ -331,6 +319,25 @@ WERROR dsdb_convert_object_ex(struct ldb_context *ldb, if (rdn_m) { struct ldb_message_element *el; + const char *rdn_name = NULL; + const struct ldb_val *rdn_value = NULL; + const struct dsdb_attribute *rdn_attr = NULL; + uint32_t rdn_attid; + + /* + * We only need the schema calls for the RDN in this + * codepath, and by doing this we avoid needing to + * have the dsdb_attribute_by_lDAPDisplayName accessor + * working during the schema load. + */ + rdn_name = ldb_dn_get_rdn_name(msg->dn); + rdn_attr = dsdb_attribute_by_lDAPDisplayName(schema, rdn_name); + if (!rdn_attr) { + return WERR_FOOBAR; + } + rdn_attid = rdn_attr->attributeID_id; + rdn_value = ldb_dn_get_rdn_val(msg->dn); + el = ldb_msg_find_element(msg, rdn_attr->lDAPDisplayName); if (!el) { ret = ldb_msg_add_value(msg, rdn_attr->lDAPDisplayName, rdn_value, NULL); -- 1.7.9.5 From 3b10d8349f96dcde0ab8662de4e7d3e77be9cc44 Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Mon, 21 Jan 2013 22:27:10 -0800 Subject: [PATCH 03/16] dsdb-schema: remove looping on all schema classes for system_possible_inferrior The logic to populate possible inferriors and system possible inferriors is the same so instead of looping twice we do both attributes (depending on the type of the class) in the same loop Reviewed-by: Andrew Bartlett (cherry picked from commit cd7f3fd07215a7b8372b6b623faed02ae1310cb1) --- source4/dsdb/schema/schema_inferiors.c | 53 ++++++++++++-------------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/source4/dsdb/schema/schema_inferiors.c b/source4/dsdb/schema/schema_inferiors.c index 2f7d461..4d095216 100644 --- a/source4/dsdb/schema/schema_inferiors.c +++ b/source4/dsdb/schema/schema_inferiors.c @@ -201,45 +201,31 @@ static void schema_fill_possible_inferiors(const struct dsdb_schema *schema, struct dsdb_class *schema_class) { struct dsdb_class *c2; + const char** poss_inf = schema_class->possibleInferiors; + const char** sys_poss_inf = schema_class->systemPossibleInferiors; - schema_class->possibleInferiors = NULL; - - for (c2=schema->classes; c2; c2=c2->next) { + for (c2 = schema->classes; c2; c2 = c2->next) { const char **superiors = schema_posssuperiors(schema, c2); - if (c2->systemOnly == false - && c2->objectClassCategory != 2 - && c2->objectClassCategory != 3 - && str_list_check(superiors, schema_class->lDAPDisplayName)) { - if (schema_class->possibleInferiors == NULL) { - schema_class->possibleInferiors = const_str_list(str_list_make_empty(schema_class)); + if (c2->objectClassCategory != 2 && + c2->objectClassCategory != 3 && + str_list_check(superiors, schema_class->lDAPDisplayName)) + { + if (c2->systemOnly == false) { + if (poss_inf == NULL) { + poss_inf = const_str_list(str_list_make_empty(schema_class)); + } + poss_inf = str_list_add_const(poss_inf, + c2->lDAPDisplayName); } - schema_class->possibleInferiors = str_list_add_const(schema_class->possibleInferiors, - c2->lDAPDisplayName); - } - } - schema_class->possibleInferiors = str_list_unique(schema_class->possibleInferiors); -} - -static void schema_fill_system_possible_inferiors(const struct dsdb_schema *schema, - struct dsdb_class *schema_class) -{ - struct dsdb_class *c2; - - schema_class->systemPossibleInferiors = NULL; - - for (c2=schema->classes; c2; c2=c2->next) { - const char **superiors = schema_posssuperiors(schema, c2); - if (c2->objectClassCategory != 2 - && c2->objectClassCategory != 3 - && str_list_check(superiors, schema_class->lDAPDisplayName)) { - if (schema_class->systemPossibleInferiors == NULL) { - schema_class->systemPossibleInferiors = const_str_list(str_list_make_empty(schema_class)); + if (sys_poss_inf == NULL) { + sys_poss_inf = const_str_list(str_list_make_empty(schema_class)); } - schema_class->systemPossibleInferiors = str_list_add_const(schema_class->systemPossibleInferiors, - c2->lDAPDisplayName); + sys_poss_inf = str_list_add_const(sys_poss_inf, + c2->lDAPDisplayName); } } - schema_class->systemPossibleInferiors = str_list_unique(schema_class->systemPossibleInferiors); + schema_class->systemPossibleInferiors = str_list_unique(sys_poss_inf); + schema_class->possibleInferiors = str_list_unique(poss_inf); } /* @@ -347,7 +333,6 @@ int schema_fill_constructed(const struct dsdb_schema *schema) for (schema_class=schema->classes; schema_class; schema_class=schema_class->next) { schema_fill_possible_inferiors(schema, schema_class); - schema_fill_system_possible_inferiors(schema, schema_class); } /* free up our internal cache elements */ -- 1.7.9.5 From 968046740875df975efe38fac618e03b6a0fb90e Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Fri, 5 Oct 2012 02:09:47 -0700 Subject: [PATCH 04/16] s4-drs: Remove unused var Signed-off-by: Matthieu Patou (cherry picked from commit f8c5f9836415ce0891624b170410e807abcfb706) --- source4/libnet/libnet_vampire.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source4/libnet/libnet_vampire.c b/source4/libnet/libnet_vampire.c index 48fdc89..4091346 100644 --- a/source4/libnet/libnet_vampire.c +++ b/source4/libnet/libnet_vampire.c @@ -527,7 +527,6 @@ NTSTATUS libnet_vampire_cb_schema_chunk(void *private_data, struct drsuapi_DsReplicaObjectListItemEx *cur; uint32_t nc_linked_attributes_count; uint32_t linked_attributes_count; - struct drsuapi_DsReplicaLinkedAttribute *linked_attributes; switch (c->ctr_level) { case 1: @@ -537,7 +536,6 @@ NTSTATUS libnet_vampire_cb_schema_chunk(void *private_data, first_object = c->ctr1->first_object; nc_linked_attributes_count = 0; linked_attributes_count = 0; - linked_attributes = NULL; break; case 6: mapping_ctr = &c->ctr6->mapping_ctr; @@ -546,7 +544,6 @@ NTSTATUS libnet_vampire_cb_schema_chunk(void *private_data, first_object = c->ctr6->first_object; nc_linked_attributes_count = c->ctr6->nc_linked_attributes_count; linked_attributes_count = c->ctr6->linked_attributes_count; - linked_attributes = c->ctr6->linked_attributes; break; default: return NT_STATUS_INVALID_PARAMETER; -- 1.7.9.5 From 0fb5f1e3b84272b2c580d06ea885f9652f38b088 Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Mon, 31 Dec 2012 02:12:23 -0800 Subject: [PATCH 05/16] libnet: set the invocation_id earlier in order to avoid annoying messages At that moment we have all the information to set the invocation id so let's set it, it will avoid useless messages about missing invocation id. Signed-off-by: Matthieu Patou Reviewed-By: Andrew Bartlett (cherry picked from commit 0c86126d166c8f75bd3593fce077f26bca51f8aa) --- source4/libnet/libnet_vampire.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source4/libnet/libnet_vampire.c b/source4/libnet/libnet_vampire.c index 4091346..a354d41 100644 --- a/source4/libnet/libnet_vampire.c +++ b/source4/libnet/libnet_vampire.c @@ -288,6 +288,19 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s default: return NT_STATUS_INVALID_PARAMETER; } + /* We must set these up to ensure the replMetaData is written + * correctly, before our NTDS Settings entry is replicated */ + ok = samdb_set_ntds_invocation_id(s->ldb, &c->dest_dsa->invocation_id); + if (!ok) { + DEBUG(0,("Failed to set cached ntds invocationId\n")); + return NT_STATUS_FOOBAR; + } + ok = samdb_set_ntds_objectGUID(s->ldb, &c->dest_dsa->ntds_guid); + if (!ok) { + DEBUG(0,("Failed to set cached ntds objectGUID\n")); + return NT_STATUS_FOOBAR; + } + status = dsdb_schema_pfm_from_drsuapi_pfm(mapping_ctr, true, s, &pfm_remote, NULL); @@ -492,19 +505,6 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s talloc_free(s_dsa); talloc_free(schema_objs); - /* We must set these up to ensure the replMetaData is written - * correctly, before our NTDS Settings entry is replicated */ - ok = samdb_set_ntds_invocation_id(s->ldb, &c->dest_dsa->invocation_id); - if (!ok) { - DEBUG(0,("Failed to set cached ntds invocationId\n")); - return NT_STATUS_FOOBAR; - } - ok = samdb_set_ntds_objectGUID(s->ldb, &c->dest_dsa->ntds_guid); - if (!ok) { - DEBUG(0,("Failed to set cached ntds objectGUID\n")); - return NT_STATUS_FOOBAR; - } - s->schema = dsdb_get_schema(s->ldb, s); if (!s->schema) { DEBUG(0,("Failed to get loaded dsdb_schema\n")); -- 1.7.9.5 From 5ccd3a5713dd1d65b24e12a2bcc03f583eda067e Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Mon, 31 Dec 2012 15:38:50 -0800 Subject: [PATCH 06/16] libnet-vampire: reports Exops as they rather than sync on some partitions Instead of showing: Partition[CN=RODC,OU=Domain Controllers,DC=samba,DC=example,DC=com] objects[1] linked_values[8] Report a exop based on CN=RODC,OU=Domain Controllers,DC=samba,DC=example,DC=com as Exop on CN=RODC,OU=Domain Controllers,DC=samba,DC=example,DC=com, ... Signed-off-by: Matthieu Patou Reviewed-By: Andrew Bartlett Autobuild-User(master): Matthieu Patou Autobuild-Date(master): Wed Jan 9 09:01:30 CET 2013 on sn-devel-104 (cherry picked from commit 2cc6f9ce7f8068440ef527b0aebd3ad5ad105a9d) --- source4/libnet/libnet_vampire.c | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/source4/libnet/libnet_vampire.c b/source4/libnet/libnet_vampire.c index a354d41..599119f 100644 --- a/source4/libnet/libnet_vampire.c +++ b/source4/libnet/libnet_vampire.c @@ -643,6 +643,7 @@ NTSTATUS libnet_vampire_cb_store_chunk(void *private_data, char *tmp_dns_name; uint32_t i; uint64_t seq_num; + bool is_exop = false; s_dsa = talloc_zero(s, struct repsFromTo1); NT_STATUS_HAVE_NO_MEMORY(s_dsa); @@ -686,12 +687,21 @@ NTSTATUS libnet_vampire_cb_store_chunk(void *private_data, req_replica_flags = 0; break; case 5: + if (c->req5->extended_op != DRSUAPI_EXOP_NONE) { + is_exop = true; + } req_replica_flags = c->req5->replica_flags; break; case 8: + if (c->req8->extended_op != DRSUAPI_EXOP_NONE) { + is_exop = true; + } req_replica_flags = c->req8->replica_flags; break; case 10: + if (c->req10->extended_op != DRSUAPI_EXOP_NONE) { + is_exop = true; + } req_replica_flags = c->req10->replica_flags; break; default: @@ -728,13 +738,24 @@ NTSTATUS libnet_vampire_cb_store_chunk(void *private_data, } s->total_objects += object_count; - if (nc_object_count) { - DEBUG(0,("Partition[%s] objects[%u/%u] linked_values[%u/%u]\n", - c->partition->nc.dn, s->total_objects, nc_object_count, - linked_attributes_count, nc_linked_attributes_count)); + if (is_exop) { + if (nc_object_count) { + DEBUG(0,("Exop on[%s] objects[%u/%u] linked_values[%u/%u]\n", + c->partition->nc.dn, s->total_objects, nc_object_count, + linked_attributes_count, nc_linked_attributes_count)); + } else { + DEBUG(0,("Exop on[%s] objects[%u] linked_values[%u]\n", + c->partition->nc.dn, s->total_objects, linked_attributes_count)); + } } else { - DEBUG(0,("Partition[%s] objects[%u] linked_values[%u]\n", - c->partition->nc.dn, s->total_objects, linked_attributes_count)); + if (nc_object_count) { + DEBUG(0,("Partition[%s] objects[%u/%u] linked_values[%u/%u]\n", + c->partition->nc.dn, s->total_objects, nc_object_count, + linked_attributes_count, nc_linked_attributes_count)); + } else { + DEBUG(0,("Partition[%s] objects[%u] linked_values[%u]\n", + c->partition->nc.dn, s->total_objects, linked_attributes_count)); + } } -- 1.7.9.5 From 498ddede32356d83b70bce6e9e26201f8a1f9930 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 23 May 2013 08:46:31 +0200 Subject: [PATCH 07/16] dsdb-schema: schema_fill_possible_inferiors() should rebuild everthing commit cd7f3fd07215a7b8372b6b623faed02ae1310cb1 reverted the change of commit c2853f55fc603d4875bb1e50a1cbf409df0421ea. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit c524be17815e92ce9fcdd0565e76b026e483cc9d) --- source4/dsdb/schema/schema_inferiors.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/dsdb/schema/schema_inferiors.c b/source4/dsdb/schema/schema_inferiors.c index 4d095216..56f5733 100644 --- a/source4/dsdb/schema/schema_inferiors.c +++ b/source4/dsdb/schema/schema_inferiors.c @@ -201,8 +201,8 @@ static void schema_fill_possible_inferiors(const struct dsdb_schema *schema, struct dsdb_class *schema_class) { struct dsdb_class *c2; - const char** poss_inf = schema_class->possibleInferiors; - const char** sys_poss_inf = schema_class->systemPossibleInferiors; + const char **poss_inf = NULL; + const char **sys_poss_inf = NULL; for (c2 = schema->classes; c2; c2 = c2->next) { const char **superiors = schema_posssuperiors(schema, c2); -- 1.7.9.5 From e940460d0117b69c70175a1f40ec7dd6a83094a3 Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Sat, 26 Jan 2013 23:42:10 -0800 Subject: [PATCH 08/16] dsdb-schema: make deduplication of class and schema possible (bug #8680) When a class or an attribute is replicated it might already exists in the existing schema, so while replicating the new version of this object we want to get rid of the old version of the object is the current validating schema so that we don't end up having duplicates. Pair-Programmed-With: Stefan Metzmacher Signed-off-by: Matthieu Patou Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit fe85bc1fb9ab2f891a9fd46bd8e00325622d39cf) --- source4/dsdb/schema/schema.h | 5 +++ source4/dsdb/schema/schema_init.c | 72 ++++++++++++++++++++++++++++++++++--- source4/dsdb/schema/schema_set.c | 46 +++++++++++++++++++++--- 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/source4/dsdb/schema/schema.h b/source4/dsdb/schema/schema.h index 66df1c5..538b858 100644 --- a/source4/dsdb/schema/schema.h +++ b/source4/dsdb/schema/schema.h @@ -221,6 +221,11 @@ struct dsdb_schema { struct dsdb_attribute *attributes; struct dsdb_class *classes; + struct dsdb_attribute **attributes_to_remove; + uint32_t attributes_to_remove_size; + struct dsdb_class **classes_to_remove; + uint32_t classes_to_remove_size; + /* lists of classes sorted by various attributes, for faster access */ uint32_t num_classes; diff --git a/source4/dsdb/schema/schema_init.c b/source4/dsdb/schema/schema_init.c index 752d4f5..efbd38a 100644 --- a/source4/dsdb/schema/schema_init.c +++ b/source4/dsdb/schema/schema_init.c @@ -699,9 +699,10 @@ WERROR dsdb_attribute_from_ldb(const struct dsdb_schema *schema, return WERR_OK; } -WERROR dsdb_set_attribute_from_ldb(struct ldb_context *ldb, - struct dsdb_schema *schema, - struct ldb_message *msg) +WERROR dsdb_set_attribute_from_ldb_dups(struct ldb_context *ldb, + struct dsdb_schema *schema, + struct ldb_message *msg, + bool checkdups) { WERROR status; struct dsdb_attribute *attr = talloc_zero(schema, struct dsdb_attribute); @@ -729,12 +730,44 @@ WERROR dsdb_set_attribute_from_ldb(struct ldb_context *ldb, return WERR_DS_ATT_SCHEMA_REQ_SYNTAX; } + if (checkdups) { + const struct dsdb_attribute *a2; + struct dsdb_attribute **a; + uint32_t i; + + a2 = dsdb_attribute_by_attributeID_id(schema, + attr->attributeID_id); + if (a2 == NULL) { + goto done; + } + + i = schema->attributes_to_remove_size; + a = talloc_realloc(schema, schema->attributes_to_remove, + struct dsdb_attribute *, i + 1); + if (a == NULL) { + return WERR_NOMEM; + } + /* Mark the old attribute as to be removed */ + a[i] = discard_const_p(struct dsdb_attribute, a2); + schema->attributes_to_remove = a; + schema->attributes_to_remove_size++; + } + +done: DLIST_ADD(schema->attributes, attr); return WERR_OK; } -WERROR dsdb_set_class_from_ldb(struct dsdb_schema *schema, - struct ldb_message *msg) +WERROR dsdb_set_attribute_from_ldb(struct ldb_context *ldb, + struct dsdb_schema *schema, + struct ldb_message *msg) +{ + return dsdb_set_attribute_from_ldb_dups(ldb, schema, msg, false); +} + +WERROR dsdb_set_class_from_ldb_dups(struct dsdb_schema *schema, + struct ldb_message *msg, + bool checkdups) { WERROR status; struct dsdb_class *obj = talloc_zero(schema, struct dsdb_class); @@ -792,10 +825,39 @@ WERROR dsdb_set_class_from_ldb(struct dsdb_schema *schema, GET_BOOL_LDB(msg, "isDefunct", obj, isDefunct, false); GET_BOOL_LDB(msg, "systemOnly", obj, systemOnly, false); + if (checkdups) { + const struct dsdb_class *c2; + struct dsdb_class **c; + uint32_t i; + + c2 = dsdb_class_by_governsID_id(schema, obj->governsID_id); + if (c2 == NULL) { + goto done; + } + + i = schema->classes_to_remove_size; + c = talloc_realloc(schema, schema->classes_to_remove, + struct dsdb_class *, i + 1); + if (c == NULL) { + return WERR_NOMEM; + } + /* Mark the old class to be removed */ + c[i] = discard_const_p(struct dsdb_class, c2); + schema->classes_to_remove = c; + schema->classes_to_remove_size++; + } + +done: DLIST_ADD(schema->classes, obj); return WERR_OK; } +WERROR dsdb_set_class_from_ldb(struct dsdb_schema *schema, + struct ldb_message *msg) +{ + return dsdb_set_class_from_ldb_dups(schema, msg, false); +} + #define dsdb_oom(error_string, mem_ctx) *error_string = talloc_asprintf(mem_ctx, "dsdb out of memory at %s:%d\n", __FILE__, __LINE__) /* diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c index 31862a4..73264f9 100644 --- a/source4/dsdb/schema/schema_set.c +++ b/source4/dsdb/schema/schema_set.c @@ -329,6 +329,18 @@ int dsdb_setup_sorted_accessors(struct ldb_context *ldb, unsigned int num_int_id; int ret; + for (i=0; i < schema->classes_to_remove_size; i++) { + DLIST_REMOVE(schema->classes, schema->classes_to_remove[i]); + TALLOC_FREE(schema->classes_to_remove[i]); + } + for (i=0; i < schema->attributes_to_remove_size; i++) { + DLIST_REMOVE(schema->attributes, schema->attributes_to_remove[i]); + TALLOC_FREE(schema->attributes_to_remove[i]); + } + + TALLOC_FREE(schema->attributes_to_remove); + TALLOC_FREE(schema->classes_to_remove); + /* free all caches */ dsdb_sorted_accessors_free(schema); @@ -669,10 +681,26 @@ int dsdb_schema_fill_extended_dn(struct ldb_context *ldb, struct dsdb_schema *sc } /** - * Add an element to the schema (attribute or class) from an LDB message + * @brief Add a new element to the schema and checks if it's a duplicate + * + * This function will add a new element to the schema and checks for existing + * duplicates. + * + * @param[in] ldb A pointer to an LDB context + * + * @param[in] schema A pointer to the dsdb_schema where the element + * will be added. + * + * @param[in] msg The ldb_message object representing the element + * to add. + * + * @param[in] checkdups A boolean to indicate if checks for duplicates + * should be done. + * + * @return A WERROR code */ -WERROR dsdb_schema_set_el_from_ldb_msg(struct ldb_context *ldb, struct dsdb_schema *schema, - struct ldb_message *msg) +WERROR dsdb_schema_set_el_from_ldb_msg_dups(struct ldb_context *ldb, struct dsdb_schema *schema, + struct ldb_message *msg, bool checkdups) { const char* tstring; time_t ts; @@ -686,15 +714,23 @@ WERROR dsdb_schema_set_el_from_ldb_msg(struct ldb_context *ldb, struct dsdb_sche } if (samdb_find_attribute(ldb, msg, "objectclass", "attributeSchema") != NULL) { - return dsdb_set_attribute_from_ldb(ldb, schema, msg); + + return dsdb_set_attribute_from_ldb_dups(ldb, schema, msg, checkdups); } else if (samdb_find_attribute(ldb, msg, "objectclass", "classSchema") != NULL) { - return dsdb_set_class_from_ldb(schema, msg); + return dsdb_set_class_from_ldb_dups(schema, msg, checkdups); } /* Don't fail on things not classes or attributes */ return WERR_OK; } +WERROR dsdb_schema_set_el_from_ldb_msg(struct ldb_context *ldb, + struct dsdb_schema *schema, + struct ldb_message *msg) +{ + return dsdb_schema_set_el_from_ldb_msg_dups(ldb, schema, msg, false); +} + /** * Rather than read a schema from the LDB itself, read it from an ldif * file. This allows schema to be loaded and used while adding the -- 1.7.9.5 From d917a0ca37f32c175af59d2085ec13e9fd85a256 Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Sun, 7 Oct 2012 21:46:38 -0700 Subject: [PATCH 09/16] libnet-vampire: add attributes and classes from the replicated schema to the bootstrap schema (bug #8680) Replicated schema might have attributes and auxilary classes on some critical classes (ie. top, user, computer ) that are not in the bootstrap schema. Without those new attributes and classes, bootstrap schema is unable to translate those critical classes in the schema constructed from the replicated data. Without thoses classes new schema is useless and can't be indexed properly. In order to overcome this problem, we put all new attributes and classes definitions into the bootstrap schema so that foundations classes can be translated. Pair-Programmed-With: Stefan Metzmacher Signed-off-by: Matthieu Patou Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit c7d4b87512eabbff5172716a755a3cd61fe5476b) --- source4/libnet/libnet_vampire.c | 126 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 7 deletions(-) diff --git a/source4/libnet/libnet_vampire.c b/source4/libnet/libnet_vampire.c index 599119f..0f20b15 100644 --- a/source4/libnet/libnet_vampire.c +++ b/source4/libnet/libnet_vampire.c @@ -216,6 +216,77 @@ NTSTATUS libnet_vampire_cb_check_options(void *private_data, return NT_STATUS_OK; } +static NTSTATUS libnet_vampire_merge_schema(struct ldb_context *ldb, + struct dsdb_schema *dest_schema, + const struct dsdb_schema *ref_schema) +{ + const struct dsdb_class *cur_class = NULL; + const struct dsdb_attribute *cur_attr = NULL; + int ret; + + for (cur_class = ref_schema->classes; + cur_class; + cur_class = cur_class->next) + { + const struct dsdb_class *tmp1; + struct dsdb_class *tmp2; + + tmp1 = dsdb_class_by_governsID_id(dest_schema, + cur_class->governsID_id); + if (tmp1 != NULL) { + continue; + } + + /* + * Do a shallow copy so that original next and prev are + * not modified, we don't need to do a deep copy + * as the rest won't be modified and this is for + * a short lived object. + */ + tmp2 = talloc(dest_schema->classes, struct dsdb_class); + if (tmp2 == NULL) { + return NT_STATUS_NO_MEMORY; + } + *tmp2 = *cur_class; + DLIST_ADD(dest_schema->classes, tmp2); + } + + for (cur_attr = ref_schema->attributes; + cur_attr; + cur_attr = cur_attr->next) + { + const struct dsdb_attribute *tmp1; + struct dsdb_attribute *tmp2; + + tmp1 = dsdb_attribute_by_attributeID_id(dest_schema, + cur_attr->attributeID_id); + if (tmp1 != NULL) { + continue; + } + + /* + * Do a shallow copy so that original next and prev are + * not modified, we don't need to do a deep copy + * as the rest won't be modified and this is for + * a short lived object. + */ + tmp2 = talloc(dest_schema->attributes, struct dsdb_attribute); + if (tmp2 == NULL) { + return NT_STATUS_NO_MEMORY; + } + *tmp2 = *cur_attr; + DLIST_ADD(dest_schema->attributes, tmp2); + } + + ret = dsdb_setup_sorted_accessors(ldb, dest_schema); + if (LDB_SUCCESS != ret) { + DEBUG(0,("Failed to add new attribute to reference schema!\n")); + return NT_STATUS_INTERNAL_ERROR; + } + + return NT_STATUS_OK; +} + static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s, const struct libnet_BecomeDC_StoreChunk *c) { @@ -321,6 +392,11 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s NT_STATUS_HAVE_NO_MEMORY(tmp_dns_name); s_dsa->other_info->dns_name = tmp_dns_name; + if (s->self_made_schema == NULL) { + DEBUG(0,("libnet_vampire_cb_apply_schema: called with out self_made_schema\n")); + return NT_STATUS_INTERNAL_ERROR; + } + schema_ldb = provision_get_schema(s, s->lp_ctx, c->forest->schema_dn_str, &s->prefixmap_blob); @@ -352,9 +428,31 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s while (schema_list) { uint32_t converted_obj_count = 0; uint32_t failed_obj_count = 0; + int cycle_before_switching = 0; TALLOC_CTX *tmp_ctx = talloc_new(s); NT_STATUS_HAVE_NO_MEMORY(tmp_ctx); + if (s->self_made_schema != working_schema) { + /* + * If the selfmade schema is not the schema used to + * translate and validate replicated object, + * Which means that we are using the bootstrap schema + * Then we add attributes and classes that were already + * translated to the working schema, the idea is that + * we might need to add new attributes and classes + * to be able to translate critical replicated objects + * and without that we wouldn't be able to translate them + */ + NTSTATUS nt_status; + + nt_status = libnet_vampire_merge_schema(s->ldb, + working_schema, + s->self_made_schema); + if (!NT_STATUS_IS_OK(nt_status)) { + return nt_status; + } + } + for (schema_list_item = schema_list; schema_list_item; schema_list_item=schema_list_next_item) { struct dsdb_extended_replicated_object object; @@ -385,16 +483,22 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s * Convert the schema from ldb_message format * (OIDs as OID strings) into schema, using * the remote prefixMap + * + * It's not likely, but possible to get the + * same object twice and we should keep + * the last instance. */ - status = dsdb_schema_set_el_from_ldb_msg(s->ldb, - s->self_made_schema, - object.msg); + status = dsdb_schema_set_el_from_ldb_msg_dups(s->ldb, + s->self_made_schema, + object.msg, true); if (!W_ERROR_IS_OK(status)) { DEBUG(1,("Warning: failed to convert object %s into a schema element: %s\n", ldb_dn_get_linearized(object.msg->dn), win_errstr(status))); failed_obj_count++; } else { + DEBUG(8,("Converted object %s into a schema element\n", + ldb_dn_get_linearized(object.msg->dn))); DLIST_REMOVE(schema_list, schema_list_item); converted_obj_count++; } @@ -402,9 +506,8 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s } talloc_free(tmp_ctx); - DEBUG(4,("Schema load pass %d: %d/%d of %d objects left to be converted.\n", - pass_no, failed_obj_count, converted_obj_count, object_count)); - pass_no++; + DEBUG(4,("Schema load pass %d: converted %d, %d of %d objects left to be converted.\n", + pass_no, converted_obj_count, failed_obj_count, object_count)); /* check if we converted any objects in this pass */ if (converted_obj_count == 0) { @@ -412,7 +515,15 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s return NT_STATUS_INTERNAL_ERROR; } - if (schema_list) { + /* + * Don't try to load the schema if there is missing object + * _and_ we are on the first pass as some critical objects + * might be missing. + */ + cycle_before_switching = lpcfg_parm_int(s->lp_ctx, NULL, + "become dc", + "schema convert retrial", 1); + if (failed_obj_count == 0 || pass_no > cycle_before_switching) { /* prepare for another cycle */ working_schema = s->self_made_schema; @@ -422,6 +533,7 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s return NT_STATUS_INTERNAL_ERROR; } } + pass_no++; }; /* free temp objects for 1st conversion phase */ -- 1.7.9.5 From 849575116938e9e0b762cbcabb15595f74275e26 Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Sun, 27 Jan 2013 15:43:07 -0800 Subject: [PATCH 10/16] dsdb-drs: when replicating schema object checks ask for removal of previous version if exists (bug #8680) Signed-off-by: Matthieu Patou Reviewed-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 640c2ff57518a5711e795e6cc9f48ae29d379a01) --- source4/dsdb/repl/replicated_objects.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index 29b494b..6c96b41 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -133,9 +133,10 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, * (OIDs as OID strings) into schema, using * the remote prefixMap */ - werr = dsdb_schema_set_el_from_ldb_msg(ldb, - working_schema, - object.msg); + werr = dsdb_schema_set_el_from_ldb_msg_dups(ldb, + working_schema, + object.msg, + true); if (!W_ERROR_IS_OK(werr)) { DEBUG(4,("debug: failed to convert object %s into a schema element, will try during next loop: %s\n", ldb_dn_get_linearized(object.msg->dn), -- 1.7.9.5 From f47ead43a16307710927f30c66d96791a374164c Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Thu, 18 Apr 2013 22:03:23 -0700 Subject: [PATCH 11/16] selftest: Improve test coverage of DRS (bug #8680) Pair-Programmed-With: Andrew Bartlett Signed-off-by: Matthieu Patou Reviewed-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit d36e9118cbf9deb01c6ee9af6790ce35bb3b936a) --- selftest/skip | 2 -- source4/selftest/tests.py | 51 +++++++++++++++++++++++---------------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/selftest/skip b/selftest/skip index 48d9ba6..61d8fef 100644 --- a/selftest/skip +++ b/selftest/skip @@ -101,9 +101,7 @@ bench # don't run benchmarks in our selftest # ktutil might not be installed or from mit... # we should build a samba4ktutil and use that instead ^samba4.blackbox.ktpass # this test isn't portable ... -^samba4.drs.repl_schema.python # flakey test ^samba4.smb2.ioctl # snapshots not supported by default -^samba4.drs.delete_object.python # flakey test ^samba4.rpc.unixinfo # This contains a server-side getpwuid call which hangs the server when nss_winbindd is in use ^samba.tests.dcerpc.unix # This contains a server-side getpwuid call which hangs the server when nss_winbindd is in use base.dir2 # This test spins on modern ext4, so we have to skip it diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index a469432..ec8a97b 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -458,31 +458,6 @@ plantestsuite("samba4.blackbox.group.py", "none", ["PYTHON=%s" % python, os.path plantestsuite("samba4.blackbox.spn.py(dc:local)", "dc:local", ["PYTHON=%s" % python, os.path.join(samba4srcdir, "setup/tests/blackbox_spn.sh"), '$PREFIX/dc']) plantestsuite("samba4.ldap.bind(dc)", "dc", [python, os.path.join(srcdir(), "auth/credentials/tests/bind.py"), '$SERVER', '-U"$USERNAME%$PASSWORD"']) -# DRS python tests -planoldpythontestsuite("vampire_dc", "samba.tests.blackbox.samba_tool_drs", - environ={'DC1': '$DC_SERVER', 'DC2': '$VAMPIRE_DC_SERVER'}, - extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) -planoldpythontestsuite("vampire_dc:local", "replica_sync", - extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], - name="samba4.drs.replica_sync.python(vampire_dc)", - environ={'DC1': '$DC_SERVER', 'DC2': '$VAMPIRE_DC_SERVER'}, - extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) -planoldpythontestsuite("vampire_dc", "delete_object", - extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], - name="samba4.drs.delete_object.python(vampire_dc)", - environ={'DC1': '$DC_SERVER', 'DC2': '$VAMPIRE_DC_SERVER'}, - extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) -planoldpythontestsuite("vampire_dc", "fsmo", - name="samba4.drs.fsmo.python(vampire_dc)", - extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], - environ={'DC1': "$DC_SERVER", 'DC2': "$VAMPIRE_DC_SERVER"}, - extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) -planoldpythontestsuite("vampire_dc", "repl_schema", - extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], - name="samba4.drs.repl_schema.python(vampire_dc)", - environ={'DC1': "$DC_SERVER", 'DC2': '$VAMPIRE_DC_SERVER'}, - extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) - # This makes sure we test the rid allocation code t = "rpc.samr.large-dc" plansmbtorture4testsuite(t, "vampire_dc", ['$SERVER', '-U$USERNAME%$PASSWORD', '--workgroup=$DOMAIN'], modname=("samba4.%s.one" % t)) @@ -499,6 +474,32 @@ plantestsuite("samba4.blackbox.renamedc.sh", "none", ["PYTHON=%s" % python, os.p # Demote the vampire DC, it must be the last test on the VAMPIRE DC for env in ['vampire_dc', 'promoted_dc']: + + # DRS python tests + planoldpythontestsuite(env, "samba.tests.blackbox.samba_tool_drs", + environ={'DC1': '$DC_SERVER', 'DC2': '$%s_SERVER' % env.upper()}, + extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) + planoldpythontestsuite("%s:local" % env, "replica_sync", + extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], + name="samba4.drs.replica_sync.python(%s)" % env, + environ={'DC1': '$DC_SERVER', 'DC2': '$%s_SERVER' % env.upper()}, + extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) + planoldpythontestsuite(env, "delete_object", + extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], + name="samba4.drs.delete_object.python(%s)" % env, + environ={'DC1': '$DC_SERVER', 'DC2': '$%s_SERVER' % env.upper()}, + extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) + planoldpythontestsuite(env, "fsmo", + name="samba4.drs.fsmo.python(%s)" % env, + extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], + environ={'DC1': "$DC_SERVER", 'DC2': '$%s_SERVER' % env.upper()}, + extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) + planoldpythontestsuite(env, "repl_schema", + extra_path=[os.path.join(samba4srcdir, 'torture/drs/python')], + name="samba4.drs.repl_schema.python(%s)" % env, + environ={'DC1': "$DC_SERVER", 'DC2': '$%s_SERVER' % env.upper()}, + extra_args=['-U$DOMAIN/$DC_USERNAME%$DC_PASSWORD']) + plantestsuite("samba4.blackbox.samba_tool_demote(%s)" % env, env, [os.path.join(samba4srcdir, "utils/tests/test_demote.sh"), '$SERVER', '$SERVER_IP', '$USERNAME', '$PASSWORD', '$DOMAIN', '$DC_SERVER', '$PREFIX/%s' % env, smbclient4]) # TODO: Verifying the databases really should be a part of the # environment teardown. -- 1.7.9.5 From e7441b7cc52595d86a8e26b5b5135bc3292f502d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 17 May 2013 23:02:03 +0200 Subject: [PATCH 12/16] dsdb-repl: split out dsdb_repl_resolve_working_schema This can be reused later in other places. Pair-Programmed-With: Matthieu Patou Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 9af430ec0be46d0a616faee3552600219bc57097) --- source4/dsdb/repl/replicated_objects.c | 147 ++++++++++++++++++++------------ 1 file changed, 94 insertions(+), 53 deletions(-) diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index 6c96b41..82a4667 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -31,35 +31,23 @@ #include "libcli/auth/libcli_auth.h" #include "param/param.h" -/** - * Multi-pass working schema creation - * Function will: - * - shallow copy initial schema supplied - * - create a working schema in multiple passes - * until all objects are resolved - * Working schema is a schema with Attributes, Classes - * and indexes, but w/o subClassOf, possibleSupperiors etc. - * It is to be used just us cache for converting attribute values. - */ -WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, - const struct dsdb_schema *initial_schema, - const struct drsuapi_DsReplicaOIDMapping_Ctr *mapping_ctr, - uint32_t object_count, - const struct drsuapi_DsReplicaObjectListItemEx *first_object, - const DATA_BLOB *gensec_skey, - TALLOC_CTX *mem_ctx, - struct dsdb_schema **_schema_out) +WERROR dsdb_repl_resolve_working_schema(struct ldb_context *ldb, + TALLOC_CTX *mem_ctx, + struct dsdb_schema_prefixmap *pfm_remote, + struct dsdb_schema *initial_schema, + struct dsdb_schema *resulting_schema, + uint32_t object_count, + const struct drsuapi_DsReplicaObjectListItemEx *first_object) { struct schema_list { struct schema_list *next, *prev; const struct drsuapi_DsReplicaObjectListItemEx *obj; }; - - WERROR werr; - struct dsdb_schema_prefixmap *pfm_remote; struct schema_list *schema_list = NULL, *schema_list_item, *schema_list_next_item; + WERROR werr; struct dsdb_schema *working_schema; const struct drsuapi_DsReplicaObjectListItemEx *cur; + DATA_BLOB empty_key = data_blob_null; int ret, pass_no; uint32_t ignore_attids[] = { DRSUAPI_ATTID_auxiliaryClass, @@ -70,46 +58,37 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, DRSUAPI_ATTID_INVALID }; - /* make a copy of the iniatial_scheam so we don't mess with it */ - working_schema = dsdb_schema_copy_shallow(mem_ctx, ldb, initial_schema); - if (!working_schema) { - DEBUG(0,(__location__ ": schema copy failed!\n")); - return WERR_NOMEM; - } - - /* we are going to need remote prefixMap for decoding */ - werr = dsdb_schema_pfm_from_drsuapi_pfm(mapping_ctr, true, - mem_ctx, &pfm_remote, NULL); - if (!W_ERROR_IS_OK(werr)) { - DEBUG(0,(__location__ ": Failed to decode remote prefixMap: %s", - win_errstr(werr))); - return werr; - } - /* create a list of objects yet to be converted */ for (cur = first_object; cur; cur = cur->next_object) { schema_list_item = talloc(mem_ctx, struct schema_list); + if (schema_list_item == NULL) { + return WERR_NOMEM; + } + schema_list_item->obj = cur; DLIST_ADD_END(schema_list, schema_list_item, struct schema_list); } /* resolve objects until all are resolved and in local schema */ pass_no = 1; + working_schema = initial_schema; while (schema_list) { uint32_t converted_obj_count = 0; uint32_t failed_obj_count = 0; - TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx); - W_ERROR_HAVE_NO_MEMORY(tmp_ctx); - for (schema_list_item = schema_list; schema_list_item; schema_list_item=schema_list_next_item) { + for (schema_list_item = schema_list; + schema_list_item; + schema_list_item=schema_list_next_item) { struct dsdb_extended_replicated_object object; cur = schema_list_item->obj; - /* Save the next item, now we have saved out + /* + * Save the next item, now we have saved out * the current one, so we can DLIST_REMOVE it - * safely */ + * safely + */ schema_list_next_item = schema_list_item->next; /* @@ -117,14 +96,17 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, * schema we have so far. It's ok if we fail to convert * an object. We should convert more objects on next pass. */ - werr = dsdb_convert_object_ex(ldb, working_schema, pfm_remote, - cur, gensec_skey, + werr = dsdb_convert_object_ex(ldb, working_schema, + pfm_remote, + cur, &empty_key, ignore_attids, 0, - tmp_ctx, &object); + schema_list_item, &object); if (!W_ERROR_IS_OK(werr)) { - DEBUG(4,("debug: Failed to convert schema object %s into ldb msg, will try during next loop\n", - cur->object.identifier->dn)); + DEBUG(4,("debug: Failed to convert schema " + "object %s into ldb msg, " + "will try during next loop\n", + cur->object.identifier->dn)); failed_obj_count++; } else { @@ -134,22 +116,23 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, * the remote prefixMap */ werr = dsdb_schema_set_el_from_ldb_msg_dups(ldb, - working_schema, + resulting_schema, object.msg, true); if (!W_ERROR_IS_OK(werr)) { - DEBUG(4,("debug: failed to convert object %s into a schema element, will try during next loop: %s\n", + DEBUG(4,("debug: failed to convert " + "object %s into a schema element, " + "will try during next loop: %s\n", ldb_dn_get_linearized(object.msg->dn), win_errstr(werr))); failed_obj_count++; } else { DLIST_REMOVE(schema_list, schema_list_item); - talloc_free(schema_list_item); + TALLOC_FREE(schema_list_item); converted_obj_count++; } } } - talloc_free(tmp_ctx); DEBUG(4,("Schema load pass %d: converted %d, %d of %d objects left to be converted.\n", pass_no, converted_obj_count, failed_obj_count, object_count)); @@ -157,7 +140,11 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, /* check if we converted any objects in this pass */ if (converted_obj_count == 0) { - DEBUG(0,("Can't continue Schema load: didn't manage to convert any objects: all %d remaining of %d objects failed to convert\n", failed_obj_count, object_count)); + DEBUG(0,("Can't continue Schema load: " + "didn't manage to convert any objects: " + "all %d remaining of %d objects " + "failed to convert\n", + failed_obj_count, object_count)); return WERR_INTERNAL_ERROR; } @@ -167,7 +154,61 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, DEBUG(0,("Failed to create schema-cache indexes!\n")); return WERR_INTERNAL_ERROR; } - }; + } + + return WERR_OK; +} + +/** + * Multi-pass working schema creation + * Function will: + * - shallow copy initial schema supplied + * - create a working schema in multiple passes + * until all objects are resolved + * Working schema is a schema with Attributes, Classes + * and indexes, but w/o subClassOf, possibleSupperiors etc. + * It is to be used just us cache for converting attribute values. + */ +WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, + const struct dsdb_schema *initial_schema, + const struct drsuapi_DsReplicaOIDMapping_Ctr *mapping_ctr, + uint32_t object_count, + const struct drsuapi_DsReplicaObjectListItemEx *first_object, + const DATA_BLOB *gensec_skey, + TALLOC_CTX *mem_ctx, + struct dsdb_schema **_schema_out) +{ + WERROR werr; + struct dsdb_schema_prefixmap *pfm_remote; + struct dsdb_schema *working_schema; + + /* make a copy of the iniatial_scheam so we don't mess with it */ + working_schema = dsdb_schema_copy_shallow(mem_ctx, ldb, initial_schema); + if (!working_schema) { + DEBUG(0,(__location__ ": schema copy failed!\n")); + return WERR_NOMEM; + } + + /* we are going to need remote prefixMap for decoding */ + werr = dsdb_schema_pfm_from_drsuapi_pfm(mapping_ctr, true, + mem_ctx, &pfm_remote, NULL); + if (!W_ERROR_IS_OK(werr)) { + DEBUG(0,(__location__ ": Failed to decode remote prefixMap: %s", + win_errstr(werr))); + return werr; + } + + werr = dsdb_repl_resolve_working_schema(ldb, mem_ctx, + pfm_remote, + working_schema, + working_schema, + object_count, + first_object); + if (!W_ERROR_IS_OK(werr)) { + DEBUG(0, ("%s: dsdb_repl_resolve_working_schema() failed: %s", + __location__, win_errstr(werr))); + return werr; + } *_schema_out = working_schema; -- 1.7.9.5 From 70e64c954423e9f1688f212a85b9469e46bf1550 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 17 May 2013 23:18:41 +0200 Subject: [PATCH 13/16] dsdb-repl: merge the logic from libnet_vampire_cb_apply_schema() This way libnet_vampire_cb_apply_schema() is able to use dsdb_repl_resolve_working_schema(). Pair-Programmed-With: Matthieu Patou Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 24fb281ea7983a42ba94b664530170e2401523f7) --- source4/dsdb/repl/replicated_objects.c | 119 ++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 6 deletions(-) diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index 82a4667..b0abc1a 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -31,9 +31,81 @@ #include "libcli/auth/libcli_auth.h" #include "param/param.h" +static WERROR dsdb_repl_merge_working_schema(struct ldb_context *ldb, + struct dsdb_schema *dest_schema, + const struct dsdb_schema *ref_schema) +{ + const struct dsdb_class *cur_class = NULL; + const struct dsdb_attribute *cur_attr = NULL; + int ret; + + for (cur_class = ref_schema->classes; + cur_class; + cur_class = cur_class->next) + { + const struct dsdb_class *tmp1; + struct dsdb_class *tmp2; + + tmp1 = dsdb_class_by_governsID_id(dest_schema, + cur_class->governsID_id); + if (tmp1 != NULL) { + continue; + } + + /* + * Do a shallow copy so that original next and prev are + * not modified, we don't need to do a deep copy + * as the rest won't be modified and this is for + * a short lived object. + */ + tmp2 = talloc(dest_schema->classes, struct dsdb_class); + if (tmp2 == NULL) { + return WERR_NOMEM; + } + *tmp2 = *cur_class; + DLIST_ADD(dest_schema->classes, tmp2); + } + + for (cur_attr = ref_schema->attributes; + cur_attr; + cur_attr = cur_attr->next) + { + const struct dsdb_attribute *tmp1; + struct dsdb_attribute *tmp2; + + tmp1 = dsdb_attribute_by_attributeID_id(dest_schema, + cur_attr->attributeID_id); + if (tmp1 != NULL) { + continue; + } + + /* + * Do a shallow copy so that original next and prev are + * not modified, we don't need to do a deep copy + * as the rest won't be modified and this is for + * a short lived object. + */ + tmp2 = talloc(dest_schema->attributes, struct dsdb_attribute); + if (tmp2 == NULL) { + return WERR_NOMEM; + } + *tmp2 = *cur_attr; + DLIST_ADD(dest_schema->attributes, tmp2); + } + + ret = dsdb_setup_sorted_accessors(ldb, dest_schema); + if (LDB_SUCCESS != ret) { + DEBUG(0,("Failed to add new attribute to reference schema!\n")); + return WERR_INTERNAL_ERROR; + } + + return WERR_OK; +} + WERROR dsdb_repl_resolve_working_schema(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, struct dsdb_schema_prefixmap *pfm_remote, + uint32_t cycle_before_switching, struct dsdb_schema *initial_schema, struct dsdb_schema *resulting_schema, uint32_t object_count, @@ -77,6 +149,25 @@ WERROR dsdb_repl_resolve_working_schema(struct ldb_context *ldb, uint32_t converted_obj_count = 0; uint32_t failed_obj_count = 0; + if (resulting_schema != working_schema) { + /* + * If the selfmade schema is not the schema used to + * translate and validate replicated object, + * Which means that we are using the bootstrap schema + * Then we add attributes and classes that were already + * translated to the working schema, the idea is that + * we might need to add new attributes and classes + * to be able to translate critical replicated objects + * and without that we wouldn't be able to translate them + */ + werr = dsdb_repl_merge_working_schema(ldb, + working_schema, + resulting_schema); + if (!W_ERROR_IS_OK(werr)) { + return werr; + } + } + for (schema_list_item = schema_list; schema_list_item; schema_list_item=schema_list_next_item) { @@ -114,6 +205,10 @@ WERROR dsdb_repl_resolve_working_schema(struct ldb_context *ldb, * Convert the schema from ldb_message format * (OIDs as OID strings) into schema, using * the remote prefixMap + * + * It's not likely, but possible to get the + * same object twice and we should keep + * the last instance. */ werr = dsdb_schema_set_el_from_ldb_msg_dups(ldb, resulting_schema, @@ -127,6 +222,8 @@ WERROR dsdb_repl_resolve_working_schema(struct ldb_context *ldb, win_errstr(werr))); failed_obj_count++; } else { + DEBUG(8,("Converted object %s into a schema element\n", + ldb_dn_get_linearized(object.msg->dn))); DLIST_REMOVE(schema_list, schema_list_item); TALLOC_FREE(schema_list_item); converted_obj_count++; @@ -136,7 +233,6 @@ WERROR dsdb_repl_resolve_working_schema(struct ldb_context *ldb, DEBUG(4,("Schema load pass %d: converted %d, %d of %d objects left to be converted.\n", pass_no, converted_obj_count, failed_obj_count, object_count)); - pass_no++; /* check if we converted any objects in this pass */ if (converted_obj_count == 0) { @@ -148,12 +244,22 @@ WERROR dsdb_repl_resolve_working_schema(struct ldb_context *ldb, return WERR_INTERNAL_ERROR; } - /* rebuild indexes */ - ret = dsdb_setup_sorted_accessors(ldb, working_schema); - if (LDB_SUCCESS != ret) { - DEBUG(0,("Failed to create schema-cache indexes!\n")); - return WERR_INTERNAL_ERROR; + /* + * Don't try to load the schema if there is missing object + * _and_ we are on the first pass as some critical objects + * might be missing. + */ + if (failed_obj_count == 0 || pass_no > cycle_before_switching) { + /* prepare for another cycle */ + working_schema = resulting_schema; + + ret = dsdb_setup_sorted_accessors(ldb, working_schema); + if (LDB_SUCCESS != ret) { + DEBUG(0,("Failed to create schema-cache indexes!\n")); + return WERR_INTERNAL_ERROR; + } } + pass_no++; } return WERR_OK; @@ -200,6 +306,7 @@ WERROR dsdb_repl_make_working_schema(struct ldb_context *ldb, werr = dsdb_repl_resolve_working_schema(ldb, mem_ctx, pfm_remote, + 0, /* cycle_before_switching */ working_schema, working_schema, object_count, -- 1.7.9.5 From 1c875dc345ed5ec906bbcafcfa6dd979b1fdc247 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 17 May 2013 23:18:55 +0200 Subject: [PATCH 14/16] libnet-vampire: make use of dsdb_repl_resolve_working_schema() Pair-Programmed-With: Matthieu Patou Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Thu May 23 14:18:03 CEST 2013 on sn-devel-104 (cherry picked from commit e24fe5705e3c4d33705ebb584ea2009bb4a1a82c) --- source4/libnet/libnet_vampire.c | 227 +++------------------------------------ 1 file changed, 17 insertions(+), 210 deletions(-) diff --git a/source4/libnet/libnet_vampire.c b/source4/libnet/libnet_vampire.c index 0f20b15..9489f0b 100644 --- a/source4/libnet/libnet_vampire.c +++ b/source4/libnet/libnet_vampire.c @@ -216,94 +216,15 @@ NTSTATUS libnet_vampire_cb_check_options(void *private_data, return NT_STATUS_OK; } -static NTSTATUS libnet_vampire_merge_schema(struct ldb_context *ldb, - struct dsdb_schema *dest_schema, - const struct dsdb_schema *ref_schema) -{ - const struct dsdb_class *cur_class = NULL; - const struct dsdb_attribute *cur_attr = NULL; - int ret; - - for (cur_class = ref_schema->classes; - cur_class; - cur_class = cur_class->next) - { - const struct dsdb_class *tmp1; - struct dsdb_class *tmp2; - - tmp1 = dsdb_class_by_governsID_id(dest_schema, - cur_class->governsID_id); - if (tmp1 != NULL) { - continue; - } - - /* - * Do a shallow copy so that original next and prev are - * not modified, we don't need to do a deep copy - * as the rest won't be modified and this is for - * a short lived object. - */ - tmp2 = talloc(dest_schema->classes, struct dsdb_class); - if (tmp2 == NULL) { - return NT_STATUS_NO_MEMORY; - } - *tmp2 = *cur_class; - DLIST_ADD(dest_schema->classes, tmp2); - } - - for (cur_attr = ref_schema->attributes; - cur_attr; - cur_attr = cur_attr->next) - { - const struct dsdb_attribute *tmp1; - struct dsdb_attribute *tmp2; - - tmp1 = dsdb_attribute_by_attributeID_id(dest_schema, - cur_attr->attributeID_id); - if (tmp1 != NULL) { - continue; - } - - /* - * Do a shallow copy so that original next and prev are - * not modified, we don't need to do a deep copy - * as the rest won't be modified and this is for - * a short lived object. - */ - tmp2 = talloc(dest_schema->attributes, struct dsdb_attribute); - if (tmp2 == NULL) { - return NT_STATUS_NO_MEMORY; - } - *tmp2 = *cur_attr; - DLIST_ADD(dest_schema->attributes, tmp2); - } - - ret = dsdb_setup_sorted_accessors(ldb, dest_schema); - if (LDB_SUCCESS != ret) { - DEBUG(0,("Failed to add new attribute to reference schema!\n")); - return NT_STATUS_INTERNAL_ERROR; - } - - return NT_STATUS_OK; -} - static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s, const struct libnet_BecomeDC_StoreChunk *c) { - struct schema_list { - struct schema_list *next, *prev; - const struct drsuapi_DsReplicaObjectListItemEx *obj; - }; - WERROR status; struct dsdb_schema_prefixmap *pfm_remote; const struct drsuapi_DsReplicaOIDMapping_Ctr *mapping_ctr; - struct schema_list *schema_list = NULL, *schema_list_item, *schema_list_next_item; - struct dsdb_schema *working_schema; struct dsdb_schema *provision_schema; uint32_t object_count = 0; struct drsuapi_DsReplicaObjectListItemEx *first_object; - const struct drsuapi_DsReplicaObjectListItemEx *cur; uint32_t linked_attributes_count; struct drsuapi_DsReplicaLinkedAttribute *linked_attributes; const struct drsuapi_DsReplicaCursor2CtrEx *uptodateness_vector; @@ -314,17 +235,10 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s struct ldb_message *msg; struct ldb_message_element *prefixMap_el; uint32_t i; - int ret, pass_no; + int ret; bool ok; uint64_t seq_num; - uint32_t ignore_attids[] = { - DRSUAPI_ATTID_auxiliaryClass, - DRSUAPI_ATTID_mayContain, - DRSUAPI_ATTID_mustContain, - DRSUAPI_ATTID_possSuperiors, - DRSUAPI_ATTID_systemPossSuperiors, - DRSUAPI_ATTID_INVALID - }; + uint32_t cycle_before_switching; DEBUG(0,("Analyze and apply schema objects\n")); @@ -372,7 +286,6 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s return NT_STATUS_FOOBAR; } - status = dsdb_schema_pfm_from_drsuapi_pfm(mapping_ctr, true, s, &pfm_remote, NULL); if (!W_ERROR_IS_OK(status)) { @@ -414,131 +327,25 @@ static NTSTATUS libnet_vampire_cb_apply_schema(struct libnet_vampire_cb_state *s talloc_free(schema_ldb); } - /* create a list of objects yet to be converted */ - for (cur = first_object; cur; cur = cur->next_object) { - schema_list_item = talloc(s, struct schema_list); - schema_list_item->obj = cur; - DLIST_ADD_END(schema_list, schema_list_item, struct schema_list); + cycle_before_switching = lpcfg_parm_long(s->lp_ctx, NULL, + "become dc", + "schema convert retrial", 1); + + status = dsdb_repl_resolve_working_schema(s->ldb, s, + pfm_remote, + cycle_before_switching, + provision_schema, + s->self_made_schema, + object_count, + first_object); + if (!W_ERROR_IS_OK(status)) { + DEBUG(0, ("%s: dsdb_repl_resolve_working_schema() failed: %s", + __location__, win_errstr(status))); + return werror_to_ntstatus(status); } - /* resolve objects until all are resolved and in local schema */ - pass_no = 1; - working_schema = provision_schema; - - while (schema_list) { - uint32_t converted_obj_count = 0; - uint32_t failed_obj_count = 0; - int cycle_before_switching = 0; - TALLOC_CTX *tmp_ctx = talloc_new(s); - NT_STATUS_HAVE_NO_MEMORY(tmp_ctx); - - if (s->self_made_schema != working_schema) { - /* - * If the selfmade schema is not the schema used to - * translate and validate replicated object, - * Which means that we are using the bootstrap schema - * Then we add attributes and classes that were already - * translated to the working schema, the idea is that - * we might need to add new attributes and classes - * to be able to translate critical replicated objects - * and without that we wouldn't be able to translate them - */ - NTSTATUS nt_status; - - nt_status = libnet_vampire_merge_schema(s->ldb, - working_schema, - s->self_made_schema); - if (!NT_STATUS_IS_OK(nt_status)) { - return nt_status; - } - } - - for (schema_list_item = schema_list; schema_list_item; schema_list_item=schema_list_next_item) { - struct dsdb_extended_replicated_object object; - - cur = schema_list_item->obj; - - /* Save the next item, now we have saved out - * the current one, so we can DLIST_REMOVE it - * safely */ - schema_list_next_item = schema_list_item->next; - - /* - * Convert the objects into LDB messages using the - * schema we have so far. It's ok if we fail to convert - * an object. We should convert more objects on next pass. - */ - status = dsdb_convert_object_ex(s->ldb, working_schema, pfm_remote, - cur, c->gensec_skey, - ignore_attids, - 0, - tmp_ctx, &object); - if (!W_ERROR_IS_OK(status)) { - DEBUG(1,("Warning: Failed to convert schema object %s into ldb msg\n", - cur->object.identifier->dn)); - - failed_obj_count++; - } else { - /* - * Convert the schema from ldb_message format - * (OIDs as OID strings) into schema, using - * the remote prefixMap - * - * It's not likely, but possible to get the - * same object twice and we should keep - * the last instance. - */ - status = dsdb_schema_set_el_from_ldb_msg_dups(s->ldb, - s->self_made_schema, - object.msg, true); - if (!W_ERROR_IS_OK(status)) { - DEBUG(1,("Warning: failed to convert object %s into a schema element: %s\n", - ldb_dn_get_linearized(object.msg->dn), - win_errstr(status))); - failed_obj_count++; - } else { - DEBUG(8,("Converted object %s into a schema element\n", - ldb_dn_get_linearized(object.msg->dn))); - DLIST_REMOVE(schema_list, schema_list_item); - converted_obj_count++; - } - } - } - talloc_free(tmp_ctx); - - DEBUG(4,("Schema load pass %d: converted %d, %d of %d objects left to be converted.\n", - pass_no, converted_obj_count, failed_obj_count, object_count)); - - /* check if we converted any objects in this pass */ - if (converted_obj_count == 0) { - DEBUG(0,("Can't continue Schema load: didn't manage to convert any objects: all %d remaining of %d objects failed to convert\n", failed_obj_count, object_count)); - return NT_STATUS_INTERNAL_ERROR; - } - - /* - * Don't try to load the schema if there is missing object - * _and_ we are on the first pass as some critical objects - * might be missing. - */ - cycle_before_switching = lpcfg_parm_int(s->lp_ctx, NULL, - "become dc", - "schema convert retrial", 1); - if (failed_obj_count == 0 || pass_no > cycle_before_switching) { - /* prepare for another cycle */ - working_schema = s->self_made_schema; - - ret = dsdb_setup_sorted_accessors(s->ldb, working_schema); - if (LDB_SUCCESS != ret) { - DEBUG(0,("Failed to create schema-cache indexes!\n")); - return NT_STATUS_INTERNAL_ERROR; - } - } - pass_no++; - }; - /* free temp objects for 1st conversion phase */ talloc_unlink(s, provision_schema); - TALLOC_FREE(schema_list); /* * attach the schema we just brought over DRS to the ldb, -- 1.7.9.5 From 279a8e3e10d15a1c5dab1fd7f2d27f660201a5e4 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 10 Jun 2013 10:45:25 +0200 Subject: [PATCH 15/16] dsdb: use the correct talloc parent in dsdb_repl_merge_working_schema() schema->{classes,attributes} are the DLIST pointer not an array. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit d2f847149d3d1310c829169564704b45ac43e978) --- source4/dsdb/repl/replicated_objects.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index b0abc1a..e018aa4 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -58,7 +58,7 @@ static WERROR dsdb_repl_merge_working_schema(struct ldb_context *ldb, * as the rest won't be modified and this is for * a short lived object. */ - tmp2 = talloc(dest_schema->classes, struct dsdb_class); + tmp2 = talloc(dest_schema, struct dsdb_class); if (tmp2 == NULL) { return WERR_NOMEM; } @@ -85,7 +85,7 @@ static WERROR dsdb_repl_merge_working_schema(struct ldb_context *ldb, * as the rest won't be modified and this is for * a short lived object. */ - tmp2 = talloc(dest_schema->attributes, struct dsdb_attribute); + tmp2 = talloc(dest_schema, struct dsdb_attribute); if (tmp2 == NULL) { return WERR_NOMEM; } -- 1.7.9.5 From 90cb62cedd8072ec934e4cb855e2b5458b2c8693 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 10 Jun 2013 10:46:41 +0200 Subject: [PATCH 16/16] dsdb: reset schema->{classes,attributes}_to_remove_size to 0 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Stefan Metzmacher Autobuild-Date(master): Tue Jun 11 11:40:39 CEST 2013 on sn-devel-104 (cherry picked from commit 3fba9ba7ea85e33faac2718d2463c5d0cd2d85f4) --- source4/dsdb/schema/schema_set.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c index 73264f9..ce8facb 100644 --- a/source4/dsdb/schema/schema_set.c +++ b/source4/dsdb/schema/schema_set.c @@ -338,8 +338,10 @@ int dsdb_setup_sorted_accessors(struct ldb_context *ldb, TALLOC_FREE(schema->attributes_to_remove[i]); } - TALLOC_FREE(schema->attributes_to_remove); TALLOC_FREE(schema->classes_to_remove); + schema->classes_to_remove_size = 0; + TALLOC_FREE(schema->attributes_to_remove); + schema->attributes_to_remove_size = 0; /* free all caches */ dsdb_sorted_accessors_free(schema); -- 1.7.9.5