From f6f71aab9ec15997f2f1f8ed4632efc53784a55f Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 6 Jan 2022 15:11:20 -0800 Subject: [PATCH 1/5] tests: Add 2 tests for unique fileid's with top bit set (generated from itime) for files and directories. smb2.fileid_unique.fileid_unique smb2.fileid_unique.fileid_unique-dir Create 100 files or directories as fast as we can against a "normal" share, then read info on them and ensure (a) top bit is set (generated from itime) and (b) uniqueness across all generated objects (checks poor timestamp resolution doesn't create duplicate fileids). Add knownfail.d/fileid-unique BUG: https://bugzilla.samba.org/show_bug.cgi?id=14928 Signed-off-by: Jeremy Allison --- selftest/knownfail.d/fileid-unique | 2 + source3/selftest/tests.py | 2 + source4/selftest/tests.py | 1 + source4/torture/smb2/create.c | 175 +++++++++++++++++++++++++++++ source4/torture/smb2/smb2.c | 1 + 5 files changed, 181 insertions(+) create mode 100644 selftest/knownfail.d/fileid-unique diff --git a/selftest/knownfail.d/fileid-unique b/selftest/knownfail.d/fileid-unique new file mode 100644 index 00000000000..a29c4a0bb55 --- /dev/null +++ b/selftest/knownfail.d/fileid-unique @@ -0,0 +1,2 @@ +^samba3.smb2.fileid_unique.fileid_unique\(fileserver\) +^samba3.smb2.fileid_unique.fileid_unique-dir\(fileserver\) diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index df39fdaa53a..69cdc5b7f85 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -953,6 +953,8 @@ for t in tests: plansmbtorture4testsuite(t, "ad_dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD') elif t == "smb2.fileid": plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/vfs_fruit_xattr -U$USERNAME%$PASSWORD') + elif t == "smb2.fileid_unique": + plansmbtorture4testsuite(t, "fileserver", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD') elif t == "smb2.acls_non_canonical": plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/acls_non_canonical -U$USERNAME%$PASSWORD') elif t == "rpc.wkssvc": diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index e496499da23..a13a7ced0e5 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -377,6 +377,7 @@ smb2_s3only = [ "smb2.durable-v2-delay", "smb2.aio_delay", "smb2.fileid", + "smb2.fileid_unique", "smb2.timestamps", ] smb2 = [x for x in smbtorture4_testsuites("smb2.") if x not in smb2_s3only] diff --git a/source4/torture/smb2/create.c b/source4/torture/smb2/create.c index aba3f69a28a..b0e56d81db7 100644 --- a/source4/torture/smb2/create.c +++ b/source4/torture/smb2/create.c @@ -2707,6 +2707,161 @@ done: return ret; } +static bool test_fileid_unique_object( + struct torture_context *tctx, + struct smb2_tree *tree, + unsigned int num_objs, + bool create_dirs) +{ + TALLOC_CTX *mem_ctx = talloc_new(tctx); + char *fname = NULL; + struct smb2_handle testdirh; + struct smb2_handle h1; + struct smb2_create create; + unsigned int i; + uint64_t fileid_array[num_objs]; + NTSTATUS status; + bool ret = true; + + smb2_deltree(tree, DNAME); + + status = torture_smb2_testdir(tree, DNAME, &testdirh); + torture_assert_ntstatus_ok_goto(tctx, status, ret, done, + "test_fileid_unique failed\n"); + smb2_util_close(tree, testdirh); + + /* Create num_obj files as rapidly as we can. */ + for (i = 0; i < num_objs; i++) { + fname = talloc_asprintf(mem_ctx, + "%s\\testfile.%u", + DNAME, + i); + torture_assert_goto(tctx, + fname != NULL, + ret, + done, + "talloc failed\n"); + + create = (struct smb2_create) { + .in.desired_access = SEC_FILE_READ_ATTRIBUTE, + .in.share_access = NTCREATEX_SHARE_ACCESS_MASK, + .in.file_attributes = FILE_ATTRIBUTE_NORMAL, + .in.create_disposition = NTCREATEX_DISP_CREATE, + .in.fname = fname, + }; + + if (create_dirs) { + create.in.file_attributes = FILE_ATTRIBUTE_DIRECTORY; + create.in.create_options = FILE_DIRECTORY_FILE; + } + + status = smb2_create(tree, tctx, &create); + torture_assert_ntstatus_ok_goto(tctx, + status, + ret, + done, + "test file could not be created\n"); + h1 = create.out.file.handle; + smb2_util_close(tree, h1); + TALLOC_FREE(fname); + } + + /* + * Get the file ids. + */ + for (i = 0; i < num_objs; i++) { + union smb_fileinfo finfo; + + fname = talloc_asprintf(mem_ctx, + "%s\\testfile.%u", + DNAME, + i); + torture_assert_goto(tctx, + fname != NULL, + ret, + done, + "talloc failed\n"); + + create = (struct smb2_create) { + .in.desired_access = SEC_FILE_READ_ATTRIBUTE, + .in.share_access = NTCREATEX_SHARE_ACCESS_MASK, + .in.file_attributes = FILE_ATTRIBUTE_NORMAL, + .in.create_disposition = NTCREATEX_DISP_OPEN, + .in.fname = fname, + }; + + if (create_dirs) { + create.in.file_attributes = FILE_ATTRIBUTE_DIRECTORY; + create.in.create_options = FILE_DIRECTORY_FILE; + } + + status = smb2_create(tree, tctx, &create); + torture_assert_ntstatus_ok_goto(tctx, + status, + ret, + done, + "test file could not be opened\n"); + h1 = create.out.file.handle; + + finfo = (union smb_fileinfo) { + .generic.level = RAW_FILEINFO_SMB2_ALL_INFORMATION, + .generic.in.file.handle = h1, + }; + + status = smb2_getinfo_file(tree, tctx, &finfo); + torture_assert_ntstatus_ok_goto(tctx, + status, + ret, + done, + "failed to get fileid\n"); + smb2_util_close(tree, h1); + /* + * Samba created files on a "normal" share + * using itime should have the top bit of the fileid set. + */ + fileid_array[i] = finfo.all_info2.out.file_id; + torture_assert_goto(tctx, + ((fileid_array[i] & 0x8000000000000000) != 0), + ret, + done, + "fileid top bit not set\n"); + TALLOC_FREE(fname); + } + + /* All returned fileids must be unique. 100 is small so brute force. */ + for (i = 0; i < num_objs - 1; i++) { + unsigned int j; + for (j = i + 1; j < num_objs; j++) { + torture_assert_goto(tctx, + (fileid_array[i] != fileid_array[j]), + ret, + done, + "fileid top bit not set\n"); + } + } + +done: + + smb2_util_close(tree, testdirh); + smb2_deltree(tree, DNAME); + talloc_free(mem_ctx); + return ret; +} + +static bool test_fileid_unique( + struct torture_context *tctx, + struct smb2_tree *tree) +{ + return test_fileid_unique_object(tctx, tree, 100, false); +} + +static bool test_fileid_unique_dir( + struct torture_context *tctx, + struct smb2_tree *tree) +{ + return test_fileid_unique_object(tctx, tree, 100, true); +} + /* test opening quota fakefile handle and returned attributes */ @@ -2823,3 +2978,23 @@ struct torture_suite *torture_smb2_fileid_init(TALLOC_CTX *ctx) return suite; } + +/* + Testing for uniqueness of SMB2 File-IDs +*/ +struct torture_suite *torture_smb2_fileid_unique_init(TALLOC_CTX *ctx) +{ + struct torture_suite *suite = torture_suite_create(ctx, + "fileid_unique"); + + torture_suite_add_1smb2_test(suite, + "fileid_unique", + test_fileid_unique); + torture_suite_add_1smb2_test(suite, + "fileid_unique-dir", + test_fileid_unique_dir); + + suite->description = talloc_strdup(suite, "SMB2-FILEID-UNIQUE tests"); + + return suite; +} diff --git a/source4/torture/smb2/smb2.c b/source4/torture/smb2/smb2.c index f3a5c8ac875..95a7b49952f 100644 --- a/source4/torture/smb2/smb2.c +++ b/source4/torture/smb2/smb2.c @@ -213,6 +213,7 @@ NTSTATUS torture_smb2_init(TALLOC_CTX *ctx) torture_suite_add_1smb2_test(suite, "secleak", torture_smb2_sec_leak); torture_suite_add_1smb2_test(suite, "session-id", run_sessidtest); torture_suite_add_suite(suite, torture_smb2_deny_init(suite)); + torture_suite_add_suite(suite, torture_smb2_fileid_unique_init(suite)); suite->description = talloc_strdup(suite, "SMB2-specific tests"); -- 2.30.2 From f82781ed5ed3d901b155f1657594a9b392a1e743 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 6 Jan 2022 13:58:20 -0800 Subject: [PATCH 2/5] lib: util: Add a function nt_time_to_unix_timespec_constraints(), that allows low and high NTTIME value checks to be selected by the caller. Not yet used. This will allow us to move nt_time_to_unix_timespec() and nt_time_to_full_timespec() to use common code. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14928 Signed-off-by: Jeremy Allison --- lib/util/time.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/util/time.h | 5 +++++ 2 files changed, 55 insertions(+) diff --git a/lib/util/time.c b/lib/util/time.c index cec91c14791..6808a6c7ae1 100644 --- a/lib/util/time.c +++ b/lib/util/time.c @@ -867,6 +867,56 @@ _PUBLIC_ int get_time_zone(time_t t) return tm_diff(&tm_utc,tm); } +/* + * Convert an NTTIME to a unix timespec, giving the + * option of checking for the calculated value being + * less than TIME_T_MIN or greater than TIME_T_MAX. + */ + +struct timespec nt_time_to_unix_timespec_constraints( + NTTIME nt, + bool check_low, + bool check_high) +{ + int64_t d; + struct timespec ret; + + d = (int64_t)nt; + /* d is now in 100ns units, since jan 1st 1601". + Save off the ns fraction. */ + + /* + * Take the last seven decimal digits and multiply by 100. + * to convert from 100ns units to 1ns units. + */ + ret.tv_nsec = (long) ((d % (1000 * 1000 * 10)) * 100); + + /* Convert to seconds */ + d /= 1000*1000*10; + + /* Now adjust by 369 years to make the secs since 1970 */ + d -= TIME_FIXUP_CONSTANT_INT; + + if (check_low) { + if (d <= (int64_t)TIME_T_MIN) { + ret.tv_sec = TIME_T_MIN; + ret.tv_nsec = 0; + return ret; + } + } + + if (check_high) { + if (d >= (int64_t)TIME_T_MAX) { + ret.tv_sec = TIME_T_MAX; + ret.tv_nsec = 0; + return ret; + } + } + + ret.tv_sec = (time_t)d; + return ret; +} + struct timespec nt_time_to_unix_timespec(NTTIME nt) { int64_t d; diff --git a/lib/util/time.h b/lib/util/time.h index 72347b39b99..b2ce7528a4b 100644 --- a/lib/util/time.h +++ b/lib/util/time.h @@ -343,6 +343,11 @@ bool nt_time_equal(NTTIME *t1, NTTIME *t2); void interpret_dos_date(uint32_t date,int *year,int *month,int *day,int *hour,int *minute,int *second); +struct timespec nt_time_to_unix_timespec_constraints( + NTTIME nt, + bool check_low, + bool check_high); + struct timespec nt_time_to_unix_timespec(NTTIME nt); time_t convert_timespec_to_time_t(struct timespec ts); -- 2.30.2 From 5dfdde9a45c34c587674a97a4ae92b676e675ef8 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 6 Jan 2022 14:05:41 -0800 Subject: [PATCH 3/5] lib: util: Convert nt_time_to_unix_timespec() to use nt_time_to_unix_timespec_constraints(). Removes common code. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14928 Signed-off-by: Jeremy Allison --- lib/util/time.c | 34 +++------------------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/lib/util/time.c b/lib/util/time.c index 6808a6c7ae1..a8740cbb6e2 100644 --- a/lib/util/time.c +++ b/lib/util/time.c @@ -919,7 +919,6 @@ struct timespec nt_time_to_unix_timespec_constraints( struct timespec nt_time_to_unix_timespec(NTTIME nt) { - int64_t d; struct timespec ret; if (nt == 0 || nt == (int64_t)-1) { @@ -928,36 +927,9 @@ struct timespec nt_time_to_unix_timespec(NTTIME nt) return ret; } - d = (int64_t)nt; - /* d is now in 100ns units, since jan 1st 1601". - Save off the ns fraction. */ - - /* - * Take the last seven decimal digits and multiply by 100. - * to convert from 100ns units to 1ns units. - */ - ret.tv_nsec = (long) ((d % (1000 * 1000 * 10)) * 100); - - /* Convert to seconds */ - d /= 1000*1000*10; - - /* Now adjust by 369 years to make the secs since 1970 */ - d -= TIME_FIXUP_CONSTANT_INT; - - if (d <= (int64_t)TIME_T_MIN) { - ret.tv_sec = TIME_T_MIN; - ret.tv_nsec = 0; - return ret; - } - - if (d >= (int64_t)TIME_T_MAX) { - ret.tv_sec = TIME_T_MAX; - ret.tv_nsec = 0; - return ret; - } - - ret.tv_sec = (time_t)d; - return ret; + return nt_time_to_unix_timespec_constraints(nt, + true, /* check low. */ + true /* check high */); } -- 2.30.2 From 0b1ebe64ad1f413758e184cbeb15b032e1d3a519 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 6 Jan 2022 14:09:08 -0800 Subject: [PATCH 4/5] lib: util: Convert nt_time_to_full_timespec() to use nt_time_to_unix_timespec_constraints(). Removes common code. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14928 Signed-off-by: Jeremy Allison --- lib/util/time.c | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/lib/util/time.c b/lib/util/time.c index a8740cbb6e2..4d50786a2f6 100644 --- a/lib/util/time.c +++ b/lib/util/time.c @@ -1144,9 +1144,6 @@ NTTIME full_timespec_to_nt_time(const struct timespec *_ts) **/ struct timespec nt_time_to_full_timespec(NTTIME nt) { - int64_t d; - struct timespec ret; - if (nt == NTTIME_OMIT) { return make_omit_timespec(); } @@ -1161,30 +1158,9 @@ struct timespec nt_time_to_full_timespec(NTTIME nt) nt = NTTIME_MAX; } - d = (int64_t)nt; - /* d is now in 100ns units, since jan 1st 1601". - Save off the ns fraction. */ - - /* - * Take the last seven decimal digits and multiply by 100. - * to convert from 100ns units to 1ns units. - */ - ret.tv_nsec = (long) ((d % (1000 * 1000 * 10)) * 100); - - /* Convert to seconds */ - d /= 1000*1000*10; - - /* Now adjust by 369 years to make the secs since 1970 */ - d -= TIME_FIXUP_CONSTANT_INT; - - if (d >= (int64_t)TIME_T_MAX) { - ret.tv_sec = TIME_T_MAX; - ret.tv_nsec = 0; - return ret; - } - - ret.tv_sec = (time_t)d; - return ret; + return nt_time_to_unix_timespec_constraints(nt, + false, /* check low. */ + true /* check high */); } /** -- 2.30.2 From eb415317c20647aa3e4f7f3d4de3f1194743ea10 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Wed, 5 Jan 2022 11:40:46 -0800 Subject: [PATCH 5/5] s3: smbd: Create and use a common function for generating a fileid - create_clock_itime(). This first gets the clock_gettime_mono() value, converts to an NTTIME (as this is what is stored in the dos attribute EA), then mixes in 8 bits of randomness shifted up by 55 bits to cope with poor resolution clocks to avoid duplicate inodes. Using 8 bits of randomness on top of an NTTIME gives us around 114 years headroom. We can now guarentee returning a itime-based fileid in a normal share (storing dos attributes in an EA). Remove knownfail.d/fileid-unique BUG: https://bugzilla.samba.org/show_bug.cgi?id=14928 Signed-off-by: Jeremy Allison --- selftest/knownfail.d/fileid-unique | 2 -- source3/include/proto.h | 1 + source3/lib/system.c | 54 ++++++++++++++++++++++++++++++ source3/smbd/open.c | 6 ++-- 4 files changed, 58 insertions(+), 5 deletions(-) delete mode 100644 selftest/knownfail.d/fileid-unique diff --git a/selftest/knownfail.d/fileid-unique b/selftest/knownfail.d/fileid-unique deleted file mode 100644 index a29c4a0bb55..00000000000 --- a/selftest/knownfail.d/fileid-unique +++ /dev/null @@ -1,2 +0,0 @@ -^samba3.smb2.fileid_unique.fileid_unique\(fileserver\) -^samba3.smb2.fileid_unique.fileid_unique-dir\(fileserver\) diff --git a/source3/include/proto.h b/source3/include/proto.h index 6154b1ab26d..dba728b3d86 100644 --- a/source3/include/proto.h +++ b/source3/include/proto.h @@ -175,6 +175,7 @@ void update_stat_ex_create_time(struct stat_ex *dst, struct timespec create_time void update_stat_ex_file_id(struct stat_ex *dst, uint64_t file_id); void update_stat_ex_from_saved_stat(struct stat_ex *dst, const struct stat_ex *src); +void create_clock_itime(struct stat_ex *dst); int sys_stat(const char *fname, SMB_STRUCT_STAT *sbuf, bool fake_dir_create_times); int sys_fstat(int fd, SMB_STRUCT_STAT *sbuf, diff --git a/source3/lib/system.c b/source3/lib/system.c index 671fc2760a0..fea96dc358e 100644 --- a/source3/lib/system.c +++ b/source3/lib/system.c @@ -310,6 +310,60 @@ void init_stat_ex_from_stat (struct stat_ex *dst, dst->st_ex_iflags |= ST_EX_IFLAG_CALCULATED_FILE_ID; } +/******************************************************************* + Create a clock-derived itime (imaginary) time. Used to generate + the fileid. +********************************************************************/ + +void create_clock_itime(struct stat_ex *dst) +{ + NTTIME tval; + struct timespec itime; + uint64_t mixin; + uint8_t rval; + + /* Start with the system clock. */ + clock_gettime_mono(&itime); + + /* Convert to NTTIME. */ + tval = unix_timespec_to_nt_time(itime); + + /* + * In case the system clock is poor granularity + * (happens on VM or docker images) then mix in + * 8 bits of randomness. + */ + generate_random_buffer((unsigned char *)&rval, 1); + mixin = rval; + + /* + * Shift up by 55 bits. This gives us approx 114 years + * of headroom. + */ + mixin <<= 55; + + /* And OR into the nttime. */ + tval |= mixin; + + /* + * Convert to a unix timespec, ignoring any + * constraints on seconds being higher than + * TIME_T_MAX or lower than TIME_T_MIN. These + * are only needed to allow unix display time functions + * to work correctly, and this is being used to + * generate a fileid. All we care about is the + * NTTIME being valid across all NTTIME ranges + * (which we carefully ensured above). + */ + + itime = nt_time_to_unix_timespec_constraints(tval, + false, /* check low. */ + false /* check high */); + + /* And set as a generated itime. */ + update_stat_ex_itime(dst, itime); +} + /******************************************************************* A stat() wrapper. ********************************************************************/ diff --git a/source3/smbd/open.c b/source3/smbd/open.c index e3bb304292e..cf5c620fe21 100644 --- a/source3/smbd/open.c +++ b/source3/smbd/open.c @@ -4129,13 +4129,13 @@ static NTSTATUS open_file_ntcreate(connection_struct *conn, * If we created a file and it's not a stream, this is the point where * we set the itime (aka invented time) that get's stored in the DOS * attribute xattr. The value is going to be either what the filesystem - * provided or a copy of the creation date. + * provided or a generated itime value. * * Either way, we turn the itime into a File-ID, unless the filesystem * provided one (unlikely). */ if (info == FILE_WAS_CREATED && !is_named_stream(smb_fname)) { - smb_fname->st.st_ex_iflags &= ~ST_EX_IFLAG_CALCULATED_ITIME; + create_clock_itime(&smb_fname->st); if (lp_store_dos_attributes(SNUM(conn)) && smb_fname->st.st_ex_iflags & ST_EX_IFLAG_CALCULATED_FILE_ID) @@ -4317,7 +4317,7 @@ static NTSTATUS mkdir_internal(connection_struct *conn, return NT_STATUS_NOT_A_DIRECTORY; } - smb_dname->st.st_ex_iflags &= ~ST_EX_IFLAG_CALCULATED_ITIME; + create_clock_itime(&smb_dname->st); if (lp_store_dos_attributes(SNUM(conn))) { if (smb_dname->st.st_ex_iflags & ST_EX_IFLAG_CALCULATED_FILE_ID) -- 2.30.2