From 4a48d2b63021c43df01868c0751ea37c43eb276a Mon Sep 17 00:00:00 2001 From: Chiaki ISHIKAWA Date: Wed, 29 Apr 2015 04:57:06 +0900 Subject: [PATCH] Support for -gsplit-dwarf (bug 10005) --- ccache.c | 432 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- compopt.c | 1 - 2 files changed, 427 insertions(+), 6 deletions(-) diff --git a/ccache.c b/ccache.c index c1f4a11..d1f6400 100644 --- a/ccache.c +++ b/ccache.c @@ -98,9 +98,12 @@ static char *output_dep; /* The path to the coverage file (implicit when using -ftest-coverage). */ static char *output_cov; -/* Diagnostic generation information (clang). */ +/* Diagnostic generation information (clang). Contains pathname if not NULL. */ static char *output_dia = NULL; +/* -gsplit-dwarf support: Split dwarf information (GCC 4.8 and up). Contains pathname if not NULL. */ +static char *output_dwo = NULL; + /* * Name (represented as a struct file_hash) of the file containing the cached * object code. @@ -138,6 +141,21 @@ static char *cached_cov; static char *cached_dia; /* + * -gsplit-dwarf support: + * Full path to the file containing the split dwarf (for GCC 4.8 and above) + * (cachedir/a/b/cdef[...]-size.dwo). + * + * contains NULL if -gsplit-dwarf is not given. + */ +static char *cached_dwo; + +/* + * -gsplit-dwarf support: + * split_dwarf_p is true if "-gsplit-dwarf" is given to the compiler (GCC 4.8 and up). + */ +bool split_dwarf_p = false; + +/* * Full path to the file containing the manifest * (cachedir/a/b/cdef[...]-size.manifest). */ @@ -765,6 +783,7 @@ send_cached_stderr(void) } /* Create or update the manifest file. */ +/* Note: we do not need to put .dwo file in MANIFEST file. */ void update_manifest_file(void) { struct stat st; @@ -786,6 +805,10 @@ void update_manifest_file(void) if (x_stat(manifest_path, &st) == 0) { stats_update_size(file_size(&st) - old_size, old_size == 0 ? 1 : 0); } + else { + cc_log("stat on manifest (manifest_path) %s failed (%s), so size statistics is incorrect.", + manifest_path, strerror(errno)); + } } else { cc_log("Failed to add object file hash to %s", manifest_path); } @@ -796,10 +819,13 @@ static void to_cache(struct args *args) { char *tmp_stdout, *tmp_stderr, *tmp_aux, *tmp_cov; + char *tmp_dwo; /* for supporting -gsplit-dwarf */ struct stat st; int status, tmp_stdout_fd, tmp_stderr_fd; FILE *f; + /* Here cached_obj contains a file name for object */ + tmp_stdout = format("%s.tmp.stdout", cached_obj); tmp_stdout_fd = create_tmp_fd(&tmp_stdout); tmp_stderr = format("%s.tmp.stderr", cached_obj); @@ -818,6 +844,41 @@ to_cache(struct args *args) tmp_cov = NULL; } + /* -gsplit-dwarf support: creating .dwo file name + * + * Question: What is the reliable way of creating the .dwo file name + * in the case of using |-o output_obj| to the compiler ? + * Answer: Heuristics based on one run of GCC 4.8 is placed below, and + * it works with gcc 4.9 as well. + * + * BTW, no matter whether -gsplit-warning is placed on the command + * line, or if so, whether it has been seen or not, we need to set + * tmp_dwo in this part of the code to reflect the value of the + * current output_obj to tmp_dwo. + * If -gsplit-dwarf is specified on the command line eventually, this + * tmp_dwo will be used. + */ + + // Creation of .dwo file name. + // In the extreme simple case, + // Try replacing .o with .dwo. + // + // Fact: gcc-4.8 produced the following .dwo file for the command + // below: + // t-test.o.tmp.dwo + // + // /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.c -o t-test.o.tmp.gazonk + // + // So here is the heuristics (works with gcc 4.9 also). + // Strip anything after the final '.' and add "dwo" instead. + + { /* code suggested by Joel. */ + char *base_name = remove_extension(output_obj); + tmp_dwo = format("%s.dwo", base_name); + free(base_name); + } + cc_log("Setting tmp_dwo to %s", tmp_dwo); + args_add(args, "-o"); args_add(args, output_obj); @@ -851,6 +912,9 @@ to_cache(struct args *args) if (tmp_cov) { tmp_unlink(tmp_cov); } + /* -gsplit-dwarf support requires the removal of a few more files.*/ + if (tmp_dwo) + tmp_unlink(tmp_dwo); failed(); } if (st.st_size != 0) { @@ -861,10 +925,19 @@ to_cache(struct args *args) if (tmp_cov) { tmp_unlink(tmp_cov); } + /* gsplit-support modification */ + if (tmp_dwo) { + tmp_unlink(tmp_dwo); + } failed(); } tmp_unlink(tmp_stdout); + /* + * OBSERVATION: processing stderr does not have to be changed in the + * presence of "-gsplit-dwarf". + */ + /* * Merge stderr from the preprocessor (if any) and stderr from the real * compiler into tmp_stderr. @@ -925,6 +998,10 @@ to_cache(struct args *args) if (tmp_cov) { tmp_unlink(tmp_cov); } + /* gsplit-support modification */ + if (tmp_dwo) { + tmp_unlink(tmp_dwo); + } failed(); } @@ -939,6 +1016,28 @@ to_cache(struct args *args) failed(); } + /* -gsplit-dwarf support: + * The code immediately above created object file (output_obj). + * We should also repeat the similar procedurek for .dwo file + * when "-gsplit-dwarf is given. + * Because of the order in which tmp_dwo is set when command line is + * scanned, we can have non-null tmp_dwo even when split_swarf_p is + * false. Check only when split_dwarf_p is true. + */ + + if (tmp_dwo && split_dwarf_p) { + if (x_stat(tmp_dwo, &st) != 0) { + cc_log("Compiler didn't produce a split dwarf file"); + stats_update(STATS_NOOUTPUT); + failed(); + } + if (st.st_size == 0) { + cc_log("(tmp_dwo) Compiler produced an empty split dwarf file"); + stats_update(STATS_EMPTYOUTPUT); + failed(); + } + } + if (x_stat(tmp_stderr, &st) != 0) { stats_update(STATS_ERROR); failed(); @@ -995,6 +1094,50 @@ to_cache(struct args *args) } put_file_in_cache(output_obj, cached_obj); + + cc_log("(cached_obj) Stored in cache: %s", cached_obj); + + /* Description of processing below.: + * Perform the copy of .dwo file if --gsplit-dwarf is used. + */ + + /* stats_update(STATS_TOCACHE) is now called AFTER the processing of + * .dwo file below. + */ + + /* + * Caution: if we output to real object file first, then + * output_obj *IS* the real object, and so we should not remove it! + * Also, in this case, we should NOT copy back the cached file to the + * final destination object. This is a waste of time. + * Same can be said of .dwo file: real and cached one below. + * + * Somehow dependency file was not copied this way. + */ + + /* -gsplit-dwarf support: + * if -gsplit-dwarf is given on the compiler command line, + * .dwo file ought to be put into cache as well. So let's do it here. + */ + if (split_dwarf_p) + { + assert(tmp_dwo); + assert(cached_dwo); + put_file_in_cache(tmp_dwo, cached_dwo); + cc_log("(cached_dwo) Stored in cache: %s", cached_dwo); + + /* internal sanity check */ + if (!generating_dependencies) + assert(tmp_dwo && cached_dwo); + + /* Caution: if we output to real object file first, then + * tmp_dwo *IS* the real .dwo file, and so we should not + * remove it! The original code is left as a comment line + * below to stress this important point. + */ + /* tmp_unlink(tmp_dwo); */ + } + if (generating_dependencies) { put_file_in_cache(output_dep, cached_dep); } @@ -1030,6 +1173,8 @@ to_cache(struct args *args) free(tmp_stderr); free(tmp_stdout); free(tmp_cov); + if (tmp_dwo) + free(tmp_dwo); } /* @@ -1154,6 +1299,32 @@ update_cached_result_globals(struct file_hash *hash) cached_dep = get_path_in_cache(object_name, ".d"); cached_cov = get_path_in_cache(object_name, ".gcno"); cached_dia = get_path_in_cache(object_name, ".dia"); + + /* + * --gsplit-dwarf support. + * We set cached_dwo to NULL to signify --gsplit-dwarf is not used. + */ + if (split_dwarf_p) + cached_dwo = get_path_in_cache(object_name, ".dwo"); + else + cached_dwo = NULL; + + /* TODO/NOTE: Sanity Check. + * If split_dwarf_p is NOT true and we get non-null + * get_path_in_cache(object_name, ".dwo"), we probably should purge + * that file! Unlikely to happen, but non-zero probability. + * But this is not so critical. We will purge the file eventually by + * LRU algorithm. + * So just leave a log message. + */ + if (!split_dwarf_p) { + char *cp; + cp = get_path_in_cache(object_name, ".dwo"); + if (!cp) { + cc_log("We are not using split dwarf but, we have .dwo. Maybe we should purge it.:%s", cp); + } + } + stats_file = format("%s/%c/stats", conf->cache_dir, object_name[0]); free(object_name); } @@ -1470,6 +1641,12 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) hash_delimiter(hash, "-fprofile-dir"); hash_string(hash, profile_dir); } + /* OBSERVATION: processing for profile_use is a good example + * which we can mimic to implement the support for + * --gsplit-dwarf. + * So the support for -gsplit-dwarf was modeled after the + * processing to support profile. + */ if (profile_use) { /* Calculate gcda name */ char *gcda_name; @@ -1562,14 +1739,21 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) return; } - /* Check if the object file is there. */ + /* + * Check if the object file is there. + * For -gsplit-dwarf support, check for .dwo as well. See below. + * cached_obj contains the path for cached object file. + */ + if (stat(cached_obj, &st) != 0) { cc_log("Object file %s not in cache", cached_obj); return; } - /* Check if the diagnostic file is there. */ - if (output_dia && stat(cached_dia, &st) != 0) { + /* Check if the diagnostic file is there. + * Notice the logic. We only check when output_dia is not NULL. + */ + if (output_dia && x_stat(cached_dia, &st) != 0) { cc_log("Diagnostic file %s not in cache", cached_dia); return; } @@ -1584,6 +1768,28 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) return; } + /* -gsplit-dwarf support + * We should repeat the above check for cached_dwo. + */ + /* First, sanity checks. */ + if (split_dwarf_p && !generating_dependencies) + assert(output_dwo); + if (output_dwo) + assert(cached_dwo); + + if (output_dwo) { + if (x_stat(cached_dwo, &st) != 0) { + cc_log("Split dwarf file %s not in cache", cached_dwo); + return; + } + if (st.st_size == 0) { + cc_log("(cached_dwo) Invalid (empty) dwo file %s in cache", cached_dwo); + x_unlink(cached_dwo); + x_unlink(cached_obj); /* to really invalidate */ + return; + } + } + /* * (If mode != FROMCACHE_DIRECT_MODE, the dependency file is created by * gcc.) @@ -1596,11 +1802,57 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) return; } + /* + * Copy object file from cache. Do so also for FissionDwarf file, + * cached_dwo, when -gsplit-dwarf is specified. + */ if (!str_eq(output_obj, "/dev/null")) { + /* we get file(s) from cache unless we generate object in direct mode */ + /* sanity check for -gsplit-dwarf support */ + assert( (mode == FROMCACHE_CPP_MODE || mode == FROMCACHE_DIRECT_MODE)); + get_file_from_cache(cached_obj, output_obj); + + /* We repeat the same procedure as above for .dwo file if split_dwarf_p */ + if (split_dwarf_p) { + assert(output_dwo); + /* + * we don't need to get file from cache when we called the + * compiler in this run of ccache since the output_dwo would + * have been created by the compiler. + */ + /* internal sanity check: the following means compiler + * was not called. We are in from_cache() remember?_*/ + assert(mode == FROMCACHE_CPP_MODE || mode == FROMCACHE_DIRECT_MODE); + + get_file_from_cache(cached_dwo, output_dwo); + /* + * Note: if something failed internal to get_file_from_cache + * we may want to remove a flurry of files. + * There were calls to x_unlinks in the original code + * circa 2014 summer. + */ + } } + + /* NOTE: The error processing above assumed we only do a copy/hardlink + * for cached_obj. We may need to repeat the copy and error + * processing for cached_dwo in a similar manner instead of doing the + * copy/link only. + * + * But do note that the unlinking of related files + * MUST BE REPEATED in the both cases of the failures of copy/link + * of cached_obj -> output_obj + * and cached_dwo -> output_dwo if (split_dwarf_p). + */ + if (produce_dep_file) { get_file_from_cache(cached_dep, output_dep); + /* + * Note: if something failed internal to get_file_from_cache + * we may want to remove a flurry of files. + * There were calls to x_unlinks in the original code circa 2014 summer. + */ } if (generating_coverage && stat(cached_cov, &st) == 0 && st.st_size > 0) { /* gcc won't generate notes if there is no code */ @@ -1610,6 +1862,10 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) get_file_from_cache(cached_dia, output_dia); } + /* OBSERVATION: The above seems to be the best place where we + * can copy object file and dwo file safely (-gsplit-dwarf support). + */ + /* Update modification timestamps to save files from LRU cleanup. Also gives files a sensible mtime when hard-linking. */ update_mtime(cached_obj); @@ -1628,8 +1884,13 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) put_file_in_cache(output_dep, cached_dep); } - send_cached_stderr(); + /* -gsplit-dwarf support */ + if (cached_dwo) + update_mtime(cached_dwo); + + send_cached_stderr(); + /* Note: we do not need to put .dwo file in MANIFEST file. */ if (put_object_in_manifest) { update_manifest_file(); } @@ -1645,6 +1906,9 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) cc_log("Succeeded getting cached result"); stats_update(STATS_CACHEHIT_CPP); break; + default: + /* -gsplit-dwarf support: internal sanity check */ + assert(0); } /* and exit with the right status code */ @@ -1870,6 +2134,12 @@ cc_process_args(struct args *args, struct args **preprocessor_args, goto out; } output_obj = make_relative_path(x_strdup(argv[i+1])); + + /* OBSERVATION: We need to create output_dwo if + -gsplit-dwarf is given. But we should not do here + YET since --gsplit-dwarf may be given later on the + command line !*/ + i++; continue; } @@ -1880,6 +2150,19 @@ cc_process_args(struct args *args, struct args **preprocessor_args, continue; } + /* -gsplit-dwarf support + * This place is chosen as the place to handle -gsplit-dwarf. + */ + + if (str_eq(argv[i], "-gsplit-dwarf")) { + cc_log("split_dwarf_p is set to true due to -gsplit-dwarf."); + split_dwarf_p = true; + /* we should add this option to the compiler option + when the real compiler is invoked. */ + args_add(stripped_args, argv[i]); + continue; + } + /* debugging is handled specially, so that we know if we can strip line number info */ @@ -2296,6 +2579,21 @@ cc_process_args(struct args *args, struct args **preprocessor_args, /* Rewrite to relative to increase hit rate. */ input_file = make_relative_path(x_strdup(argv[i])); + } /* for */ + + /* found_S_opt needs to be handled with caution with split_dwarf_p */ + + if (found_S_opt /* || generating_dependencies*/ ) { + /* even if -gsplit-dwarf is given, .dwo file is not generated when + * found_S_opt is true (-S is also given). + * So split_dwarf_p is set to false. + * + * Note: Just a tidbit. When -S is given on the command line, + * the cached assember output file in the cache spool actually + * has .o suffix. Interesting, isn't it? + */ + split_dwarf_p = false; + cc_log("split_dwarf_p is reset to false due to found_S_opt"); } if (!input_file) { @@ -2383,6 +2681,16 @@ cc_process_args(struct args *args, struct args **preprocessor_args, conf->cpp_extension = x_strdup(extension_for_language(p_language) + 1); } + /* Naive question: Is there an angle in which split_dwarf_p processing + * has anything to do with "-" output that is handled specialy below? + * For example, what do we want to do with .dwo file (produce it + * somewhere, but what path name to use)? + * => + * As it turns out, GCC does not work with output with "-" and + * since -gsplit-dwarf is only for GCC at this moment (March, 2015), + * we don't have to worry about this. + */ + /* don't try to second guess the compilers heuristics for stdout handling */ if (output_obj && str_eq(output_obj, "-")) { stats_update(STATS_OUTSTDOUT); @@ -2391,6 +2699,50 @@ cc_process_args(struct args *args, struct args **preprocessor_args, goto out; } + /* -gsplit-dwarf support: + * We have probably already seen and check for + * -gsplit-dwarf by the time we reach here. (We did it somewhere above.) + * Here is the chance to create output_dwo based on the value of output_obj. + * The basic outline would be + * output_obj = basename(input_file) + * add ".dwo" to it with minor tweaks to handle special cases. + */ + + /* + * Hueristics is based on the following observation. + * Special case 1: What do we do if we need to output .s file + * for output and -gsplit-dwarf is specified? + * What file(s) does GCC 4.8 produce exactly? + * Answer: + * /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.c -S + * generated + * t-test.s + * that contains assembler directives to generate data (dwarf data). + * This means that if we are only producing assembler source + * we do not need to handle ".dwo" file at all. + * + * /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.c -S -o t-test.s.gazonk + * produced + * t-test.s.gazonk + * + * DONE: + * The above special cases need to be handled. + * The modification below more or less assumed that .dwo file is produced + * always (together with the paired .o file) unless "-S" is specified. + * + * File-scope global variable |found_S_opt| is set to true + * when "-S" is specified. + * + * Special case 2: /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.s + * produced + * t-test.o + * t-test.o.dwo. + * + * So let us disable split_dwarf_p to false if + * found_S_opt is true. + * This is done at the end of a loop in argument processing. + */ + if (!output_obj) { if (output_is_precompiled_header) { output_obj = format("%s.gch", input_file); @@ -2409,6 +2761,32 @@ cc_process_args(struct args *args, struct args **preprocessor_args, } } + /* -gsplit-dwarf support: create output_dwo */ + if (split_dwarf_p && !found_S_opt) { + char *p; + p = strrchr(output_obj, '.'); + if (!p || !p[1]) { + cc_log("Badly formed object filename"); + stats_update(STATS_ARGS); + result = false; + goto out; + } + { /* code suggested by Joel. */ + char *base_name = remove_extension(output_obj); + output_dwo = format("%s.dwo", base_name); + free(base_name); + } + cc_log("setting output_dwo to %s", output_dwo); + } + + /* Long-term question: Do we need to cope with strange output_obj file + * name in case split_dwarf_p? "-gsplit-dwarf" support may encounter + * corner cases when strange file names are given. + * Answer: Don't worry too much. GCC itself may not run with strange + * path name in the first placbe. We may find the answer in the long + * run. + */ + /* cope with -o /dev/null */ if (!str_eq(output_obj,"/dev/null") && stat(output_obj, &st) == 0 @@ -2459,6 +2837,20 @@ cc_process_args(struct args *args, struct args **preprocessor_args, } } + /* -gsplit-dwarf support: Ordering of dependency generation. + * + * -gsplit-dwarf processing should not affect dependency generation. + * + * Note, given the command line below, dependency file is generated + * first and then ccache.o is generated. + * Observing the ordering and take care of that. + * + * ccache /usr/bin/gcc-4.8 -gsplit-dwarf -DHAVE_CONFIG_H \ + * -DSYSCONFDIR=/usr/local/etc -I. -I. -MD -MP -MF .deps/ccache.c.d \ + * -g -O2 -Wall -W -Werror -c -o ccache.o ccache.c + * + */ + /* * Add flags for dependency generation only to the preprocessor command line. */ @@ -2655,11 +3047,13 @@ cc_reset(void) args_free(orig_args); orig_args = NULL; free(input_file); input_file = NULL; free(output_obj); output_obj = NULL; + if (output_dwo) free(output_dwo); output_dwo = NULL; free(output_dep); output_dep = NULL; free(output_cov); output_cov = NULL; free(output_dia); output_dia = NULL; free(cached_obj_hash); cached_obj_hash = NULL; free(cached_obj); cached_obj = NULL; + if (cached_dwo) free(cached_dwo); cached_dwo = NULL; free(cached_stderr); cached_stderr = NULL; free(cached_dep); cached_dep = NULL; free(cached_cov); cached_cov = NULL; @@ -2680,6 +3074,7 @@ cc_reset(void) output_is_precompiled_header = false; conf = conf_create(); + split_dwarf_p = false; } /* Make a copy of stderr that will not be cached, so things like @@ -2778,6 +3173,33 @@ ccache(int argc, char *argv[]) if (output_dia) { cc_log("Diagnostic file: %s", output_dia); } + + /* + * -gsplit-dwarf support + * Note: cc_process_args ought to have generated/set output_dwo. But + * we may not want to do so before generating dependencies (or set + * generating_dependencies) because it has been the implicit ordering + * that the dependency is generated first and then object file is + * generated. + * (CI failed to record the particular command line, etc. which necessitated him + * to insert the assert() statement below.) + * See a block comment that starts with + * "-gsplit-dwarf support: Ordering of dependency generation." + * in the earlier part of this file. + */ + + /* Sanity Check */ + if (split_dwarf_p ) { + if (!generating_dependencies) + assert (output_dwo); + } else { + assert (!output_dwo); + } + + if (output_dwo) { + cc_log("Split dwarf file: %s", output_dwo); + } + cc_log("Object file: %s", output_obj); hash_start(&common_hash); diff --git a/compopt.c b/compopt.c index 41f710d..26e1591 100644 --- a/compopt.c +++ b/compopt.c @@ -60,7 +60,6 @@ static const struct compopt compopts[] = { {"-fplugin=libcc1plugin", TOO_HARD}, /* interaction with GDB */ {"-frepo", TOO_HARD}, {"-fworking-directory", AFFECTS_CPP}, - {"-gsplit-dwarf", TOO_HARD}, /* generates a .dwo file at the same time */ {"-idirafter", AFFECTS_CPP | TAKES_ARG | TAKES_PATH}, {"-iframework", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-imacros", AFFECTS_CPP | TAKES_ARG | TAKES_PATH}, -- 2.1.4