From 31f03011c55700573fe6b94c6cba42f717bc8a09 Mon Sep 17 00:00:00 2001 From: Chiaki ISHIKAWA Date: Wed, 11 Mar 2015 00:08:02 +0900 Subject: [PATCH 7/8] Support for -gsplit-dwarf --- ccache.c | 430 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- compopt.c | 1 - 2 files changed, 426 insertions(+), 5 deletions(-) diff --git a/ccache.c b/ccache.c index 92bfb21..2ad5e80 100644 --- a/ccache.c +++ b/ccache.c @@ -95,9 +95,12 @@ static char *output_obj; /* The path to the dependency file (implicit or specified with -MF). */ static char *output_dep; -/* Diagnostic generation information (clang). */ +/* Diagnostic generation information (clang). Contains pathname if not NULL. */ static char *output_dia = NULL; +/* -gsplit-dwarf support: Split dwarf information (GCC 4.8 and up). Contains pathname if not NULL. */ +static char *output_dwo = NULL; + /* * Name (represented as a struct file_hash) of the file containing the cached * object code. @@ -129,6 +132,21 @@ static char *cached_dep; static char *cached_dia; /* + * -gsplit-dwarf support: + * Full path to the file containing the split dwarf (for GCC 4.8 and above) + * (cachedir/a/b/cdef[...]-size.dwo). + * + * contains NULL if -gsplit-dwarf is not given. + */ +static char *cached_dwo; + +/* + * -gsplit-dwarf support: + * split_dwarf_p is true if "-gsplit-dwarf" is given to the compiler (GCC 4.8 and up). + */ +bool split_dwarf_p = false; + +/* * Full path to the file containing the manifest * (cachedir/a/b/cdef[...]-size.manifest). */ @@ -741,6 +759,7 @@ send_cached_stderr(void) } /* Create or update the manifest file. */ +/* Note: we do not need to put .dwo file in MANIFEST file. */ void update_manifest_file(void) { struct stat st; @@ -778,14 +797,52 @@ static void to_cache(struct args *args) { char *tmp_stdout, *tmp_stderr; + char *tmp_dwo; /* for supporting -gsplit-dwarf */ struct stat st; int status, tmp_stdout_fd, tmp_stderr_fd; + /* Here cached_obj contains a file name for object */ + tmp_stdout = format("%s.tmp.stdout", cached_obj); tmp_stdout_fd = create_tmp_fd(&tmp_stdout); tmp_stderr = format("%s.tmp.stderr", cached_obj); tmp_stderr_fd = create_tmp_fd(&tmp_stderr); + /* -gsplit-dwarf support: creating .dwo file name + * + * Question: What is the reliable way of creating the .dwo file name + * in the case of using |-o output_obj| to the compiler ? + * Answer: Heuristics based on one run of GCC 4.8 is placed below, and + * it works with gcc 4.9 as well. + * + * BTW, no matter whether -gsplit-warning is placed on the command + * line, or if so, whether it has been seen or not, we need to set + * tmp_dwo in this part of the code to reflect the value of the + * current output_obj to tmp_dwo. + * If -gsplit-dwarf is specified on the command line eventually, this + * tmp_dwo will be used. + */ + + // Creation of .dwo file name. + // In the extreme simple case, + // Try replacing .o with .dwo. + // + // Fact: gcc-4.8 produced the following .dwo file for the command + // below: + // t-test.o.tmp.dwo + // + // /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.c -o t-test.o.tmp.gazonk + // + // So here is the heuristics (works with gcc 4.9 also). + // Strip anything after the final '.' and add "dwo" instead. + + { /* code suggested by Joel. */ + char *base_name = remove_extension(output_obj); + tmp_dwo = format("%s.dwo", base_name); + free(base_name); + } + cc_log("Setting tmp_dwo to %s", tmp_dwo); + args_add(args, "-o"); args_add(args, output_obj); @@ -818,6 +875,9 @@ to_cache(struct args *args) stats_update(STATS_MISSING); tmp_unlink(tmp_stdout); tmp_unlink(tmp_stderr); + /* -gsplit-dwarf support requires the removal of a few more files.*/ + if (tmp_dwo) + tmp_unlink(tmp_dwo); failed(); } if (st.st_size != 0) { @@ -825,11 +885,20 @@ to_cache(struct args *args) stats_update(STATS_STDOUT); tmp_unlink(tmp_stdout); tmp_unlink(tmp_stderr); + /* -gsplit-dwarf support requires the removal of a few more files. */ + if (tmp_dwo) { + tmp_unlink(tmp_dwo); + } failed(); } tmp_unlink(tmp_stdout); /* + * OBSERVATION: processing stderr does not have to be changed in the + * presence of "-gsplit-dwarf". + */ + + /* * Merge stderr from the preprocessor (if any) and stderr from the real * compiler into tmp_stderr. */ @@ -887,6 +956,11 @@ to_cache(struct args *args) cc_log("verbose: (tmp_stderr) %s being removed", tmp_stderr); tmp_unlink(tmp_stderr); + + /* gsplit-support modification */ + if (tmp_dwo) { + tmp_unlink(tmp_dwo); + } failed(); } @@ -901,6 +975,30 @@ to_cache(struct args *args) failed(); } + + /* -gsplit-dwarf support: + * The code immediately above created object file (output_obj). + * We should also repeat the similar procedurek for .dwo file + * when "-gsplit-dwarf is given. + * Because of the order in which tmp_dwo is set when command line is + * scanned, we can have non-null tmp_dwo even when split_swarf_p is + * false. Check only when split_dwarf_p is true. + */ + + if (tmp_dwo && split_dwarf_p) { + /* cc_log("tmp_dwo = %s", tmp_dwo); */ + if (stat(tmp_dwo, &st) != 0) { + cc_log("(tmp_dwo) Compiler didn't produce a split dwarf file"); + stats_update(STATS_NOOUTPUT); + failed(); + } + if (st.st_size == 0) { + cc_log("(tmp_dwo) Compiler produced an empty split dwarf file"); + stats_update(STATS_EMPTYOUTPUT); + failed(); + } + } + if (stat(tmp_stderr, &st) != 0) { cc_log("Failed to stat %s: %s", tmp_stderr, strerror(errno)); stats_update(STATS_ERROR); @@ -954,6 +1052,50 @@ to_cache(struct args *args) } put_file_in_cache(output_obj, cached_obj); + + cc_log("(cached_obj) Stored in cache: %s", cached_obj); + + /* Description of processing below.: + * Perform the copy of .dwo file if --gsplit-dwarf is used. + */ + + /* stats_update(STATS_TOCACHE) is now called AFTER the processing of + * .dwo file below. + */ + + /* + * Caution: if we output to real object file first, then + * output_obj *IS* the real object, and so we should not remove it! + * Also, in this case, we should NOT copy back the cached file to the + * final destination object. This is a waste of time. + * Same can be said of .dwo file: real and cached one below. + * + * Somehow dependency file was not copied this way. + */ + + /* -gsplit-dwarf support: + * if -gsplit-dwarf is given on the compiler command line, + * .dwo file ought to be put into cache as well. So let's do it here. + */ + if (split_dwarf_p) + { + assert(tmp_dwo); + assert(cached_dwo); + put_file_in_cache(tmp_dwo, cached_dwo); + cc_log("(cached_dwo) Stored in cache: %s", cached_dwo); + + /* internal sanity check */ + if (!generating_dependencies) + assert(tmp_dwo && cached_dwo); + + /* Caution: if we output to real object file first, then + * tmp_dwo *IS* the real .dwo file, and so we should not + * remove it! The original code is left as a comment line + * below to stress this important point. + */ + /* tmp_unlink(tmp_dwo); */ + } + if (generating_dependencies) { put_file_in_cache(output_dep, cached_dep); } @@ -988,6 +1130,8 @@ to_cache(struct args *args) free(tmp_stderr); free(tmp_stdout); + if (tmp_dwo) + free(tmp_dwo); } /* @@ -1111,6 +1255,32 @@ update_cached_result_globals(struct file_hash *hash) cached_stderr = get_path_in_cache(object_name, ".stderr"); cached_dep = get_path_in_cache(object_name, ".d"); cached_dia = get_path_in_cache(object_name, ".dia"); + + /* + * --gsplit-dwarf support. + * We set cached_dwo to NULL to signify --gsplit-dwarf is not used. + */ + if (split_dwarf_p) + cached_dwo = get_path_in_cache(object_name, ".dwo"); + else + cached_dwo = NULL; + + /* TODO/NOTE: Sanity Check. + * If split_dwarf_p is NOT true and we get non-null + * get_path_in_cache(object_name, ".dwo"), we probably should purge + * that file! Unlikely to happen, but non-zero probability. + * But this is not so critical. We will purge the file eventually by + * LRU algorithm. + * So just leave a log message. + */ + if (!split_dwarf_p) { + char *cp; + cp = get_path_in_cache(object_name, ".dwo"); + if (!cp) { + cc_log("We are not using split dwarf but, we have .dwo. Maybe we should purge it.:%s", cp); + } + } + stats_file = format("%s/%c/stats", conf->cache_dir, object_name[0]); free(object_name); } @@ -1404,6 +1574,12 @@ calculate_object_hash(struct args *args, struct mdfour *hash, int direct_mode) hash_delimiter(hash, "-fprofile-dir"); hash_string(hash, profile_dir); } + /* OBSERVATION: processing for profile_use is a good example + * which we can mimic to implement the support for + * --gsplit-dwarf. + * So the support for -gsplit-dwarf was modeled after the + * processing to support profile. + */ if (profile_use) { /* Calculate gcda name */ char *gcda_name; @@ -1496,13 +1672,20 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) return; } - /* Check if the object file is there. */ + /* + * Check if the object file is there. + * For -gsplit-dwarf support, check for .dwo as well. See below. + * cached_obj contains the path for cached object file. + */ + if (stat(cached_obj, &st) != 0) { cc_log("(cached_obj) Object file %s not in cache", cached_obj); return; } - /* Check if the diagnostic file is there. */ + /* Check if the diagnostic file is there. + * Notice the logic. We only check when output_dia is not NULL. + */ if (output_dia && stat(cached_dia, &st) != 0) { cc_log("(cached_dia) Diagnostic file %s not in cache", cached_dia); return; @@ -1518,6 +1701,30 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) return; } + /* -gsplit-dwarf support + * We should repeat the above check for cached_dwo + */ + if (split_dwarf_p && !generating_dependencies) + assert(output_dwo); + + if (output_dwo) + assert(cached_dwo); + + if (output_dwo) { + + if (stat(cached_dwo, &st) != 0) { + cc_log("(cached_dwo) Split dwarf file %s not in cache", cached_dwo); + return; + } + + if (st.st_size == 0) { + cc_log("(cached_dwo) Invalid (empty) dwo file %s in cache", cached_dwo); + x_unlink(cached_dwo); + x_unlink(cached_obj); /* to really invalidate */ + return; + } + } + /* * (If mode != FROMCACHE_DIRECT_MODE, the dependency file is created by * gcc.) @@ -1530,7 +1737,15 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) return; } + /* + * Copy object file from cache. Do so also for FissionDwarf file, + * cached_dwo, when -gsplit-dwarf is specified. + */ if (!str_eq(output_obj, "/dev/null")) { + /* we get file(s) from cache unless we generate object in direct mode */ + /* sanity check for -gsplit-dwarf support */ + assert( (mode == FROMCACHE_CPP_MODE || mode == FROMCACHE_DIRECT_MODE)); + get_file_from_cache(cached_obj, output_obj); cc_log("Created (output_obj) %s from (cached_obj) %s", output_obj, cached_obj); /* @@ -1551,7 +1766,41 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) * x_unlink(cached_dep); *x_unlink(cached_dia); */ + + /* We repeat the same procedure as above for .dwo file if split_dwarf_p */ + if (split_dwarf_p) { + assert(output_dwo); + /* + * we don't need to get file from cache when we called the + * compiler in this run of ccache since the output_dwo would + * have been created by the compiler. + */ + /* internal sanity check: the following means compiler + * was not called. We are in from_cache() remember?_*/ + assert(mode == FROMCACHE_CPP_MODE || mode == FROMCACHE_DIRECT_MODE); + + get_file_from_cache(cached_dwo, output_dwo); + cc_log("Created (output_dwo) %s from (cached_dwo) %s", + output_dwo, cached_dwo); + /* + * Note: if something failed internal to get_file_from_cache + * we may want to remove a flurry of files. + * There were calls to x_unlinks in the original code + * circa 2014 summer. + */ + } } + /* NOTE: The error processing above assumed we only do a copy/hardlink + * for cached_obj. We may need to repeat the copy and error + * processing for cached_dwo in a similar manner instead of doing the + * copy/link only. + * + * But do note that the unlinking of related files + * MUST BE REPEATED in the both cases of the failures of copy/link + * of cached_obj -> output_obj + * and cached_dwo -> output_dwo if (split_dwarf_p). + */ + if (produce_dep_file) { get_file_from_cache(cached_dep, output_dep); cc_log("Created (output_dep) %s from (cached_dep) %s", output_dep, cached_dep); @@ -1571,6 +1820,10 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) */ } + /* OBSERVATION: The above seems to be the best place where we + * can copy object file and dwo file safely (-gsplit-dwarf support). + */ + /* Update modification timestamps to save files from LRU cleanup. Also gives files a sensible mtime when hard-linking. */ update_mtime(cached_obj); @@ -1586,8 +1839,13 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) put_file_in_cache(output_dep, cached_dep); } - send_cached_stderr(); + /* -gsplit-dwarf support */ + if (cached_dwo) + update_mtime(cached_dwo); + send_cached_stderr(); + + /* Note: we do not need to put .dwo file in MANIFEST file. */ if (put_object_in_manifest) { update_manifest_file(); } @@ -1603,6 +1861,9 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) cc_log("Succeeded getting cached result"); stats_update(STATS_CACHEHIT_CPP); break; + default: + /* -gsplit-dwarf support: internal sanity check */ + assert(0); } /* and exit with the right status code */ @@ -1828,6 +2089,12 @@ cc_process_args(struct args *args, struct args **preprocessor_args, goto out; } output_obj = make_relative_path(x_strdup(argv[i+1])); + + /* OBSERVATION: We need to create output_dwo if + -gsplit-dwarf is given. But we should not do here + YET since --gsplit-dwarf may be given later on the + command line !*/ + i++; continue; } @@ -1838,6 +2105,19 @@ cc_process_args(struct args *args, struct args **preprocessor_args, continue; } + /* -gsplit-dwarf support + * This place is chosen as the place to handle -gsplit-dwarf. + */ + + if (str_eq(argv[i], "-gsplit-dwarf")) { + cc_log("split_dwarf_p is set to true due to -gsplit-dwarf."); + split_dwarf_p = true; + /* we should add this option to the compiler option + when the real compiler is invoked. */ + args_add(stripped_args, argv[i]); + continue; + } + /* debugging is handled specially, so that we know if we can strip line number info */ @@ -2227,6 +2507,21 @@ cc_process_args(struct args *args, struct args **preprocessor_args, /* Rewrite to relative to increase hit rate. */ input_file = make_relative_path(x_strdup(argv[i])); + } /* for */ + + /* found_S_opt needs to be handled with caution with split_dwarf_p */ + + if (found_S_opt /* || generating_dependencies*/ ) { + /* even if -gsplit-dwarf is given, .dwo file is not generated when + * found_S_opt is true (-S is also given). + * So split_dwarf_p is set to false. + * + * Note: Just a tidbit. When -S is given on the command line, + * the cached assember output file in the cache spool actually + * has .o suffix. Interesting, isn't it? + */ + split_dwarf_p = false; + cc_log("split_dwarf_p is reset to false due to found_S_opt"); } if (!input_file) { @@ -2314,6 +2609,16 @@ cc_process_args(struct args *args, struct args **preprocessor_args, conf->cpp_extension = x_strdup(extension_for_language(p_language) + 1); } + /* Naive question: Is there an angle in which split_dwarf_p processing + * has anything to do with "-" output that is handled specialy below? + * For example, what do we want to do with .dwo file (produce it + * somewhere, but what path name to use)? + * => + * As it turns out, GCC does not work with output with "-" and + * since -gsplit-dwarf is only for GCC at this moment (March, 2015), + * we don't have to worry about this. + */ + /* don't try to second guess the compilers heuristics for stdout handling */ if (output_obj && str_eq(output_obj, "-")) { stats_update(STATS_OUTSTDOUT); @@ -2322,6 +2627,50 @@ cc_process_args(struct args *args, struct args **preprocessor_args, goto out; } + /* -gsplit-dwarf support: + * We have probably already seen and check for + * -gsplit-dwarf by the time we reach here. (We did it somewhere above.) + * Here is the chance to create output_dwo based on the value of output_obj. + * The basic outline would be + * output_obj = basename(input_file) + * add ".dwo" to it with minor tweaks to handle special cases. + */ + + /* + * Hueristics is based on the following observation. + * Special case 1: What do we do if we need to output .s file + * for output and -gsplit-dwarf is specified? + * What file(s) does GCC 4.8 produce exactly? + * Answer: + * /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.c -S + * generated + * t-test.s + * that contains assembler directives to generate data (dwarf data). + * This means that if we are only producing assembler source + * we do not need to handle ".dwo" file at all. + * + * /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.c -S -o t-test.s.gazonk + * produced + * t-test.s.gazonk + * + * DONE: + * The above special cases need to be handled. + * The modification below more or less assumed that .dwo file is produced + * always (together with the paired .o file) unless "-S" is specified. + * + * File-scope global variable |found_S_opt| is set to true + * when "-S" is specified. + * + * Special case 2: /usr/bin/gcc-4.8 -gsplit-dwarf -c t-test.s + * produced + * t-test.o + * t-test.o.dwo. + * + * So let us disable split_dwarf_p to false if + * found_S_opt is true. + * This is done at the end of a loop in argument processing. + */ + if (!output_obj) { if (output_is_precompiled_header) { output_obj = format("%s.gch", input_file); @@ -2340,6 +2689,32 @@ cc_process_args(struct args *args, struct args **preprocessor_args, } } + /* -gsplit-dwarf support: create output_dwo */ + if (split_dwarf_p && !found_S_opt) { + char *p; + p = strrchr(output_obj, '.'); + if (!p || !p[1]) { + cc_log("Badly formed object filename"); + stats_update(STATS_ARGS); + result = false; + goto out; + } + { /* code suggested by Joel. */ + char *base_name = remove_extension(output_obj); + output_dwo = format("%s.dwo", base_name); + free(base_name); + } + cc_log("setting output_dwo to %s", output_dwo); + } + + /* Long-term question: Do we need to cope with strange output_obj file + * name in case split_dwarf_p? "-gsplit-dwarf" support may encounter + * corner cases when strange file names are given. + * Answer: Don't worry too much. GCC itself may not run with strange + * path name in the first placbe. We may find the answer in the long + * run. + */ + /* cope with -o /dev/null */ if (!str_eq(output_obj,"/dev/null") && stat(output_obj, &st) == 0 @@ -2390,6 +2765,20 @@ cc_process_args(struct args *args, struct args **preprocessor_args, } } + /* -gsplit-dwarf support: Ordering of dependency generation. + * + * -gsplit-dwarf processing should not affect dependency generation. + * + * Note, given the command line below, dependency file is generated + * first and then ccache.o is generated. + * Observing the ordering and take care of that. + * + * ccache /usr/bin/gcc-4.8 -gsplit-dwarf -DHAVE_CONFIG_H \ + * -DSYSCONFDIR=/usr/local/etc -I. -I. -MD -MP -MF .deps/ccache.c.d \ + * -g -O2 -Wall -W -Werror -c -o ccache.o ccache.c + * + */ + /* * Add flags for dependency generation only to the preprocessor command line. */ @@ -2577,10 +2966,12 @@ cc_reset(void) args_free(orig_args); orig_args = NULL; free(input_file); input_file = NULL; free(output_obj); output_obj = NULL; + if (output_dwo) free(output_dwo); output_dwo = NULL; free(output_dep); output_dep = NULL; free(output_dia); output_dia = NULL; free(cached_obj_hash); cached_obj_hash = NULL; free(cached_obj); cached_obj = NULL; + if (cached_dwo) free(cached_dwo); cached_dwo = NULL; free(cached_stderr); cached_stderr = NULL; free(cached_dep); cached_dep = NULL; free(cached_dia); cached_dia = NULL; @@ -2597,6 +2988,7 @@ cc_reset(void) output_is_precompiled_header = false; conf = conf_create(); + split_dwarf_p = false; } /* Make a copy of stderr that will not be cached, so things like @@ -2692,6 +3084,36 @@ ccache(int argc, char *argv[]) if (output_dia) { cc_log("Diagnostic file: %s", output_dia); } + + /* + * -gsplit-dwarf support + * Note: cc_process_args ought to have generated/set output_dwo. But + * we may not want to do so before generating dependencies (or set + * generating_dependencies) because it has been the implicit ordering + * that the dependency is generated first and then object file is + * generated. + * (CI failed to record the particular command line, etc. which necessitated him + * to insert the assert() statement below.) + * See a block comment that starts with + * "-gsplit-dwarf support: Ordering of dependency generation." + * in the earlier part of this file. + */ + + if (split_dwarf_p ) { + if (output_dia) + cc_log("output_dia = %s", output_dia); + if (output_dwo) + cc_log("output_dwo = %s", output_dwo); + fflush(stdout); + if (!generating_dependencies) + assert (output_dwo); + } else { + assert (!output_dwo); + } + + if (!generating_dependencies && output_dwo) { + cc_log("Split dwarf file: %s", output_dwo); + } cc_log("(output_obj) Object file: %s", output_obj); hash_start(&common_hash); diff --git a/compopt.c b/compopt.c index 71b9685..9bfb25a 100644 --- a/compopt.c +++ b/compopt.c @@ -62,7 +62,6 @@ static const struct compopt compopts[] = { {"-frepo", TOO_HARD}, {"-ftest-coverage", TOO_HARD}, /* generates a .gcno file at the same time */ {"-fworking-directory", AFFECTS_CPP}, - {"-gsplit-dwarf", TOO_HARD}, /* generates a .dwo file at the same time */ {"-idirafter", AFFECTS_CPP | TAKES_ARG | TAKES_PATH}, {"-iframework", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-imacros", AFFECTS_CPP | TAKES_ARG | TAKES_PATH}, -- 2.1.4