diff -rup binutils.orig/gold/gold.cc binutils-2.25.1/gold/gold.cc --- binutils.orig/gold/gold.cc 2017-10-18 12:37:20.539605249 +0100 +++ binutils-2.25.1/gold/gold.cc 2017-10-18 12:37:41.293359925 +0100 @@ -877,14 +877,27 @@ queue_final_tasks(const General_options& } // Create tasks for tree-style build ID computation, if necessary. - final_blocker = layout->queue_build_id_tasks(workqueue, final_blocker, of); + if (strcmp(options.build_id(), "tree") == 0) + { + // Queue a task to compute the build id. This will be blocked by + // FINAL_BLOCKER, and will in turn schedule the task to close + // the output file. + workqueue->queue(new Task_function(new Build_id_task_runner(&options, + layout, + of), + final_blocker, + "Task_function Build_id_task_runner")); + } + else + { + // Queue a task to close the output file. This will be blocked by + // FINAL_BLOCKER. + workqueue->queue(new Task_function(new Close_task_runner(&options, layout, + of, NULL, 0), + final_blocker, + "Task_function Close_task_runner")); + } - // Queue a task to close the output file. This will be blocked by - // FINAL_BLOCKER. - workqueue->queue(new Task_function(new Close_task_runner(&options, layout, - of), - final_blocker, - "Task_function Close_task_runner")); } } // End namespace gold. diff -rup binutils.orig/gold/layout.cc binutils-2.25.1/gold/layout.cc --- binutils.orig/gold/layout.cc 2017-10-18 12:37:20.539605249 +0100 +++ binutils-2.25.1/gold/layout.cc 2017-10-18 12:37:41.294359913 +0100 @@ -236,27 +236,31 @@ Free_list::print_stats() } // A Hash_task computes the MD5 checksum of an array of char. -// It has a blocker on either side (i.e., the task cannot run until -// the first is unblocked, and it unblocks the second after running). class Hash_task : public Task { public: - Hash_task(const unsigned char* src, + Hash_task(Output_file* of, + size_t offset, size_t size, unsigned char* dst, - Task_token* build_id_blocker, Task_token* final_blocker) - : src_(src), size_(size), dst_(dst), build_id_blocker_(build_id_blocker), + : of_(of), offset_(offset), size_(size), dst_(dst), final_blocker_(final_blocker) { } void run(Workqueue*) - { md5_buffer(reinterpret_cast(src_), size_, dst_); } + { + const unsigned char* iv = + this->of_->get_input_view(this->offset_, this->size_); + md5_buffer(reinterpret_cast(iv), this->size_, this->dst_); + this->of_->free_input_view(this->offset_, this->size_, iv); + } Task_token* - is_runnable(); + is_runnable() + { return NULL; } // Unblock FINAL_BLOCKER_ when done. void @@ -268,21 +272,13 @@ class Hash_task : public Task { return "Hash_task"; } private: - const unsigned char* const src_; + Output_file* of_; + const size_t offset_; const size_t size_; unsigned char* const dst_; - Task_token* const build_id_blocker_; Task_token* const final_blocker_; }; -Task_token* -Hash_task::is_runnable() -{ - if (this->build_id_blocker_->is_blocked()) - return this->build_id_blocker_; - return NULL; -} - // Layout::Relaxation_debug_check methods. // Check that sections and special data are in reset states. @@ -449,9 +445,6 @@ Layout::Layout(int number_of_input_files eh_frame_hdr_section_(NULL), gdb_index_data_(NULL), build_id_note_(NULL), - array_of_hashes_(NULL), - size_of_array_of_hashes_(0), - input_view_(NULL), debug_abbrev_(NULL), debug_info_(NULL), group_signatures_(), @@ -5349,56 +5342,13 @@ Layout::write_sections_after_input_secti this->section_headers_->write(of); } -// Build IDs can be computed as a "flat" sha1 or md5 of a string of bytes, -// or as a "tree" where each chunk of the string is hashed and then those -// hashes are put into a (much smaller) string which is hashed with sha1. -// We compute a checksum over the entire file because that is simplest. - -Task_token* -Layout::queue_build_id_tasks(Workqueue* workqueue, Task_token* build_id_blocker, - Output_file* of) -{ - const size_t filesize = (this->output_file_size() <= 0 ? 0 - : static_cast(this->output_file_size())); - if (this->build_id_note_ != NULL - && strcmp(parameters->options().build_id(), "tree") == 0 - && parameters->options().build_id_chunk_size_for_treehash() > 0 - && filesize > 0 - && (filesize >= - parameters->options().build_id_min_file_size_for_treehash())) - { - static const size_t MD5_OUTPUT_SIZE_IN_BYTES = 16; - const size_t chunk_size = - parameters->options().build_id_chunk_size_for_treehash(); - const size_t num_hashes = ((filesize - 1) / chunk_size) + 1; - Task_token* post_hash_tasks_blocker = new Task_token(true); - post_hash_tasks_blocker->add_blockers(num_hashes); - this->size_of_array_of_hashes_ = num_hashes * MD5_OUTPUT_SIZE_IN_BYTES; - const unsigned char* src = of->get_input_view(0, filesize); - this->input_view_ = src; - unsigned char *dst = new unsigned char[this->size_of_array_of_hashes_]; - this->array_of_hashes_ = dst; - for (size_t i = 0, src_offset = 0; i < num_hashes; - i++, dst += MD5_OUTPUT_SIZE_IN_BYTES, src_offset += chunk_size) - { - size_t size = std::min(chunk_size, filesize - src_offset); - workqueue->queue(new Hash_task(src + src_offset, - size, - dst, - build_id_blocker, - post_hash_tasks_blocker)); - } - return post_hash_tasks_blocker; - } - return build_id_blocker; -} - // If a tree-style build ID was requested, the parallel part of that computation // is already done, and the final hash-of-hashes is computed here. For other // types of build IDs, all the work is done here. void -Layout::write_build_id(Output_file* of) const +Layout::write_build_id(Output_file* of, unsigned char* array_of_hashes, + size_t size_of_hashes) const { if (this->build_id_note_ == NULL) return; @@ -5406,7 +5356,7 @@ Layout::write_build_id(Output_file* of) unsigned char* ov = of->get_output_view(this->build_id_note_->offset(), this->build_id_note_->data_size()); - if (this->array_of_hashes_ == NULL) + if (array_of_hashes == NULL) { const size_t output_file_size = this->output_file_size(); const unsigned char* iv = of->get_input_view(0, output_file_size); @@ -5427,10 +5377,9 @@ Layout::write_build_id(Output_file* of) { // Non-overlapping substrings of the output file have been hashed. // Compute SHA-1 hash of the hashes. - sha1_buffer(reinterpret_cast(this->array_of_hashes_), - this->size_of_array_of_hashes_, ov); - delete[] this->array_of_hashes_; - of->free_input_view(0, this->output_file_size(), this->input_view_); + sha1_buffer(reinterpret_cast(array_of_hashes), + size_of_hashes, ov); + delete[] array_of_hashes; } of->write_output_view(this->build_id_note_->offset(), @@ -5629,6 +5578,57 @@ Write_after_input_sections_task::run(Wor this->layout_->write_sections_after_input_sections(this->of_); } +// Build IDs can be computed as a "flat" sha1 or md5 of a string of bytes, +// or as a "tree" where each chunk of the string is hashed and then those +// hashes are put into a (much smaller) string which is hashed with sha1. +// We compute a checksum over the entire file because that is simplest. + +void +Build_id_task_runner::run(Workqueue* workqueue, const Task*) +{ + Task_token* post_hash_tasks_blocker = new Task_token(true); + const Layout* layout = this->layout_; + Output_file* of = this->of_; + const size_t filesize = (layout->output_file_size() <= 0 ? 0 + : static_cast(layout->output_file_size())); + unsigned char* array_of_hashes = NULL; + size_t size_of_hashes = 0; + + if (strcmp(this->options_->build_id(), "tree") == 0 + && this->options_->build_id_chunk_size_for_treehash() > 0 + && filesize > 0 + && (filesize >= this->options_->build_id_min_file_size_for_treehash())) + { + static const size_t MD5_OUTPUT_SIZE_IN_BYTES = 16; + const size_t chunk_size = + this->options_->build_id_chunk_size_for_treehash(); + const size_t num_hashes = ((filesize - 1) / chunk_size) + 1; + post_hash_tasks_blocker->add_blockers(num_hashes); + size_of_hashes = num_hashes * MD5_OUTPUT_SIZE_IN_BYTES; + array_of_hashes = new unsigned char[size_of_hashes]; + unsigned char *dst = array_of_hashes; + for (size_t i = 0, src_offset = 0; i < num_hashes; + i++, dst += MD5_OUTPUT_SIZE_IN_BYTES, src_offset += chunk_size) + { + size_t size = std::min(chunk_size, filesize - src_offset); + workqueue->queue(new Hash_task(of, + src_offset, + size, + dst, + post_hash_tasks_blocker)); + } + } + + // Queue the final task to write the build id and close the output file. + workqueue->queue(new Task_function(new Close_task_runner(this->options_, + layout, + of, + array_of_hashes, + size_of_hashes), + post_hash_tasks_blocker, + "Task_function Close_task_runner")); +} + // Close_task_runner methods. // Finish up the build ID computation, if necessary, and write a binary file, @@ -5638,8 +5638,9 @@ void Close_task_runner::run(Workqueue*, const Task*) { // At this point the multi-threaded part of the build ID computation, - // if any, is done. See queue_build_id_tasks(). - this->layout_->write_build_id(this->of_); + // if any, is done. See Build_id_task_runner. + this->layout_->write_build_id(this->of_, this->array_of_hashes_, + this->size_of_hashes_); // If we've been asked to create a binary file, we do so here. if (this->options_->oformat_enum() != General_options::OBJECT_FORMAT_ELF) diff -rup binutils.orig/gold/layout.h binutils-2.25.1/gold/layout.h --- binutils.orig/gold/layout.h 2017-10-18 12:37:20.539605249 +0100 +++ binutils-2.25.1/gold/layout.h 2017-10-18 12:37:41.294359913 +0100 @@ -891,16 +891,9 @@ class Layout const Output_data_reloc_generic* dyn_rel, bool add_debug, bool dynrel_includes_plt); - // If a treehash is necessary to compute the build ID, then queue - // the necessary tasks and return a blocker that will unblock when - // they finish. Otherwise return BUILD_ID_BLOCKER. - Task_token* - queue_build_id_tasks(Workqueue* workqueue, Task_token* build_id_blocker, - Output_file* of); - // Compute and write out the build ID if needed. void - write_build_id(Output_file*) const; + write_build_id(Output_file*, unsigned char*, size_t) const; // Rewrite output file in binary format. void @@ -1374,12 +1367,6 @@ class Layout Gdb_index* gdb_index_data_; // The space for the build ID checksum if there is one. Output_section_data* build_id_note_; - // Temporary storage for tree hash of build ID. - unsigned char* array_of_hashes_; - // Size of array_of_hashes_ (in bytes). - size_t size_of_array_of_hashes_; - // Input view for computing tree hash of build ID. Freed in write_build_id(). - const unsigned char* input_view_; // The output section containing dwarf abbreviations Output_reduced_debug_abbrev_section* debug_abbrev_; // The output section containing the dwarf debug info tree @@ -1596,14 +1583,40 @@ class Write_after_input_sections_task : Task_token* final_blocker_; }; +// This task function handles computation of the build id. +// When using --build-id=tree, it schedules the tasks that +// compute the hashes for each chunk of the file. This task +// cannot run until we have finalized the size of the output +// file, after the completion of Write_after_input_sections_task. + +class Build_id_task_runner : public Task_function_runner +{ + public: + Build_id_task_runner(const General_options* options, const Layout* layout, + Output_file* of) + : options_(options), layout_(layout), of_(of) + { } + + // Run the operation. + void + run(Workqueue*, const Task*); + + private: + const General_options* options_; + const Layout* layout_; + Output_file* of_; +}; + // This task function handles closing the file. class Close_task_runner : public Task_function_runner { public: Close_task_runner(const General_options* options, const Layout* layout, - Output_file* of) - : options_(options), layout_(layout), of_(of) + Output_file* of, unsigned char* array_of_hashes, + size_t size_of_hashes) + : options_(options), layout_(layout), of_(of), + array_of_hashes_(array_of_hashes), size_of_hashes_(size_of_hashes) { } // Run the operation. @@ -1614,6 +1627,8 @@ class Close_task_runner : public Task_fu const General_options* options_; const Layout* layout_; Output_file* of_; + unsigned char* const array_of_hashes_; + const size_t size_of_hashes_; }; // A small helper function to align an address. diff -rup binutils.orig/gold/testsuite/Makefile.am binutils-2.25.1/gold/testsuite/Makefile.am --- binutils.orig/gold/testsuite/Makefile.am 2017-10-18 12:37:20.541605225 +0100 +++ binutils-2.25.1/gold/testsuite/Makefile.am 2017-10-18 12:37:41.294359913 +0100 @@ -1144,6 +1144,14 @@ flagstest_compress_debug_sections: flags $(CXXLINK) -Bgcctestdir/ -o $@ $< -Wl,--compress-debug-sections=zlib test -s $@ +# Test --compress-debug-sections with --build-id=tree. +check_PROGRAMS += flagstest_compress_debug_sections_and_build_id_tree +flagstest_compress_debug_sections_and_build_id_tree: flagstest_debug.o gcctestdir/ld + $(CXXLINK) -Bgcctestdir/ -o $@ $< -Wl,--compress-debug-sections=zlib \ + -Wl,--build-id=tree \ + -Wl,--build-id-chunk-size-for-treehash=4096 \ + -Wl,--build-id-min-file-size-for-treehash=0 + test -s $@ # The specialfile output has a tricky case when we also compress debug # sections, because it requires output-file resizing. diff -rup binutils.orig/gold/testsuite/Makefile.in binutils-2.25.1/gold/testsuite/Makefile.in --- binutils.orig/gold/testsuite/Makefile.in 2017-10-18 12:37:20.543605201 +0100 +++ binutils-2.25.1/gold/testsuite/Makefile.in 2017-10-18 12:45:11.181041939 +0100 @@ -300,9 +300,12 @@ check_PROGRAMS = $(am__EXEEXT_1) $(am__E # Test --compress-debug-sections. FIXME: check we actually compress. +# Test --compress-debug-sections with --build-id=tree. + # The specialfile output has a tricky case when we also compress debug # sections, because it requires output-file resizing. @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@am__append_31 = flagstest_compress_debug_sections \ +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ flagstest_compress_debug_sections_and_build_id_tree \ @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ flagstest_o_specialfile_and_compress_debug_sections # Test -TText and -Tdata. @@ -850,6 +853,7 @@ libgoldtest_a_OBJECTS = $(am_libgoldtest @GCC_TRUE@@NATIVE_LINKER_TRUE@ initpri3a$(EXEEXT) \ @GCC_TRUE@@NATIVE_LINKER_TRUE@ flagstest_o_specialfile$(EXEEXT) @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@am__EXEEXT_21 = flagstest_compress_debug_sections$(EXEEXT) \ +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ flagstest_compress_debug_sections_and_build_id_tree$(EXEEXT) \ @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ flagstest_o_specialfile_and_compress_debug_sections$(EXEEXT) @GCC_TRUE@@NATIVE_LINKER_TRUE@am__EXEEXT_22 = \ @GCC_TRUE@@NATIVE_LINKER_TRUE@ flagstest_o_ttext_1$(EXEEXT) \ @@ -1112,6 +1116,14 @@ flagstest_compress_debug_sections_LDADD flagstest_compress_debug_sections_DEPENDENCIES = libgoldtest.a \ ../libgold.a ../../libiberty/libiberty.a $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) +flagstest_compress_debug_sections_and_build_id_tree_SOURCES = \ + flagstest_compress_debug_sections_and_build_id_tree.c +flagstest_compress_debug_sections_and_build_id_tree_OBJECTS = \ + flagstest_compress_debug_sections_and_build_id_tree.$(OBJEXT) +flagstest_compress_debug_sections_and_build_id_tree_LDADD = $(LDADD) +flagstest_compress_debug_sections_and_build_id_tree_DEPENDENCIES = \ + libgoldtest.a ../libgold.a ../../libiberty/libiberty.a \ + $(am__DEPENDENCIES_1) flagstest_o_specialfile_SOURCES = flagstest_o_specialfile.c flagstest_o_specialfile_OBJECTS = flagstest_o_specialfile.$(OBJEXT) flagstest_o_specialfile_LDADD = $(LDADD) @@ -1877,7 +1889,9 @@ SOURCES = $(libgoldtest_a_SOURCES) basic $(exception_static_test_SOURCES) $(exception_test_SOURCES) \ $(exception_x86_64_bnd_test_SOURCES) \ $(exclude_libs_test_SOURCES) \ - flagstest_compress_debug_sections.c flagstest_o_specialfile.c \ + flagstest_compress_debug_sections.c \ + flagstest_compress_debug_sections_and_build_id_tree.c \ + flagstest_o_specialfile.c \ flagstest_o_specialfile_and_compress_debug_sections.c \ flagstest_o_ttext_1.c icf_virtual_function_folding_test.c \ $(ifuncmain1_SOURCES) ifuncmain1pic.c ifuncmain1picstatic.c \ @@ -2900,6 +2914,12 @@ exclude_libs_test$(EXEEXT): $(exclude_li @NATIVE_LINKER_FALSE@flagstest_compress_debug_sections$(EXEEXT): $(flagstest_compress_debug_sections_OBJECTS) $(flagstest_compress_debug_sections_DEPENDENCIES) @NATIVE_LINKER_FALSE@ @rm -f flagstest_compress_debug_sections$(EXEEXT) @NATIVE_LINKER_FALSE@ $(LINK) $(flagstest_compress_debug_sections_OBJECTS) $(flagstest_compress_debug_sections_LDADD) $(LIBS) +@GCC_FALSE@flagstest_compress_debug_sections_and_build_id_tree$(EXEEXT): $(flagstest_compress_debug_sections_and_build_id_tree_OBJECTS) $(flagstest_compress_debug_sections_and_build_id_tree_DEPENDENCIES) +@GCC_FALSE@ @rm -f flagstest_compress_debug_sections_and_build_id_tree$(EXEEXT) +@GCC_FALSE@ $(LINK) $(flagstest_compress_debug_sections_and_build_id_tree_OBJECTS) $(flagstest_compress_debug_sections_and_build_id_tree_LDADD) $(LIBS) +@NATIVE_LINKER_FALSE@flagstest_compress_debug_sections_and_build_id_tree$(EXEEXT): $(flagstest_compress_debug_sections_and_build_id_tree_OBJECTS) $(flagstest_compress_debug_sections_and_build_id_tree_DEPENDENCIES) +@NATIVE_LINKER_FALSE@ @rm -f flagstest_compress_debug_sections_and_build_id_tree$(EXEEXT) +@NATIVE_LINKER_FALSE@ $(LINK) $(flagstest_compress_debug_sections_and_build_id_tree_OBJECTS) $(flagstest_compress_debug_sections_and_build_id_tree_LDADD) $(LIBS) @GCC_FALSE@flagstest_o_specialfile$(EXEEXT): $(flagstest_o_specialfile_OBJECTS) $(flagstest_o_specialfile_DEPENDENCIES) @GCC_FALSE@ @rm -f flagstest_o_specialfile$(EXEEXT) @GCC_FALSE@ $(LINK) $(flagstest_o_specialfile_OBJECTS) $(flagstest_o_specialfile_LDADD) $(LIBS) @@ -3628,6 +3648,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/exception_test_main.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/exclude_libs_test.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/flagstest_compress_debug_sections.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/flagstest_compress_debug_sections_and_build_id_tree.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/flagstest_o_specialfile.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/flagstest_o_specialfile_and_compress_debug_sections.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/flagstest_o_ttext_1.Po@am__quote@ @@ -4361,6 +4382,8 @@ flagstest_o_specialfile.log: flagstest_o @p='flagstest_o_specialfile$(EXEEXT)'; $(am__check_pre) $(LOG_COMPILE) "$$tst" $(am__check_post) flagstest_compress_debug_sections.log: flagstest_compress_debug_sections$(EXEEXT) @p='flagstest_compress_debug_sections$(EXEEXT)'; $(am__check_pre) $(LOG_COMPILE) "$$tst" $(am__check_post) +flagstest_compress_debug_sections_and_build_id_tree.log: flagstest_compress_debug_sections_and_build_id_tree$(EXEEXT) + @p='flagstest_compress_debug_sections_and_build_id_tree$(EXEEXT)'; $(am__check_pre) $(LOG_COMPILE) "$$tst" $(am__check_post) flagstest_o_specialfile_and_compress_debug_sections.log: flagstest_o_specialfile_and_compress_debug_sections$(EXEEXT) @p='flagstest_o_specialfile_and_compress_debug_sections$(EXEEXT)'; $(am__check_pre) $(LOG_COMPILE) "$$tst" $(am__check_post) flagstest_o_ttext_1.log: flagstest_o_ttext_1$(EXEEXT) @@ -5128,6 +5151,12 @@ uninstall-am: @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@flagstest_compress_debug_sections: flagstest_debug.o gcctestdir/ld @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ $(CXXLINK) -Bgcctestdir/ -o $@ $< -Wl,--compress-debug-sections=zlib @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ test -s $@ +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@flagstest_compress_debug_sections_and_build_id_tree: flagstest_debug.o gcctestdir/ld +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ $(CXXLINK) -Bgcctestdir/ -o $@ $< -Wl,--compress-debug-sections=zlib \ +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ -Wl,--build-id=tree \ +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ -Wl,--build-id-chunk-size-for-treehash=4096 \ +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ -Wl,--build-id-min-file-size-for-treehash=0 +@GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ test -s $@ @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@flagstest_o_specialfile_and_compress_debug_sections: flagstest_debug.o \ @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ gcctestdir/ld @GCC_TRUE@@HAVE_ZLIB_TRUE@@NATIVE_LINKER_TRUE@ $(CXXLINK) -Bgcctestdir/ -o /dev/stdout $< -Wl,--compress-debug-sections=zlib 2>&1 | cat > $@ Only in binutils-2.25.1/gold/testsuite: Makefile.in.orig Only in binutils-2.25.1/gold/testsuite: Makefile.in.rej