From b0b1317dea360e5a93389f6c17bdcc0c9f0a4dc0 Mon Sep 17 00:00:00 2001 From: Michael Olbrich Date: Fri, 25 May 2018 08:42:39 +0200 Subject: ninja: update jobserver patches from github issue Signed-off-by: Michael Olbrich --- ...enPool-monitoring-to-SubprocessSet-DoWork.patch | 111 +++++++++--- ...bserver-when-jN-is-forced-on-command-line.patch | 192 +++++++++++++++++++++ .../ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch | 128 ++++++++++++++ ...05-Use-LinePrinter-for-TokenPool-messages.patch | 122 +++++++++++++ .../ninja-1.8.2/0006-Prepare-PR-for-merging.patch | 151 ++++++++++++++++ patches/ninja-1.8.2/series | 6 +- scripts/lib/ptxd_make_world_common.sh | 4 +- 7 files changed, 690 insertions(+), 24 deletions(-) create mode 100644 patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch create mode 100644 patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch create mode 100644 patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch create mode 100644 patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch diff --git a/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch b/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch index caf8a5aac..f8ee646d7 100644 --- a/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch +++ b/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch @@ -6,37 +6,49 @@ Improve on the original jobserver client implementation. This makes ninja a more aggressive GNU make jobserver client. - add monitor interface to TokenPool -- TokenPool passed down when plan indicates more work is ready to start +- TokenPool is passed down when main loop indicates that more work is + ready and would be allowed to start if a token becomes available - posix: update DoWork() to monitor TokenPool read file descriptor - WaitForCommand() exits when DoWork() sets token flag - Main loop starts over when WaitForCommand() sets token exit status --- - src/build.cc | 27 ++++++++++++++++++++------- - src/build.h | 2 +- - src/build_test.cc | 4 ++-- + src/build.cc | 53 +++++++++++++++++++++++++++++++++++------------ + src/build.h | 3 ++- + src/build_test.cc | 9 ++++++-- src/exit_status.h | 3 ++- - src/subprocess-posix.cc | 33 +++++++++++++++++++++++++++++++-- + src/subprocess-posix.cc | 33 +++++++++++++++++++++++++++-- src/subprocess-win32.cc | 2 +- - src/subprocess.h | 8 +++++++- - src/subprocess_test.cc | 47 +++++++++++++++++++++++++++++++++-------------- + src/subprocess.h | 8 ++++++- + src/subprocess_test.cc | 47 ++++++++++++++++++++++++++++------------- src/tokenpool-gnu-make.cc | 5 +++++ src/tokenpool.h | 6 ++++++ - 10 files changed, 108 insertions(+), 29 deletions(-) + 10 files changed, 134 insertions(+), 35 deletions(-) diff --git a/src/build.cc b/src/build.cc -index cc796ff838fa..ccd67697786d 100644 +index cc796ff838fa..219bb9f1ff48 100644 --- a/src/build.cc +++ b/src/build.cc -@@ -50,7 +50,7 @@ struct DryRunCommandRunner : public CommandRunner { +@@ -49,8 +49,9 @@ struct DryRunCommandRunner : public CommandRunner { + // Overridden from CommandRunner: virtual bool CanRunMore(); ++ virtual bool AcquireToken(); virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); + virtual bool WaitForCommand(Result* result, bool more_ready); private: queue finished_; -@@ -65,7 +65,7 @@ bool DryRunCommandRunner::StartCommand(Edge* edge) { +@@ -60,12 +61,16 @@ bool DryRunCommandRunner::CanRunMore() { + return true; + } + ++bool DryRunCommandRunner::AcquireToken() { ++ return true; ++} ++ + bool DryRunCommandRunner::StartCommand(Edge* edge) { + finished_.push(edge); return true; } @@ -45,16 +57,34 @@ index cc796ff838fa..ccd67697786d 100644 if (finished_.empty()) return false; -@@ -490,7 +490,7 @@ struct RealCommandRunner : public CommandRunner { +@@ -489,8 +494,9 @@ struct RealCommandRunner : public CommandRunner { + explicit RealCommandRunner(const BuildConfig& config); virtual ~RealCommandRunner(); virtual bool CanRunMore(); ++ virtual bool AcquireToken(); virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); + virtual bool WaitForCommand(Result* result, bool more_ready); virtual vector GetActiveEdges(); virtual void Abort(); -@@ -544,14 +544,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) { +@@ -527,9 +533,12 @@ bool RealCommandRunner::CanRunMore() { + subprocs_.running_.size() + subprocs_.finished_.size(); + return (int)subproc_number < config_.parallelism + && (subprocs_.running_.empty() || +- ((config_.max_load_average <= 0.0f || +- GetLoadAverage() < config_.max_load_average) +- && (!tokens_ || tokens_->Acquire()))); ++ (config_.max_load_average <= 0.0f || ++ GetLoadAverage() < config_.max_load_average)); ++} ++ ++bool RealCommandRunner::AcquireToken() { ++ return (!tokens_ || tokens_->Acquire()); + } + + bool RealCommandRunner::StartCommand(Edge* edge) { +@@ -544,14 +553,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) { return true; } @@ -81,16 +111,34 @@ index cc796ff838fa..ccd67697786d 100644 if (tokens_) tokens_->Release(); -@@ -685,7 +694,7 @@ bool Builder::Build(string* err) { +@@ -662,9 +680,14 @@ bool Builder::Build(string* err) { + // command runner. + // Second, we attempt to wait for / reap the next finished command. + while (plan_.more_to_do()) { +- // See if we can start any more commands. +- if (failures_allowed && plan_.more_ready() && +- command_runner_->CanRunMore()) { ++ // See if we can start any more commands... ++ bool can_run_more = ++ failures_allowed && ++ plan_.more_ready() && ++ command_runner_->CanRunMore(); ++ ++ // ... but we also need a token to do that. ++ if (can_run_more && command_runner_->AcquireToken()) { + Edge* edge = plan_.FindWork(); + if (!StartEdge(edge, err)) { + Cleanup(); +@@ -685,7 +708,7 @@ bool Builder::Build(string* err) { // See if we can reap any finished commands. if (pending_commands) { CommandRunner::Result result; - if (!command_runner_->WaitForCommand(&result) || -+ if (!command_runner_->WaitForCommand(&result, plan_.more_ready()) || ++ if (!command_runner_->WaitForCommand(&result, can_run_more) || result.status == ExitInterrupted) { Cleanup(); status_->BuildFinished(); -@@ -693,6 +702,10 @@ bool Builder::Build(string* err) { +@@ -693,6 +716,10 @@ bool Builder::Build(string* err) { return false; } @@ -102,10 +150,18 @@ index cc796ff838fa..ccd67697786d 100644 if (!FinishCommand(&result, err)) { Cleanup(); diff --git a/src/build.h b/src/build.h -index cca7e8d8181d..ec6594f714c0 100644 +index cca7e8d8181d..ca605e62e0e3 100644 --- a/src/build.h +++ b/src/build.h -@@ -119,7 +119,7 @@ struct CommandRunner { +@@ -108,6 +108,7 @@ private: + struct CommandRunner { + virtual ~CommandRunner() {} + virtual bool CanRunMore() = 0; ++ virtual bool AcquireToken() = 0; + virtual bool StartCommand(Edge* edge) = 0; + + /// The result of waiting for a command. +@@ -119,7 +120,7 @@ struct CommandRunner { bool success() const { return status == ExitSuccess; } }; /// Wait for a command to complete, or return false if interrupted. @@ -115,19 +171,32 @@ index cca7e8d8181d..ec6594f714c0 100644 virtual vector GetActiveEdges() { return vector(); } virtual void Abort() {} diff --git a/src/build_test.cc b/src/build_test.cc -index 46ab33ef86c8..c1d3b047271a 100644 +index 46ab33ef86c8..a1022edcf546 100644 --- a/src/build_test.cc +++ b/src/build_test.cc -@@ -446,7 +446,7 @@ struct FakeCommandRunner : public CommandRunner { +@@ -445,8 +445,9 @@ struct FakeCommandRunner : public CommandRunner { + // CommandRunner impl virtual bool CanRunMore(); ++ virtual bool AcquireToken(); virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); + virtual bool WaitForCommand(Result* result, bool more_ready); virtual vector GetActiveEdges(); virtual void Abort(); -@@ -575,7 +575,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { +@@ -547,6 +548,10 @@ bool FakeCommandRunner::CanRunMore() { + return last_command_ == NULL; + } + ++bool FakeCommandRunner::AcquireToken() { ++ return true; ++} ++ + bool FakeCommandRunner::StartCommand(Edge* edge) { + assert(!last_command_); + commands_ran_.push_back(edge->EvaluateCommand()); +@@ -575,7 +580,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { return true; } diff --git a/patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch b/patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch new file mode 100644 index 000000000..cd4b78f69 --- /dev/null +++ b/patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch @@ -0,0 +1,192 @@ +From: Stefan Becker +Date: Sun, 12 Nov 2017 16:58:55 +0200 +Subject: [PATCH] Ignore jobserver when -jN is forced on command line + +This emulates the behaviour of GNU make. + +- add parallelism_from_cmdline flag to build configuration +- set the flag when -jN is given on command line +- pass the flag to TokenPool::Get() +- GNUmakeTokenPool::Setup() + * prints a warning when the flag is true and jobserver was detected + * returns false, i.e. jobserver will be ignored +- ignore config.parallelism in CanRunMore() when we have a valid + TokenPool, because it gets always initialized to a default when not + given on the command line +--- + src/build.cc | 10 ++++++---- + src/build.h | 4 +++- + src/ninja.cc | 1 + + src/tokenpool-gnu-make.cc | 34 +++++++++++++++++++--------------- + src/tokenpool-none.cc | 4 ++-- + src/tokenpool.h | 4 ++-- + 6 files changed, 33 insertions(+), 24 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index 219bb9f1ff48..bc26bdade61b 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -507,7 +507,7 @@ struct RealCommandRunner : public CommandRunner { + }; + + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { +- tokens_ = TokenPool::Get(); ++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); + } + + RealCommandRunner::~RealCommandRunner() { +@@ -529,9 +529,11 @@ void RealCommandRunner::Abort() { + } + + bool RealCommandRunner::CanRunMore() { +- size_t subproc_number = +- subprocs_.running_.size() + subprocs_.finished_.size(); +- return (int)subproc_number < config_.parallelism ++ bool parallelism_limit_not_reached = ++ tokens_ || // ignore config_.parallelism ++ ((int) (subprocs_.running_.size() + ++ subprocs_.finished_.size()) < config_.parallelism); ++ return parallelism_limit_not_reached + && (subprocs_.running_.empty() || + (config_.max_load_average <= 0.0f || + GetLoadAverage() < config_.max_load_average)); +diff --git a/src/build.h b/src/build.h +index ca605e62e0e3..6bc6fea26e94 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -128,7 +128,8 @@ struct CommandRunner { + + /// Options (e.g. verbosity, parallelism) passed to a build. + struct BuildConfig { +- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), ++ BuildConfig() : verbosity(NORMAL), dry_run(false), ++ parallelism(1), parallelism_from_cmdline(false), + failures_allowed(1), max_load_average(-0.0f) {} + + enum Verbosity { +@@ -139,6 +140,7 @@ struct BuildConfig { + Verbosity verbosity; + bool dry_run; + int parallelism; ++ bool parallelism_from_cmdline; + int failures_allowed; + /// The maximum load average we must not exceed. A negative value + /// means that we do not have any limit. +diff --git a/src/ninja.cc b/src/ninja.cc +index ed004ac8f1fe..4332636c1b64 100644 +--- a/src/ninja.cc ++++ b/src/ninja.cc +@@ -1063,6 +1063,7 @@ int ReadFlags(int* argc, char*** argv, + if (*end != 0 || value <= 0) + Fatal("invalid -j parameter"); + config->parallelism = value; ++ config->parallelism_from_cmdline = true; + break; + } + case 'k': { +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index 396bb7d87443..af4be05a31cf 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(); ++ bool Setup(bool ignore); + + private: + int available_; +@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup() { ++bool GNUmakeTokenPool::Setup(bool ignore) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -109,16 +109,20 @@ bool GNUmakeTokenPool::Setup() { + if (!jobserver) + jobserver = strstr(value, "--jobserver-auth="); + if (jobserver) { +- int rfd = -1; +- int wfd = -1; +- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && +- CheckFd(rfd) && +- CheckFd(wfd) && +- SetAlarmHandler()) { +- printf("ninja: using GNU make jobserver.\n"); +- rfd_ = rfd; +- wfd_ = wfd; +- return true; ++ if (ignore) { ++ printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ } else { ++ int rfd = -1; ++ int wfd = -1; ++ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && ++ CheckFd(rfd) && ++ CheckFd(wfd) && ++ SetAlarmHandler()) { ++ printf("ninja: using GNU make jobserver.\n"); ++ rfd_ = rfd; ++ wfd_ = wfd; ++ return true; ++ } + } + } + } +@@ -206,9 +210,9 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(void) { ++struct TokenPool *TokenPool::Get(bool ignore) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup()) ++ if (tokenpool->Setup(ignore)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 602b3316f54d..199b22264bc6 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -22,6 +22,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(void) { ++struct TokenPool *TokenPool::Get(bool ignore) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 301e1998ee8e..878a0933c2f0 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -28,5 +28,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(void); ++ static struct TokenPool *Get(bool ignore); + }; diff --git a/patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch b/patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch new file mode 100644 index 000000000..2a2505001 --- /dev/null +++ b/patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch @@ -0,0 +1,128 @@ +From: Stefan Becker +Date: Sun, 12 Nov 2017 18:04:12 +0200 +Subject: [PATCH] Honor -lN from MAKEFLAGS + +This emulates the behaviour of GNU make. + +- build: make a copy of max_load_average and pass it to TokenPool. +- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in + MAKEFLAGS then set max_load_average to N. +--- + src/build.cc | 10 +++++++--- + src/tokenpool-gnu-make.cc | 19 +++++++++++++++---- + src/tokenpool-none.cc | 2 +- + src/tokenpool.h | 2 +- + 4 files changed, 24 insertions(+), 9 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index bc26bdade61b..6eaf299caeec 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -501,13 +501,17 @@ struct RealCommandRunner : public CommandRunner { + virtual void Abort(); + + const BuildConfig& config_; ++ // copy of config_.max_load_average; can be modified by TokenPool setup ++ double max_load_average_; + SubprocessSet subprocs_; + TokenPool *tokens_; + map subproc_to_edge_; + }; + + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { +- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); ++ max_load_average_ = config.max_load_average; ++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, ++ max_load_average_); + } + + RealCommandRunner::~RealCommandRunner() { +@@ -535,8 +539,8 @@ bool RealCommandRunner::CanRunMore() { + subprocs_.finished_.size()) < config_.parallelism); + return parallelism_limit_not_reached + && (subprocs_.running_.empty() || +- (config_.max_load_average <= 0.0f || +- GetLoadAverage() < config_.max_load_average)); ++ (max_load_average_ <= 0.0f || ++ GetLoadAverage() < max_load_average_)); + } + + bool RealCommandRunner::AcquireToken() { +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index af4be05a31cf..fb654c4d88ba 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(bool ignore); ++ bool Setup(bool ignore, double& max_load_average); + + private: + int available_; +@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup(bool ignore) { ++bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -118,9 +118,20 @@ bool GNUmakeTokenPool::Setup(bool ignore) { + CheckFd(rfd) && + CheckFd(wfd) && + SetAlarmHandler()) { ++ const char *l_arg = strstr(value, " -l"); ++ int load_limit = -1; ++ + printf("ninja: using GNU make jobserver.\n"); + rfd_ = rfd; + wfd_ = wfd; ++ ++ // translate GNU make -lN to ninja -lN ++ if (l_arg && ++ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && ++ (load_limit > 0)) { ++ max_load_average = load_limit; ++ } ++ + return true; + } + } +@@ -210,9 +221,9 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(bool ignore) { ++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore)) ++ if (tokenpool->Setup(ignore, max_load_average)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 199b22264bc6..e8e25426c39f 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -22,6 +22,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore) { ++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 878a0933c2f0..f9e8cc2ee081 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -28,5 +28,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore); ++ static struct TokenPool *Get(bool ignore, double& max_load_average); + }; diff --git a/patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch b/patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch new file mode 100644 index 000000000..1894be5a1 --- /dev/null +++ b/patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch @@ -0,0 +1,122 @@ +From: Stefan Becker +Date: Wed, 6 Dec 2017 22:14:21 +0200 +Subject: [PATCH] Use LinePrinter for TokenPool messages + +- replace printf() with calls to LinePrinter +- print GNU make jobserver message only when verbose build is requested +--- + src/build.cc | 1 + + src/tokenpool-gnu-make.cc | 22 ++++++++++++++++------ + src/tokenpool-none.cc | 4 +++- + src/tokenpool.h | 4 +++- + 4 files changed, 23 insertions(+), 8 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index 6eaf299caeec..754d362c7162 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -511,6 +511,7 @@ struct RealCommandRunner : public CommandRunner { + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { + max_load_average_ = config.max_load_average; + tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, ++ config_.verbosity == BuildConfig::VERBOSE, + max_load_average_); + } + +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index fb654c4d88ba..b0d3e6ebc463 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -23,6 +23,8 @@ + #include + #include + ++#include "line_printer.h" ++ + // TokenPool implementation for GNU make jobserver + // (http://make.mad-scientist.net/papers/jobserver-implementation/) + struct GNUmakeTokenPool : public TokenPool { +@@ -35,7 +37,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(bool ignore, double& max_load_average); ++ bool Setup(bool ignore, bool verbose, double& max_load_average); + + private: + int available_; +@@ -100,7 +102,9 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { ++bool GNUmakeTokenPool::Setup(bool ignore, ++ bool verbose, ++ double& max_load_average) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -109,8 +113,10 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + if (!jobserver) + jobserver = strstr(value, "--jobserver-auth="); + if (jobserver) { ++ LinePrinter printer; ++ + if (ignore) { +- printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); + } else { + int rfd = -1; + int wfd = -1; +@@ -121,7 +127,9 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + const char *l_arg = strstr(value, " -l"); + int load_limit = -1; + +- printf("ninja: using GNU make jobserver.\n"); ++ if (verbose) { ++ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); ++ } + rfd_ = rfd; + wfd_ = wfd; + +@@ -221,9 +229,11 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { ++struct TokenPool *TokenPool::Get(bool ignore, ++ bool verbose, ++ double& max_load_average) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore, max_load_average)) ++ if (tokenpool->Setup(ignore, verbose, max_load_average)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index e8e25426c39f..1c1c499c8d9c 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -22,6 +22,8 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { ++struct TokenPool *TokenPool::Get(bool ignore, ++ bool verbose, ++ double& max_load_average) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index f9e8cc2ee081..4bf477f20c8a 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -28,5 +28,7 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore, double& max_load_average); ++ static struct TokenPool *Get(bool ignore, ++ bool verbose, ++ double& max_load_average); + }; diff --git a/patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch b/patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch new file mode 100644 index 000000000..bf44d5f4b --- /dev/null +++ b/patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch @@ -0,0 +1,151 @@ +From: Stefan Becker +Date: Sat, 7 Apr 2018 17:11:21 +0300 +Subject: [PATCH] Prepare PR for merging + +- fix Windows build error in no-op TokenPool implementation +- improve Acquire() to block for a maximum of 100ms +- address review comments +--- + src/build.h | 2 ++ + src/tokenpool-gnu-make.cc | 53 ++++++++++++++++++++++++++++++++++++++++------- + src/tokenpool-none.cc | 7 +------ + 3 files changed, 49 insertions(+), 13 deletions(-) + +diff --git a/src/build.h b/src/build.h +index 6bc6fea26e94..530812bb9a2a 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -120,6 +120,8 @@ struct CommandRunner { + bool success() const { return status == ExitSuccess; } + }; + /// Wait for a command to complete, or return false if interrupted. ++ /// If more_ready is true then the optional TokenPool is monitored too ++ /// and we return when a token becomes available. + virtual bool WaitForCommand(Result* result, bool more_ready) = 0; + + virtual vector GetActiveEdges() { return vector(); } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index b0d3e6ebc463..4132bb06d9dd 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. ++// Copyright 2016-2018 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -153,6 +154,15 @@ bool GNUmakeTokenPool::Acquire() { + if (available_ > 0) + return true; + ++ // Please read ++ // ++ // http://make.mad-scientist.net/papers/jobserver-implementation/ ++ // ++ // for the reasoning behind the following code. ++ // ++ // Try to read one character from the pipe. Returns true on success. ++ // ++ // First check if read() would succeed without blocking. + #ifdef USE_PPOLL + pollfd pollfds[] = {{rfd_, POLLIN, 0}}; + int ret = poll(pollfds, 1, 0); +@@ -164,33 +174,62 @@ bool GNUmakeTokenPool::Acquire() { + int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); + #endif + if (ret > 0) { ++ // Handle potential race condition: ++ // - the above check succeeded, i.e. read() should not block ++ // - the character disappears before we call read() ++ // ++ // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_ ++ // can safely be closed by signal handlers without affecting rfd_. + dup_rfd_ = dup(rfd_); + + if (dup_rfd_ != -1) { + struct sigaction act, old_act; + int ret = 0; + ++ // Temporarily replace SIGCHLD handler with our own + memset(&act, 0, sizeof(act)); + act.sa_handler = CloseDupRfd; + if (sigaction(SIGCHLD, &act, &old_act) == 0) { +- char buf; +- +- // block until token read, child exits or timeout +- alarm(1); +- ret = read(dup_rfd_, &buf, 1); +- alarm(0); ++ struct itimerval timeout; ++ ++ // install a 100ms timeout that generates SIGALARM on expiration ++ memset(&timeout, 0, sizeof(timeout)); ++ timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] ++ if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { ++ char buf; ++ ++ // Now try to read() from dup_rfd_. Return values from read(): ++ // ++ // 1. token read -> 1 ++ // 2. pipe closed -> 0 ++ // 3. alarm expires -> -1 (EINTR) ++ // 4. child exits -> -1 (EINTR) ++ // 5. alarm expired before entering read() -> -1 (EBADF) ++ // 6. child exited before entering read() -> -1 (EBADF) ++ // 7. child exited before handler is installed -> go to 1 - 3 ++ ret = read(dup_rfd_, &buf, 1); ++ ++ // disarm timer ++ memset(&timeout, 0, sizeof(timeout)); ++ setitimer(ITIMER_REAL, &timeout, NULL); ++ } + + sigaction(SIGCHLD, &old_act, NULL); + } + + CloseDupRfd(0); + ++ // Case 1 from above list + if (ret > 0) { + available_++; + return true; + } + } + } ++ ++ // read() would block, i.e. no token available, ++ // cases 2-6 from above list or ++ // select() / poll() / dup() / sigaction() / setitimer() failed + return false; + } + +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 1c1c499c8d9c..4c592875b4ad 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. ++// Copyright 2016-2018 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -14,11 +14,6 @@ + + #include "tokenpool.h" + +-#include +-#include +-#include +-#include +-#include + #include + + // No-op TokenPool implementation diff --git a/patches/ninja-1.8.2/series b/patches/ninja-1.8.2/series index aec7db323..4494b5b79 100644 --- a/patches/ninja-1.8.2/series +++ b/patches/ninja-1.8.2/series @@ -2,4 +2,8 @@ #tag:base --start-number 1 0001-Add-GNU-make-jobserver-client-support.patch 0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch -# 0034c2e369003b7ed18a6396de19c997 - git-ptx-patches magic +0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch +0004-Honor-lN-from-MAKEFLAGS.patch +0005-Use-LinePrinter-for-TokenPool-messages.patch +0006-Prepare-PR-for-merging.patch +# d3f8b2e9621f8f2490c54704a2621fb2 - git-ptx-patches magic diff --git a/scripts/lib/ptxd_make_world_common.sh b/scripts/lib/ptxd_make_world_common.sh index 90cac76c0..c8fc03282 100644 --- a/scripts/lib/ptxd_make_world_common.sh +++ b/scripts/lib/ptxd_make_world_common.sh @@ -333,9 +333,9 @@ ptxd_make_world_init() { pkg_make_opt="-v ${pkg_make_opt}" pkg_install_opt="-v ${pkg_install_opt}" fi - # both jobserver and argument limit parallelism so both are needed + # pass jobserver via MAKEFLAGS to ninja pkg_env="${pkg_env} MAKEFLAGS='${PTXDIST_JOBSERVER_FLAGS}'" - PTXDIST_PARALLELMFLAGS_INTERN="${PTXDIST_PARALLEL_FLAGS}" + PTXDIST_PARALLELMFLAGS_INTERN="" unset conf_opt_ptr ;; -- cgit v1.2.3