summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichael Olbrich <m.olbrich@pengutronix.de>2018-05-25 08:42:39 +0200
committerMichael Olbrich <m.olbrich@pengutronix.de>2018-05-25 20:59:29 +0200
commitb0b1317dea360e5a93389f6c17bdcc0c9f0a4dc0 (patch)
tree2dd80c501bf52ab019561d6cf600437e3330438b
parente399eeb480607055ec22d4409ce4739a3e4ef101 (diff)
downloadptxdist-b0b1317dea360e5a93389f6c17bdcc0c9f0a4dc0.tar.gz
ptxdist-b0b1317dea360e5a93389f6c17bdcc0c9f0a4dc0.tar.xz
ninja: update jobserver patches from github issue
Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
-rw-r--r--patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch111
-rw-r--r--patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch192
-rw-r--r--patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch128
-rw-r--r--patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch122
-rw-r--r--patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch151
-rw-r--r--patches/ninja-1.8.2/series6
-rw-r--r--scripts/lib/ptxd_make_world_common.sh4
7 files changed, 690 insertions, 24 deletions
diff --git a/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch b/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch
index caf8a5aac..f8ee646d7 100644
--- a/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch
+++ b/patches/ninja-1.8.2/0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch
@@ -6,37 +6,49 @@ Improve on the original jobserver client implementation. This makes
ninja a more aggressive GNU make jobserver client.
- add monitor interface to TokenPool
-- TokenPool passed down when plan indicates more work is ready to start
+- TokenPool is passed down when main loop indicates that more work is
+ ready and would be allowed to start if a token becomes available
- posix: update DoWork() to monitor TokenPool read file descriptor
- WaitForCommand() exits when DoWork() sets token flag
- Main loop starts over when WaitForCommand() sets token exit status
---
- src/build.cc | 27 ++++++++++++++++++++-------
- src/build.h | 2 +-
- src/build_test.cc | 4 ++--
+ src/build.cc | 53 +++++++++++++++++++++++++++++++++++------------
+ src/build.h | 3 ++-
+ src/build_test.cc | 9 ++++++--
src/exit_status.h | 3 ++-
- src/subprocess-posix.cc | 33 +++++++++++++++++++++++++++++++--
+ src/subprocess-posix.cc | 33 +++++++++++++++++++++++++++--
src/subprocess-win32.cc | 2 +-
- src/subprocess.h | 8 +++++++-
- src/subprocess_test.cc | 47 +++++++++++++++++++++++++++++++++--------------
+ src/subprocess.h | 8 ++++++-
+ src/subprocess_test.cc | 47 ++++++++++++++++++++++++++++-------------
src/tokenpool-gnu-make.cc | 5 +++++
src/tokenpool.h | 6 ++++++
- 10 files changed, 108 insertions(+), 29 deletions(-)
+ 10 files changed, 134 insertions(+), 35 deletions(-)
diff --git a/src/build.cc b/src/build.cc
-index cc796ff838fa..ccd67697786d 100644
+index cc796ff838fa..219bb9f1ff48 100644
--- a/src/build.cc
+++ b/src/build.cc
-@@ -50,7 +50,7 @@ struct DryRunCommandRunner : public CommandRunner {
+@@ -49,8 +49,9 @@ struct DryRunCommandRunner : public CommandRunner {
+
// Overridden from CommandRunner:
virtual bool CanRunMore();
++ virtual bool AcquireToken();
virtual bool StartCommand(Edge* edge);
- virtual bool WaitForCommand(Result* result);
+ virtual bool WaitForCommand(Result* result, bool more_ready);
private:
queue<Edge*> finished_;
-@@ -65,7 +65,7 @@ bool DryRunCommandRunner::StartCommand(Edge* edge) {
+@@ -60,12 +61,16 @@ bool DryRunCommandRunner::CanRunMore() {
+ return true;
+ }
+
++bool DryRunCommandRunner::AcquireToken() {
++ return true;
++}
++
+ bool DryRunCommandRunner::StartCommand(Edge* edge) {
+ finished_.push(edge);
return true;
}
@@ -45,16 +57,34 @@ index cc796ff838fa..ccd67697786d 100644
if (finished_.empty())
return false;
-@@ -490,7 +490,7 @@ struct RealCommandRunner : public CommandRunner {
+@@ -489,8 +494,9 @@ struct RealCommandRunner : public CommandRunner {
+ explicit RealCommandRunner(const BuildConfig& config);
virtual ~RealCommandRunner();
virtual bool CanRunMore();
++ virtual bool AcquireToken();
virtual bool StartCommand(Edge* edge);
- virtual bool WaitForCommand(Result* result);
+ virtual bool WaitForCommand(Result* result, bool more_ready);
virtual vector<Edge*> GetActiveEdges();
virtual void Abort();
-@@ -544,14 +544,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) {
+@@ -527,9 +533,12 @@ bool RealCommandRunner::CanRunMore() {
+ subprocs_.running_.size() + subprocs_.finished_.size();
+ return (int)subproc_number < config_.parallelism
+ && (subprocs_.running_.empty() ||
+- ((config_.max_load_average <= 0.0f ||
+- GetLoadAverage() < config_.max_load_average)
+- && (!tokens_ || tokens_->Acquire())));
++ (config_.max_load_average <= 0.0f ||
++ GetLoadAverage() < config_.max_load_average));
++}
++
++bool RealCommandRunner::AcquireToken() {
++ return (!tokens_ || tokens_->Acquire());
+ }
+
+ bool RealCommandRunner::StartCommand(Edge* edge) {
+@@ -544,14 +553,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) {
return true;
}
@@ -81,16 +111,34 @@ index cc796ff838fa..ccd67697786d 100644
if (tokens_)
tokens_->Release();
-@@ -685,7 +694,7 @@ bool Builder::Build(string* err) {
+@@ -662,9 +680,14 @@ bool Builder::Build(string* err) {
+ // command runner.
+ // Second, we attempt to wait for / reap the next finished command.
+ while (plan_.more_to_do()) {
+- // See if we can start any more commands.
+- if (failures_allowed && plan_.more_ready() &&
+- command_runner_->CanRunMore()) {
++ // See if we can start any more commands...
++ bool can_run_more =
++ failures_allowed &&
++ plan_.more_ready() &&
++ command_runner_->CanRunMore();
++
++ // ... but we also need a token to do that.
++ if (can_run_more && command_runner_->AcquireToken()) {
+ Edge* edge = plan_.FindWork();
+ if (!StartEdge(edge, err)) {
+ Cleanup();
+@@ -685,7 +708,7 @@ bool Builder::Build(string* err) {
// See if we can reap any finished commands.
if (pending_commands) {
CommandRunner::Result result;
- if (!command_runner_->WaitForCommand(&result) ||
-+ if (!command_runner_->WaitForCommand(&result, plan_.more_ready()) ||
++ if (!command_runner_->WaitForCommand(&result, can_run_more) ||
result.status == ExitInterrupted) {
Cleanup();
status_->BuildFinished();
-@@ -693,6 +702,10 @@ bool Builder::Build(string* err) {
+@@ -693,6 +716,10 @@ bool Builder::Build(string* err) {
return false;
}
@@ -102,10 +150,18 @@ index cc796ff838fa..ccd67697786d 100644
if (!FinishCommand(&result, err)) {
Cleanup();
diff --git a/src/build.h b/src/build.h
-index cca7e8d8181d..ec6594f714c0 100644
+index cca7e8d8181d..ca605e62e0e3 100644
--- a/src/build.h
+++ b/src/build.h
-@@ -119,7 +119,7 @@ struct CommandRunner {
+@@ -108,6 +108,7 @@ private:
+ struct CommandRunner {
+ virtual ~CommandRunner() {}
+ virtual bool CanRunMore() = 0;
++ virtual bool AcquireToken() = 0;
+ virtual bool StartCommand(Edge* edge) = 0;
+
+ /// The result of waiting for a command.
+@@ -119,7 +120,7 @@ struct CommandRunner {
bool success() const { return status == ExitSuccess; }
};
/// Wait for a command to complete, or return false if interrupted.
@@ -115,19 +171,32 @@ index cca7e8d8181d..ec6594f714c0 100644
virtual vector<Edge*> GetActiveEdges() { return vector<Edge*>(); }
virtual void Abort() {}
diff --git a/src/build_test.cc b/src/build_test.cc
-index 46ab33ef86c8..c1d3b047271a 100644
+index 46ab33ef86c8..a1022edcf546 100644
--- a/src/build_test.cc
+++ b/src/build_test.cc
-@@ -446,7 +446,7 @@ struct FakeCommandRunner : public CommandRunner {
+@@ -445,8 +445,9 @@ struct FakeCommandRunner : public CommandRunner {
+
// CommandRunner impl
virtual bool CanRunMore();
++ virtual bool AcquireToken();
virtual bool StartCommand(Edge* edge);
- virtual bool WaitForCommand(Result* result);
+ virtual bool WaitForCommand(Result* result, bool more_ready);
virtual vector<Edge*> GetActiveEdges();
virtual void Abort();
-@@ -575,7 +575,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) {
+@@ -547,6 +548,10 @@ bool FakeCommandRunner::CanRunMore() {
+ return last_command_ == NULL;
+ }
+
++bool FakeCommandRunner::AcquireToken() {
++ return true;
++}
++
+ bool FakeCommandRunner::StartCommand(Edge* edge) {
+ assert(!last_command_);
+ commands_ran_.push_back(edge->EvaluateCommand());
+@@ -575,7 +580,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) {
return true;
}
diff --git a/patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch b/patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch
new file mode 100644
index 000000000..cd4b78f69
--- /dev/null
+++ b/patches/ninja-1.8.2/0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch
@@ -0,0 +1,192 @@
+From: Stefan Becker <chemobejk@gmail.com>
+Date: Sun, 12 Nov 2017 16:58:55 +0200
+Subject: [PATCH] Ignore jobserver when -jN is forced on command line
+
+This emulates the behaviour of GNU make.
+
+- add parallelism_from_cmdline flag to build configuration
+- set the flag when -jN is given on command line
+- pass the flag to TokenPool::Get()
+- GNUmakeTokenPool::Setup()
+ * prints a warning when the flag is true and jobserver was detected
+ * returns false, i.e. jobserver will be ignored
+- ignore config.parallelism in CanRunMore() when we have a valid
+ TokenPool, because it gets always initialized to a default when not
+ given on the command line
+---
+ src/build.cc | 10 ++++++----
+ src/build.h | 4 +++-
+ src/ninja.cc | 1 +
+ src/tokenpool-gnu-make.cc | 34 +++++++++++++++++++---------------
+ src/tokenpool-none.cc | 4 ++--
+ src/tokenpool.h | 4 ++--
+ 6 files changed, 33 insertions(+), 24 deletions(-)
+
+diff --git a/src/build.cc b/src/build.cc
+index 219bb9f1ff48..bc26bdade61b 100644
+--- a/src/build.cc
++++ b/src/build.cc
+@@ -507,7 +507,7 @@ struct RealCommandRunner : public CommandRunner {
+ };
+
+ RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
+- tokens_ = TokenPool::Get();
++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline);
+ }
+
+ RealCommandRunner::~RealCommandRunner() {
+@@ -529,9 +529,11 @@ void RealCommandRunner::Abort() {
+ }
+
+ bool RealCommandRunner::CanRunMore() {
+- size_t subproc_number =
+- subprocs_.running_.size() + subprocs_.finished_.size();
+- return (int)subproc_number < config_.parallelism
++ bool parallelism_limit_not_reached =
++ tokens_ || // ignore config_.parallelism
++ ((int) (subprocs_.running_.size() +
++ subprocs_.finished_.size()) < config_.parallelism);
++ return parallelism_limit_not_reached
+ && (subprocs_.running_.empty() ||
+ (config_.max_load_average <= 0.0f ||
+ GetLoadAverage() < config_.max_load_average));
+diff --git a/src/build.h b/src/build.h
+index ca605e62e0e3..6bc6fea26e94 100644
+--- a/src/build.h
++++ b/src/build.h
+@@ -128,7 +128,8 @@ struct CommandRunner {
+
+ /// Options (e.g. verbosity, parallelism) passed to a build.
+ struct BuildConfig {
+- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
++ BuildConfig() : verbosity(NORMAL), dry_run(false),
++ parallelism(1), parallelism_from_cmdline(false),
+ failures_allowed(1), max_load_average(-0.0f) {}
+
+ enum Verbosity {
+@@ -139,6 +140,7 @@ struct BuildConfig {
+ Verbosity verbosity;
+ bool dry_run;
+ int parallelism;
++ bool parallelism_from_cmdline;
+ int failures_allowed;
+ /// The maximum load average we must not exceed. A negative value
+ /// means that we do not have any limit.
+diff --git a/src/ninja.cc b/src/ninja.cc
+index ed004ac8f1fe..4332636c1b64 100644
+--- a/src/ninja.cc
++++ b/src/ninja.cc
+@@ -1063,6 +1063,7 @@ int ReadFlags(int* argc, char*** argv,
+ if (*end != 0 || value <= 0)
+ Fatal("invalid -j parameter");
+ config->parallelism = value;
++ config->parallelism_from_cmdline = true;
+ break;
+ }
+ case 'k': {
+diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
+index 396bb7d87443..af4be05a31cf 100644
+--- a/src/tokenpool-gnu-make.cc
++++ b/src/tokenpool-gnu-make.cc
+@@ -1,4 +1,4 @@
+-// Copyright 2016 Google Inc. All Rights Reserved.
++// Copyright 2016-2017 Google Inc. All Rights Reserved.
+ //
+ // Licensed under the Apache License, Version 2.0 (the "License");
+ // you may not use this file except in compliance with the License.
+@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool {
+ virtual void Clear();
+ virtual int GetMonitorFd();
+
+- bool Setup();
++ bool Setup(bool ignore);
+
+ private:
+ int available_;
+@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() {
+ }
+ }
+
+-bool GNUmakeTokenPool::Setup() {
++bool GNUmakeTokenPool::Setup(bool ignore) {
+ const char *value = getenv("MAKEFLAGS");
+ if (value) {
+ // GNU make <= 4.1
+@@ -109,16 +109,20 @@ bool GNUmakeTokenPool::Setup() {
+ if (!jobserver)
+ jobserver = strstr(value, "--jobserver-auth=");
+ if (jobserver) {
+- int rfd = -1;
+- int wfd = -1;
+- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
+- CheckFd(rfd) &&
+- CheckFd(wfd) &&
+- SetAlarmHandler()) {
+- printf("ninja: using GNU make jobserver.\n");
+- rfd_ = rfd;
+- wfd_ = wfd;
+- return true;
++ if (ignore) {
++ printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
++ } else {
++ int rfd = -1;
++ int wfd = -1;
++ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
++ CheckFd(rfd) &&
++ CheckFd(wfd) &&
++ SetAlarmHandler()) {
++ printf("ninja: using GNU make jobserver.\n");
++ rfd_ = rfd;
++ wfd_ = wfd;
++ return true;
++ }
+ }
+ }
+ }
+@@ -206,9 +210,9 @@ int GNUmakeTokenPool::GetMonitorFd() {
+ return(rfd_);
+ }
+
+-struct TokenPool *TokenPool::Get(void) {
++struct TokenPool *TokenPool::Get(bool ignore) {
+ GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
+- if (tokenpool->Setup())
++ if (tokenpool->Setup(ignore))
+ return tokenpool;
+ else
+ delete tokenpool;
+diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
+index 602b3316f54d..199b22264bc6 100644
+--- a/src/tokenpool-none.cc
++++ b/src/tokenpool-none.cc
+@@ -1,4 +1,4 @@
+-// Copyright 2016 Google Inc. All Rights Reserved.
++// Copyright 2016-2017 Google Inc. All Rights Reserved.
+ //
+ // Licensed under the Apache License, Version 2.0 (the "License");
+ // you may not use this file except in compliance with the License.
+@@ -22,6 +22,6 @@
+ #include <stdlib.h>
+
+ // No-op TokenPool implementation
+-struct TokenPool *TokenPool::Get(void) {
++struct TokenPool *TokenPool::Get(bool ignore) {
+ return NULL;
+ }
+diff --git a/src/tokenpool.h b/src/tokenpool.h
+index 301e1998ee8e..878a0933c2f0 100644
+--- a/src/tokenpool.h
++++ b/src/tokenpool.h
+@@ -1,4 +1,4 @@
+-// Copyright 2016 Google Inc. All Rights Reserved.
++// Copyright 2016-2017 Google Inc. All Rights Reserved.
+ //
+ // Licensed under the Apache License, Version 2.0 (the "License");
+ // you may not use this file except in compliance with the License.
+@@ -28,5 +28,5 @@ struct TokenPool {
+ #endif
+
+ // returns NULL if token pool is not available
+- static struct TokenPool *Get(void);
++ static struct TokenPool *Get(bool ignore);
+ };
diff --git a/patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch b/patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch
new file mode 100644
index 000000000..2a2505001
--- /dev/null
+++ b/patches/ninja-1.8.2/0004-Honor-lN-from-MAKEFLAGS.patch
@@ -0,0 +1,128 @@
+From: Stefan Becker <chemobejk@gmail.com>
+Date: Sun, 12 Nov 2017 18:04:12 +0200
+Subject: [PATCH] Honor -lN from MAKEFLAGS
+
+This emulates the behaviour of GNU make.
+
+- build: make a copy of max_load_average and pass it to TokenPool.
+- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in
+ MAKEFLAGS then set max_load_average to N.
+---
+ src/build.cc | 10 +++++++---
+ src/tokenpool-gnu-make.cc | 19 +++++++++++++++----
+ src/tokenpool-none.cc | 2 +-
+ src/tokenpool.h | 2 +-
+ 4 files changed, 24 insertions(+), 9 deletions(-)
+
+diff --git a/src/build.cc b/src/build.cc
+index bc26bdade61b..6eaf299caeec 100644
+--- a/src/build.cc
++++ b/src/build.cc
+@@ -501,13 +501,17 @@ struct RealCommandRunner : public CommandRunner {
+ virtual void Abort();
+
+ const BuildConfig& config_;
++ // copy of config_.max_load_average; can be modified by TokenPool setup
++ double max_load_average_;
+ SubprocessSet subprocs_;
+ TokenPool *tokens_;
+ map<Subprocess*, Edge*> subproc_to_edge_;
+ };
+
+ RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
+- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline);
++ max_load_average_ = config.max_load_average;
++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline,
++ max_load_average_);
+ }
+
+ RealCommandRunner::~RealCommandRunner() {
+@@ -535,8 +539,8 @@ bool RealCommandRunner::CanRunMore() {
+ subprocs_.finished_.size()) < config_.parallelism);
+ return parallelism_limit_not_reached
+ && (subprocs_.running_.empty() ||
+- (config_.max_load_average <= 0.0f ||
+- GetLoadAverage() < config_.max_load_average));
++ (max_load_average_ <= 0.0f ||
++ GetLoadAverage() < max_load_average_));
+ }
+
+ bool RealCommandRunner::AcquireToken() {
+diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
+index af4be05a31cf..fb654c4d88ba 100644
+--- a/src/tokenpool-gnu-make.cc
++++ b/src/tokenpool-gnu-make.cc
+@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool {
+ virtual void Clear();
+ virtual int GetMonitorFd();
+
+- bool Setup(bool ignore);
++ bool Setup(bool ignore, double& max_load_average);
+
+ private:
+ int available_;
+@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() {
+ }
+ }
+
+-bool GNUmakeTokenPool::Setup(bool ignore) {
++bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
+ const char *value = getenv("MAKEFLAGS");
+ if (value) {
+ // GNU make <= 4.1
+@@ -118,9 +118,20 @@ bool GNUmakeTokenPool::Setup(bool ignore) {
+ CheckFd(rfd) &&
+ CheckFd(wfd) &&
+ SetAlarmHandler()) {
++ const char *l_arg = strstr(value, " -l");
++ int load_limit = -1;
++
+ printf("ninja: using GNU make jobserver.\n");
+ rfd_ = rfd;
+ wfd_ = wfd;
++
++ // translate GNU make -lN to ninja -lN
++ if (l_arg &&
++ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) &&
++ (load_limit > 0)) {
++ max_load_average = load_limit;
++ }
++
+ return true;
+ }
+ }
+@@ -210,9 +221,9 @@ int GNUmakeTokenPool::GetMonitorFd() {
+ return(rfd_);
+ }
+
+-struct TokenPool *TokenPool::Get(bool ignore) {
++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
+ GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
+- if (tokenpool->Setup(ignore))
++ if (tokenpool->Setup(ignore, max_load_average))
+ return tokenpool;
+ else
+ delete tokenpool;
+diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
+index 199b22264bc6..e8e25426c39f 100644
+--- a/src/tokenpool-none.cc
++++ b/src/tokenpool-none.cc
+@@ -22,6 +22,6 @@
+ #include <stdlib.h>
+
+ // No-op TokenPool implementation
+-struct TokenPool *TokenPool::Get(bool ignore) {
++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
+ return NULL;
+ }
+diff --git a/src/tokenpool.h b/src/tokenpool.h
+index 878a0933c2f0..f9e8cc2ee081 100644
+--- a/src/tokenpool.h
++++ b/src/tokenpool.h
+@@ -28,5 +28,5 @@ struct TokenPool {
+ #endif
+
+ // returns NULL if token pool is not available
+- static struct TokenPool *Get(bool ignore);
++ static struct TokenPool *Get(bool ignore, double& max_load_average);
+ };
diff --git a/patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch b/patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch
new file mode 100644
index 000000000..1894be5a1
--- /dev/null
+++ b/patches/ninja-1.8.2/0005-Use-LinePrinter-for-TokenPool-messages.patch
@@ -0,0 +1,122 @@
+From: Stefan Becker <chemobejk@gmail.com>
+Date: Wed, 6 Dec 2017 22:14:21 +0200
+Subject: [PATCH] Use LinePrinter for TokenPool messages
+
+- replace printf() with calls to LinePrinter
+- print GNU make jobserver message only when verbose build is requested
+---
+ src/build.cc | 1 +
+ src/tokenpool-gnu-make.cc | 22 ++++++++++++++++------
+ src/tokenpool-none.cc | 4 +++-
+ src/tokenpool.h | 4 +++-
+ 4 files changed, 23 insertions(+), 8 deletions(-)
+
+diff --git a/src/build.cc b/src/build.cc
+index 6eaf299caeec..754d362c7162 100644
+--- a/src/build.cc
++++ b/src/build.cc
+@@ -511,6 +511,7 @@ struct RealCommandRunner : public CommandRunner {
+ RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
+ max_load_average_ = config.max_load_average;
+ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline,
++ config_.verbosity == BuildConfig::VERBOSE,
+ max_load_average_);
+ }
+
+diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
+index fb654c4d88ba..b0d3e6ebc463 100644
+--- a/src/tokenpool-gnu-make.cc
++++ b/src/tokenpool-gnu-make.cc
+@@ -23,6 +23,8 @@
+ #include <string.h>
+ #include <stdlib.h>
+
++#include "line_printer.h"
++
+ // TokenPool implementation for GNU make jobserver
+ // (http://make.mad-scientist.net/papers/jobserver-implementation/)
+ struct GNUmakeTokenPool : public TokenPool {
+@@ -35,7 +37,7 @@ struct GNUmakeTokenPool : public TokenPool {
+ virtual void Clear();
+ virtual int GetMonitorFd();
+
+- bool Setup(bool ignore, double& max_load_average);
++ bool Setup(bool ignore, bool verbose, double& max_load_average);
+
+ private:
+ int available_;
+@@ -100,7 +102,9 @@ bool GNUmakeTokenPool::SetAlarmHandler() {
+ }
+ }
+
+-bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
++bool GNUmakeTokenPool::Setup(bool ignore,
++ bool verbose,
++ double& max_load_average) {
+ const char *value = getenv("MAKEFLAGS");
+ if (value) {
+ // GNU make <= 4.1
+@@ -109,8 +113,10 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
+ if (!jobserver)
+ jobserver = strstr(value, "--jobserver-auth=");
+ if (jobserver) {
++ LinePrinter printer;
++
+ if (ignore) {
+- printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
++ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
+ } else {
+ int rfd = -1;
+ int wfd = -1;
+@@ -121,7 +127,9 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
+ const char *l_arg = strstr(value, " -l");
+ int load_limit = -1;
+
+- printf("ninja: using GNU make jobserver.\n");
++ if (verbose) {
++ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n");
++ }
+ rfd_ = rfd;
+ wfd_ = wfd;
+
+@@ -221,9 +229,11 @@ int GNUmakeTokenPool::GetMonitorFd() {
+ return(rfd_);
+ }
+
+-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
++struct TokenPool *TokenPool::Get(bool ignore,
++ bool verbose,
++ double& max_load_average) {
+ GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
+- if (tokenpool->Setup(ignore, max_load_average))
++ if (tokenpool->Setup(ignore, verbose, max_load_average))
+ return tokenpool;
+ else
+ delete tokenpool;
+diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
+index e8e25426c39f..1c1c499c8d9c 100644
+--- a/src/tokenpool-none.cc
++++ b/src/tokenpool-none.cc
+@@ -22,6 +22,8 @@
+ #include <stdlib.h>
+
+ // No-op TokenPool implementation
+-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
++struct TokenPool *TokenPool::Get(bool ignore,
++ bool verbose,
++ double& max_load_average) {
+ return NULL;
+ }
+diff --git a/src/tokenpool.h b/src/tokenpool.h
+index f9e8cc2ee081..4bf477f20c8a 100644
+--- a/src/tokenpool.h
++++ b/src/tokenpool.h
+@@ -28,5 +28,7 @@ struct TokenPool {
+ #endif
+
+ // returns NULL if token pool is not available
+- static struct TokenPool *Get(bool ignore, double& max_load_average);
++ static struct TokenPool *Get(bool ignore,
++ bool verbose,
++ double& max_load_average);
+ };
diff --git a/patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch b/patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch
new file mode 100644
index 000000000..bf44d5f4b
--- /dev/null
+++ b/patches/ninja-1.8.2/0006-Prepare-PR-for-merging.patch
@@ -0,0 +1,151 @@
+From: Stefan Becker <chemobejk@gmail.com>
+Date: Sat, 7 Apr 2018 17:11:21 +0300
+Subject: [PATCH] Prepare PR for merging
+
+- fix Windows build error in no-op TokenPool implementation
+- improve Acquire() to block for a maximum of 100ms
+- address review comments
+---
+ src/build.h | 2 ++
+ src/tokenpool-gnu-make.cc | 53 ++++++++++++++++++++++++++++++++++++++++-------
+ src/tokenpool-none.cc | 7 +------
+ 3 files changed, 49 insertions(+), 13 deletions(-)
+
+diff --git a/src/build.h b/src/build.h
+index 6bc6fea26e94..530812bb9a2a 100644
+--- a/src/build.h
++++ b/src/build.h
+@@ -120,6 +120,8 @@ struct CommandRunner {
+ bool success() const { return status == ExitSuccess; }
+ };
+ /// Wait for a command to complete, or return false if interrupted.
++ /// If more_ready is true then the optional TokenPool is monitored too
++ /// and we return when a token becomes available.
+ virtual bool WaitForCommand(Result* result, bool more_ready) = 0;
+
+ virtual vector<Edge*> GetActiveEdges() { return vector<Edge*>(); }
+diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
+index b0d3e6ebc463..4132bb06d9dd 100644
+--- a/src/tokenpool-gnu-make.cc
++++ b/src/tokenpool-gnu-make.cc
+@@ -1,4 +1,4 @@
+-// Copyright 2016-2017 Google Inc. All Rights Reserved.
++// Copyright 2016-2018 Google Inc. All Rights Reserved.
+ //
+ // Licensed under the Apache License, Version 2.0 (the "License");
+ // you may not use this file except in compliance with the License.
+@@ -19,6 +19,7 @@
+ #include <poll.h>
+ #include <unistd.h>
+ #include <signal.h>
++#include <sys/time.h>
+ #include <stdio.h>
+ #include <string.h>
+ #include <stdlib.h>
+@@ -153,6 +154,15 @@ bool GNUmakeTokenPool::Acquire() {
+ if (available_ > 0)
+ return true;
+
++ // Please read
++ //
++ // http://make.mad-scientist.net/papers/jobserver-implementation/
++ //
++ // for the reasoning behind the following code.
++ //
++ // Try to read one character from the pipe. Returns true on success.
++ //
++ // First check if read() would succeed without blocking.
+ #ifdef USE_PPOLL
+ pollfd pollfds[] = {{rfd_, POLLIN, 0}};
+ int ret = poll(pollfds, 1, 0);
+@@ -164,33 +174,62 @@ bool GNUmakeTokenPool::Acquire() {
+ int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
+ #endif
+ if (ret > 0) {
++ // Handle potential race condition:
++ // - the above check succeeded, i.e. read() should not block
++ // - the character disappears before we call read()
++ //
++ // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_
++ // can safely be closed by signal handlers without affecting rfd_.
+ dup_rfd_ = dup(rfd_);
+
+ if (dup_rfd_ != -1) {
+ struct sigaction act, old_act;
+ int ret = 0;
+
++ // Temporarily replace SIGCHLD handler with our own
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = CloseDupRfd;
+ if (sigaction(SIGCHLD, &act, &old_act) == 0) {
+- char buf;
+-
+- // block until token read, child exits or timeout
+- alarm(1);
+- ret = read(dup_rfd_, &buf, 1);
+- alarm(0);
++ struct itimerval timeout;
++
++ // install a 100ms timeout that generates SIGALARM on expiration
++ memset(&timeout, 0, sizeof(timeout));
++ timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec]
++ if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) {
++ char buf;
++
++ // Now try to read() from dup_rfd_. Return values from read():
++ //
++ // 1. token read -> 1
++ // 2. pipe closed -> 0
++ // 3. alarm expires -> -1 (EINTR)
++ // 4. child exits -> -1 (EINTR)
++ // 5. alarm expired before entering read() -> -1 (EBADF)
++ // 6. child exited before entering read() -> -1 (EBADF)
++ // 7. child exited before handler is installed -> go to 1 - 3
++ ret = read(dup_rfd_, &buf, 1);
++
++ // disarm timer
++ memset(&timeout, 0, sizeof(timeout));
++ setitimer(ITIMER_REAL, &timeout, NULL);
++ }
+
+ sigaction(SIGCHLD, &old_act, NULL);
+ }
+
+ CloseDupRfd(0);
+
++ // Case 1 from above list
+ if (ret > 0) {
+ available_++;
+ return true;
+ }
+ }
+ }
++
++ // read() would block, i.e. no token available,
++ // cases 2-6 from above list or
++ // select() / poll() / dup() / sigaction() / setitimer() failed
+ return false;
+ }
+
+diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
+index 1c1c499c8d9c..4c592875b4ad 100644
+--- a/src/tokenpool-none.cc
++++ b/src/tokenpool-none.cc
+@@ -1,4 +1,4 @@
+-// Copyright 2016-2017 Google Inc. All Rights Reserved.
++// Copyright 2016-2018 Google Inc. All Rights Reserved.
+ //
+ // Licensed under the Apache License, Version 2.0 (the "License");
+ // you may not use this file except in compliance with the License.
+@@ -14,11 +14,6 @@
+
+ #include "tokenpool.h"
+
+-#include <fcntl.h>
+-#include <poll.h>
+-#include <unistd.h>
+-#include <stdio.h>
+-#include <string.h>
+ #include <stdlib.h>
+
+ // No-op TokenPool implementation
diff --git a/patches/ninja-1.8.2/series b/patches/ninja-1.8.2/series
index aec7db323..4494b5b79 100644
--- a/patches/ninja-1.8.2/series
+++ b/patches/ninja-1.8.2/series
@@ -2,4 +2,8 @@
#tag:base --start-number 1
0001-Add-GNU-make-jobserver-client-support.patch
0002-Add-TokenPool-monitoring-to-SubprocessSet-DoWork.patch
-# 0034c2e369003b7ed18a6396de19c997 - git-ptx-patches magic
+0003-Ignore-jobserver-when-jN-is-forced-on-command-line.patch
+0004-Honor-lN-from-MAKEFLAGS.patch
+0005-Use-LinePrinter-for-TokenPool-messages.patch
+0006-Prepare-PR-for-merging.patch
+# d3f8b2e9621f8f2490c54704a2621fb2 - git-ptx-patches magic
diff --git a/scripts/lib/ptxd_make_world_common.sh b/scripts/lib/ptxd_make_world_common.sh
index 90cac76c0..c8fc03282 100644
--- a/scripts/lib/ptxd_make_world_common.sh
+++ b/scripts/lib/ptxd_make_world_common.sh
@@ -333,9 +333,9 @@ ptxd_make_world_init() {
pkg_make_opt="-v ${pkg_make_opt}"
pkg_install_opt="-v ${pkg_install_opt}"
fi
- # both jobserver and argument limit parallelism so both are needed
+ # pass jobserver via MAKEFLAGS to ninja
pkg_env="${pkg_env} MAKEFLAGS='${PTXDIST_JOBSERVER_FLAGS}'"
- PTXDIST_PARALLELMFLAGS_INTERN="${PTXDIST_PARALLEL_FLAGS}"
+ PTXDIST_PARALLELMFLAGS_INTERN=""
unset conf_opt_ptr
;;