[openwrt/openwrt] tools/ninja: update to 1.11.1

LEDE Commits lede-commits at lists.infradead.org
Tue Jan 24 14:15:49 PST 2023


hauke pushed a commit to openwrt/openwrt.git, branch master:
https://git.openwrt.org/58a8ad38016cd8a27ab3c95ab38cff7b417c0cff

commit 58a8ad38016cd8a27ab3c95ab38cff7b417c0cff
Author: Nick Hainke <vincent at systemli.org>
AuthorDate: Sat Oct 29 10:22:36 2022 +0200

    tools/ninja: update to 1.11.1
    
    Release Notes:
    https://github.com/ninja-build/ninja/releases/tag/v1.11.1
    
    Sync the "100-make_jobserver_support.patch" with the PR:
    https://github.com/ninja-build/ninja/pull/1140
    
    Signed-off-by: Nick Hainke <vincent at systemli.org>
---
 tools/ninja/Makefile                               |    4 +-
 .../ninja/patches/100-make_jobserver_support.patch | 5364 ++++++--------------
 2 files changed, 1625 insertions(+), 3743 deletions(-)

diff --git a/tools/ninja/Makefile b/tools/ninja/Makefile
index c5c83d9b14..4763e759d8 100644
--- a/tools/ninja/Makefile
+++ b/tools/ninja/Makefile
@@ -1,12 +1,12 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ninja
-PKG_VERSION:=1.11.0
+PKG_VERSION:=1.11.1
 PKG_RELEASE:=1
 
 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
 PKG_SOURCE_URL:=https://codeload.github.com/ninja-build/ninja/tar.gz/v$(PKG_VERSION)?
-PKG_HASH:=3c6ba2e66400fe3f1ae83deb4b235faf3137ec20bd5b08c29bfc368db143e4c6
+PKG_HASH:=31747ae633213f1eda3842686f83c2aa1412e0f5691d1c14dbbcc67fe7400cea
 
 include $(INCLUDE_DIR)/host-build.mk
 
diff --git a/tools/ninja/patches/100-make_jobserver_support.patch b/tools/ninja/patches/100-make_jobserver_support.patch
index 7dac8ef814..34d2b6c431 100644
--- a/tools/ninja/patches/100-make_jobserver_support.patch
+++ b/tools/ninja/patches/100-make_jobserver_support.patch
@@ -1,4 +1,4 @@
-From 17d13fd7881fd3ce9f9b9d44ce435d6caf4b8f28 Mon Sep 17 00:00:00 2001
+From afec30f5caf4b051827ffdd822ebd27c58219fee Mon Sep 17 00:00:00 2001
 From: Stefan Becker <stefanb at gpartner-nvidia.com>
 Date: Tue, 22 Mar 2016 13:48:07 +0200
 Subject: [PATCH 01/11] Add GNU make jobserver client support
@@ -31,28 +31,41 @@ Fixes https://github.com/ninja-build/ninja/issues/1139
  create mode 100644 src/tokenpool-none.cc
  create mode 100644 src/tokenpool.h
 
-diff --git a/configure.py b/configure.py
-index 43904349a8..db3492c93c 100755
 --- a/configure.py
 +++ b/configure.py
-@@ -522,6 +522,7 @@ def has_re2c():
+@@ -517,11 +517,13 @@ for name in ['build',
+              'state',
+              'status',
+              'string_piece_util',
++             'tokenpool-gnu-make',
+              'util',
+              'version']:
      objs += cxx(name, variables=cxxvariables)
  if platform.is_windows():
      for name in ['subprocess-win32',
-+                 'tokenpool-none',
++                 'tokenpool-gnu-make-win32',
                   'includes_normalize-win32',
                   'msvc_helper-win32',
                   'msvc_helper_main-win32']:
-@@ -531,6 +532,7 @@ def has_re2c():
+@@ -530,7 +532,9 @@ if platform.is_windows():
+         objs += cxx('minidump-win32', variables=cxxvariables)
      objs += cc('getopt')
  else:
-     objs += cxx('subprocess-posix')
-+    objs += cxx('tokenpool-gnu-make')
+-    objs += cxx('subprocess-posix')
++    for name in ['subprocess-posix',
++                 'tokenpool-gnu-make-posix']:
++        objs += cxx(name)
  if platform.is_aix():
      objs += cc('getopt')
  if platform.is_msvc():
-diff --git a/src/build.cc b/src/build.cc
-index 6f11ed7a3c..fa096eac33 100644
+@@ -588,6 +592,7 @@ for name in ['build_log_test',
+              'string_piece_util_test',
+              'subprocess_test',
+              'test',
++             'tokenpool_test',
+              'util_test']:
+     objs += cxx(name, variables=cxxvariables)
+ if platform.is_windows():
 --- a/src/build.cc
 +++ b/src/build.cc
 @@ -35,6 +35,7 @@
@@ -63,7 +76,36 @@ index 6f11ed7a3c..fa096eac33 100644
  #include "util.h"
  
  using namespace std;
-@@ -149,7 +150,7 @@ void Plan::EdgeWanted(const Edge* edge) {
+@@ -47,8 +48,9 @@ struct DryRunCommandRunner : public Comm
+ 
+   // Overridden from CommandRunner:
+   virtual bool CanRunMore() const;
++  virtual bool AcquireToken();
+   virtual bool StartCommand(Edge* edge);
+-  virtual bool WaitForCommand(Result* result);
++  virtual bool WaitForCommand(Result* result, bool more_ready);
+ 
+  private:
+   queue<Edge*> finished_;
+@@ -58,12 +60,16 @@ bool DryRunCommandRunner::CanRunMore() c
+   return true;
+ }
+ 
++bool DryRunCommandRunner::AcquireToken() {
++  return true;
++}
++
+ bool DryRunCommandRunner::StartCommand(Edge* edge) {
+   finished_.push(edge);
+   return true;
+ }
+ 
+-bool DryRunCommandRunner::WaitForCommand(Result* result) {
++bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) {
+    if (finished_.empty())
+      return false;
+ 
+@@ -149,7 +155,7 @@ void Plan::EdgeWanted(const Edge* edge)
  }
  
  Edge* Plan::FindWork() {
@@ -72,7 +114,7 @@ index 6f11ed7a3c..fa096eac33 100644
      return NULL;
    EdgeSet::iterator e = ready_.begin();
    Edge* edge = *e;
-@@ -448,8 +449,8 @@ void Plan::Dump() const {
+@@ -448,19 +454,39 @@ void Plan::Dump() const {
  }
  
  struct RealCommandRunner : public CommandRunner {
@@ -81,18 +123,31 @@ index 6f11ed7a3c..fa096eac33 100644
 +  explicit RealCommandRunner(const BuildConfig& config);
 +  virtual ~RealCommandRunner();
    virtual bool CanRunMore() const;
++  virtual bool AcquireToken();
    virtual bool StartCommand(Edge* edge);
-   virtual bool WaitForCommand(Result* result);
-@@ -458,9 +459,18 @@ struct RealCommandRunner : public CommandRunner {
+-  virtual bool WaitForCommand(Result* result);
++  virtual bool WaitForCommand(Result* result, bool more_ready);
+   virtual vector<Edge*> GetActiveEdges();
+   virtual void Abort();
  
    const BuildConfig& config_;
++  // copy of config_.max_load_average; can be modified by TokenPool setup
++  double max_load_average_;
    SubprocessSet subprocs_;
-+  TokenPool *tokens_;
++  TokenPool* tokens_;
    map<const Subprocess*, Edge*> subproc_to_edge_;
  };
  
 +RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
-+  tokens_ = TokenPool::Get();
++  max_load_average_ = config.max_load_average;
++  if ((tokens_ = TokenPool::Get()) != NULL) {
++    if (!tokens_->Setup(config_.parallelism_from_cmdline,
++                        config_.verbosity == BuildConfig::VERBOSE,
++                        max_load_average_)) {
++      delete tokens_;
++      tokens_ = NULL;
++    }
++  }
 +}
 +
 +RealCommandRunner::~RealCommandRunner() {
@@ -102,7 +157,7 @@ index 6f11ed7a3c..fa096eac33 100644
  vector<Edge*> RealCommandRunner::GetActiveEdges() {
    vector<Edge*> edges;
    for (map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin();
-@@ -471,14 +481,18 @@ vector<Edge*> RealCommandRunner::GetActiveEdges() {
+@@ -471,14 +497,23 @@ vector<Edge*> RealCommandRunner::GetActi
  
  void RealCommandRunner::Abort() {
    subprocs_.Clear();
@@ -111,19 +166,27 @@ index 6f11ed7a3c..fa096eac33 100644
  }
  
  bool RealCommandRunner::CanRunMore() const {
-   size_t subproc_number =
-       subprocs_.running_.size() + subprocs_.finished_.size();
-   return (int)subproc_number < config_.parallelism
+-  size_t subproc_number =
+-      subprocs_.running_.size() + subprocs_.finished_.size();
+-  return (int)subproc_number < config_.parallelism
 -    && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f)
 -        || GetLoadAverage() < config_.max_load_average);
++  bool parallelism_limit_not_reached =
++    tokens_ || // ignore config_.parallelism
++    ((int) (subprocs_.running_.size() +
++            subprocs_.finished_.size()) < config_.parallelism);
++  return parallelism_limit_not_reached
 +    && (subprocs_.running_.empty() ||
-+        ((config_.max_load_average <= 0.0f ||
-+          GetLoadAverage() < config_.max_load_average)
-+      && (!tokens_ || tokens_->Acquire())));
++        (max_load_average_ <= 0.0f ||
++         GetLoadAverage() < max_load_average_));
++}
++
++bool RealCommandRunner::AcquireToken() {
++  return (!tokens_ || tokens_->Acquire());
  }
  
  bool RealCommandRunner::StartCommand(Edge* edge) {
-@@ -486,6 +500,8 @@ bool RealCommandRunner::StartCommand(Edge* edge) {
+@@ -486,19 +521,33 @@ bool RealCommandRunner::StartCommand(Edg
    Subprocess* subproc = subprocs_.Add(command, edge->use_console());
    if (!subproc)
      return false;
@@ -132,25 +195,50 @@ index 6f11ed7a3c..fa096eac33 100644
    subproc_to_edge_.insert(make_pair(subproc, edge));
  
    return true;
-@@ -499,6 +515,9 @@ bool RealCommandRunner::WaitForCommand(Result* result) {
+ }
+ 
+-bool RealCommandRunner::WaitForCommand(Result* result) {
++bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) {
+   Subprocess* subproc;
+-  while ((subproc = subprocs_.NextFinished()) == NULL) {
+-    bool interrupted = subprocs_.DoWork();
++  subprocs_.ResetTokenAvailable();
++  while (((subproc = subprocs_.NextFinished()) == NULL) &&
++         !subprocs_.IsTokenAvailable()) {
++    bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL);
+     if (interrupted)
        return false;
    }
  
++  // token became available
++  if (subproc == NULL) {
++    result->status = ExitTokenAvailable;
++    return true;
++  }
++
++  // command completed
 +  if (tokens_)
 +    tokens_->Release();
 +
    result->status = subproc->Finish();
    result->output = subproc->GetOutput();
  
-@@ -621,31 +640,31 @@ bool Builder::Build(string* err) {
+@@ -620,38 +669,43 @@ bool Builder::Build(string* err) {
+   // command runner.
    // Second, we attempt to wait for / reap the next finished command.
    while (plan_.more_to_do()) {
-     // See if we can start any more commands.
+-    // See if we can start any more commands.
 -    if (failures_allowed && command_runner_->CanRunMore()) {
 -      if (Edge* edge = plan_.FindWork()) {
 -        if (edge->GetBindingBool("generator")) {
-+    if (failures_allowed && plan_.more_ready() &&
-+        command_runner_->CanRunMore()) {
++    // See if we can start any more commands...
++    bool can_run_more =
++        failures_allowed   &&
++        plan_.more_ready() &&
++        command_runner_->CanRunMore();
++
++    // ... but we also need a token to do that.
++    if (can_run_more && command_runner_->AcquireToken()) {
 +      Edge* edge = plan_.FindWork();
 +      if (edge->GetBindingBool("generator")) {
            scan_.build_log()->Close();
@@ -191,8 +279,24 @@ index 6f11ed7a3c..fa096eac33 100644
      }
  
      // See if we can reap any finished commands.
-diff --git a/src/build.h b/src/build.h
-index d697dfb89e..7dcd111e61 100644
+     if (pending_commands) {
+       CommandRunner::Result result;
+-      if (!command_runner_->WaitForCommand(&result) ||
++      if (!command_runner_->WaitForCommand(&result, can_run_more) ||
+           result.status == ExitInterrupted) {
+         Cleanup();
+         status_->BuildFinished();
+@@ -659,6 +713,10 @@ bool Builder::Build(string* err) {
+         return false;
+       }
+ 
++      // We might be able to start another command; start the main loop over.
++      if (result.status == ExitTokenAvailable)
++        continue;
++
+       --pending_commands;
+       if (!FinishCommand(&result, err)) {
+         Cleanup();
 --- a/src/build.h
 +++ b/src/build.h
 @@ -52,6 +52,9 @@ struct Plan {
@@ -205,13 +309,47 @@ index d697dfb89e..7dcd111e61 100644
    /// Dumps the current state of the plan.
    void Dump() const;
  
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-new file mode 100644
-index 0000000000..a8f9b7139d
+@@ -136,6 +139,7 @@ private:
+ struct CommandRunner {
+   virtual ~CommandRunner() {}
+   virtual bool CanRunMore() const = 0;
++  virtual bool AcquireToken() = 0;
+   virtual bool StartCommand(Edge* edge) = 0;
+ 
+   /// The result of waiting for a command.
+@@ -147,7 +151,9 @@ struct CommandRunner {
+     bool success() const { return status == ExitSuccess; }
+   };
+   /// Wait for a command to complete, or return false if interrupted.
+-  virtual bool WaitForCommand(Result* result) = 0;
++  /// If more_ready is true then the optional TokenPool is monitored too
++  /// and we return when a token becomes available.
++  virtual bool WaitForCommand(Result* result, bool more_ready) = 0;
+ 
+   virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
+   virtual void Abort() {}
+@@ -155,7 +161,8 @@ struct CommandRunner {
+ 
+ /// Options (e.g. verbosity, parallelism) passed to a build.
+ struct BuildConfig {
+-  BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
++  BuildConfig() : verbosity(NORMAL), dry_run(false),
++                  parallelism(1), parallelism_from_cmdline(false),
+                   failures_allowed(1), max_load_average(-0.0f) {}
+ 
+   enum Verbosity {
+@@ -167,6 +174,7 @@ struct BuildConfig {
+   Verbosity verbosity;
+   bool dry_run;
+   int parallelism;
++  bool parallelism_from_cmdline;
+   int failures_allowed;
+   /// The maximum load average we must not exceed. A negative value
+   /// means that we do not have any limit.
 --- /dev/null
 +++ b/src/tokenpool-gnu-make.cc
-@@ -0,0 +1,211 @@
-+// Copyright 2016 Google Inc. All Rights Reserved.
+@@ -0,0 +1,108 @@
++// Copyright 2016-2018 Google Inc. All Rights Reserved.
 +//
 +// Licensed under the Apache License, Version 2.0 (the "License");
 +// you may not use this file except in compliance with the License.
@@ -225,111 +363,55 @@ index 0000000000..a8f9b7139d
 +// See the License for the specific language governing permissions and
 +// limitations under the License.
 +
-+#include "tokenpool.h"
++#include "tokenpool-gnu-make.h"
 +
-+#include <errno.h>
-+#include <fcntl.h>
-+#include <poll.h>
-+#include <unistd.h>
-+#include <signal.h>
++#include <stdlib.h>
 +#include <stdio.h>
 +#include <string.h>
-+#include <stdlib.h>
-+
-+// TokenPool implementation for GNU make jobserver
-+// (http://make.mad-scientist.net/papers/jobserver-implementation/)
-+struct GNUmakeTokenPool : public TokenPool {
-+  GNUmakeTokenPool();
-+  virtual ~GNUmakeTokenPool();
-+
-+  virtual bool Acquire();
-+  virtual void Reserve();
-+  virtual void Release();
-+  virtual void Clear();
-+
-+  bool Setup();
-+
-+ private:
-+  int available_;
-+  int used_;
-+
-+#ifdef _WIN32
-+  // @TODO
-+#else
-+  int rfd_;
-+  int wfd_;
-+
-+  struct sigaction old_act_;
-+  bool restore_;
-+
-+  static int dup_rfd_;
-+  static void CloseDupRfd(int signum);
-+
-+  bool CheckFd(int fd);
-+  bool SetAlarmHandler();
-+#endif
 +
-+  void Return();
-+};
++#include "line_printer.h"
 +
++// TokenPool implementation for GNU make jobserver - common bits
 +// every instance owns an implicit token -> available_ == 1
-+GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0),
-+                                       rfd_(-1), wfd_(-1), restore_(false) {
++GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) {
 +}
 +
 +GNUmakeTokenPool::~GNUmakeTokenPool() {
-+  Clear();
-+  if (restore_)
-+    sigaction(SIGALRM, &old_act_, NULL);
 +}
 +
-+bool GNUmakeTokenPool::CheckFd(int fd) {
-+  if (fd < 0)
-+    return false;
-+  int ret = fcntl(fd, F_GETFD);
-+  if (ret < 0)
++bool GNUmakeTokenPool::Setup(bool ignore,
++                             bool verbose,
++                             double& max_load_average) {
++  const char* value = GetEnv("MAKEFLAGS");
++  if (!value)
 +    return false;
-+  return true;
-+}
 +
-+int GNUmakeTokenPool::dup_rfd_ = -1;
++  // GNU make <= 4.1
++  const char* jobserver = strstr(value, "--jobserver-fds=");
++  if (!jobserver)
++    // GNU make => 4.2
++    jobserver = strstr(value, "--jobserver-auth=");
++  if (jobserver) {
++    LinePrinter printer;
 +
-+void GNUmakeTokenPool::CloseDupRfd(int signum) {
-+  close(dup_rfd_);
-+  dup_rfd_ = -1;
-+}
++    if (ignore) {
++      printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
++    } else {
++      if (ParseAuth(jobserver)) {
++        const char* l_arg = strstr(value, " -l");
++        int load_limit = -1;
 +
-+bool GNUmakeTokenPool::SetAlarmHandler() {
-+  struct sigaction act;
-+  memset(&act, 0, sizeof(act));
-+  act.sa_handler = CloseDupRfd;
-+  if (sigaction(SIGALRM, &act, &old_act_) < 0) {
-+    perror("sigaction:");
-+    return(false);
-+  } else {
-+    restore_ = true;
-+    return(true);
-+  }
-+}
++        if (verbose) {
++          printer.PrintOnNewLine("ninja: using GNU make jobserver.\n");
++        }
++
++        // translate GNU make -lN to ninja -lN
++        if (l_arg &&
++            (sscanf(l_arg + 3, "%d ", &load_limit) == 1) &&
++            (load_limit > 0)) {
++          max_load_average = load_limit;
++        }
 +
-+bool GNUmakeTokenPool::Setup() {
-+  const char *value = getenv("MAKEFLAGS");
-+  if (value) {
-+    // GNU make <= 4.1
-+    const char *jobserver = strstr(value, "--jobserver-fds=");
-+    // GNU make => 4.2
-+    if (!jobserver)
-+      jobserver = strstr(value, "--jobserver-auth=");
-+    if (jobserver) {
-+      int rfd = -1;
-+      int wfd = -1;
-+      if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
-+          CheckFd(rfd) &&
-+          CheckFd(wfd) &&
-+          SetAlarmHandler()) {
-+        printf("ninja: using GNU make jobserver.\n");
-+        rfd_ = rfd;
-+        wfd_ = wfd;
 +        return true;
 +      }
 +    }
@@ -342,44 +424,13 @@ index 0000000000..a8f9b7139d
 +  if (available_ > 0)
 +    return true;
 +
-+#ifdef USE_PPOLL
-+  pollfd pollfds[] = {{rfd_, POLLIN, 0}};
-+  int ret = poll(pollfds, 1, 0);
-+#else
-+  fd_set set;
-+  struct timeval timeout = { 0, 0 };
-+  FD_ZERO(&set);
-+  FD_SET(rfd_, &set);
-+  int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
-+#endif
-+  if (ret > 0) {
-+    dup_rfd_ = dup(rfd_);
-+
-+    if (dup_rfd_ != -1) {
-+      struct sigaction act, old_act;
-+      int ret = 0;
-+
-+      memset(&act, 0, sizeof(act));
-+      act.sa_handler = CloseDupRfd;
-+      if (sigaction(SIGCHLD, &act, &old_act) == 0) {
-+        char buf;
-+
-+        // block until token read, child exits or timeout
-+        alarm(1);
-+        ret = read(dup_rfd_, &buf, 1);
-+        alarm(0);
-+
-+        sigaction(SIGCHLD, &old_act, NULL);
-+      }
-+
-+      CloseDupRfd(0);
-+
-+      if (ret > 0) {
-+        available_++;
-+        return true;
-+      }
-+    }
++  if (AcquireToken()) {
++    // token acquired
++    available_++;
++    return true;
 +  }
++
++  // no token available
 +  return false;
 +}
 +
@@ -389,15 +440,8 @@ index 0000000000..a8f9b7139d
 +}
 +
 +void GNUmakeTokenPool::Return() {
-+  const char buf = '+';
-+  while (1) {
-+    int ret = write(wfd_, &buf, 1);
-+    if (ret > 0)
-+      available_--;
-+    if ((ret != -1) || (errno != EINTR))
-+      return;
-+    // write got interrupted - retry
-+  }
++  if (ReturnToken())
++    available_--;
 +}
 +
 +void GNUmakeTokenPool::Release() {
@@ -413,55 +457,10 @@ index 0000000000..a8f9b7139d
 +  while (available_ > 1)
 +    Return();
 +}
-+
-+struct TokenPool *TokenPool::Get(void) {
-+  GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
-+  if (tokenpool->Setup())
-+    return tokenpool;
-+  else
-+    delete tokenpool;
-+  return NULL;
-+}
-diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
-new file mode 100644
-index 0000000000..602b3316f5
---- /dev/null
-+++ b/src/tokenpool-none.cc
-@@ -0,0 +1,27 @@
-+// Copyright 2016 Google Inc. All Rights Reserved.
-+//
-+// Licensed under the Apache License, Version 2.0 (the "License");
-+// you may not use this file except in compliance with the License.
-+// You may obtain a copy of the License at
-+//
-+//     http://www.apache.org/licenses/LICENSE-2.0
-+//
-+// Unless required by applicable law or agreed to in writing, software
-+// distributed under the License is distributed on an "AS IS" BASIS,
-+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+// See the License for the specific language governing permissions and
-+// limitations under the License.
-+
-+#include "tokenpool.h"
-+
-+#include <fcntl.h>
-+#include <poll.h>
-+#include <unistd.h>
-+#include <stdio.h>
-+#include <string.h>
-+#include <stdlib.h>
-+
-+// No-op TokenPool implementation
-+struct TokenPool *TokenPool::Get(void) {
-+  return NULL;
-+}
-diff --git a/src/tokenpool.h b/src/tokenpool.h
-new file mode 100644
-index 0000000000..f560b1083b
 --- /dev/null
 +++ b/src/tokenpool.h
-@@ -0,0 +1,26 @@
-+// Copyright 2016 Google Inc. All Rights Reserved.
+@@ -0,0 +1,42 @@
++// Copyright 2016-2018 Google Inc. All Rights Reserved.
 +//
 +// Licensed under the Apache License, Version 2.0 (the "License");
 +// you may not use this file except in compliance with the License.
@@ -475,6 +474,10 @@ index 0000000000..f560b1083b
 +// See the License for the specific language governing permissions and
 +// limitations under the License.
 +
++#ifdef _WIN32
++#include <windows.h>
++#endif
++
 +// interface to token pool
 +struct TokenPool {
 +  virtual ~TokenPool() {}
@@ -484,188 +487,32 @@ index 0000000000..f560b1083b
 +  virtual void Release() = 0;
 +  virtual void Clear() = 0;
 +
-+  // returns NULL if token pool is not available
-+  static struct TokenPool *Get(void);
-+};
-
-From ccaccc610cd456f6068758f82e72006364c7380b Mon Sep 17 00:00:00 2001
-From: Stefan Becker <stefanb at gpartner-nvidia.com>
-Date: Fri, 27 May 2016 16:47:10 +0300
-Subject: [PATCH 02/11] Add TokenPool monitoring to SubprocessSet::DoWork()
-
-Improve on the original jobserver client implementation. This makes
-ninja a more aggressive GNU make jobserver client.
-
-- add monitor interface to TokenPool
-- TokenPool is passed down when main loop indicates that more work is
-  ready and would be allowed to start if a token becomes available
-- posix: update DoWork() to monitor TokenPool read file descriptor
-- WaitForCommand() exits when DoWork() sets token flag
-- Main loop starts over when WaitForCommand() sets token exit status
----
- src/build.cc              | 53 +++++++++++++++++++++++++++++----------
- src/build.h               |  3 ++-
- src/build_test.cc         |  9 +++++--
- src/exit_status.h         |  3 ++-
- src/subprocess-posix.cc   | 33 ++++++++++++++++++++++--
- src/subprocess-win32.cc   |  2 +-
- src/subprocess.h          |  8 +++++-
- src/subprocess_test.cc    | 47 +++++++++++++++++++++++-----------
- src/tokenpool-gnu-make.cc |  5 ++++
- src/tokenpool.h           |  6 +++++
- 10 files changed, 134 insertions(+), 35 deletions(-)
-
-diff --git a/src/build.cc b/src/build.cc
-index fa096eac33..a25c349050 100644
---- a/src/build.cc
-+++ b/src/build.cc
-@@ -48,8 +48,9 @@ struct DryRunCommandRunner : public CommandRunner {
- 
-   // Overridden from CommandRunner:
-   virtual bool CanRunMore() const;
-+  virtual bool AcquireToken();
-   virtual bool StartCommand(Edge* edge);
--  virtual bool WaitForCommand(Result* result);
-+  virtual bool WaitForCommand(Result* result, bool more_ready);
- 
-  private:
-   queue<Edge*> finished_;
-@@ -59,12 +60,16 @@ bool DryRunCommandRunner::CanRunMore() const {
-   return true;
- }
- 
-+bool DryRunCommandRunner::AcquireToken() {
-+  return true;
-+}
-+
- bool DryRunCommandRunner::StartCommand(Edge* edge) {
-   finished_.push(edge);
-   return true;
- }
- 
--bool DryRunCommandRunner::WaitForCommand(Result* result) {
-+bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) {
-    if (finished_.empty())
-      return false;
- 
-@@ -452,8 +457,9 @@ struct RealCommandRunner : public CommandRunner {
-   explicit RealCommandRunner(const BuildConfig& config);
-   virtual ~RealCommandRunner();
-   virtual bool CanRunMore() const;
-+  virtual bool AcquireToken();
-   virtual bool StartCommand(Edge* edge);
--  virtual bool WaitForCommand(Result* result);
-+  virtual bool WaitForCommand(Result* result, bool more_ready);
-   virtual vector<Edge*> GetActiveEdges();
-   virtual void Abort();
- 
-@@ -490,9 +496,12 @@ bool RealCommandRunner::CanRunMore() const {
-       subprocs_.running_.size() + subprocs_.finished_.size();
-   return (int)subproc_number < config_.parallelism
-     && (subprocs_.running_.empty() ||
--        ((config_.max_load_average <= 0.0f ||
--          GetLoadAverage() < config_.max_load_average)
--      && (!tokens_ || tokens_->Acquire())));
-+        (config_.max_load_average <= 0.0f ||
-+         GetLoadAverage() < config_.max_load_average));
-+}
-+
-+bool RealCommandRunner::AcquireToken() {
-+  return (!tokens_ || tokens_->Acquire());
- }
- 
- bool RealCommandRunner::StartCommand(Edge* edge) {
-@@ -507,14 +516,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) {
-   return true;
- }
- 
--bool RealCommandRunner::WaitForCommand(Result* result) {
-+bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) {
-   Subprocess* subproc;
--  while ((subproc = subprocs_.NextFinished()) == NULL) {
--    bool interrupted = subprocs_.DoWork();
-+  subprocs_.ResetTokenAvailable();
-+  while (((subproc = subprocs_.NextFinished()) == NULL) &&
-+         !subprocs_.IsTokenAvailable()) {
-+    bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL);
-     if (interrupted)
-       return false;
-   }
- 
-+  // token became available
-+  if (subproc == NULL) {
-+    result->status = ExitTokenAvailable;
-+    return true;
-+  }
-+
-+  // command completed
-   if (tokens_)
-     tokens_->Release();
- 
-@@ -639,9 +657,14 @@ bool Builder::Build(string* err) {
-   // command runner.
-   // Second, we attempt to wait for / reap the next finished command.
-   while (plan_.more_to_do()) {
--    // See if we can start any more commands.
--    if (failures_allowed && plan_.more_ready() &&
--        command_runner_->CanRunMore()) {
-+    // See if we can start any more commands...
-+    bool can_run_more =
-+        failures_allowed   &&
-+        plan_.more_ready() &&
-+        command_runner_->CanRunMore();
++  // returns false if token pool setup failed
++  virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0;
 +
-+    // ... but we also need a token to do that.
-+    if (can_run_more && command_runner_->AcquireToken()) {
-       Edge* edge = plan_.FindWork();
-       if (edge->GetBindingBool("generator")) {
-           scan_.build_log()->Close();
-@@ -670,7 +693,7 @@ bool Builder::Build(string* err) {
-     // See if we can reap any finished commands.
-     if (pending_commands) {
-       CommandRunner::Result result;
--      if (!command_runner_->WaitForCommand(&result) ||
-+      if (!command_runner_->WaitForCommand(&result, can_run_more) ||
-           result.status == ExitInterrupted) {
-         Cleanup();
-         status_->BuildFinished();
-@@ -678,6 +701,10 @@ bool Builder::Build(string* err) {
-         return false;
-       }
- 
-+      // We might be able to start another command; start the main loop over.
-+      if (result.status == ExitTokenAvailable)
-+        continue;
++#ifdef _WIN32
++  virtual void WaitForTokenAvailability(HANDLE ioport) = 0;
++  // returns true if a token has become available
++  // key is result from GetQueuedCompletionStatus()
++  virtual bool TokenIsAvailable(ULONG_PTR key) = 0;
++#else
++  virtual int GetMonitorFd() = 0;
++#endif
 +
-       --pending_commands;
-       if (!FinishCommand(&result, err)) {
-         Cleanup();
-diff --git a/src/build.h b/src/build.h
-index 7dcd111e61..35c7b97d12 100644
---- a/src/build.h
-+++ b/src/build.h
-@@ -139,6 +139,7 @@ struct Plan {
- struct CommandRunner {
-   virtual ~CommandRunner() {}
-   virtual bool CanRunMore() const = 0;
-+  virtual bool AcquireToken() = 0;
-   virtual bool StartCommand(Edge* edge) = 0;
- 
-   /// The result of waiting for a command.
-@@ -150,7 +151,7 @@ struct CommandRunner {
-     bool success() const { return status == ExitSuccess; }
-   };
-   /// Wait for a command to complete, or return false if interrupted.
--  virtual bool WaitForCommand(Result* result) = 0;
-+  virtual bool WaitForCommand(Result* result, bool more_ready) = 0;
- 
-   virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
-   virtual void Abort() {}
-diff --git a/src/build_test.cc b/src/build_test.cc
-index 4ef62b2113..7a5ff4015a 100644
++  // returns NULL if token pool is not available
++  static TokenPool* Get();
++};
 --- a/src/build_test.cc
 +++ b/src/build_test.cc
-@@ -474,8 +474,9 @@ struct FakeCommandRunner : public CommandRunner {
+@@ -15,6 +15,7 @@
+ #include "build.h"
+ 
+ #include <assert.h>
++#include <stdarg.h>
+ 
+ #include "build_log.h"
+ #include "deps_log.h"
+@@ -474,8 +475,9 @@ struct FakeCommandRunner : public Comman
  
    // CommandRunner impl
    virtual bool CanRunMore() const;
@@ -676,7 +523,7 @@ index 4ef62b2113..7a5ff4015a 100644
    virtual vector<Edge*> GetActiveEdges();
    virtual void Abort();
  
-@@ -578,6 +579,10 @@ bool FakeCommandRunner::CanRunMore() const {
+@@ -578,6 +580,10 @@ bool FakeCommandRunner::CanRunMore() con
    return active_edges_.size() < max_active_edges_;
  }
  
@@ -687,7 +534,7 @@ index 4ef62b2113..7a5ff4015a 100644
  bool FakeCommandRunner::StartCommand(Edge* edge) {
    assert(active_edges_.size() < max_active_edges_);
    assert(find(active_edges_.begin(), active_edges_.end(), edge)
-@@ -649,7 +654,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) {
+@@ -649,7 +655,7 @@ bool FakeCommandRunner::StartCommand(Edg
    return true;
  }
  
@@ -696,2651 +543,820 @@ index 4ef62b2113..7a5ff4015a 100644
    if (active_edges_.empty())
      return false;
  
-diff --git a/src/exit_status.h b/src/exit_status.h
-index a714ece791..75ebf6a7a0 100644
---- a/src/exit_status.h
-+++ b/src/exit_status.h
-@@ -18,7 +18,8 @@
- enum ExitStatus {
-   ExitSuccess,
-   ExitFailure,
--  ExitInterrupted
-+  ExitTokenAvailable,
-+  ExitInterrupted,
- };
- 
- #endif  // NINJA_EXIT_STATUS_H_
-diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc
-index 8e785406c9..74451b0be2 100644
---- a/src/subprocess-posix.cc
-+++ b/src/subprocess-posix.cc
-@@ -13,6 +13,7 @@
- // limitations under the License.
- 
- #include "subprocess.h"
-+#include "tokenpool.h"
- 
- #include <sys/select.h>
- #include <assert.h>
-@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
+@@ -3985,3 +3991,356 @@ TEST_F(BuildTest, ValidationWithCircular
+   EXPECT_FALSE(builder_.AddTarget("out", &err));
+   EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err);
  }
- 
- #ifdef USE_PPOLL
--bool SubprocessSet::DoWork() {
-+bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-   vector<pollfd> fds;
-   nfds_t nfds = 0;
- 
-@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() {
-     ++nfds;
-   }
- 
-+  if (tokens) {
-+    pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 };
-+    fds.push_back(pfd);
-+    ++nfds;
-+  }
 +
-   interrupted_ = 0;
-   int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_);
-   if (ret == -1) {
-@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() {
-     ++i;
-   }
- 
-+  if (tokens) {
-+    pollfd *pfd = &fds[nfds - 1];
-+    if (pfd->fd >= 0) {
-+      assert(pfd->fd == tokens->GetMonitorFd());
-+      if (pfd->revents != 0)
-+        token_available_ = true;
-+    }
-+  }
-+
-   return IsInterrupted();
- }
- 
- #else  // !defined(USE_PPOLL)
--bool SubprocessSet::DoWork() {
-+bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-   fd_set set;
-   int nfds = 0;
-   FD_ZERO(&set);
-@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() {
-     }
-   }
- 
-+  if (tokens) {
-+    int fd = tokens->GetMonitorFd();
-+    FD_SET(fd, &set);
-+    if (nfds < fd+1)
-+      nfds = fd+1;
-+  }
-+
-   interrupted_ = 0;
-   int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_);
-   if (ret == -1) {
-@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() {
-     ++i;
-   }
- 
-+  if (tokens) {
-+    int fd = tokens->GetMonitorFd();
-+    if ((fd >= 0) && FD_ISSET(fd, &set))
-+    token_available_ = true;
-+  }
-+
-   return IsInterrupted();
- }
- #endif  // !defined(USE_PPOLL)
-diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc
-index ff3baaca7f..66d2c2c430 100644
---- a/src/subprocess-win32.cc
-+++ b/src/subprocess-win32.cc
-@@ -251,7 +251,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
-   return subprocess;
- }
- 
--bool SubprocessSet::DoWork() {
-+bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-   DWORD bytes_read;
-   Subprocess* subproc;
-   OVERLAPPED* overlapped;
-diff --git a/src/subprocess.h b/src/subprocess.h
-index 9e3d2ee98f..9ea67ea477 100644
---- a/src/subprocess.h
-+++ b/src/subprocess.h
-@@ -76,6 +76,8 @@ struct Subprocess {
-   friend struct SubprocessSet;
- };
- 
-+struct TokenPool;
-+
- /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses.
- /// DoWork() waits for any state change in subprocesses; finished_
- /// is a queue of subprocesses as they finish.
-@@ -84,13 +86,17 @@ struct SubprocessSet {
-   ~SubprocessSet();
- 
-   Subprocess* Add(const std::string& command, bool use_console = false);
--  bool DoWork();
-+  bool DoWork(struct TokenPool* tokens);
-   Subprocess* NextFinished();
-   void Clear();
- 
-   std::vector<Subprocess*> running_;
-   std::queue<Subprocess*> finished_;
- 
-+  bool token_available_;
-+  bool IsTokenAvailable() { return token_available_; }
-+  void ResetTokenAvailable() { token_available_ = false; }
-+
- #ifdef _WIN32
-   static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType);
-   static HANDLE ioport_;
-diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
-index 073fe86931..4bc8083e26 100644
---- a/src/subprocess_test.cc
-+++ b/src/subprocess_test.cc
-@@ -45,10 +45,12 @@ TEST_F(SubprocessTest, BadCommandStderr) {
-   Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command");
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-+  subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
-     // Pretend we discovered that stderr was ready for writing.
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitFailure, subproc->Finish());
-   EXPECT_NE("", subproc->GetOutput());
-@@ -59,10 +61,12 @@ TEST_F(SubprocessTest, NoSuchCommand) {
-   Subprocess* subproc = subprocs_.Add("ninja_no_such_command");
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-+  subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
-     // Pretend we discovered that stderr was ready for writing.
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitFailure, subproc->Finish());
-   EXPECT_NE("", subproc->GetOutput());
-@@ -78,9 +82,11 @@ TEST_F(SubprocessTest, InterruptChild) {
-   Subprocess* subproc = subprocs_.Add("kill -INT $$");
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-+  subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitInterrupted, subproc->Finish());
- }
-@@ -90,7 +96,7 @@ TEST_F(SubprocessTest, InterruptParent) {
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-   while (!subproc->Done()) {
--    bool interrupted = subprocs_.DoWork();
-+    bool interrupted = subprocs_.DoWork(NULL);
-     if (interrupted)
-       return;
-   }
-@@ -102,9 +108,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) {
-   Subprocess* subproc = subprocs_.Add("kill -TERM $$");
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-+  subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitInterrupted, subproc->Finish());
- }
-@@ -114,7 +122,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigTerm) {
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-   while (!subproc->Done()) {
--    bool interrupted = subprocs_.DoWork();
-+    bool interrupted = subprocs_.DoWork(NULL);
-     if (interrupted)
-       return;
-   }
-@@ -126,9 +134,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) {
-   Subprocess* subproc = subprocs_.Add("kill -HUP $$");
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-+  subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitInterrupted, subproc->Finish());
- }
-@@ -138,7 +148,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigHup) {
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-   while (!subproc->Done()) {
--    bool interrupted = subprocs_.DoWork();
-+    bool interrupted = subprocs_.DoWork(NULL);
-     if (interrupted)
-       return;
-   }
-@@ -153,9 +163,11 @@ TEST_F(SubprocessTest, Console) {
-         subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true);
-     ASSERT_NE((Subprocess*)0, subproc);
- 
-+    subprocs_.ResetTokenAvailable();
-     while (!subproc->Done()) {
--      subprocs_.DoWork();
-+      subprocs_.DoWork(NULL);
-     }
-+    ASSERT_EQ(false, subprocs_.IsTokenAvailable());
- 
-     EXPECT_EQ(ExitSuccess, subproc->Finish());
-   }
-@@ -167,9 +179,11 @@ TEST_F(SubprocessTest, SetWithSingle) {
-   Subprocess* subproc = subprocs_.Add(kSimpleCommand);
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-+  subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-   ASSERT_EQ(ExitSuccess, subproc->Finish());
-   ASSERT_NE("", subproc->GetOutput());
- 
-@@ -200,12 +214,13 @@ TEST_F(SubprocessTest, SetWithMulti) {
-     ASSERT_EQ("", processes[i]->GetOutput());
-   }
- 
-+  subprocs_.ResetTokenAvailable();
-   while (!processes[0]->Done() || !processes[1]->Done() ||
-          !processes[2]->Done()) {
-     ASSERT_GT(subprocs_.running_.size(), 0u);
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
--
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-   ASSERT_EQ(0u, subprocs_.running_.size());
-   ASSERT_EQ(3u, subprocs_.finished_.size());
- 
-@@ -237,8 +252,10 @@ TEST_F(SubprocessTest, SetWithLots) {
-     ASSERT_NE((Subprocess *) 0, subproc);
-     procs.push_back(subproc);
-   }
-+  subprocs_.ResetTokenAvailable();
-   while (!subprocs_.running_.empty())
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-   for (size_t i = 0; i < procs.size(); ++i) {
-     ASSERT_EQ(ExitSuccess, procs[i]->Finish());
-     ASSERT_NE("", procs[i]->GetOutput());
-@@ -254,9 +271,11 @@ TEST_F(SubprocessTest, SetWithLots) {
- // that stdin is closed.
- TEST_F(SubprocessTest, ReadStdin) {
-   Subprocess* subproc = subprocs_.Add("cat -");
-+  subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
--    subprocs_.DoWork();
-+    subprocs_.DoWork(NULL);
-   }
-+  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-   ASSERT_EQ(ExitSuccess, subproc->Finish());
-   ASSERT_EQ(1u, subprocs_.finished_.size());
- }
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-index a8f9b7139d..396bb7d874 100644
---- a/src/tokenpool-gnu-make.cc
-+++ b/src/tokenpool-gnu-make.cc
-@@ -33,6 +33,7 @@ struct GNUmakeTokenPool : public TokenPool {
-   virtual void Reserve();
-   virtual void Release();
-   virtual void Clear();
-+  virtual int GetMonitorFd();
- 
-   bool Setup();
- 
-@@ -201,6 +202,10 @@ void GNUmakeTokenPool::Clear() {
-     Return();
- }
- 
-+int GNUmakeTokenPool::GetMonitorFd() {
-+  return(rfd_);
-+}
-+
- struct TokenPool *TokenPool::Get(void) {
-   GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
-   if (tokenpool->Setup())
-diff --git a/src/tokenpool.h b/src/tokenpool.h
-index f560b1083b..301e1998ee 100644
---- a/src/tokenpool.h
-+++ b/src/tokenpool.h
-@@ -21,6 +21,12 @@ struct TokenPool {
-   virtual void Release() = 0;
-   virtual void Clear() = 0;
- 
-+#ifdef _WIN32
-+  // @TODO
-+#else
-+  virtual int GetMonitorFd() = 0;
-+#endif
-+
-   // returns NULL if token pool is not available
-   static struct TokenPool *Get(void);
- };
-
-From d09f3d77821b3b1fdf09fc0ef8e814907675eafb Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Sun, 12 Nov 2017 16:58:55 +0200
-Subject: [PATCH 03/11] Ignore jobserver when -jN is forced on command line
-
-This emulates the behaviour of GNU make.
-
-- add parallelism_from_cmdline flag to build configuration
-- set the flag when -jN is given on command line
-- pass the flag to TokenPool::Get()
-- GNUmakeTokenPool::Setup()
-  * prints a warning when the flag is true and jobserver was detected
-  * returns false, i.e. jobserver will be ignored
-- ignore config.parallelism in CanRunMore() when we have a valid
-  TokenPool, because it gets always initialized to a default when not
-  given on the command line
----
- src/build.cc              | 10 ++++++----
- src/build.h               |  4 +++-
- src/ninja.cc              |  1 +
- src/tokenpool-gnu-make.cc | 34 +++++++++++++++++++---------------
- src/tokenpool-none.cc     |  4 ++--
- src/tokenpool.h           |  4 ++--
- 6 files changed, 33 insertions(+), 24 deletions(-)
-
-diff --git a/src/build.cc b/src/build.cc
-index a25c349050..406a84ec39 100644
---- a/src/build.cc
-+++ b/src/build.cc
-@@ -470,7 +470,7 @@ struct RealCommandRunner : public CommandRunner {
- };
- 
- RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
--  tokens_ = TokenPool::Get();
-+  tokens_ = TokenPool::Get(config_.parallelism_from_cmdline);
- }
- 
- RealCommandRunner::~RealCommandRunner() {
-@@ -492,9 +492,11 @@ void RealCommandRunner::Abort() {
- }
- 
- bool RealCommandRunner::CanRunMore() const {
--  size_t subproc_number =
--      subprocs_.running_.size() + subprocs_.finished_.size();
--  return (int)subproc_number < config_.parallelism
-+  bool parallelism_limit_not_reached =
-+    tokens_ || // ignore config_.parallelism
-+    ((int) (subprocs_.running_.size() +
-+            subprocs_.finished_.size()) < config_.parallelism);
-+  return parallelism_limit_not_reached
-     && (subprocs_.running_.empty() ||
-         (config_.max_load_average <= 0.0f ||
-          GetLoadAverage() < config_.max_load_average));
-diff --git a/src/build.h b/src/build.h
-index 35c7b97d12..dfde576573 100644
---- a/src/build.h
-+++ b/src/build.h
-@@ -159,7 +159,8 @@ struct CommandRunner {
- 
- /// Options (e.g. verbosity, parallelism) passed to a build.
- struct BuildConfig {
--  BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
-+  BuildConfig() : verbosity(NORMAL), dry_run(false),
-+                  parallelism(1), parallelism_from_cmdline(false),
-                   failures_allowed(1), max_load_average(-0.0f) {}
- 
-   enum Verbosity {
-@@ -171,6 +172,7 @@ struct BuildConfig {
-   Verbosity verbosity;
-   bool dry_run;
-   int parallelism;
-+  bool parallelism_from_cmdline;
-   int failures_allowed;
-   /// The maximum load average we must not exceed. A negative value
-   /// means that we do not have any limit.
-diff --git a/src/ninja.cc b/src/ninja.cc
-index df39ba92d1..d904c56c4e 100644
---- a/src/ninja.cc
-+++ b/src/ninja.cc
-@@ -1447,6 +1447,7 @@ int ReadFlags(int* argc, char*** argv,
-         // We want to run N jobs in parallel. For N = 0, INT_MAX
-         // is close enough to infinite for most sane builds.
-         config->parallelism = value > 0 ? value : INT_MAX;
-+        config->parallelism_from_cmdline = true;
-         deferGuessParallelism.needGuess = false;
-         break;
-       }
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-index 396bb7d874..af4be05a31 100644
---- a/src/tokenpool-gnu-make.cc
-+++ b/src/tokenpool-gnu-make.cc
-@@ -1,4 +1,4 @@
--// Copyright 2016 Google Inc. All Rights Reserved.
-+// Copyright 2016-2017 Google Inc. All Rights Reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
-@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool {
-   virtual void Clear();
-   virtual int GetMonitorFd();
- 
--  bool Setup();
-+  bool Setup(bool ignore);
- 
-  private:
-   int available_;
-@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() {
-   }
- }
- 
--bool GNUmakeTokenPool::Setup() {
-+bool GNUmakeTokenPool::Setup(bool ignore) {
-   const char *value = getenv("MAKEFLAGS");
-   if (value) {
-     // GNU make <= 4.1
-@@ -109,16 +109,20 @@ bool GNUmakeTokenPool::Setup() {
-     if (!jobserver)
-       jobserver = strstr(value, "--jobserver-auth=");
-     if (jobserver) {
--      int rfd = -1;
--      int wfd = -1;
--      if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
--          CheckFd(rfd) &&
--          CheckFd(wfd) &&
--          SetAlarmHandler()) {
--        printf("ninja: using GNU make jobserver.\n");
--        rfd_ = rfd;
--        wfd_ = wfd;
--        return true;
-+      if (ignore) {
-+        printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
-+      } else {
-+        int rfd = -1;
-+        int wfd = -1;
-+        if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
-+            CheckFd(rfd) &&
-+            CheckFd(wfd) &&
-+            SetAlarmHandler()) {
-+          printf("ninja: using GNU make jobserver.\n");
-+          rfd_ = rfd;
-+          wfd_ = wfd;
-+          return true;
-+        }
-       }
-     }
-   }
-@@ -206,9 +210,9 @@ int GNUmakeTokenPool::GetMonitorFd() {
-   return(rfd_);
- }
- 
--struct TokenPool *TokenPool::Get(void) {
-+struct TokenPool *TokenPool::Get(bool ignore) {
-   GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
--  if (tokenpool->Setup())
-+  if (tokenpool->Setup(ignore))
-     return tokenpool;
-   else
-     delete tokenpool;
-diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
-index 602b3316f5..199b22264b 100644
---- a/src/tokenpool-none.cc
-+++ b/src/tokenpool-none.cc
-@@ -1,4 +1,4 @@
--// Copyright 2016 Google Inc. All Rights Reserved.
-+// Copyright 2016-2017 Google Inc. All Rights Reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
-@@ -22,6 +22,6 @@
- #include <stdlib.h>
- 
- // No-op TokenPool implementation
--struct TokenPool *TokenPool::Get(void) {
-+struct TokenPool *TokenPool::Get(bool ignore) {
-   return NULL;
- }
-diff --git a/src/tokenpool.h b/src/tokenpool.h
-index 301e1998ee..878a0933c2 100644
---- a/src/tokenpool.h
-+++ b/src/tokenpool.h
-@@ -1,4 +1,4 @@
--// Copyright 2016 Google Inc. All Rights Reserved.
-+// Copyright 2016-2017 Google Inc. All Rights Reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
-@@ -28,5 +28,5 @@ struct TokenPool {
- #endif
- 
-   // returns NULL if token pool is not available
--  static struct TokenPool *Get(void);
-+  static struct TokenPool *Get(bool ignore);
- };
-
-From dfe4ca753caee65bf9041e2b4e883dfa172a5c6a Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Sun, 12 Nov 2017 18:04:12 +0200
-Subject: [PATCH 04/11] Honor -lN from MAKEFLAGS
-
-This emulates the behaviour of GNU make.
-
-- build: make a copy of max_load_average and pass it to TokenPool.
-- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in
-  MAKEFLAGS then set max_load_average to N.
----
- src/build.cc              | 10 +++++++---
- src/tokenpool-gnu-make.cc | 19 +++++++++++++++----
- src/tokenpool-none.cc     |  2 +-
- src/tokenpool.h           |  2 +-
- 4 files changed, 24 insertions(+), 9 deletions(-)
-
-diff --git a/src/build.cc b/src/build.cc
-index 406a84ec39..9e6272d035 100644
---- a/src/build.cc
-+++ b/src/build.cc
-@@ -464,13 +464,17 @@ struct RealCommandRunner : public CommandRunner {
-   virtual void Abort();
- 
-   const BuildConfig& config_;
-+  // copy of config_.max_load_average; can be modified by TokenPool setup
-+  double max_load_average_;
-   SubprocessSet subprocs_;
-   TokenPool *tokens_;
-   map<const Subprocess*, Edge*> subproc_to_edge_;
- };
- 
- RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
--  tokens_ = TokenPool::Get(config_.parallelism_from_cmdline);
-+  max_load_average_ = config.max_load_average;
-+  tokens_ = TokenPool::Get(config_.parallelism_from_cmdline,
-+                           max_load_average_);
- }
- 
- RealCommandRunner::~RealCommandRunner() {
-@@ -498,8 +502,8 @@ bool RealCommandRunner::CanRunMore() const {
-             subprocs_.finished_.size()) < config_.parallelism);
-   return parallelism_limit_not_reached
-     && (subprocs_.running_.empty() ||
--        (config_.max_load_average <= 0.0f ||
--         GetLoadAverage() < config_.max_load_average));
-+        (max_load_average_ <= 0.0f ||
-+         GetLoadAverage() < max_load_average_));
- }
- 
- bool RealCommandRunner::AcquireToken() {
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-index af4be05a31..fb654c4d88 100644
---- a/src/tokenpool-gnu-make.cc
-+++ b/src/tokenpool-gnu-make.cc
-@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool {
-   virtual void Clear();
-   virtual int GetMonitorFd();
- 
--  bool Setup(bool ignore);
-+  bool Setup(bool ignore, double& max_load_average);
- 
-  private:
-   int available_;
-@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() {
-   }
- }
- 
--bool GNUmakeTokenPool::Setup(bool ignore) {
-+bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
-   const char *value = getenv("MAKEFLAGS");
-   if (value) {
-     // GNU make <= 4.1
-@@ -118,9 +118,20 @@ bool GNUmakeTokenPool::Setup(bool ignore) {
-             CheckFd(rfd) &&
-             CheckFd(wfd) &&
-             SetAlarmHandler()) {
-+          const char *l_arg = strstr(value, " -l");
-+          int load_limit = -1;
-+
-           printf("ninja: using GNU make jobserver.\n");
-           rfd_ = rfd;
-           wfd_ = wfd;
-+
-+          // translate GNU make -lN to ninja -lN
-+          if (l_arg &&
-+              (sscanf(l_arg + 3, "%d ", &load_limit) == 1) &&
-+              (load_limit > 0)) {
-+            max_load_average = load_limit;
-+          }
-+
-           return true;
-         }
-       }
-@@ -210,9 +221,9 @@ int GNUmakeTokenPool::GetMonitorFd() {
-   return(rfd_);
- }
- 
--struct TokenPool *TokenPool::Get(bool ignore) {
-+struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
-   GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
--  if (tokenpool->Setup(ignore))
-+  if (tokenpool->Setup(ignore, max_load_average))
-     return tokenpool;
-   else
-     delete tokenpool;
-diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
-index 199b22264b..e8e25426c3 100644
---- a/src/tokenpool-none.cc
-+++ b/src/tokenpool-none.cc
-@@ -22,6 +22,6 @@
- #include <stdlib.h>
- 
- // No-op TokenPool implementation
--struct TokenPool *TokenPool::Get(bool ignore) {
-+struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
-   return NULL;
- }
-diff --git a/src/tokenpool.h b/src/tokenpool.h
-index 878a0933c2..f9e8cc2ee0 100644
---- a/src/tokenpool.h
-+++ b/src/tokenpool.h
-@@ -28,5 +28,5 @@ struct TokenPool {
- #endif
- 
-   // returns NULL if token pool is not available
--  static struct TokenPool *Get(bool ignore);
-+  static struct TokenPool *Get(bool ignore, double& max_load_average);
- };
-
-From 1c10047fc6a3269ba42839da19361e09cbc06ff0 Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Wed, 6 Dec 2017 22:14:21 +0200
-Subject: [PATCH 05/11] Use LinePrinter for TokenPool messages
-
-- replace printf() with calls to LinePrinter
-- print GNU make jobserver message only when verbose build is requested
----
- src/build.cc              |  1 +
- src/tokenpool-gnu-make.cc | 22 ++++++++++++++++------
- src/tokenpool-none.cc     |  4 +++-
- src/tokenpool.h           |  4 +++-
- 4 files changed, 23 insertions(+), 8 deletions(-)
-
-diff --git a/src/build.cc b/src/build.cc
-index 9e6272d035..662e4bd7be 100644
---- a/src/build.cc
-+++ b/src/build.cc
-@@ -474,6 +474,7 @@ struct RealCommandRunner : public CommandRunner {
- RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
-   max_load_average_ = config.max_load_average;
-   tokens_ = TokenPool::Get(config_.parallelism_from_cmdline,
-+                           config_.verbosity == BuildConfig::VERBOSE,
-                            max_load_average_);
- }
- 
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-index fb654c4d88..b0d3e6ebc4 100644
---- a/src/tokenpool-gnu-make.cc
-+++ b/src/tokenpool-gnu-make.cc
-@@ -23,6 +23,8 @@
- #include <string.h>
- #include <stdlib.h>
- 
-+#include "line_printer.h"
-+
- // TokenPool implementation for GNU make jobserver
- // (http://make.mad-scientist.net/papers/jobserver-implementation/)
- struct GNUmakeTokenPool : public TokenPool {
-@@ -35,7 +37,7 @@ struct GNUmakeTokenPool : public TokenPool {
-   virtual void Clear();
-   virtual int GetMonitorFd();
- 
--  bool Setup(bool ignore, double& max_load_average);
-+  bool Setup(bool ignore, bool verbose, double& max_load_average);
- 
-  private:
-   int available_;
-@@ -100,7 +102,9 @@ bool GNUmakeTokenPool::SetAlarmHandler() {
-   }
- }
- 
--bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
-+bool GNUmakeTokenPool::Setup(bool ignore,
-+                             bool verbose,
-+                             double& max_load_average) {
-   const char *value = getenv("MAKEFLAGS");
-   if (value) {
-     // GNU make <= 4.1
-@@ -109,8 +113,10 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
-     if (!jobserver)
-       jobserver = strstr(value, "--jobserver-auth=");
-     if (jobserver) {
-+      LinePrinter printer;
-+
-       if (ignore) {
--        printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
-+        printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
-       } else {
-         int rfd = -1;
-         int wfd = -1;
-@@ -121,7 +127,9 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) {
-           const char *l_arg = strstr(value, " -l");
-           int load_limit = -1;
- 
--          printf("ninja: using GNU make jobserver.\n");
-+          if (verbose) {
-+            printer.PrintOnNewLine("ninja: using GNU make jobserver.\n");
-+          }
-           rfd_ = rfd;
-           wfd_ = wfd;
- 
-@@ -221,9 +229,11 @@ int GNUmakeTokenPool::GetMonitorFd() {
-   return(rfd_);
- }
- 
--struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
-+struct TokenPool *TokenPool::Get(bool ignore,
-+                                 bool verbose,
-+                                 double& max_load_average) {
-   GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
--  if (tokenpool->Setup(ignore, max_load_average))
-+  if (tokenpool->Setup(ignore, verbose, max_load_average))
-     return tokenpool;
-   else
-     delete tokenpool;
-diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
-index e8e25426c3..1c1c499c8d 100644
---- a/src/tokenpool-none.cc
-+++ b/src/tokenpool-none.cc
-@@ -22,6 +22,8 @@
- #include <stdlib.h>
- 
- // No-op TokenPool implementation
--struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) {
-+struct TokenPool *TokenPool::Get(bool ignore,
-+                                 bool verbose,
-+                                 double& max_load_average) {
-   return NULL;
- }
-diff --git a/src/tokenpool.h b/src/tokenpool.h
-index f9e8cc2ee0..4bf477f20c 100644
---- a/src/tokenpool.h
-+++ b/src/tokenpool.h
-@@ -28,5 +28,7 @@ struct TokenPool {
- #endif
- 
-   // returns NULL if token pool is not available
--  static struct TokenPool *Get(bool ignore, double& max_load_average);
-+  static struct TokenPool *Get(bool ignore,
-+                               bool verbose,
-+                               double& max_load_average);
- };
-
-From fdbf68416e3574add3bffd0b637d0694fbaba320 Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Sat, 7 Apr 2018 17:11:21 +0300
-Subject: [PATCH 06/11] Prepare PR for merging
-
-- fix Windows build error in no-op TokenPool implementation
-- improve Acquire() to block for a maximum of 100ms
-- address review comments
----
- src/build.h               |  2 ++
- src/tokenpool-gnu-make.cc | 53 +++++++++++++++++++++++++++++++++------
- src/tokenpool-none.cc     |  7 +-----
- 3 files changed, 49 insertions(+), 13 deletions(-)
-
-diff --git a/src/build.h b/src/build.h
-index dfde576573..66ddefb888 100644
---- a/src/build.h
-+++ b/src/build.h
-@@ -151,6 +151,8 @@ struct CommandRunner {
-     bool success() const { return status == ExitSuccess; }
-   };
-   /// Wait for a command to complete, or return false if interrupted.
-+  /// If more_ready is true then the optional TokenPool is monitored too
-+  /// and we return when a token becomes available.
-   virtual bool WaitForCommand(Result* result, bool more_ready) = 0;
- 
-   virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-index b0d3e6ebc4..4132bb06d9 100644
---- a/src/tokenpool-gnu-make.cc
-+++ b/src/tokenpool-gnu-make.cc
-@@ -1,4 +1,4 @@
--// Copyright 2016-2017 Google Inc. All Rights Reserved.
-+// Copyright 2016-2018 Google Inc. All Rights Reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
-@@ -19,6 +19,7 @@
- #include <poll.h>
- #include <unistd.h>
- #include <signal.h>
-+#include <sys/time.h>
- #include <stdio.h>
- #include <string.h>
- #include <stdlib.h>
-@@ -153,6 +154,15 @@ bool GNUmakeTokenPool::Acquire() {
-   if (available_ > 0)
-     return true;
- 
-+  // Please read
-+  //
-+  //   http://make.mad-scientist.net/papers/jobserver-implementation/
-+  //
-+  // for the reasoning behind the following code.
-+  //
-+  // Try to read one character from the pipe. Returns true on success.
-+  //
-+  // First check if read() would succeed without blocking.
- #ifdef USE_PPOLL
-   pollfd pollfds[] = {{rfd_, POLLIN, 0}};
-   int ret = poll(pollfds, 1, 0);
-@@ -164,33 +174,62 @@ bool GNUmakeTokenPool::Acquire() {
-   int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
- #endif
-   if (ret > 0) {
-+    // Handle potential race condition:
-+    //  - the above check succeeded, i.e. read() should not block
-+    //  - the character disappears before we call read()
-+    //
-+    // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_
-+    // can safely be closed by signal handlers without affecting rfd_.
-     dup_rfd_ = dup(rfd_);
- 
-     if (dup_rfd_ != -1) {
-       struct sigaction act, old_act;
-       int ret = 0;
- 
-+      // Temporarily replace SIGCHLD handler with our own
-       memset(&act, 0, sizeof(act));
-       act.sa_handler = CloseDupRfd;
-       if (sigaction(SIGCHLD, &act, &old_act) == 0) {
--        char buf;
--
--        // block until token read, child exits or timeout
--        alarm(1);
--        ret = read(dup_rfd_, &buf, 1);
--        alarm(0);
-+        struct itimerval timeout;
-+
-+        // install a 100ms timeout that generates SIGALARM on expiration
-+        memset(&timeout, 0, sizeof(timeout));
-+        timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec]
-+        if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) {
-+          char buf;
-+
-+          // Now try to read() from dup_rfd_. Return values from read():
-+          //
-+          // 1. token read                               ->  1
-+          // 2. pipe closed                              ->  0
-+          // 3. alarm expires                            -> -1 (EINTR)
-+          // 4. child exits                              -> -1 (EINTR)
-+          // 5. alarm expired before entering read()     -> -1 (EBADF)
-+          // 6. child exited before entering read()      -> -1 (EBADF)
-+          // 7. child exited before handler is installed -> go to 1 - 3
-+          ret = read(dup_rfd_, &buf, 1);
-+
-+          // disarm timer
-+          memset(&timeout, 0, sizeof(timeout));
-+          setitimer(ITIMER_REAL, &timeout, NULL);
-+        }
- 
-         sigaction(SIGCHLD, &old_act, NULL);
-       }
- 
-       CloseDupRfd(0);
- 
-+      // Case 1 from above list
-       if (ret > 0) {
-         available_++;
-         return true;
-       }
-     }
-   }
-+
-+  // read() would block, i.e. no token available,
-+  // cases 2-6 from above list or
-+  // select() / poll() / dup() / sigaction() / setitimer() failed
-   return false;
- }
- 
-diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
-index 1c1c499c8d..4c592875b4 100644
---- a/src/tokenpool-none.cc
-+++ b/src/tokenpool-none.cc
-@@ -1,4 +1,4 @@
--// Copyright 2016-2017 Google Inc. All Rights Reserved.
-+// Copyright 2016-2018 Google Inc. All Rights Reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
-@@ -14,11 +14,6 @@
- 
- #include "tokenpool.h"
- 
--#include <fcntl.h>
--#include <poll.h>
--#include <unistd.h>
--#include <stdio.h>
--#include <string.h>
- #include <stdlib.h>
- 
- // No-op TokenPool implementation
-
-From ec6220a0baf7d3a6eaf1a2b75bf8960ddfe24c2f Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Fri, 25 May 2018 00:17:07 +0300
-Subject: [PATCH 07/11] Add tests for TokenPool
-
-- TokenPool setup
-- GetMonitorFd() API
-- implicit token and tokens in jobserver pipe
-- Acquire() / Reserve() / Release() protocol
-- Clear() API
----
- configure.py          |   1 +
- src/tokenpool_test.cc | 198 ++++++++++++++++++++++++++++++++++++++++++
- 2 files changed, 199 insertions(+)
- create mode 100644 src/tokenpool_test.cc
-
-diff --git a/configure.py b/configure.py
-index db3492c93c..dc8a0066b7 100755
---- a/configure.py
-+++ b/configure.py
-@@ -590,6 +590,7 @@ def has_re2c():
-              'string_piece_util_test',
-              'subprocess_test',
-              'test',
-+             'tokenpool_test',
-              'util_test']:
-     objs += cxx(name, variables=cxxvariables)
- if platform.is_windows():
-diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc
-new file mode 100644
-index 0000000000..6c89064ca4
---- /dev/null
-+++ b/src/tokenpool_test.cc
-@@ -0,0 +1,198 @@
-+// Copyright 2018 Google Inc. All Rights Reserved.
-+//
-+// Licensed under the Apache License, Version 2.0 (the "License");
-+// you may not use this file except in compliance with the License.
-+// You may obtain a copy of the License at
-+//
-+//     http://www.apache.org/licenses/LICENSE-2.0
-+//
-+// Unless required by applicable law or agreed to in writing, software
-+// distributed under the License is distributed on an "AS IS" BASIS,
-+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+// See the License for the specific language governing permissions and
-+// limitations under the License.
-+
-+#include "tokenpool.h"
-+
-+#include "test.h"
-+
-+#ifndef _WIN32
-+#include <stdio.h>
-+#include <stdlib.h>
-+#include <unistd.h>
-+
-+#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS")
-+#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true);
-+#endif
-+
-+namespace {
-+
-+const double kLoadAverageDefault = -1.23456789;
-+
-+struct TokenPoolTest : public testing::Test {
-+  double load_avg_;
-+  TokenPool *tokens_;
-+#ifndef _WIN32
-+  char buf_[1024];
-+  int fds_[2];
-+#endif
-+
-+  virtual void SetUp() {
-+    load_avg_ = kLoadAverageDefault;
-+    tokens_ = NULL;
-+#ifndef _WIN32
-+    ENVIRONMENT_CLEAR();
-+    if (pipe(fds_) < 0)
-+      ASSERT_TRUE(false);
-+#endif
-+  }
-+
-+  void CreatePool(const char *format, bool ignore_jobserver) {
-+#ifndef _WIN32
-+    if (format) {
-+      sprintf(buf_, format, fds_[0], fds_[1]);
-+      ENVIRONMENT_INIT(buf_);
-+    }
-+#endif
-+    tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_);
-+  }
-+
-+  void CreateDefaultPool() {
-+    CreatePool("foo --jobserver-auth=%d,%d bar", false);
-+  }
-+
-+  virtual void TearDown() {
-+    if (tokens_)
-+      delete tokens_;
-+#ifndef _WIN32
-+    close(fds_[0]);
-+    close(fds_[1]);
-+    ENVIRONMENT_CLEAR();
-+#endif
-+  }
-+};
-+
-+} // anonymous namespace
-+
-+// verifies none implementation
-+TEST_F(TokenPoolTest, NoTokenPool) {
-+  CreatePool(NULL, false);
-+
-+  EXPECT_EQ(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+}
-+
-+#ifndef _WIN32
-+TEST_F(TokenPoolTest, SuccessfulOldSetup) {
-+  // GNUmake <= 4.1
-+  CreatePool("foo --jobserver-fds=%d,%d bar", false);
-+
-+  EXPECT_NE(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+}
-+
-+TEST_F(TokenPoolTest, SuccessfulNewSetup) {
-+  // GNUmake => 4.2
-+  CreateDefaultPool();
-+
-+  EXPECT_NE(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+}
-+
-+TEST_F(TokenPoolTest, IgnoreWithJN) {
-+  CreatePool("foo --jobserver-auth=%d,%d bar", true);
-+
-+  EXPECT_EQ(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+}
-+
-+TEST_F(TokenPoolTest, HonorLN) {
-+  CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false);
-+
-+  EXPECT_NE(NULL, tokens_);
-+  EXPECT_EQ(9.0, load_avg_);
-+}
-+
-+TEST_F(TokenPoolTest, MonitorFD) {
-+  CreateDefaultPool();
-+
-+  ASSERT_NE(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+
-+  EXPECT_EQ(fds_[0], tokens_->GetMonitorFd());
-+}
-+
-+TEST_F(TokenPoolTest, ImplicitToken) {
-+  CreateDefaultPool();
-+
-+  ASSERT_NE(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+
-+  EXPECT_TRUE(tokens_->Acquire());
-+  tokens_->Reserve();
-+  EXPECT_FALSE(tokens_->Acquire());
-+  tokens_->Release();
-+  EXPECT_TRUE(tokens_->Acquire());
-+}
-+
-+TEST_F(TokenPoolTest, TwoTokens) {
-+  CreateDefaultPool();
-+
-+  ASSERT_NE(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+
-+  // implicit token
-+  EXPECT_TRUE(tokens_->Acquire());
-+  tokens_->Reserve();
-+  EXPECT_FALSE(tokens_->Acquire());
-+
-+  // jobserver offers 2nd token
-+  ASSERT_EQ(1u, write(fds_[1], "T", 1));
-+  EXPECT_TRUE(tokens_->Acquire());
-+  tokens_->Reserve();
-+  EXPECT_FALSE(tokens_->Acquire());
-+
-+  // release 2nd token
-+  tokens_->Release();
-+  EXPECT_TRUE(tokens_->Acquire());
-+
-+  // release implict token - must return 2nd token back to jobserver
-+  tokens_->Release();
-+  EXPECT_TRUE(tokens_->Acquire());
-+
-+  // there must be one token in the pipe
-+  EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_)));
-+
-+  // implicit token
-+  EXPECT_TRUE(tokens_->Acquire());
-+}
-+
-+TEST_F(TokenPoolTest, Clear) {
-+  CreateDefaultPool();
-+
-+  ASSERT_NE(NULL, tokens_);
-+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
-+
-+  // implicit token
-+  EXPECT_TRUE(tokens_->Acquire());
-+  tokens_->Reserve();
-+  EXPECT_FALSE(tokens_->Acquire());
-+
-+  // jobserver offers 2nd & 3rd token
-+  ASSERT_EQ(2u, write(fds_[1], "TT", 2));
-+  EXPECT_TRUE(tokens_->Acquire());
-+  tokens_->Reserve();
-+  EXPECT_TRUE(tokens_->Acquire());
-+  tokens_->Reserve();
-+  EXPECT_FALSE(tokens_->Acquire());
-+
-+  tokens_->Clear();
-+  EXPECT_TRUE(tokens_->Acquire());
-+
-+  // there must be two tokens in the pipe
-+  EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_)));
-+
-+  // implicit token
-+  EXPECT_TRUE(tokens_->Acquire());
-+}
-+#endif
-
-From e59d8858327126d1593fd0b8e607975a79072e92 Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Thu, 24 May 2018 18:52:45 +0300
-Subject: [PATCH 08/11] Add tests for subprocess module
-
-- add TokenPoolTest stub to provide TokenPool::GetMonitorFd()
-- add two tests
-  * both tests set up a dummy GNUmake jobserver pipe
-  * both tests call DoWork() with TokenPoolTest
-  * test 1: verify that DoWork() detects when a token is available
-  * test 2: verify that DoWork() works as before without a token
-- the tests are not compiled in under Windows
----
- src/subprocess_test.cc | 76 ++++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 76 insertions(+)
-
-diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
-index 4bc8083e26..6264c8bf11 100644
---- a/src/subprocess_test.cc
-+++ b/src/subprocess_test.cc
-@@ -13,6 +13,7 @@
- // limitations under the License.
- 
- #include "subprocess.h"
-+#include "tokenpool.h"
- 
- #include "test.h"
- 
-@@ -34,8 +35,23 @@ const char* kSimpleCommand = "cmd /c dir \\";
- const char* kSimpleCommand = "ls /";
- #endif
- 
-+struct TokenPoolTest : public TokenPool {
-+  bool Acquire()     { return false; }
-+  void Reserve()     {}
-+  void Release()     {}
-+  void Clear()       {}
-+
-+#ifdef _WIN32
-+  // @TODO
-+#else
-+  int _fd;
-+  int GetMonitorFd() { return _fd; }
-+#endif
-+};
-+
- struct SubprocessTest : public testing::Test {
-   SubprocessSet subprocs_;
-+  TokenPoolTest tokens_;
- };
- 
- }  // anonymous namespace
-@@ -280,3 +296,63 @@ TEST_F(SubprocessTest, ReadStdin) {
-   ASSERT_EQ(1u, subprocs_.finished_.size());
- }
- #endif  // _WIN32
-+
-+// @TODO: remove once TokenPool implementation for Windows is available
-+#ifndef _WIN32
-+TEST_F(SubprocessTest, TokenAvailable) {
-+  Subprocess* subproc = subprocs_.Add(kSimpleCommand);
-+  ASSERT_NE((Subprocess *) 0, subproc);
-+
-+  // simulate GNUmake jobserver pipe with 1 token
-+  int fds[2];
-+  ASSERT_EQ(0u, pipe(fds));
-+  tokens_._fd = fds[0];
-+  ASSERT_EQ(1u, write(fds[1], "T", 1));
-+
-+  subprocs_.ResetTokenAvailable();
-+  subprocs_.DoWork(&tokens_);
-+
-+  EXPECT_TRUE(subprocs_.IsTokenAvailable());
-+  EXPECT_EQ(0u, subprocs_.finished_.size());
-+
-+  // remove token to let DoWork() wait for command again
-+  char token;
-+  ASSERT_EQ(1u, read(fds[0], &token, 1));
-+
-+  while (!subproc->Done()) {
-+    subprocs_.DoWork(&tokens_);
-+  }
-+
-+  close(fds[1]);
-+  close(fds[0]);
-+
-+  EXPECT_EQ(ExitSuccess, subproc->Finish());
-+  EXPECT_NE("", subproc->GetOutput());
-+
-+  EXPECT_EQ(1u, subprocs_.finished_.size());
-+}
-+
-+TEST_F(SubprocessTest, TokenNotAvailable) {
-+  Subprocess* subproc = subprocs_.Add(kSimpleCommand);
-+  ASSERT_NE((Subprocess *) 0, subproc);
-+
-+  // simulate GNUmake jobserver pipe with 0 tokens
-+  int fds[2];
-+  ASSERT_EQ(0u, pipe(fds));
-+  tokens_._fd = fds[0];
-+
-+  subprocs_.ResetTokenAvailable();
-+  while (!subproc->Done()) {
-+    subprocs_.DoWork(&tokens_);
-+  }
-+
-+  close(fds[1]);
-+  close(fds[0]);
-+
-+  EXPECT_FALSE(subprocs_.IsTokenAvailable());
-+  EXPECT_EQ(ExitSuccess, subproc->Finish());
-+  EXPECT_NE("", subproc->GetOutput());
-+
-+  EXPECT_EQ(1u, subprocs_.finished_.size());
-+}
-+#endif  // _WIN32
-
-From 0145e2d4db64ea6c21aeb371928e4071f65164eb Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Sat, 26 May 2018 23:17:51 +0300
-Subject: [PATCH 09/11] Add tests for build module
-
-Add tests that verify the token functionality of the builder main loop.
-We replace the default fake command runner with a special version where
-the tests can control each call to AcquireToken(), CanRunMore() and
-WaitForCommand().
----
- src/build_test.cc | 364 ++++++++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 364 insertions(+)
-
-diff --git a/src/build_test.cc b/src/build_test.cc
-index 7a5ff4015a..dd41dfbe1d 100644
---- a/src/build_test.cc
-+++ b/src/build_test.cc
-@@ -15,6 +15,7 @@
- #include "build.h"
- 
- #include <assert.h>
-+#include <stdarg.h>
- 
- #include "build_log.h"
- #include "deps_log.h"
-@@ -3990,3 +3991,366 @@ TEST_F(BuildTest, ValidationWithCircularDependency) {
-   EXPECT_FALSE(builder_.AddTarget("out", &err));
-   EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err);
- }
-+
-+/// The token tests are concerned with the main loop functionality when
-+// the CommandRunner has an active TokenPool. It is therefore intentional
-+// that the plan doesn't complete and that builder_.Build() returns false!
-+
-+/// Fake implementation of CommandRunner that simulates a TokenPool
-+struct FakeTokenCommandRunner : public CommandRunner {
-+  explicit FakeTokenCommandRunner() {}
-+
-+  // CommandRunner impl
-+  virtual bool CanRunMore() const;
-+  virtual bool AcquireToken();
-+  virtual bool StartCommand(Edge* edge);
-+  virtual bool WaitForCommand(Result* result, bool more_ready);
-+  virtual vector<Edge*> GetActiveEdges();
-+  virtual void Abort();
-+
-+  vector<string> commands_ran_;
-+  vector<Edge *> edges_;
-+
-+  vector<bool> acquire_token_;
-+  vector<bool> can_run_more_;
-+  vector<bool> wait_for_command_;
-+};
-+
-+bool FakeTokenCommandRunner::CanRunMore() const {
-+  if (can_run_more_.size() == 0) {
-+    EXPECT_FALSE("unexpected call to CommandRunner::CanRunMore()");
-+    return false;
-+  }
-+
-+  bool result = can_run_more_[0];
-+
-+  // Unfortunately CanRunMore() isn't "const" for tests
-+  const_cast<FakeTokenCommandRunner*>(this)->can_run_more_.erase(
-+    const_cast<FakeTokenCommandRunner*>(this)->can_run_more_.begin()
-+  );
-+
-+  return result;
-+}
-+
-+bool FakeTokenCommandRunner::AcquireToken() {
-+  if (acquire_token_.size() == 0) {
-+    EXPECT_FALSE("unexpected call to CommandRunner::AcquireToken()");
-+    return false;
-+  }
-+
-+  bool result = acquire_token_[0];
-+  acquire_token_.erase(acquire_token_.begin());
-+  return result;
-+}
-+
-+bool FakeTokenCommandRunner::StartCommand(Edge* edge) {
-+  commands_ran_.push_back(edge->EvaluateCommand());
-+  edges_.push_back(edge);
-+  return true;
-+}
-+
-+bool FakeTokenCommandRunner::WaitForCommand(Result* result, bool more_ready) {
-+  if (wait_for_command_.size() == 0) {
-+    EXPECT_FALSE("unexpected call to CommandRunner::WaitForCommand()");
-+    return false;
-+  }
-+
-+  bool expected = wait_for_command_[0];
-+  if (expected != more_ready) {
-+    EXPECT_EQ(expected, more_ready);
-+    return false;
-+  }
-+  wait_for_command_.erase(wait_for_command_.begin());
-+
-+  if (edges_.size() == 0)
-+    return false;
-+
-+  Edge* edge = edges_[0];
-+  result->edge = edge;
-+
-+  if (more_ready &&
-+      (edge->rule().name() == "token-available")) {
-+    result->status = ExitTokenAvailable;
-+  } else {
-+    edges_.erase(edges_.begin());
-+    result->status = ExitSuccess;
-+  }
-+
-+  return true;
-+}
-+
-+vector<Edge*> FakeTokenCommandRunner::GetActiveEdges() {
-+  return edges_;
-+}
-+
-+void FakeTokenCommandRunner::Abort() {
-+  edges_.clear();
-+}
-+
-+struct BuildTokenTest : public BuildTest {
-+  virtual void SetUp();
-+  virtual void TearDown();
-+
-+  FakeTokenCommandRunner token_command_runner_;
-+
-+  void ExpectAcquireToken(int count, ...);
-+  void ExpectCanRunMore(int count, ...);
-+  void ExpectWaitForCommand(int count, ...);
-+
-+private:
-+  void EnqueueBooleans(vector<bool>& booleans, int count, va_list ao);
-+};
-+
-+void BuildTokenTest::SetUp() {
-+  BuildTest::SetUp();
-+
-+  // replace FakeCommandRunner with FakeTokenCommandRunner
-+  builder_.command_runner_.release();
-+  builder_.command_runner_.reset(&token_command_runner_);
-+}
-+void BuildTokenTest::TearDown() {
-+  EXPECT_EQ(0u, token_command_runner_.acquire_token_.size());
-+  EXPECT_EQ(0u, token_command_runner_.can_run_more_.size());
-+  EXPECT_EQ(0u, token_command_runner_.wait_for_command_.size());
-+
-+  BuildTest::TearDown();
-+}
-+
-+void BuildTokenTest::ExpectAcquireToken(int count, ...) {
-+  va_list ap;
-+  va_start(ap, count);
-+  EnqueueBooleans(token_command_runner_.acquire_token_, count, ap);
-+  va_end(ap);
-+}
-+
-+void BuildTokenTest::ExpectCanRunMore(int count, ...) {
-+  va_list ap;
-+  va_start(ap, count);
-+  EnqueueBooleans(token_command_runner_.can_run_more_, count, ap);
-+  va_end(ap);
-+}
-+
-+void BuildTokenTest::ExpectWaitForCommand(int count, ...) {
-+  va_list ap;
-+  va_start(ap, count);
-+  EnqueueBooleans(token_command_runner_.wait_for_command_, count, ap);
-+  va_end(ap);
-+}
-+
-+void BuildTokenTest::EnqueueBooleans(vector<bool>& booleans, int count, va_list ap) {
-+  while (count--) {
-+    int value = va_arg(ap, int);
-+    booleans.push_back(!!value); // force bool
-+  }
-+}
-+
-+TEST_F(BuildTokenTest, CompleteNoWork) {
-+  // plan should not execute anything
-+  string err;
-+
-+  EXPECT_TRUE(builder_.Build(&err));
-+  EXPECT_EQ("", err);
-+
-+  EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
-+}
-+
-+TEST_F(BuildTokenTest, DoNotAquireToken) {
-+  // plan should execute one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("cat1", &err));
-+  ASSERT_EQ("", err);
-+
-+  // pretend we can't run anything
-+  ExpectCanRunMore(1, false);
-+
-+  EXPECT_FALSE(builder_.Build(&err));
-+  EXPECT_EQ("stuck [this is a bug]", err);
-+
-+  EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
-+}
-+
-+TEST_F(BuildTokenTest, DoNotStartWithoutToken) {
-+  // plan should execute one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("cat1", &err));
-+  ASSERT_EQ("", err);
-+
-+  // we could run a command but do not have a token for it
-+  ExpectCanRunMore(1,   true);
-+  ExpectAcquireToken(1, false);
-+
-+  EXPECT_FALSE(builder_.Build(&err));
-+  EXPECT_EQ("stuck [this is a bug]", err);
-+
-+  EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
-+}
-+
-+TEST_F(BuildTokenTest, CompleteOneStep) {
-+  // plan should execute one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("cat1", &err));
-+  ASSERT_EQ("", err);
-+
-+  // allow running of one command
-+  ExpectCanRunMore(1,   true);
-+  ExpectAcquireToken(1, true);
-+  // block and wait for command to finalize
-+  ExpectWaitForCommand(1, false);
-+
-+  EXPECT_TRUE(builder_.Build(&err));
-+  EXPECT_EQ("", err);
-+
-+  EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
-+  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1");
-+}
-+
-+TEST_F(BuildTokenTest, AcquireOneToken) {
-+  // plan should execute more than one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("cat12", &err));
-+  ASSERT_EQ("", err);
-+
-+  // allow running of one command
-+  ExpectCanRunMore(3,     true, false, false);
-+  ExpectAcquireToken(1,   true);
-+  // block and wait for command to finalize
-+  ExpectWaitForCommand(1, false);
-+
-+  EXPECT_FALSE(builder_.Build(&err));
-+  EXPECT_EQ("stuck [this is a bug]", err);
-+
-+  EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
-+  // any of the two dependencies could have been executed
-+  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1" ||
-+              token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2");
-+}
-+
-+TEST_F(BuildTokenTest, WantTwoTokens) {
-+  // plan should execute more than one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("cat12", &err));
-+  ASSERT_EQ("", err);
-+
-+  // allow running of one command
-+  ExpectCanRunMore(3,     true, true, false);
-+  ExpectAcquireToken(2,   true, false);
-+  // wait for command to finalize or token to become available
-+  ExpectWaitForCommand(1, true);
-+
-+  EXPECT_FALSE(builder_.Build(&err));
-+  EXPECT_EQ("stuck [this is a bug]", err);
-+
-+  EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
-+  // any of the two dependencies could have been executed
-+  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1" ||
-+              token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2");
-+}
-+
-+TEST_F(BuildTokenTest, CompleteTwoSteps) {
-+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
-+"build out1: cat in1\n"
-+"build out2: cat out1\n"));
-+
-+  // plan should execute more than one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("out2", &err));
-+  ASSERT_EQ("", err);
-+
-+  // allow running of two commands
-+  ExpectCanRunMore(2,     true, true);
-+  ExpectAcquireToken(2,   true, true);
-+  // wait for commands to finalize
-+  ExpectWaitForCommand(2, false, false);
-+
-+  EXPECT_TRUE(builder_.Build(&err));
-+  EXPECT_EQ("", err);
-+
-+  EXPECT_EQ(2u, token_command_runner_.commands_ran_.size());
-+  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > out1");
-+  EXPECT_TRUE(token_command_runner_.commands_ran_[1] == "cat out1 > out2");
-+}
-+
-+TEST_F(BuildTokenTest, TwoCommandsInParallel) {
-+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
-+"rule token-available\n"
-+"  command = cat $in > $out\n"
-+"build out1: token-available in1\n"
-+"build out2: token-available in2\n"
-+"build out12: cat out1 out2\n"));
-+
-+  // plan should execute more than one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("out12", &err));
-+  ASSERT_EQ("", err);
-+
-+  // 1st command: token available -> allow running
-+  // 2nd command: no token available but becomes available later
-+  ExpectCanRunMore(4,     true, true,  true,  false);
-+  ExpectAcquireToken(3,   true, false, true);
-+  // 1st call waits for command to finalize or token to become available
-+  // 2nd call waits for command to finalize
-+  // 3rd call waits for command to finalize
-+  ExpectWaitForCommand(3, true, false, false);
-+
-+  EXPECT_FALSE(builder_.Build(&err));
-+  EXPECT_EQ("stuck [this is a bug]", err);
-+
-+  EXPECT_EQ(2u, token_command_runner_.commands_ran_.size());
-+  EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > out1" &&
-+               token_command_runner_.commands_ran_[1] == "cat in2 > out2") ||
-+              (token_command_runner_.commands_ran_[0] == "cat in2 > out2" &&
-+               token_command_runner_.commands_ran_[1] == "cat in1 > out1"));
-+}
-+
-+TEST_F(BuildTokenTest, CompleteThreeStepsSerial) {
-+  // plan should execute more than one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("cat12", &err));
-+  ASSERT_EQ("", err);
-+
-+  // allow running of all commands
-+  ExpectCanRunMore(4,     true, true,  true,  true);
-+  ExpectAcquireToken(4,   true, false, true,  true);
-+  // wait for commands to finalize
-+  ExpectWaitForCommand(3, true, false, false);
-+
-+  EXPECT_TRUE(builder_.Build(&err));
-+  EXPECT_EQ("", err);
-+
-+  EXPECT_EQ(3u, token_command_runner_.commands_ran_.size());
-+  EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > cat1"     &&
-+               token_command_runner_.commands_ran_[1] == "cat in1 in2 > cat2") ||
-+              (token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2" &&
-+               token_command_runner_.commands_ran_[1] == "cat in1 > cat1"    ));
-+  EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat cat1 cat2 > cat12");
-+}
-+
-+TEST_F(BuildTokenTest, CompleteThreeStepsParallel) {
-+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
-+"rule token-available\n"
-+"  command = cat $in > $out\n"
-+"build out1: token-available in1\n"
-+"build out2: token-available in2\n"
-+"build out12: cat out1 out2\n"));
-+
-+  // plan should execute more than one command
-+  string err;
-+  EXPECT_TRUE(builder_.AddTarget("out12", &err));
-+  ASSERT_EQ("", err);
-+
-+  // allow running of all commands
-+  ExpectCanRunMore(4,     true, true,  true,  true);
-+  ExpectAcquireToken(4,   true, false, true,  true);
-+  // wait for commands to finalize
-+  ExpectWaitForCommand(4, true, false, false, false);
-+
-+  EXPECT_TRUE(builder_.Build(&err));
-+  EXPECT_EQ("", err);
-+
-+  EXPECT_EQ(3u, token_command_runner_.commands_ran_.size());
-+  EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > out1" &&
-+               token_command_runner_.commands_ran_[1] == "cat in2 > out2") ||
-+              (token_command_runner_.commands_ran_[0] == "cat in2 > out2" &&
-+               token_command_runner_.commands_ran_[1] == "cat in1 > out1"));
-+  EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat out1 out2 > out12");
-+}
-
-From f016e5430c9123d34a73ea7ad28693b20ee59d6d Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Mon, 8 Oct 2018 17:47:50 +0300
-Subject: [PATCH 10/11] Add Win32 implementation for GNUmakeTokenPool
-
-GNU make uses a semaphore as jobserver protocol on Win32. See also
-
-   https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html
-
-Usage is pretty simple and straightforward, i.e. WaitForSingleObject()
-to obtain a token and ReleaseSemaphore() to return it.
-
-Unfortunately subprocess-win32.cc uses an I/O completion port (IOCP).
-IOCPs aren't waitable objects, i.e. we can't use WaitForMultipleObjects()
-to wait on the IOCP and the token semaphore at the same time.
-
-Therefore GNUmakeTokenPoolWin32 creates a child thread that waits on the
-token semaphore and posts a dummy I/O completion status on the IOCP when
-it was able to obtain a token. That unblocks SubprocessSet::DoWork() and
-it can then check if a token became available or not.
-
-- split existing GNUmakeTokenPool into common and platform bits
-- add GNUmakeTokenPool interface
-- move the Posix bits to GNUmakeTokenPoolPosix
-- add the Win32 bits as GNUmakeTokenPoolWin32
-- move Setup() method up to TokenPool interface
-- update Subprocess & TokenPool tests accordingly
----
- configure.py                    |   8 +-
- src/build.cc                    |  11 +-
- src/subprocess-win32.cc         |   9 ++
- src/subprocess_test.cc          |  34 ++++-
- src/tokenpool-gnu-make-posix.cc | 203 +++++++++++++++++++++++++++
- src/tokenpool-gnu-make-win32.cc | 237 ++++++++++++++++++++++++++++++++
- src/tokenpool-gnu-make.cc       | 203 ++-------------------------
- src/tokenpool-gnu-make.h        |  40 ++++++
- src/tokenpool-none.cc           |   4 +-
- src/tokenpool.h                 |  18 ++-
- src/tokenpool_test.cc           | 113 ++++++++++++---
- 11 files changed, 653 insertions(+), 227 deletions(-)
- create mode 100644 src/tokenpool-gnu-make-posix.cc
- create mode 100644 src/tokenpool-gnu-make-win32.cc
- create mode 100644 src/tokenpool-gnu-make.h
-
-diff --git a/configure.py b/configure.py
-index dc8a0066b7..a239b90eef 100755
---- a/configure.py
-+++ b/configure.py
-@@ -517,12 +517,13 @@ def has_re2c():
-              'state',
-              'status',
-              'string_piece_util',
-+             'tokenpool-gnu-make',
-              'util',
-              'version']:
-     objs += cxx(name, variables=cxxvariables)
- if platform.is_windows():
-     for name in ['subprocess-win32',
--                 'tokenpool-none',
-+                 'tokenpool-gnu-make-win32',
-                  'includes_normalize-win32',
-                  'msvc_helper-win32',
-                  'msvc_helper_main-win32']:
-@@ -531,8 +532,9 @@ def has_re2c():
-         objs += cxx('minidump-win32', variables=cxxvariables)
-     objs += cc('getopt')
- else:
--    objs += cxx('subprocess-posix')
--    objs += cxx('tokenpool-gnu-make')
-+    for name in ['subprocess-posix',
-+                 'tokenpool-gnu-make-posix']:
-+        objs += cxx(name)
- if platform.is_aix():
-     objs += cc('getopt')
- if platform.is_msvc():
-diff --git a/src/build.cc b/src/build.cc
-index 662e4bd7be..20c3bdc2a0 100644
---- a/src/build.cc
-+++ b/src/build.cc
-@@ -473,9 +473,14 @@ struct RealCommandRunner : public CommandRunner {
- 
- RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
-   max_load_average_ = config.max_load_average;
--  tokens_ = TokenPool::Get(config_.parallelism_from_cmdline,
--                           config_.verbosity == BuildConfig::VERBOSE,
--                           max_load_average_);
-+  if ((tokens_ = TokenPool::Get()) != NULL) {
-+    if (!tokens_->Setup(config_.parallelism_from_cmdline,
-+                        config_.verbosity == BuildConfig::VERBOSE,
-+                        max_load_average_)) {
-+      delete tokens_;
-+      tokens_ = NULL;
-+    }
-+  }
- }
- 
- RealCommandRunner::~RealCommandRunner() {
-diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc
-index 66d2c2c430..ce3e2c20a4 100644
---- a/src/subprocess-win32.cc
-+++ b/src/subprocess-win32.cc
-@@ -13,6 +13,7 @@
- // limitations under the License.
- 
- #include "subprocess.h"
-+#include "tokenpool.h"
- 
- #include <assert.h>
- #include <stdio.h>
-@@ -256,6 +257,9 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-   Subprocess* subproc;
-   OVERLAPPED* overlapped;
- 
-+  if (tokens)
-+    tokens->WaitForTokenAvailability(ioport_);
-+
-   if (!GetQueuedCompletionStatus(ioport_, &bytes_read, (PULONG_PTR)&subproc,
-                                  &overlapped, INFINITE)) {
-     if (GetLastError() != ERROR_BROKEN_PIPE)
-@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-                 // delivered by NotifyInterrupted above.
-     return true;
- 
-+  if (tokens && tokens->TokenIsAvailable((ULONG_PTR)subproc)) {
-+    token_available_ = true;
-+    return false;
-+  }
-+
-   subproc->OnPipeReady();
- 
-   if (subproc->Done()) {
-diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
-index 6264c8bf11..f625963462 100644
---- a/src/subprocess_test.cc
-+++ b/src/subprocess_test.cc
-@@ -40,9 +40,16 @@ struct TokenPoolTest : public TokenPool {
-   void Reserve()     {}
-   void Release()     {}
-   void Clear()       {}
-+  bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; }
- 
- #ifdef _WIN32
--  // @TODO
-+  bool _token_available;
-+  void WaitForTokenAvailability(HANDLE ioport) {
-+    if (_token_available)
-+      // unblock GetQueuedCompletionStatus()
-+      PostQueuedCompletionStatus(ioport, 0, (ULONG_PTR) this, NULL);
-+  }
-+  bool TokenIsAvailable(ULONG_PTR key) { return key == (ULONG_PTR) this; }
- #else
-   int _fd;
-   int GetMonitorFd() { return _fd; }
-@@ -297,34 +304,48 @@ TEST_F(SubprocessTest, ReadStdin) {
- }
- #endif  // _WIN32
- 
--// @TODO: remove once TokenPool implementation for Windows is available
--#ifndef _WIN32
- TEST_F(SubprocessTest, TokenAvailable) {
-   Subprocess* subproc = subprocs_.Add(kSimpleCommand);
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-   // simulate GNUmake jobserver pipe with 1 token
-+#ifdef _WIN32
-+  tokens_._token_available = true;
-+#else
-   int fds[2];
-   ASSERT_EQ(0u, pipe(fds));
-   tokens_._fd = fds[0];
-   ASSERT_EQ(1u, write(fds[1], "T", 1));
-+#endif
- 
-   subprocs_.ResetTokenAvailable();
-   subprocs_.DoWork(&tokens_);
-+#ifdef _WIN32
-+  tokens_._token_available = false;
-+  // we need to loop here as we have no conrol where the token
-+  // I/O completion post ends up in the queue
-+  while (!subproc->Done() && !subprocs_.IsTokenAvailable()) {
-+    subprocs_.DoWork(&tokens_);
-+  }
-+#endif
- 
-   EXPECT_TRUE(subprocs_.IsTokenAvailable());
-   EXPECT_EQ(0u, subprocs_.finished_.size());
- 
-   // remove token to let DoWork() wait for command again
-+#ifndef _WIN32
-   char token;
-   ASSERT_EQ(1u, read(fds[0], &token, 1));
-+#endif
- 
-   while (!subproc->Done()) {
-     subprocs_.DoWork(&tokens_);
-   }
- 
-+#ifndef _WIN32
-   close(fds[1]);
-   close(fds[0]);
-+#endif
- 
-   EXPECT_EQ(ExitSuccess, subproc->Finish());
-   EXPECT_NE("", subproc->GetOutput());
-@@ -337,17 +358,23 @@ TEST_F(SubprocessTest, TokenNotAvailable) {
-   ASSERT_NE((Subprocess *) 0, subproc);
- 
-   // simulate GNUmake jobserver pipe with 0 tokens
-+#ifdef _WIN32
-+  tokens_._token_available = false;
-+#else
-   int fds[2];
-   ASSERT_EQ(0u, pipe(fds));
-   tokens_._fd = fds[0];
-+#endif
- 
-   subprocs_.ResetTokenAvailable();
-   while (!subproc->Done()) {
-     subprocs_.DoWork(&tokens_);
-   }
- 
-+#ifndef _WIN32
-   close(fds[1]);
-   close(fds[0]);
-+#endif
- 
-   EXPECT_FALSE(subprocs_.IsTokenAvailable());
-   EXPECT_EQ(ExitSuccess, subproc->Finish());
-@@ -355,4 +382,3 @@ TEST_F(SubprocessTest, TokenNotAvailable) {
- 
-   EXPECT_EQ(1u, subprocs_.finished_.size());
- }
--#endif  // _WIN32
-diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc
-new file mode 100644
-index 0000000000..70d84bfff7
---- /dev/null
-+++ b/src/tokenpool-gnu-make-posix.cc
-@@ -0,0 +1,203 @@
-+// Copyright 2016-2018 Google Inc. All Rights Reserved.
-+//
-+// Licensed under the Apache License, Version 2.0 (the "License");
-+// you may not use this file except in compliance with the License.
-+// You may obtain a copy of the License at
-+//
-+//     http://www.apache.org/licenses/LICENSE-2.0
-+//
-+// Unless required by applicable law or agreed to in writing, software
-+// distributed under the License is distributed on an "AS IS" BASIS,
-+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+// See the License for the specific language governing permissions and
-+// limitations under the License.
++/// The token tests are concerned with the main loop functionality when
++// the CommandRunner has an active TokenPool. It is therefore intentional
++// that the plan doesn't complete and that builder_.Build() returns false!
 +
-+#include "tokenpool-gnu-make.h"
++/// Fake implementation of CommandRunner that simulates a TokenPool
++struct FakeTokenCommandRunner : public CommandRunner {
++  explicit FakeTokenCommandRunner() {}
 +
-+#include <errno.h>
-+#include <fcntl.h>
-+#include <poll.h>
-+#include <unistd.h>
-+#include <signal.h>
-+#include <sys/time.h>
-+#include <stdio.h>
-+#include <string.h>
-+#include <stdlib.h>
++  // CommandRunner impl
++  virtual bool CanRunMore() const;
++  virtual bool AcquireToken();
++  virtual bool StartCommand(Edge* edge);
++  virtual bool WaitForCommand(Result* result, bool more_ready);
++  virtual vector<Edge*> GetActiveEdges();
++  virtual void Abort();
 +
-+// TokenPool implementation for GNU make jobserver - POSIX implementation
-+// (http://make.mad-scientist.net/papers/jobserver-implementation/)
-+struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool {
-+  GNUmakeTokenPoolPosix();
-+  virtual ~GNUmakeTokenPoolPosix();
++  vector<string> commands_ran_;
++  vector<Edge *> edges_;
 +
-+  virtual int GetMonitorFd();
++  vector<bool> acquire_token_;
++  vector<bool> can_run_more_;
++  vector<bool> wait_for_command_;
++};
 +
-+  virtual const char *GetEnv(const char *name) { return getenv(name); };
-+  virtual bool ParseAuth(const char *jobserver);
-+  virtual bool AcquireToken();
-+  virtual bool ReturnToken();
++bool FakeTokenCommandRunner::CanRunMore() const {
++  if (can_run_more_.size() == 0) {
++    EXPECT_FALSE("unexpected call to CommandRunner::CanRunMore()");
++    return false;
++  }
 +
-+ private:
-+  int rfd_;
-+  int wfd_;
++  bool result = can_run_more_[0];
 +
-+  struct sigaction old_act_;
-+  bool restore_;
++  // Unfortunately CanRunMore() isn't "const" for tests
++  const_cast<FakeTokenCommandRunner*>(this)->can_run_more_.erase(
++    const_cast<FakeTokenCommandRunner*>(this)->can_run_more_.begin()
++  );
 +
-+  static int dup_rfd_;
-+  static void CloseDupRfd(int signum);
++  return result;
++}
 +
-+  bool CheckFd(int fd);
-+  bool SetAlarmHandler();
-+};
++bool FakeTokenCommandRunner::AcquireToken() {
++  if (acquire_token_.size() == 0) {
++    EXPECT_FALSE("unexpected call to CommandRunner::AcquireToken()");
++    return false;
++  }
 +
-+GNUmakeTokenPoolPosix::GNUmakeTokenPoolPosix() : rfd_(-1), wfd_(-1), restore_(false) {
++  bool result = acquire_token_[0];
++  acquire_token_.erase(acquire_token_.begin());
++  return result;
 +}
 +
-+GNUmakeTokenPoolPosix::~GNUmakeTokenPoolPosix() {
-+  Clear();
-+  if (restore_)
-+    sigaction(SIGALRM, &old_act_, NULL);
++bool FakeTokenCommandRunner::StartCommand(Edge* edge) {
++  commands_ran_.push_back(edge->EvaluateCommand());
++  edges_.push_back(edge);
++  return true;
 +}
 +
-+bool GNUmakeTokenPoolPosix::CheckFd(int fd) {
-+  if (fd < 0)
++bool FakeTokenCommandRunner::WaitForCommand(Result* result, bool more_ready) {
++  if (wait_for_command_.size() == 0) {
++    EXPECT_FALSE("unexpected call to CommandRunner::WaitForCommand()");
 +    return false;
-+  int ret = fcntl(fd, F_GETFD);
-+  if (ret < 0)
++  }
++
++  bool expected = wait_for_command_[0];
++  if (expected != more_ready) {
++    EXPECT_EQ(expected, more_ready);
 +    return false;
-+  return true;
-+}
++  }
++  wait_for_command_.erase(wait_for_command_.begin());
 +
-+int GNUmakeTokenPoolPosix::dup_rfd_ = -1;
++  if (edges_.size() == 0)
++    return false;
 +
-+void GNUmakeTokenPoolPosix::CloseDupRfd(int signum) {
-+  close(dup_rfd_);
-+  dup_rfd_ = -1;
-+}
++  Edge* edge = edges_[0];
++  result->edge = edge;
 +
-+bool GNUmakeTokenPoolPosix::SetAlarmHandler() {
-+  struct sigaction act;
-+  memset(&act, 0, sizeof(act));
-+  act.sa_handler = CloseDupRfd;
-+  if (sigaction(SIGALRM, &act, &old_act_) < 0) {
-+    perror("sigaction:");
-+    return(false);
++  if (more_ready &&
++      (edge->rule().name() == "token-available")) {
++    result->status = ExitTokenAvailable;
 +  } else {
-+    restore_ = true;
-+    return(true);
++    edges_.erase(edges_.begin());
++    result->status = ExitSuccess;
 +  }
++
++  return true;
 +}
 +
-+bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) {
-+  int rfd = -1;
-+  int wfd = -1;
-+  if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
-+      CheckFd(rfd) &&
-+      CheckFd(wfd) &&
-+      SetAlarmHandler()) {
-+    rfd_ = rfd;
-+    wfd_ = wfd;
-+    return true;
-+  }
++vector<Edge*> FakeTokenCommandRunner::GetActiveEdges() {
++  return edges_;
++}
 +
-+  return false;
++void FakeTokenCommandRunner::Abort() {
++  edges_.clear();
 +}
 +
-+bool GNUmakeTokenPoolPosix::AcquireToken() {
-+  // Please read
-+  //
-+  //   http://make.mad-scientist.net/papers/jobserver-implementation/
-+  //
-+  // for the reasoning behind the following code.
-+  //
-+  // Try to read one character from the pipe. Returns true on success.
-+  //
-+  // First check if read() would succeed without blocking.
-+#ifdef USE_PPOLL
-+  pollfd pollfds[] = {{rfd_, POLLIN, 0}};
-+  int ret = poll(pollfds, 1, 0);
-+#else
-+  fd_set set;
-+  struct timeval timeout = { 0, 0 };
-+  FD_ZERO(&set);
-+  FD_SET(rfd_, &set);
-+  int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
-+#endif
-+  if (ret > 0) {
-+    // Handle potential race condition:
-+    //  - the above check succeeded, i.e. read() should not block
-+    //  - the character disappears before we call read()
-+    //
-+    // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_
-+    // can safely be closed by signal handlers without affecting rfd_.
-+    dup_rfd_ = dup(rfd_);
++struct BuildTokenTest : public BuildTest {
++  virtual void SetUp();
++  virtual void TearDown();
 +
-+    if (dup_rfd_ != -1) {
-+      struct sigaction act, old_act;
-+      int ret = 0;
++  FakeTokenCommandRunner token_command_runner_;
 +
-+      // Temporarily replace SIGCHLD handler with our own
-+      memset(&act, 0, sizeof(act));
-+      act.sa_handler = CloseDupRfd;
-+      if (sigaction(SIGCHLD, &act, &old_act) == 0) {
-+        struct itimerval timeout;
++  void ExpectAcquireToken(int count, ...);
++  void ExpectCanRunMore(int count, ...);
++  void ExpectWaitForCommand(int count, ...);
 +
-+        // install a 100ms timeout that generates SIGALARM on expiration
-+        memset(&timeout, 0, sizeof(timeout));
-+        timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec]
-+        if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) {
-+          char buf;
++private:
++  void EnqueueBooleans(vector<bool>& booleans, int count, va_list ap);
++};
 +
-+          // Now try to read() from dup_rfd_. Return values from read():
-+          //
-+          // 1. token read                               ->  1
-+          // 2. pipe closed                              ->  0
-+          // 3. alarm expires                            -> -1 (EINTR)
-+          // 4. child exits                              -> -1 (EINTR)
-+          // 5. alarm expired before entering read()     -> -1 (EBADF)
-+          // 6. child exited before entering read()      -> -1 (EBADF)
-+          // 7. child exited before handler is installed -> go to 1 - 3
-+          ret = read(dup_rfd_, &buf, 1);
++void BuildTokenTest::SetUp() {
++  BuildTest::SetUp();
 +
-+          // disarm timer
-+          memset(&timeout, 0, sizeof(timeout));
-+          setitimer(ITIMER_REAL, &timeout, NULL);
-+        }
++  // replace FakeCommandRunner with FakeTokenCommandRunner
++  builder_.command_runner_.release();
++  builder_.command_runner_.reset(&token_command_runner_);
++}
++void BuildTokenTest::TearDown() {
++  EXPECT_EQ(0u, token_command_runner_.acquire_token_.size());
++  EXPECT_EQ(0u, token_command_runner_.can_run_more_.size());
++  EXPECT_EQ(0u, token_command_runner_.wait_for_command_.size());
 +
-+        sigaction(SIGCHLD, &old_act, NULL);
-+      }
++  BuildTest::TearDown();
++}
 +
-+      CloseDupRfd(0);
++void BuildTokenTest::ExpectAcquireToken(int count, ...) {
++  va_list ap;
++  va_start(ap, count);
++  EnqueueBooleans(token_command_runner_.acquire_token_, count, ap);
++  va_end(ap);
++}
 +
-+      // Case 1 from above list
-+      if (ret > 0)
-+        return true;
-+    }
-+  }
++void BuildTokenTest::ExpectCanRunMore(int count, ...) {
++  va_list ap;
++  va_start(ap, count);
++  EnqueueBooleans(token_command_runner_.can_run_more_, count, ap);
++  va_end(ap);
++}
 +
-+  // read() would block, i.e. no token available,
-+  // cases 2-6 from above list or
-+  // select() / poll() / dup() / sigaction() / setitimer() failed
-+  return false;
++void BuildTokenTest::ExpectWaitForCommand(int count, ...) {
++  va_list ap;
++  va_start(ap, count);
++  EnqueueBooleans(token_command_runner_.wait_for_command_, count, ap);
++  va_end(ap);
++}
++
++void BuildTokenTest::EnqueueBooleans(vector<bool>& booleans, int count, va_list ap) {
++  while (count--) {
++    int value = va_arg(ap, int);
++    booleans.push_back(!!value); // force bool
++  }
 +}
 +
-+bool GNUmakeTokenPoolPosix::ReturnToken() {
-+  const char buf = '+';
-+  while (1) {
-+    int ret = write(wfd_, &buf, 1);
-+    if (ret > 0)
-+      return true;
-+    if ((ret != -1) || (errno != EINTR))
-+      return false;
-+    // write got interrupted - retry
-+  }
++TEST_F(BuildTokenTest, DoNotAquireToken) {
++  // plan should execute one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("cat1", &err));
++  ASSERT_EQ("", err);
++
++  // pretend we can't run anything
++  ExpectCanRunMore(1, false);
++
++  EXPECT_FALSE(builder_.Build(&err));
++  EXPECT_EQ("stuck [this is a bug]", err);
++
++  EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
 +}
 +
-+int GNUmakeTokenPoolPosix::GetMonitorFd() {
-+  return(rfd_);
-+}
++TEST_F(BuildTokenTest, DoNotStartWithoutToken) {
++  // plan should execute one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("cat1", &err));
++  ASSERT_EQ("", err);
 +
-+struct TokenPool *TokenPool::Get() {
-+  return new GNUmakeTokenPoolPosix;
++  // we could run a command but do not have a token for it
++  ExpectCanRunMore(1,   true);
++  ExpectAcquireToken(1, false);
++
++  EXPECT_FALSE(builder_.Build(&err));
++  EXPECT_EQ("stuck [this is a bug]", err);
++
++  EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
 +}
-diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc
-new file mode 100644
-index 0000000000..2719f2c1fc
---- /dev/null
-+++ b/src/tokenpool-gnu-make-win32.cc
-@@ -0,0 +1,237 @@
-+// Copyright 2018 Google Inc. All Rights Reserved.
-+//
-+// Licensed under the Apache License, Version 2.0 (the "License");
-+// you may not use this file except in compliance with the License.
-+// You may obtain a copy of the License at
-+//
-+//     http://www.apache.org/licenses/LICENSE-2.0
-+//
-+// Unless required by applicable law or agreed to in writing, software
-+// distributed under the License is distributed on an "AS IS" BASIS,
-+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+// See the License for the specific language governing permissions and
-+// limitations under the License.
 +
-+#include "tokenpool-gnu-make.h"
++TEST_F(BuildTokenTest, CompleteOneStep) {
++  // plan should execute one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("cat1", &err));
++  ASSERT_EQ("", err);
 +
-+// always include first to make sure other headers do the correct thing...
-+#include <windows.h>
++  // allow running of one command
++  ExpectCanRunMore(1,   true);
++  ExpectAcquireToken(1, true);
++  // block and wait for command to finalize
++  ExpectWaitForCommand(1, false);
 +
-+#include <ctype.h>
-+#include <stdlib.h>
-+#include <string.h>
++  EXPECT_TRUE(builder_.Build(&err));
++  EXPECT_EQ("", err);
 +
-+#include "util.h"
++  EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
++  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1");
++}
 +
-+// TokenPool implementation for GNU make jobserver - Win32 implementation
-+// (https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html)
-+struct GNUmakeTokenPoolWin32 : public GNUmakeTokenPool {
-+  GNUmakeTokenPoolWin32();
-+  virtual ~GNUmakeTokenPoolWin32();
++TEST_F(BuildTokenTest, AcquireOneToken) {
++  // plan should execute more than one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("cat12", &err));
++  ASSERT_EQ("", err);
 +
-+  virtual void WaitForTokenAvailability(HANDLE ioport);
-+  virtual bool TokenIsAvailable(ULONG_PTR key);
++  // allow running of one command
++  ExpectCanRunMore(3,     true, false, false);
++  ExpectAcquireToken(1,   true);
++  // block and wait for command to finalize
++  ExpectWaitForCommand(1, false);
 +
-+  virtual const char *GetEnv(const char *name);
-+  virtual bool ParseAuth(const char *jobserver);
-+  virtual bool AcquireToken();
-+  virtual bool ReturnToken();
++  EXPECT_FALSE(builder_.Build(&err));
++  EXPECT_EQ("stuck [this is a bug]", err);
 +
-+ private:
-+  // Semaphore for GNU make jobserver protocol
-+  HANDLE semaphore_jobserver_;
-+  // Semaphore Child -> Parent
-+  // - child releases it before entering wait on jobserver semaphore
-+  // - parent blocks on it to know when child enters wait
-+  HANDLE semaphore_enter_wait_;
-+  // Semaphore Parent -> Child
-+  // - parent releases it to allow child to restart loop
-+  // - child blocks on it to know when to restart loop
-+  HANDLE semaphore_restart_;
-+  // set to false if child should exit loop and terminate thread
-+  bool running_;
-+  // child thread
-+  HANDLE child_;
-+  // I/O completion port from SubprocessSet
-+  HANDLE ioport_;
++  EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
++  // any of the two dependencies could have been executed
++  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1" ||
++              token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2");
++}
 +
++TEST_F(BuildTokenTest, WantTwoTokens) {
++  // plan should execute more than one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("cat12", &err));
++  ASSERT_EQ("", err);
 +
-+  DWORD SemaphoreThread();
-+  void ReleaseSemaphore(HANDLE semaphore);
-+  void WaitForObject(HANDLE object);
-+  static DWORD WINAPI SemaphoreThreadWrapper(LPVOID param);
-+  static void NoopAPCFunc(ULONG_PTR param);
-+};
++  // allow running of one command
++  ExpectCanRunMore(3,     true, true, false);
++  ExpectAcquireToken(2,   true, false);
++  // wait for command to finalize or token to become available
++  ExpectWaitForCommand(1, true);
 +
-+GNUmakeTokenPoolWin32::GNUmakeTokenPoolWin32() : semaphore_jobserver_(NULL),
-+                                                 semaphore_enter_wait_(NULL),
-+                                                 semaphore_restart_(NULL),
-+                                                 running_(false),
-+                                                 child_(NULL),
-+                                                 ioport_(NULL) {
-+}
++  EXPECT_FALSE(builder_.Build(&err));
++  EXPECT_EQ("stuck [this is a bug]", err);
 +
-+GNUmakeTokenPoolWin32::~GNUmakeTokenPoolWin32() {
-+  Clear();
-+  CloseHandle(semaphore_jobserver_);
-+  semaphore_jobserver_ = NULL;
++  EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
++  // any of the two dependencies could have been executed
++  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1" ||
++              token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2");
++}
 +
-+  if (child_) {
-+    // tell child thread to exit
-+    running_ = false;
-+    ReleaseSemaphore(semaphore_restart_);
++TEST_F(BuildTokenTest, CompleteTwoSteps) {
++  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
++"build out1: cat in1\n"
++"build out2: cat out1\n"));
 +
-+    // wait for child thread to exit
-+    WaitForObject(child_);
-+    CloseHandle(child_);
-+    child_ = NULL;
-+  }
++  // plan should execute more than one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("out2", &err));
++  ASSERT_EQ("", err);
 +
-+  if (semaphore_restart_) {
-+    CloseHandle(semaphore_restart_);
-+    semaphore_restart_ = NULL;
-+  }
++  // allow running of two commands
++  ExpectCanRunMore(2,     true, true);
++  ExpectAcquireToken(2,   true, true);
++  // wait for commands to finalize
++  ExpectWaitForCommand(2, false, false);
 +
-+  if (semaphore_enter_wait_) {
-+    CloseHandle(semaphore_enter_wait_);
-+    semaphore_enter_wait_ = NULL;
-+  }
-+}
++  EXPECT_TRUE(builder_.Build(&err));
++  EXPECT_EQ("", err);
 +
-+const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) {
-+  // getenv() does not work correctly together with tokenpool_tests.cc
-+  static char buffer[MAX_PATH + 1];
-+  if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0)
-+    return NULL;
-+  return(buffer);
++  EXPECT_EQ(2u, token_command_runner_.commands_ran_.size());
++  EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > out1");
++  EXPECT_TRUE(token_command_runner_.commands_ran_[1] == "cat out1 > out2");
 +}
 +
-+bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) {
-+  // match "--jobserver-auth=gmake_semaphore_<INTEGER>..."
-+  const char *start = strchr(jobserver, '=');
-+  if (start) {
-+    const char *end = start;
-+    unsigned int len;
-+    char c, *auth;
-+
-+    while ((c = *++end) != '\0')
-+      if (!(isalnum(c) || (c == '_')))
-+        break;
-+    len = end - start; // includes string terminator in count
++TEST_F(BuildTokenTest, TwoCommandsInParallel) {
++  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
++"rule token-available\n"
++"  command = cat $in > $out\n"
++"build out1: token-available in1\n"
++"build out2: token-available in2\n"
++"build out12: cat out1 out2\n"));
 +
-+    if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) {
-+      strncpy(auth, start + 1, len - 1);
-+      auth[len - 1] = '\0';
++  // plan should execute more than one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("out12", &err));
++  ASSERT_EQ("", err);
 +
-+      if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */
-+                                                FALSE,                /* Child processes DON'T inherit */
-+                                                auth                  /* Semaphore name */
-+                                                )) != NULL) {
-+        free(auth);
-+        return true;
-+      }
++  // 1st command: token available -> allow running
++  // 2nd command: no token available but becomes available later
++  ExpectCanRunMore(4,     true, true,  true,  false);
++  ExpectAcquireToken(3,   true, false, true);
++  // 1st call waits for command to finalize or token to become available
++  // 2nd call waits for command to finalize
++  // 3rd call waits for command to finalize
++  ExpectWaitForCommand(3, true, false, false);
 +
-+      free(auth);
-+    }
-+  }
++  EXPECT_FALSE(builder_.Build(&err));
++  EXPECT_EQ("stuck [this is a bug]", err);
 +
-+  return false;
++  EXPECT_EQ(2u, token_command_runner_.commands_ran_.size());
++  EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > out1" &&
++               token_command_runner_.commands_ran_[1] == "cat in2 > out2") ||
++              (token_command_runner_.commands_ran_[0] == "cat in2 > out2" &&
++               token_command_runner_.commands_ran_[1] == "cat in1 > out1"));
 +}
 +
-+bool GNUmakeTokenPoolWin32::AcquireToken() {
-+  return WaitForSingleObject(semaphore_jobserver_, 0) == WAIT_OBJECT_0;
-+}
++TEST_F(BuildTokenTest, CompleteThreeStepsSerial) {
++  // plan should execute more than one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("cat12", &err));
++  ASSERT_EQ("", err);
 +
-+bool GNUmakeTokenPoolWin32::ReturnToken() {
-+  ReleaseSemaphore(semaphore_jobserver_);
-+  return true;
-+}
++  // allow running of all commands
++  ExpectCanRunMore(4,     true, true,  true,  true);
++  ExpectAcquireToken(4,   true, false, true,  true);
++  // wait for commands to finalize
++  ExpectWaitForCommand(3, true, false, false);
 +
-+DWORD GNUmakeTokenPoolWin32::SemaphoreThread() {
-+  while (running_) {
-+    // indicate to parent that we are entering wait
-+    ReleaseSemaphore(semaphore_enter_wait_);
++  EXPECT_TRUE(builder_.Build(&err));
++  EXPECT_EQ("", err);
 +
-+    // alertable wait forever on token semaphore
-+    if (WaitForSingleObjectEx(semaphore_jobserver_, INFINITE, TRUE) == WAIT_OBJECT_0) {
-+      // release token again for AcquireToken()
-+      ReleaseSemaphore(semaphore_jobserver_);
++  EXPECT_EQ(3u, token_command_runner_.commands_ran_.size());
++  EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > cat1"     &&
++               token_command_runner_.commands_ran_[1] == "cat in1 in2 > cat2") ||
++              (token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2" &&
++               token_command_runner_.commands_ran_[1] == "cat in1 > cat1"    ));
++  EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat cat1 cat2 > cat12");
++}
 +
-+      // indicate to parent on ioport that a token might be available
-+      if (!PostQueuedCompletionStatus(ioport_, 0, (ULONG_PTR) this, NULL))
-+        Win32Fatal("PostQueuedCompletionStatus");
-+    }
++TEST_F(BuildTokenTest, CompleteThreeStepsParallel) {
++  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
++"rule token-available\n"
++"  command = cat $in > $out\n"
++"build out1: token-available in1\n"
++"build out2: token-available in2\n"
++"build out12: cat out1 out2\n"));
 +
-+    // wait for parent to allow loop restart
-+    WaitForObject(semaphore_restart_);
-+    // semaphore is now in nonsignaled state again for next run...
-+  }
++  // plan should execute more than one command
++  string err;
++  EXPECT_TRUE(builder_.AddTarget("out12", &err));
++  ASSERT_EQ("", err);
 +
-+  return 0;
-+}
++  // allow running of all commands
++  ExpectCanRunMore(4,     true, true,  true,  true);
++  ExpectAcquireToken(4,   true, false, true,  true);
++  // wait for commands to finalize
++  ExpectWaitForCommand(4, true, false, false, false);
 +
-+DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) {
-+  GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param;
-+  return This->SemaphoreThread();
-+}
++  EXPECT_TRUE(builder_.Build(&err));
++  EXPECT_EQ("", err);
 +
-+void GNUmakeTokenPoolWin32::NoopAPCFunc(ULONG_PTR param) {
++  EXPECT_EQ(3u, token_command_runner_.commands_ran_.size());
++  EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > out1" &&
++               token_command_runner_.commands_ran_[1] == "cat in2 > out2") ||
++              (token_command_runner_.commands_ran_[0] == "cat in2 > out2" &&
++               token_command_runner_.commands_ran_[1] == "cat in1 > out1"));
++  EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat out1 out2 > out12");
 +}
+--- a/src/exit_status.h
++++ b/src/exit_status.h
+@@ -18,7 +18,8 @@
+ enum ExitStatus {
+   ExitSuccess,
+   ExitFailure,
+-  ExitInterrupted
++  ExitTokenAvailable,
++  ExitInterrupted,
+ };
+ 
+ #endif  // NINJA_EXIT_STATUS_H_
+--- a/src/subprocess-posix.cc
++++ b/src/subprocess-posix.cc
+@@ -13,6 +13,7 @@
+ // limitations under the License.
+ 
+ #include "subprocess.h"
++#include "tokenpool.h"
+ 
+ #include <sys/select.h>
+ #include <assert.h>
+@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const str
+ }
+ 
+ #ifdef USE_PPOLL
+-bool SubprocessSet::DoWork() {
++bool SubprocessSet::DoWork(TokenPool* tokens) {
+   vector<pollfd> fds;
+   nfds_t nfds = 0;
+ 
+@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() {
+     ++nfds;
+   }
+ 
++  if (tokens) {
++    pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 };
++    fds.push_back(pfd);
++    ++nfds;
++  }
 +
-+void GNUmakeTokenPoolWin32::WaitForTokenAvailability(HANDLE ioport) {
-+  if (child_ == NULL) {
-+    // first invocation
-+    //
-+    // subprocess-win32.cc uses I/O completion port (IOCP) which can't be
-+    // used as a waitable object. Therefore we can't use WaitMultipleObjects()
-+    // to wait on the IOCP and the token semaphore at the same time. Create
-+    // a child thread that waits on the semaphore and posts an I/O completion
-+    ioport_ = ioport;
-+
-+    // create both semaphores in nonsignaled state
-+    if ((semaphore_enter_wait_ = CreateSemaphore(NULL, 0, 1, NULL))
-+        == NULL)
-+      Win32Fatal("CreateSemaphore/enter_wait");
-+    if ((semaphore_restart_ = CreateSemaphore(NULL, 0, 1, NULL))
-+        == NULL)
-+      Win32Fatal("CreateSemaphore/restart");
+   interrupted_ = 0;
+   int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_);
+   if (ret == -1) {
+@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() {
+     ++i;
+   }
+ 
++  if (tokens) {
++    pollfd *pfd = &fds[nfds - 1];
++    if (pfd->fd >= 0) {
++      assert(pfd->fd == tokens->GetMonitorFd());
++      if (pfd->revents != 0)
++        token_available_ = true;
++    }
++  }
 +
-+    // start child thread
-+    running_ = true;
-+    if ((child_ = CreateThread(NULL, 0, &SemaphoreThreadWrapper, this, 0, NULL))
-+        == NULL)
-+      Win32Fatal("CreateThread");
+   return IsInterrupted();
+ }
+ 
+ #else  // !defined(USE_PPOLL)
+-bool SubprocessSet::DoWork() {
++bool SubprocessSet::DoWork(TokenPool* tokens) {
+   fd_set set;
+   int nfds = 0;
+   FD_ZERO(&set);
+@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() {
+     }
+   }
+ 
++  if (tokens) {
++    int fd = tokens->GetMonitorFd();
++    FD_SET(fd, &set);
++    if (nfds < fd+1)
++      nfds = fd+1;
++  }
 +
-+  } else {
-+    // all further invocations - allow child thread to loop
-+    ReleaseSemaphore(semaphore_restart_);
+   interrupted_ = 0;
+   int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_);
+   if (ret == -1) {
+@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() {
+     ++i;
+   }
+ 
++  if (tokens) {
++    int fd = tokens->GetMonitorFd();
++    if ((fd >= 0) && FD_ISSET(fd, &set))
++    token_available_ = true;
 +  }
 +
-+  // wait for child thread to enter wait
-+  WaitForObject(semaphore_enter_wait_);
-+  // semaphore is now in nonsignaled state again for next run...
+   return IsInterrupted();
+ }
+ #endif  // !defined(USE_PPOLL)
+--- a/src/subprocess-win32.cc
++++ b/src/subprocess-win32.cc
+@@ -13,6 +13,7 @@
+ // limitations under the License.
+ 
+ #include "subprocess.h"
++#include "tokenpool.h"
+ 
+ #include <assert.h>
+ #include <stdio.h>
+@@ -251,11 +252,14 @@ Subprocess *SubprocessSet::Add(const str
+   return subprocess;
+ }
+ 
+-bool SubprocessSet::DoWork() {
++bool SubprocessSet::DoWork(TokenPool* tokens) {
+   DWORD bytes_read;
+   Subprocess* subproc;
+   OVERLAPPED* overlapped;
+ 
++  if (tokens)
++    tokens->WaitForTokenAvailability(ioport_);
 +
-+  // now SubprocessSet::DoWork() can enter GetQueuedCompletionStatus()...
-+}
+   if (!GetQueuedCompletionStatus(ioport_, &bytes_read, (PULONG_PTR)&subproc,
+                                  &overlapped, INFINITE)) {
+     if (GetLastError() != ERROR_BROKEN_PIPE)
+@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork() {
+                 // delivered by NotifyInterrupted above.
+     return true;
+ 
++  if (tokens && tokens->TokenIsAvailable((ULONG_PTR)subproc)) {
++    token_available_ = true;
++    return false;
++  }
 +
-+bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) {
-+  // alert child thread to break wait on token semaphore
-+  QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL);
+   subproc->OnPipeReady();
+ 
+   if (subproc->Done()) {
+--- a/src/subprocess.h
++++ b/src/subprocess.h
+@@ -76,6 +76,8 @@ struct Subprocess {
+   friend struct SubprocessSet;
+ };
+ 
++struct TokenPool;
 +
-+  // return true when GetQueuedCompletionStatus() returned our key
-+  return key == (ULONG_PTR) this;
-+}
+ /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses.
+ /// DoWork() waits for any state change in subprocesses; finished_
+ /// is a queue of subprocesses as they finish.
+@@ -84,13 +86,17 @@ struct SubprocessSet {
+   ~SubprocessSet();
+ 
+   Subprocess* Add(const std::string& command, bool use_console = false);
+-  bool DoWork();
++  bool DoWork(TokenPool* tokens);
+   Subprocess* NextFinished();
+   void Clear();
+ 
+   std::vector<Subprocess*> running_;
+   std::queue<Subprocess*> finished_;
+ 
++  bool token_available_;
++  bool IsTokenAvailable() { return token_available_; }
++  void ResetTokenAvailable() { token_available_ = false; }
 +
-+void GNUmakeTokenPoolWin32::ReleaseSemaphore(HANDLE semaphore) {
-+  if (!::ReleaseSemaphore(semaphore, 1, NULL))
-+    Win32Fatal("ReleaseSemaphore");
-+}
+ #ifdef _WIN32
+   static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType);
+   static HANDLE ioport_;
+--- a/src/subprocess_test.cc
++++ b/src/subprocess_test.cc
+@@ -13,6 +13,7 @@
+ // limitations under the License.
+ 
+ #include "subprocess.h"
++#include "tokenpool.h"
+ 
+ #include "test.h"
+ 
+@@ -34,8 +35,30 @@ const char* kSimpleCommand = "cmd /c dir
+ const char* kSimpleCommand = "ls /";
+ #endif
+ 
++struct TestTokenPool : public TokenPool {
++  bool Acquire()     { return false; }
++  void Reserve()     {}
++  void Release()     {}
++  void Clear()       {}
++  bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; }
 +
-+void GNUmakeTokenPoolWin32::WaitForObject(HANDLE object) {
-+  if (WaitForSingleObject(object, INFINITE) != WAIT_OBJECT_0)
-+    Win32Fatal("WaitForSingleObject");
-+}
++#ifdef _WIN32
++  bool _token_available;
++  void WaitForTokenAvailability(HANDLE ioport) {
++    if (_token_available)
++      // unblock GetQueuedCompletionStatus()
++      PostQueuedCompletionStatus(ioport, 0, (ULONG_PTR) this, NULL);
++  }
++  bool TokenIsAvailable(ULONG_PTR key) { return key == (ULONG_PTR) this; }
++#else
++  int _fd;
++  int GetMonitorFd() { return _fd; }
++#endif
++};
 +
-+struct TokenPool *TokenPool::Get() {
-+  return new GNUmakeTokenPoolWin32;
-+}
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-index 4132bb06d9..92ff611721 100644
---- a/src/tokenpool-gnu-make.cc
-+++ b/src/tokenpool-gnu-make.cc
-@@ -12,101 +12,26 @@
- // See the License for the specific language governing permissions and
- // limitations under the License.
+ struct SubprocessTest : public testing::Test {
+   SubprocessSet subprocs_;
++  TestTokenPool tokens_;
+ };
  
--#include "tokenpool.h"
-+#include "tokenpool-gnu-make.h"
+ }  // anonymous namespace
+@@ -45,10 +68,12 @@ TEST_F(SubprocessTest, BadCommandStderr)
+   Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command");
+   ASSERT_NE((Subprocess *) 0, subproc);
  
--#include <errno.h>
--#include <fcntl.h>
--#include <poll.h>
--#include <unistd.h>
--#include <signal.h>
--#include <sys/time.h>
-+#include <stdlib.h>
- #include <stdio.h>
- #include <string.h>
--#include <stdlib.h>
++  subprocs_.ResetTokenAvailable();
+   while (!subproc->Done()) {
+     // Pretend we discovered that stderr was ready for writing.
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
+   }
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
  
- #include "line_printer.h"
+   EXPECT_EQ(ExitFailure, subproc->Finish());
+   EXPECT_NE("", subproc->GetOutput());
+@@ -59,10 +84,12 @@ TEST_F(SubprocessTest, NoSuchCommand) {
+   Subprocess* subproc = subprocs_.Add("ninja_no_such_command");
+   ASSERT_NE((Subprocess *) 0, subproc);
  
--// TokenPool implementation for GNU make jobserver
--// (http://make.mad-scientist.net/papers/jobserver-implementation/)
--struct GNUmakeTokenPool : public TokenPool {
--  GNUmakeTokenPool();
--  virtual ~GNUmakeTokenPool();
--
--  virtual bool Acquire();
--  virtual void Reserve();
--  virtual void Release();
--  virtual void Clear();
--  virtual int GetMonitorFd();
--
--  bool Setup(bool ignore, bool verbose, double& max_load_average);
--
-- private:
--  int available_;
--  int used_;
--
--#ifdef _WIN32
--  // @TODO
--#else
--  int rfd_;
--  int wfd_;
--
--  struct sigaction old_act_;
--  bool restore_;
--
--  static int dup_rfd_;
--  static void CloseDupRfd(int signum);
--
--  bool CheckFd(int fd);
--  bool SetAlarmHandler();
--#endif
--
--  void Return();
--};
--
-+// TokenPool implementation for GNU make jobserver - common bits
- // every instance owns an implicit token -> available_ == 1
--GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0),
--                                       rfd_(-1), wfd_(-1), restore_(false) {
-+GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) {
- }
++  subprocs_.ResetTokenAvailable();
+   while (!subproc->Done()) {
+     // Pretend we discovered that stderr was ready for writing.
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
+   }
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
  
- GNUmakeTokenPool::~GNUmakeTokenPool() {
--  Clear();
--  if (restore_)
--    sigaction(SIGALRM, &old_act_, NULL);
--}
--
--bool GNUmakeTokenPool::CheckFd(int fd) {
--  if (fd < 0)
--    return false;
--  int ret = fcntl(fd, F_GETFD);
--  if (ret < 0)
--    return false;
--  return true;
--}
--
--int GNUmakeTokenPool::dup_rfd_ = -1;
--
--void GNUmakeTokenPool::CloseDupRfd(int signum) {
--  close(dup_rfd_);
--  dup_rfd_ = -1;
--}
--
--bool GNUmakeTokenPool::SetAlarmHandler() {
--  struct sigaction act;
--  memset(&act, 0, sizeof(act));
--  act.sa_handler = CloseDupRfd;
--  if (sigaction(SIGALRM, &act, &old_act_) < 0) {
--    perror("sigaction:");
--    return(false);
--  } else {
--    restore_ = true;
--    return(true);
--  }
- }
+   EXPECT_EQ(ExitFailure, subproc->Finish());
+   EXPECT_NE("", subproc->GetOutput());
+@@ -78,9 +105,11 @@ TEST_F(SubprocessTest, InterruptChild) {
+   Subprocess* subproc = subprocs_.Add("kill -INT $$");
+   ASSERT_NE((Subprocess *) 0, subproc);
  
- bool GNUmakeTokenPool::Setup(bool ignore,
-                              bool verbose,
-                              double& max_load_average) {
--  const char *value = getenv("MAKEFLAGS");
-+  const char *value = GetEnv("MAKEFLAGS");
-   if (value) {
-     // GNU make <= 4.1
-     const char *jobserver = strstr(value, "--jobserver-fds=");
-@@ -119,20 +44,13 @@ bool GNUmakeTokenPool::Setup(bool ignore,
-       if (ignore) {
-         printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
-       } else {
--        int rfd = -1;
--        int wfd = -1;
--        if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
--            CheckFd(rfd) &&
--            CheckFd(wfd) &&
--            SetAlarmHandler()) {
-+        if (ParseAuth(jobserver)) {
-           const char *l_arg = strstr(value, " -l");
-           int load_limit = -1;
++  subprocs_.ResetTokenAvailable();
+   while (!subproc->Done()) {
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
+   }
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
  
-           if (verbose) {
-             printer.PrintOnNewLine("ninja: using GNU make jobserver.\n");
-           }
--          rfd_ = rfd;
--          wfd_ = wfd;
+   EXPECT_EQ(ExitInterrupted, subproc->Finish());
+ }
+@@ -90,7 +119,7 @@ TEST_F(SubprocessTest, InterruptParent)
+   ASSERT_NE((Subprocess *) 0, subproc);
  
-           // translate GNU make -lN to ninja -lN
-           if (l_arg &&
-@@ -154,83 +72,14 @@ bool GNUmakeTokenPool::Acquire() {
-   if (available_ > 0)
-     return true;
+   while (!subproc->Done()) {
+-    bool interrupted = subprocs_.DoWork();
++    bool interrupted = subprocs_.DoWork(NULL);
+     if (interrupted)
+       return;
+   }
+@@ -102,9 +131,11 @@ TEST_F(SubprocessTest, InterruptChildWit
+   Subprocess* subproc = subprocs_.Add("kill -TERM $$");
+   ASSERT_NE((Subprocess *) 0, subproc);
  
--  // Please read
--  //
--  //   http://make.mad-scientist.net/papers/jobserver-implementation/
--  //
--  // for the reasoning behind the following code.
--  //
--  // Try to read one character from the pipe. Returns true on success.
--  //
--  // First check if read() would succeed without blocking.
--#ifdef USE_PPOLL
--  pollfd pollfds[] = {{rfd_, POLLIN, 0}};
--  int ret = poll(pollfds, 1, 0);
--#else
--  fd_set set;
--  struct timeval timeout = { 0, 0 };
--  FD_ZERO(&set);
--  FD_SET(rfd_, &set);
--  int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
--#endif
--  if (ret > 0) {
--    // Handle potential race condition:
--    //  - the above check succeeded, i.e. read() should not block
--    //  - the character disappears before we call read()
--    //
--    // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_
--    // can safely be closed by signal handlers without affecting rfd_.
--    dup_rfd_ = dup(rfd_);
--
--    if (dup_rfd_ != -1) {
--      struct sigaction act, old_act;
--      int ret = 0;
--
--      // Temporarily replace SIGCHLD handler with our own
--      memset(&act, 0, sizeof(act));
--      act.sa_handler = CloseDupRfd;
--      if (sigaction(SIGCHLD, &act, &old_act) == 0) {
--        struct itimerval timeout;
--
--        // install a 100ms timeout that generates SIGALARM on expiration
--        memset(&timeout, 0, sizeof(timeout));
--        timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec]
--        if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) {
--          char buf;
--
--          // Now try to read() from dup_rfd_. Return values from read():
--          //
--          // 1. token read                               ->  1
--          // 2. pipe closed                              ->  0
--          // 3. alarm expires                            -> -1 (EINTR)
--          // 4. child exits                              -> -1 (EINTR)
--          // 5. alarm expired before entering read()     -> -1 (EBADF)
--          // 6. child exited before entering read()      -> -1 (EBADF)
--          // 7. child exited before handler is installed -> go to 1 - 3
--          ret = read(dup_rfd_, &buf, 1);
--
--          // disarm timer
--          memset(&timeout, 0, sizeof(timeout));
--          setitimer(ITIMER_REAL, &timeout, NULL);
--        }
--
--        sigaction(SIGCHLD, &old_act, NULL);
--      }
--
--      CloseDupRfd(0);
--
--      // Case 1 from above list
--      if (ret > 0) {
--        available_++;
--        return true;
--      }
--    }
-+  if (AcquireToken()) {
-+    // token acquired
-+    available_++;
-+    return true;
-+  } else {
-+    // no token available
-+    return false;
++  subprocs_.ResetTokenAvailable();
+   while (!subproc->Done()) {
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
    }
--
--  // read() would block, i.e. no token available,
--  // cases 2-6 from above list or
--  // select() / poll() / dup() / sigaction() / setitimer() failed
--  return false;
- }
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
  
- void GNUmakeTokenPool::Reserve() {
-@@ -239,15 +88,8 @@ void GNUmakeTokenPool::Reserve() {
+   EXPECT_EQ(ExitInterrupted, subproc->Finish());
  }
+@@ -114,7 +145,7 @@ TEST_F(SubprocessTest, InterruptParentWi
+   ASSERT_NE((Subprocess *) 0, subproc);
  
- void GNUmakeTokenPool::Return() {
--  const char buf = '+';
--  while (1) {
--    int ret = write(wfd_, &buf, 1);
--    if (ret > 0)
--      available_--;
--    if ((ret != -1) || (errno != EINTR))
--      return;
--    // write got interrupted - retry
--  }
-+  if (ReturnToken())
-+    available_--;
- }
+   while (!subproc->Done()) {
+-    bool interrupted = subprocs_.DoWork();
++    bool interrupted = subprocs_.DoWork(NULL);
+     if (interrupted)
+       return;
+   }
+@@ -126,9 +157,11 @@ TEST_F(SubprocessTest, InterruptChildWit
+   Subprocess* subproc = subprocs_.Add("kill -HUP $$");
+   ASSERT_NE((Subprocess *) 0, subproc);
+ 
++  subprocs_.ResetTokenAvailable();
+   while (!subproc->Done()) {
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
+   }
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
  
- void GNUmakeTokenPool::Release() {
-@@ -263,18 +105,3 @@ void GNUmakeTokenPool::Clear() {
-   while (available_ > 1)
-     Return();
+   EXPECT_EQ(ExitInterrupted, subproc->Finish());
  }
+@@ -138,7 +171,7 @@ TEST_F(SubprocessTest, InterruptParentWi
+   ASSERT_NE((Subprocess *) 0, subproc);
+ 
+   while (!subproc->Done()) {
+-    bool interrupted = subprocs_.DoWork();
++    bool interrupted = subprocs_.DoWork(NULL);
+     if (interrupted)
+       return;
+   }
+@@ -153,9 +186,11 @@ TEST_F(SubprocessTest, Console) {
+         subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true);
+     ASSERT_NE((Subprocess*)0, subproc);
+ 
++    subprocs_.ResetTokenAvailable();
+     while (!subproc->Done()) {
+-      subprocs_.DoWork();
++      subprocs_.DoWork(NULL);
+     }
++    ASSERT_FALSE(subprocs_.IsTokenAvailable());
+ 
+     EXPECT_EQ(ExitSuccess, subproc->Finish());
+   }
+@@ -167,9 +202,11 @@ TEST_F(SubprocessTest, SetWithSingle) {
+   Subprocess* subproc = subprocs_.Add(kSimpleCommand);
+   ASSERT_NE((Subprocess *) 0, subproc);
+ 
++  subprocs_.ResetTokenAvailable();
+   while (!subproc->Done()) {
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
+   }
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
+   ASSERT_EQ(ExitSuccess, subproc->Finish());
+   ASSERT_NE("", subproc->GetOutput());
+ 
+@@ -200,12 +237,13 @@ TEST_F(SubprocessTest, SetWithMulti) {
+     ASSERT_EQ("", processes[i]->GetOutput());
+   }
+ 
++  subprocs_.ResetTokenAvailable();
+   while (!processes[0]->Done() || !processes[1]->Done() ||
+          !processes[2]->Done()) {
+     ASSERT_GT(subprocs_.running_.size(), 0u);
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
+   }
 -
--int GNUmakeTokenPool::GetMonitorFd() {
--  return(rfd_);
--}
--
--struct TokenPool *TokenPool::Get(bool ignore,
--                                 bool verbose,
--                                 double& max_load_average) {
--  GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
--  if (tokenpool->Setup(ignore, verbose, max_load_average))
--    return tokenpool;
--  else
--    delete tokenpool;
--  return NULL;
--}
-diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h
-new file mode 100644
-index 0000000000..d3852088e2
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
+   ASSERT_EQ(0u, subprocs_.running_.size());
+   ASSERT_EQ(3u, subprocs_.finished_.size());
+ 
+@@ -237,8 +275,10 @@ TEST_F(SubprocessTest, SetWithLots) {
+     ASSERT_NE((Subprocess *) 0, subproc);
+     procs.push_back(subproc);
+   }
++  subprocs_.ResetTokenAvailable();
+   while (!subprocs_.running_.empty())
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
+   for (size_t i = 0; i < procs.size(); ++i) {
+     ASSERT_EQ(ExitSuccess, procs[i]->Finish());
+     ASSERT_NE("", procs[i]->GetOutput());
+@@ -254,10 +294,91 @@ TEST_F(SubprocessTest, SetWithLots) {
+ // that stdin is closed.
+ TEST_F(SubprocessTest, ReadStdin) {
+   Subprocess* subproc = subprocs_.Add("cat -");
++  subprocs_.ResetTokenAvailable();
+   while (!subproc->Done()) {
+-    subprocs_.DoWork();
++    subprocs_.DoWork(NULL);
+   }
++  ASSERT_FALSE(subprocs_.IsTokenAvailable());
+   ASSERT_EQ(ExitSuccess, subproc->Finish());
+   ASSERT_EQ(1u, subprocs_.finished_.size());
+ }
+ #endif  // _WIN32
++
++TEST_F(SubprocessTest, TokenAvailable) {
++  Subprocess* subproc = subprocs_.Add(kSimpleCommand);
++  ASSERT_NE((Subprocess *) 0, subproc);
++
++  // simulate GNUmake jobserver pipe with 1 token
++#ifdef _WIN32
++  tokens_._token_available = true;
++#else
++  int fds[2];
++  ASSERT_EQ(0u, pipe(fds));
++  tokens_._fd = fds[0];
++  ASSERT_EQ(1u, write(fds[1], "T", 1));
++#endif
++
++  subprocs_.ResetTokenAvailable();
++  subprocs_.DoWork(&tokens_);
++#ifdef _WIN32
++  tokens_._token_available = false;
++  // we need to loop here as we have no control where the token
++  // I/O completion post ends up in the queue
++  while (!subproc->Done() && !subprocs_.IsTokenAvailable()) {
++    subprocs_.DoWork(&tokens_);
++  }
++#endif
++
++  EXPECT_TRUE(subprocs_.IsTokenAvailable());
++  EXPECT_EQ(0u, subprocs_.finished_.size());
++
++  // remove token to let DoWork() wait for command again
++#ifndef _WIN32
++  char token;
++  ASSERT_EQ(1u, read(fds[0], &token, 1));
++#endif
++
++  while (!subproc->Done()) {
++    subprocs_.DoWork(&tokens_);
++  }
++
++#ifndef _WIN32
++  close(fds[1]);
++  close(fds[0]);
++#endif
++
++  EXPECT_EQ(ExitSuccess, subproc->Finish());
++  EXPECT_NE("", subproc->GetOutput());
++
++  EXPECT_EQ(1u, subprocs_.finished_.size());
++}
++
++TEST_F(SubprocessTest, TokenNotAvailable) {
++  Subprocess* subproc = subprocs_.Add(kSimpleCommand);
++  ASSERT_NE((Subprocess *) 0, subproc);
++
++  // simulate GNUmake jobserver pipe with 0 tokens
++#ifdef _WIN32
++  tokens_._token_available = false;
++#else
++  int fds[2];
++  ASSERT_EQ(0u, pipe(fds));
++  tokens_._fd = fds[0];
++#endif
++
++  subprocs_.ResetTokenAvailable();
++  while (!subproc->Done()) {
++    subprocs_.DoWork(&tokens_);
++  }
++
++#ifndef _WIN32
++  close(fds[1]);
++  close(fds[0]);
++#endif
++
++  EXPECT_FALSE(subprocs_.IsTokenAvailable());
++  EXPECT_EQ(ExitSuccess, subproc->Finish());
++  EXPECT_NE("", subproc->GetOutput());
++
++  EXPECT_EQ(1u, subprocs_.finished_.size());
++}
+--- a/src/ninja.cc
++++ b/src/ninja.cc
+@@ -1447,6 +1447,7 @@ int ReadFlags(int* argc, char*** argv,
+         // We want to run N jobs in parallel. For N = 0, INT_MAX
+         // is close enough to infinite for most sane builds.
+         config->parallelism = value > 0 ? value : INT_MAX;
++        config->parallelism_from_cmdline = true;
+         deferGuessParallelism.needGuess = false;
+         break;
+       }
 --- /dev/null
-+++ b/src/tokenpool-gnu-make.h
-@@ -0,0 +1,40 @@
-+// Copyright 2016-2018 Google Inc. All Rights Reserved.
++++ b/src/tokenpool_test.cc
+@@ -0,0 +1,279 @@
++// Copyright 2018 Google Inc. All Rights Reserved.
 +//
 +// Licensed under the Apache License, Version 2.0 (the "License");
 +// you may not use this file except in compliance with the License.
@@ -3356,107 +1372,17 @@ index 0000000000..d3852088e2
 +
 +#include "tokenpool.h"
 +
-+// interface to GNU make token pool
-+struct GNUmakeTokenPool : public TokenPool {
-+  GNUmakeTokenPool();
-+  virtual ~GNUmakeTokenPool();
-+
-+  // token pool implementation
-+  virtual bool Acquire();
-+  virtual void Reserve();
-+  virtual void Release();
-+  virtual void Clear();
-+  virtual bool Setup(bool ignore, bool verbose, double& max_load_average);
-+
-+  // platform specific implementation
-+  virtual const char *GetEnv(const char *name) = 0;
-+  virtual bool ParseAuth(const char *jobserver) = 0;
-+  virtual bool AcquireToken() = 0;
-+  virtual bool ReturnToken() = 0;
-+
-+ private:
-+  int available_;
-+  int used_;
-+
-+  void Return();
-+};
-diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
-index 4c592875b4..613d16882d 100644
---- a/src/tokenpool-none.cc
-+++ b/src/tokenpool-none.cc
-@@ -17,8 +17,6 @@
- #include <stdlib.h>
- 
- // No-op TokenPool implementation
--struct TokenPool *TokenPool::Get(bool ignore,
--                                 bool verbose,
--                                 double& max_load_average) {
-+struct TokenPool *TokenPool::Get() {
-   return NULL;
- }
-diff --git a/src/tokenpool.h b/src/tokenpool.h
-index 4bf477f20c..1be8e1d5ce 100644
---- a/src/tokenpool.h
-+++ b/src/tokenpool.h
-@@ -1,4 +1,4 @@
--// Copyright 2016-2017 Google Inc. All Rights Reserved.
-+// Copyright 2016-2018 Google Inc. All Rights Reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
-@@ -12,6 +12,10 @@
- // See the License for the specific language governing permissions and
- // limitations under the License.
- 
-+#ifdef _WIN32
-+#include <windows.h>
-+#endif
-+
- // interface to token pool
- struct TokenPool {
-   virtual ~TokenPool() {}
-@@ -21,14 +25,18 @@ struct TokenPool {
-   virtual void Release() = 0;
-   virtual void Clear() = 0;
- 
-+  // returns false if token pool setup failed
-+  virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0;
++#include "test.h"
 +
- #ifdef _WIN32
--  // @TODO
-+  virtual void WaitForTokenAvailability(HANDLE ioport) = 0;
-+  // returns true if a token has become available
-+  // key is result from GetQueuedCompletionStatus()
-+  virtual bool TokenIsAvailable(ULONG_PTR key) = 0;
- #else
-   virtual int GetMonitorFd() = 0;
- #endif
- 
-   // returns NULL if token pool is not available
--  static struct TokenPool *Get(bool ignore,
--                               bool verbose,
--                               double& max_load_average);
-+  static struct TokenPool *Get();
- };
-diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc
-index 6c89064ca4..8d4fd7d33a 100644
---- a/src/tokenpool_test.cc
-+++ b/src/tokenpool_test.cc
-@@ -16,13 +16,25 @@
- 
- #include "test.h"
- 
--#ifndef _WIN32
 +#ifdef _WIN32
 +#include <windows.h>
 +#else
 +#include <unistd.h>
 +#endif
 +
- #include <stdio.h>
- #include <stdlib.h>
--#include <unistd.h>
- 
++#include <stdio.h>
++#include <stdlib.h>
++
 +#ifdef _WIN32
 +// should contain all valid characters
 +#define SEMAPHORE_NAME      "abcdefghijklmnopqrstwxyz01234567890_"
@@ -3465,45 +1391,44 @@ index 6c89064ca4..8d4fd7d33a 100644
 +#define ENVIRONMENT_INIT(v) SetEnvironmentVariable("MAKEFLAGS", v)
 +#else
 +#define AUTH_FORMAT(tmpl)   "foo " tmpl "=%d,%d bar"
- #define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS")
--#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true);
++#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS")
 +#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true)
- #endif
- 
- namespace {
-@@ -32,43 +44,60 @@ const double kLoadAverageDefault = -1.23456789;
- struct TokenPoolTest : public testing::Test {
-   double load_avg_;
-   TokenPool *tokens_;
--#ifndef _WIN32
-   char buf_[1024];
++#endif
++
++namespace {
++
++const double kLoadAverageDefault = -1.23456789;
++
++struct TokenPoolTest : public testing::Test {
++  double load_avg_;
++  TokenPool* tokens_;
++  char buf_[1024];
 +#ifdef _WIN32
-+  const char *semaphore_name_;
++  const char* semaphore_name_;
 +  HANDLE semaphore_;
 +#else
-   int fds_[2];
- #endif
- 
-   virtual void SetUp() {
-     load_avg_ = kLoadAverageDefault;
-     tokens_ = NULL;
--#ifndef _WIN32
-     ENVIRONMENT_CLEAR();
++  int fds_[2];
++
++  char random() {
++    return int((rand() / double(RAND_MAX)) * 256);
++  }
++#endif
++
++  virtual void SetUp() {
++    load_avg_ = kLoadAverageDefault;
++    tokens_ = NULL;
++    ENVIRONMENT_CLEAR();
 +#ifdef _WIN32
 +    semaphore_name_ = SEMAPHORE_NAME;
 +    if ((semaphore_ = CreateSemaphore(0, 0, 2, SEMAPHORE_NAME)) == NULL)
 +#else
-     if (pipe(fds_) < 0)
--      ASSERT_TRUE(false);
- #endif
++    if (pipe(fds_) < 0)
++#endif
 +      ASSERT_TRUE(false);
-   }
- 
--  void CreatePool(const char *format, bool ignore_jobserver) {
--#ifndef _WIN32
-+  void CreatePool(const char *format, bool ignore_jobserver = false) {
-     if (format) {
--      sprintf(buf_, format, fds_[0], fds_[1]);
++  }
++
++  void CreatePool(const char* format, bool ignore_jobserver = false) {
++    if (format) {
 +      sprintf(buf_, format,
 +#ifdef _WIN32
 +              semaphore_name_
@@ -3511,69 +1436,73 @@ index 6c89064ca4..8d4fd7d33a 100644
 +              fds_[0], fds_[1]
 +#endif
 +      );
-       ENVIRONMENT_INIT(buf_);
-     }
--#endif
--    tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_);
++      ENVIRONMENT_INIT(buf_);
++    }
 +    if ((tokens_ = TokenPool::Get()) != NULL) {
 +      if (!tokens_->Setup(ignore_jobserver, false, load_avg_)) {
 +        delete tokens_;
 +        tokens_ = NULL;
 +      }
 +    }
-   }
- 
-   void CreateDefaultPool() {
--    CreatePool("foo --jobserver-auth=%d,%d bar", false);
++  }
++
++  void CreateDefaultPool() {
 +    CreatePool(AUTH_FORMAT("--jobserver-auth"));
-   }
- 
-   virtual void TearDown() {
-     if (tokens_)
-       delete tokens_;
--#ifndef _WIN32
++  }
++
++  virtual void TearDown() {
++    if (tokens_)
++      delete tokens_;
 +#ifdef _WIN32
 +    CloseHandle(semaphore_);
 +#else
-     close(fds_[0]);
-     close(fds_[1]);
--    ENVIRONMENT_CLEAR();
- #endif
++    close(fds_[0]);
++    close(fds_[1]);
++#endif
 +    ENVIRONMENT_CLEAR();
-   }
- };
- 
-@@ -82,10 +111,9 @@ TEST_F(TokenPoolTest, NoTokenPool) {
-   EXPECT_EQ(kLoadAverageDefault, load_avg_);
- }
- 
--#ifndef _WIN32
- TEST_F(TokenPoolTest, SuccessfulOldSetup) {
-   // GNUmake <= 4.1
--  CreatePool("foo --jobserver-fds=%d,%d bar", false);
++  }
++};
++
++} // anonymous namespace
++
++// verifies none implementation
++TEST_F(TokenPoolTest, NoTokenPool) {
++  CreatePool(NULL, false);
++
++  EXPECT_EQ(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++}
++
++TEST_F(TokenPoolTest, SuccessfulOldSetup) {
++  // GNUmake <= 4.1
 +  CreatePool(AUTH_FORMAT("--jobserver-fds"));
- 
-   EXPECT_NE(NULL, tokens_);
-   EXPECT_EQ(kLoadAverageDefault, load_avg_);
-@@ -100,19 +128,37 @@ TEST_F(TokenPoolTest, SuccessfulNewSetup) {
- }
- 
- TEST_F(TokenPoolTest, IgnoreWithJN) {
--  CreatePool("foo --jobserver-auth=%d,%d bar", true);
++
++  EXPECT_NE(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++}
++
++TEST_F(TokenPoolTest, SuccessfulNewSetup) {
++  // GNUmake => 4.2
++  CreateDefaultPool();
++
++  EXPECT_NE(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++}
++
++TEST_F(TokenPoolTest, IgnoreWithJN) {
 +  CreatePool(AUTH_FORMAT("--jobserver-auth"), true);
- 
-   EXPECT_EQ(NULL, tokens_);
-   EXPECT_EQ(kLoadAverageDefault, load_avg_);
- }
- 
- TEST_F(TokenPoolTest, HonorLN) {
--  CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false);
++
++  EXPECT_EQ(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++}
++
++TEST_F(TokenPoolTest, HonorLN) {
 +  CreatePool(AUTH_FORMAT("-l9 --jobserver-auth"));
- 
-   EXPECT_NE(NULL, tokens_);
-   EXPECT_EQ(9.0, load_avg_);
- }
- 
++
++  EXPECT_NE(NULL, tokens_);
++  EXPECT_EQ(9.0, load_avg_);
++}
++
 +#ifdef _WIN32
 +TEST_F(TokenPoolTest, SemaphoreNotFound) {
 +  semaphore_name_ = SEMAPHORE_NAME "_foobar";
@@ -3592,66 +1521,104 @@ index 6c89064ca4..8d4fd7d33a 100644
 +  EXPECT_TRUE(tokens_->TokenIsAvailable((ULONG_PTR)tokens_));
 +}
 +#else
- TEST_F(TokenPoolTest, MonitorFD) {
-   CreateDefaultPool();
- 
-@@ -121,6 +167,7 @@ TEST_F(TokenPoolTest, MonitorFD) {
- 
-   EXPECT_EQ(fds_[0], tokens_->GetMonitorFd());
- }
++TEST_F(TokenPoolTest, MonitorFD) {
++  CreateDefaultPool();
++
++  ASSERT_NE(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++
++  EXPECT_EQ(fds_[0], tokens_->GetMonitorFd());
++}
 +#endif
- 
- TEST_F(TokenPoolTest, ImplicitToken) {
-   CreateDefaultPool();
-@@ -147,7 +194,13 @@ TEST_F(TokenPoolTest, TwoTokens) {
-   EXPECT_FALSE(tokens_->Acquire());
- 
-   // jobserver offers 2nd token
++
++TEST_F(TokenPoolTest, ImplicitToken) {
++  CreateDefaultPool();
++
++  ASSERT_NE(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++
++  EXPECT_TRUE(tokens_->Acquire());
++  tokens_->Reserve();
++  EXPECT_FALSE(tokens_->Acquire());
++  tokens_->Release();
++  EXPECT_TRUE(tokens_->Acquire());
++}
++
++TEST_F(TokenPoolTest, TwoTokens) {
++  CreateDefaultPool();
++
++  ASSERT_NE(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++
++  // implicit token
++  EXPECT_TRUE(tokens_->Acquire());
++  tokens_->Reserve();
++  EXPECT_FALSE(tokens_->Acquire());
++
++  // jobserver offers 2nd token
 +#ifdef _WIN32
 +  LONG previous;
 +  ASSERT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous));
 +  ASSERT_EQ(0, previous);
 +#else
-   ASSERT_EQ(1u, write(fds_[1], "T", 1));
++  char test_tokens[1] = { random() };
++  ASSERT_EQ(1u, write(fds_[1], test_tokens, sizeof(test_tokens)));
 +#endif
-   EXPECT_TRUE(tokens_->Acquire());
-   tokens_->Reserve();
-   EXPECT_FALSE(tokens_->Acquire());
-@@ -160,8 +213,14 @@ TEST_F(TokenPoolTest, TwoTokens) {
-   tokens_->Release();
-   EXPECT_TRUE(tokens_->Acquire());
- 
--  // there must be one token in the pipe
++  EXPECT_TRUE(tokens_->Acquire());
++  tokens_->Reserve();
++  EXPECT_FALSE(tokens_->Acquire());
++
++  // release 2nd token
++  tokens_->Release();
++  EXPECT_TRUE(tokens_->Acquire());
++
++  // release implicit token - must return 2nd token back to jobserver
++  tokens_->Release();
++  EXPECT_TRUE(tokens_->Acquire());
++
 +  // there must be one token available
 +#ifdef _WIN32
 +  EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0));
 +  EXPECT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous));
 +  EXPECT_EQ(0, previous);
 +#else
-   EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_)));
++  EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_)));
++  EXPECT_EQ(test_tokens[0], buf_[0]);
 +#endif
- 
-   // implicit token
-   EXPECT_TRUE(tokens_->Acquire());
-@@ -179,7 +238,13 @@ TEST_F(TokenPoolTest, Clear) {
-   EXPECT_FALSE(tokens_->Acquire());
- 
-   // jobserver offers 2nd & 3rd token
++
++  // implicit token
++  EXPECT_TRUE(tokens_->Acquire());
++}
++
++TEST_F(TokenPoolTest, Clear) {
++  CreateDefaultPool();
++
++  ASSERT_NE(NULL, tokens_);
++  EXPECT_EQ(kLoadAverageDefault, load_avg_);
++
++  // implicit token
++  EXPECT_TRUE(tokens_->Acquire());
++  tokens_->Reserve();
++  EXPECT_FALSE(tokens_->Acquire());
++
++  // jobserver offers 2nd & 3rd token
 +#ifdef _WIN32
 +  LONG previous;
 +  ASSERT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous));
 +  ASSERT_EQ(0, previous);
 +#else
-   ASSERT_EQ(2u, write(fds_[1], "TT", 2));
++  char test_tokens[2] = { random(), random() };
++  ASSERT_EQ(2u, write(fds_[1], test_tokens, sizeof(test_tokens)));
 +#endif
-   EXPECT_TRUE(tokens_->Acquire());
-   tokens_->Reserve();
-   EXPECT_TRUE(tokens_->Acquire());
-@@ -189,10 +254,16 @@ TEST_F(TokenPoolTest, Clear) {
-   tokens_->Clear();
-   EXPECT_TRUE(tokens_->Acquire());
- 
--  // there must be two tokens in the pipe
++  EXPECT_TRUE(tokens_->Acquire());
++  tokens_->Reserve();
++  EXPECT_TRUE(tokens_->Acquire());
++  tokens_->Reserve();
++  EXPECT_FALSE(tokens_->Acquire());
++
++  tokens_->Clear();
++  EXPECT_TRUE(tokens_->Acquire());
++
 +  // there must be two tokens available
 +#ifdef _WIN32
 +  EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0));
@@ -3659,632 +1626,547 @@ index 6c89064ca4..8d4fd7d33a 100644
 +  EXPECT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous));
 +  EXPECT_EQ(0, previous);
 +#else
-   EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_)));
++  EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_)));
++  // tokens are pushed onto a stack, hence returned in reverse order
++  EXPECT_EQ(test_tokens[0], buf_[1]);
++  EXPECT_EQ(test_tokens[1], buf_[0]);
 +#endif
- 
-   // implicit token
-   EXPECT_TRUE(tokens_->Acquire());
- }
--#endif
-
-From 2b9c81c0ec1226d8795e7725529f13be41eaa385 Mon Sep 17 00:00:00 2001
-From: Stefan Becker <chemobejk at gmail.com>
-Date: Fri, 14 Dec 2018 13:27:11 +0200
-Subject: [PATCH 11/11] Prepare PR for merging - part II
-
-- remove unnecessary "struct" from TokenPool
-- add PAPCFUNC cast to QueryUserAPC()
-- remove hard-coded MAKEFLAGS string from win32
-- remove useless build test CompleteNoWork
-- rename TokenPoolTest to TestTokenPool
-- add tokenpool modules to CMake build
-- remove unused no-op TokenPool implementation
-- fix errors flagged by codespell & clang-tidy
-- address review comments from
-
-https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-195195803
-https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-185089255
-https://github.com/ninja-build/ninja/pull/1140#issuecomment-473898963
-https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610
----
- CMakeLists.txt                  |  8 ++++-
- src/build.cc                    |  2 +-
- src/build_test.cc               | 12 +------
- src/subprocess-posix.cc         |  4 +--
- src/subprocess-win32.cc         |  2 +-
- src/subprocess.h                |  2 +-
- src/subprocess_test.cc          | 26 +++++++-------
- src/tokenpool-gnu-make-posix.cc | 21 +++++------
- src/tokenpool-gnu-make-win32.cc | 36 ++++++++++---------
- src/tokenpool-gnu-make.cc       | 63 +++++++++++++++++----------------
- src/tokenpool-gnu-make.h        |  6 ++--
- src/tokenpool-none.cc           | 22 ------------
- src/tokenpool.h                 |  2 +-
- src/tokenpool_test.cc           |  8 ++---
- 14 files changed, 94 insertions(+), 120 deletions(-)
- delete mode 100644 src/tokenpool-none.cc
-
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 57ae548f5b..e2876fe413 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -112,6 +112,7 @@ add_library(libninja OBJECT
- 	src/state.cc
- 	src/status.cc
- 	src/string_piece_util.cc
-+	src/tokenpool-gnu-make.cc
- 	src/util.cc
- 	src/version.cc
- )
-@@ -123,9 +124,13 @@ if(WIN32)
- 		src/msvc_helper_main-win32.cc
- 		src/getopt.c
- 		src/minidump-win32.cc
-+		src/tokenpool-gnu-make-win32.cc
- 	)
- else()
--	target_sources(libninja PRIVATE src/subprocess-posix.cc)
-+	target_sources(libninja PRIVATE
-+		src/subprocess-posix.cc
-+		src/tokenpool-gnu-make-posix.cc
-+	)
- 	if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
- 		target_sources(libninja PRIVATE src/getopt.c)
- 	endif()
-@@ -204,6 +209,7 @@ if(BUILD_TESTING)
-     src/string_piece_util_test.cc
-     src/subprocess_test.cc
-     src/test.cc
-+    src/tokenpool_test.cc
-     src/util_test.cc
-   )
-   if(WIN32)
-diff --git a/src/build.cc b/src/build.cc
-index 20c3bdc2a0..854df08c2a 100644
---- a/src/build.cc
-+++ b/src/build.cc
-@@ -467,7 +467,7 @@ struct RealCommandRunner : public CommandRunner {
-   // copy of config_.max_load_average; can be modified by TokenPool setup
-   double max_load_average_;
-   SubprocessSet subprocs_;
--  TokenPool *tokens_;
-+  TokenPool* tokens_;
-   map<const Subprocess*, Edge*> subproc_to_edge_;
- };
- 
-diff --git a/src/build_test.cc b/src/build_test.cc
-index dd41dfbe1d..8901c9518f 100644
---- a/src/build_test.cc
-+++ b/src/build_test.cc
-@@ -4098,7 +4098,7 @@ struct BuildTokenTest : public BuildTest {
-   void ExpectWaitForCommand(int count, ...);
- 
- private:
--  void EnqueueBooleans(vector<bool>& booleans, int count, va_list ao);
-+  void EnqueueBooleans(vector<bool>& booleans, int count, va_list ap);
- };
- 
- void BuildTokenTest::SetUp() {
-@@ -4144,16 +4144,6 @@ void BuildTokenTest::EnqueueBooleans(vector<bool>& booleans, int count, va_list
-   }
- }
- 
--TEST_F(BuildTokenTest, CompleteNoWork) {
--  // plan should not execute anything
--  string err;
--
--  EXPECT_TRUE(builder_.Build(&err));
--  EXPECT_EQ("", err);
--
--  EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
--}
--
- TEST_F(BuildTokenTest, DoNotAquireToken) {
-   // plan should execute one command
-   string err;
-diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc
-index 74451b0be2..31839276c4 100644
---- a/src/subprocess-posix.cc
-+++ b/src/subprocess-posix.cc
-@@ -250,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
- }
- 
- #ifdef USE_PPOLL
--bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-+bool SubprocessSet::DoWork(TokenPool* tokens) {
-   vector<pollfd> fds;
-   nfds_t nfds = 0;
- 
-@@ -315,7 +315,7 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) {
- }
- 
- #else  // !defined(USE_PPOLL)
--bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-+bool SubprocessSet::DoWork(TokenPool* tokens) {
-   fd_set set;
-   int nfds = 0;
-   FD_ZERO(&set);
-diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc
-index ce3e2c20a4..2926e9a221 100644
---- a/src/subprocess-win32.cc
-+++ b/src/subprocess-win32.cc
-@@ -252,7 +252,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
-   return subprocess;
- }
- 
--bool SubprocessSet::DoWork(struct TokenPool* tokens) {
-+bool SubprocessSet::DoWork(TokenPool* tokens) {
-   DWORD bytes_read;
-   Subprocess* subproc;
-   OVERLAPPED* overlapped;
-diff --git a/src/subprocess.h b/src/subprocess.h
-index 9ea67ea477..1ec78171e8 100644
---- a/src/subprocess.h
-+++ b/src/subprocess.h
-@@ -86,7 +86,7 @@ struct SubprocessSet {
-   ~SubprocessSet();
- 
-   Subprocess* Add(const std::string& command, bool use_console = false);
--  bool DoWork(struct TokenPool* tokens);
-+  bool DoWork(TokenPool* tokens);
-   Subprocess* NextFinished();
-   void Clear();
- 
-diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
-index f625963462..7b146f31be 100644
---- a/src/subprocess_test.cc
-+++ b/src/subprocess_test.cc
-@@ -35,7 +35,7 @@ const char* kSimpleCommand = "cmd /c dir \\";
- const char* kSimpleCommand = "ls /";
- #endif
- 
--struct TokenPoolTest : public TokenPool {
-+struct TestTokenPool : public TokenPool {
-   bool Acquire()     { return false; }
-   void Reserve()     {}
-   void Release()     {}
-@@ -58,7 +58,7 @@ struct TokenPoolTest : public TokenPool {
- 
- struct SubprocessTest : public testing::Test {
-   SubprocessSet subprocs_;
--  TokenPoolTest tokens_;
-+  TestTokenPool tokens_;
- };
- 
- }  // anonymous namespace
-@@ -73,7 +73,7 @@ TEST_F(SubprocessTest, BadCommandStderr) {
-     // Pretend we discovered that stderr was ready for writing.
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitFailure, subproc->Finish());
-   EXPECT_NE("", subproc->GetOutput());
-@@ -89,7 +89,7 @@ TEST_F(SubprocessTest, NoSuchCommand) {
-     // Pretend we discovered that stderr was ready for writing.
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitFailure, subproc->Finish());
-   EXPECT_NE("", subproc->GetOutput());
-@@ -109,7 +109,7 @@ TEST_F(SubprocessTest, InterruptChild) {
-   while (!subproc->Done()) {
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitInterrupted, subproc->Finish());
- }
-@@ -135,7 +135,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) {
-   while (!subproc->Done()) {
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitInterrupted, subproc->Finish());
- }
-@@ -161,7 +161,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) {
-   while (!subproc->Done()) {
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
- 
-   EXPECT_EQ(ExitInterrupted, subproc->Finish());
- }
-@@ -190,7 +190,7 @@ TEST_F(SubprocessTest, Console) {
-     while (!subproc->Done()) {
-       subprocs_.DoWork(NULL);
-     }
--    ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+    ASSERT_FALSE(subprocs_.IsTokenAvailable());
- 
-     EXPECT_EQ(ExitSuccess, subproc->Finish());
-   }
-@@ -206,7 +206,7 @@ TEST_F(SubprocessTest, SetWithSingle) {
-   while (!subproc->Done()) {
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
-   ASSERT_EQ(ExitSuccess, subproc->Finish());
-   ASSERT_NE("", subproc->GetOutput());
- 
-@@ -243,7 +243,7 @@ TEST_F(SubprocessTest, SetWithMulti) {
-     ASSERT_GT(subprocs_.running_.size(), 0u);
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
-   ASSERT_EQ(0u, subprocs_.running_.size());
-   ASSERT_EQ(3u, subprocs_.finished_.size());
- 
-@@ -278,7 +278,7 @@ TEST_F(SubprocessTest, SetWithLots) {
-   subprocs_.ResetTokenAvailable();
-   while (!subprocs_.running_.empty())
-     subprocs_.DoWork(NULL);
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
-   for (size_t i = 0; i < procs.size(); ++i) {
-     ASSERT_EQ(ExitSuccess, procs[i]->Finish());
-     ASSERT_NE("", procs[i]->GetOutput());
-@@ -298,7 +298,7 @@ TEST_F(SubprocessTest, ReadStdin) {
-   while (!subproc->Done()) {
-     subprocs_.DoWork(NULL);
-   }
--  ASSERT_EQ(false, subprocs_.IsTokenAvailable());
-+  ASSERT_FALSE(subprocs_.IsTokenAvailable());
-   ASSERT_EQ(ExitSuccess, subproc->Finish());
-   ASSERT_EQ(1u, subprocs_.finished_.size());
- }
-@@ -322,7 +322,7 @@ TEST_F(SubprocessTest, TokenAvailable) {
-   subprocs_.DoWork(&tokens_);
- #ifdef _WIN32
-   tokens_._token_available = false;
--  // we need to loop here as we have no conrol where the token
-+  // we need to loop here as we have no control where the token
-   // I/O completion post ends up in the queue
-   while (!subproc->Done() && !subprocs_.IsTokenAvailable()) {
-     subprocs_.DoWork(&tokens_);
-diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc
-index 70d84bfff7..353bda226a 100644
---- a/src/tokenpool-gnu-make-posix.cc
++
++  // implicit token
++  EXPECT_TRUE(tokens_->Acquire());
++}
+--- /dev/null
 +++ b/src/tokenpool-gnu-make-posix.cc
-@@ -32,8 +32,8 @@ struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool {
- 
-   virtual int GetMonitorFd();
- 
--  virtual const char *GetEnv(const char *name) { return getenv(name); };
--  virtual bool ParseAuth(const char *jobserver);
+@@ -0,0 +1,214 @@
++// Copyright 2016-2018 Google Inc. All Rights Reserved.
++//
++// Licensed under the Apache License, Version 2.0 (the "License");
++// you may not use this file except in compliance with the License.
++// You may obtain a copy of the License at
++//
++//     http://www.apache.org/licenses/LICENSE-2.0
++//
++// Unless required by applicable law or agreed to in writing, software
++// distributed under the License is distributed on an "AS IS" BASIS,
++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++// See the License for the specific language governing permissions and
++// limitations under the License.
++
++#include "tokenpool-gnu-make.h"
++
++#include <errno.h>
++#include <fcntl.h>
++#include <poll.h>
++#include <unistd.h>
++#include <signal.h>
++#include <sys/time.h>
++#include <stdio.h>
++#include <string.h>
++#include <stdlib.h>
++#include <stack>
++
++// TokenPool implementation for GNU make jobserver - POSIX implementation
++// (http://make.mad-scientist.net/papers/jobserver-implementation/)
++struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool {
++  GNUmakeTokenPoolPosix();
++  virtual ~GNUmakeTokenPoolPosix();
++
++  virtual int GetMonitorFd();
++
 +  virtual const char* GetEnv(const char* name) { return getenv(name); };
 +  virtual bool ParseAuth(const char* jobserver);
-   virtual bool AcquireToken();
-   virtual bool ReturnToken();
- 
-@@ -64,9 +64,7 @@ bool GNUmakeTokenPoolPosix::CheckFd(int fd) {
-   if (fd < 0)
-     return false;
-   int ret = fcntl(fd, F_GETFD);
--  if (ret < 0)
--    return false;
--  return true;
++  virtual bool AcquireToken();
++  virtual bool ReturnToken();
++
++ private:
++  int rfd_;
++  int wfd_;
++
++  struct sigaction old_act_;
++  bool restore_;
++
++  // See https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html
++  //
++  //   It’s important that when you release the job slot, you write back
++  //   the same character you read. Don’t assume that all tokens are the
++  //   same character different characters may have different meanings to
++  //   GNU make. The order is not important, since make has no idea in
++  //   what order jobs will complete anyway.
++  //
++  std::stack<char> tokens_;
++
++  static int dup_rfd_;
++  static void CloseDupRfd(int signum);
++
++  bool CheckFd(int fd);
++  bool SetAlarmHandler();
++};
++
++GNUmakeTokenPoolPosix::GNUmakeTokenPoolPosix() : rfd_(-1), wfd_(-1), restore_(false) {
++}
++
++GNUmakeTokenPoolPosix::~GNUmakeTokenPoolPosix() {
++  Clear();
++  if (restore_)
++    sigaction(SIGALRM, &old_act_, NULL);
++}
++
++bool GNUmakeTokenPoolPosix::CheckFd(int fd) {
++  if (fd < 0)
++    return false;
++  int ret = fcntl(fd, F_GETFD);
 +  return ret >= 0;
- }
- 
- int GNUmakeTokenPoolPosix::dup_rfd_ = -1;
-@@ -82,14 +80,13 @@ bool GNUmakeTokenPoolPosix::SetAlarmHandler() {
-   act.sa_handler = CloseDupRfd;
-   if (sigaction(SIGALRM, &act, &old_act_) < 0) {
-     perror("sigaction:");
--    return(false);
--  } else {
--    restore_ = true;
--    return(true);
++}
++
++int GNUmakeTokenPoolPosix::dup_rfd_ = -1;
++
++void GNUmakeTokenPoolPosix::CloseDupRfd(int signum) {
++  close(dup_rfd_);
++  dup_rfd_ = -1;
++}
++
++bool GNUmakeTokenPoolPosix::SetAlarmHandler() {
++  struct sigaction act;
++  memset(&act, 0, sizeof(act));
++  act.sa_handler = CloseDupRfd;
++  if (sigaction(SIGALRM, &act, &old_act_) < 0) {
++    perror("sigaction:");
 +    return false;
-   }
++  }
 +  restore_ = true;
 +  return true;
- }
- 
--bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) {
++}
++
 +bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) {
-   int rfd = -1;
-   int wfd = -1;
-   if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
-@@ -195,9 +192,9 @@ bool GNUmakeTokenPoolPosix::ReturnToken() {
- }
- 
- int GNUmakeTokenPoolPosix::GetMonitorFd() {
--  return(rfd_);
++  int rfd = -1;
++  int wfd = -1;
++  if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
++      CheckFd(rfd) &&
++      CheckFd(wfd) &&
++      SetAlarmHandler()) {
++    rfd_ = rfd;
++    wfd_ = wfd;
++    return true;
++  }
++
++  return false;
++}
++
++bool GNUmakeTokenPoolPosix::AcquireToken() {
++  // Please read
++  //
++  //   http://make.mad-scientist.net/papers/jobserver-implementation/
++  //
++  // for the reasoning behind the following code.
++  //
++  // Try to read one character from the pipe. Returns true on success.
++  //
++  // First check if read() would succeed without blocking.
++#ifdef USE_PPOLL
++  pollfd pollfds[] = {{rfd_, POLLIN, 0}};
++  int ret = poll(pollfds, 1, 0);
++#else
++  fd_set set;
++  struct timeval timeout = { 0, 0 };
++  FD_ZERO(&set);
++  FD_SET(rfd_, &set);
++  int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
++#endif
++  if (ret > 0) {
++    // Handle potential race condition:
++    //  - the above check succeeded, i.e. read() should not block
++    //  - the character disappears before we call read()
++    //
++    // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_
++    // can safely be closed by signal handlers without affecting rfd_.
++    dup_rfd_ = dup(rfd_);
++
++    if (dup_rfd_ != -1) {
++      struct sigaction act, old_act;
++      int ret = 0;
++      char buf;
++
++      // Temporarily replace SIGCHLD handler with our own
++      memset(&act, 0, sizeof(act));
++      act.sa_handler = CloseDupRfd;
++      if (sigaction(SIGCHLD, &act, &old_act) == 0) {
++        struct itimerval timeout;
++
++        // install a 100ms timeout that generates SIGALARM on expiration
++        memset(&timeout, 0, sizeof(timeout));
++        timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec]
++        if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) {
++          // Now try to read() from dup_rfd_. Return values from read():
++          //
++          // 1. token read                               ->  1
++          // 2. pipe closed                              ->  0
++          // 3. alarm expires                            -> -1 (EINTR)
++          // 4. child exits                              -> -1 (EINTR)
++          // 5. alarm expired before entering read()     -> -1 (EBADF)
++          // 6. child exited before entering read()      -> -1 (EBADF)
++          // 7. child exited before handler is installed -> go to 1 - 3
++          ret = read(dup_rfd_, &buf, 1);
++
++          // disarm timer
++          memset(&timeout, 0, sizeof(timeout));
++          setitimer(ITIMER_REAL, &timeout, NULL);
++        }
++
++        sigaction(SIGCHLD, &old_act, NULL);
++      }
++
++      CloseDupRfd(0);
++
++      // Case 1 from above list
++      if (ret > 0) {
++        tokens_.push(buf);
++        return true;
++      }
++    }
++  }
++
++  // read() would block, i.e. no token available,
++  // cases 2-6 from above list or
++  // select() / poll() / dup() / sigaction() / setitimer() failed
++  return false;
++}
++
++bool GNUmakeTokenPoolPosix::ReturnToken() {
++  const char buf = tokens_.top();
++  while (1) {
++    int ret = write(wfd_, &buf, 1);
++    if (ret > 0) {
++      tokens_.pop();
++      return true;
++    }
++    if ((ret != -1) || (errno != EINTR))
++      return false;
++    // write got interrupted - retry
++  }
++}
++
++int GNUmakeTokenPoolPosix::GetMonitorFd() {
 +  return rfd_;
- }
- 
--struct TokenPool *TokenPool::Get() {
++}
++
 +TokenPool* TokenPool::Get() {
-   return new GNUmakeTokenPoolPosix;
- }
-diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc
-index 2719f2c1fc..b2bb52fadb 100644
---- a/src/tokenpool-gnu-make-win32.cc
++  return new GNUmakeTokenPoolPosix;
++}
+--- /dev/null
 +++ b/src/tokenpool-gnu-make-win32.cc
-@@ -14,7 +14,8 @@
- 
- #include "tokenpool-gnu-make.h"
- 
--// always include first to make sure other headers do the correct thing...
+@@ -0,0 +1,239 @@
++// Copyright 2018 Google Inc. All Rights Reserved.
++//
++// Licensed under the Apache License, Version 2.0 (the "License");
++// you may not use this file except in compliance with the License.
++// You may obtain a copy of the License at
++//
++//     http://www.apache.org/licenses/LICENSE-2.0
++//
++// Unless required by applicable law or agreed to in writing, software
++// distributed under the License is distributed on an "AS IS" BASIS,
++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++// See the License for the specific language governing permissions and
++// limitations under the License.
++
++#include "tokenpool-gnu-make.h"
++
 +// Always include this first.
 +// Otherwise the other system headers don't work correctly under Win32
- #include <windows.h>
- 
- #include <ctype.h>
-@@ -32,8 +33,8 @@ struct GNUmakeTokenPoolWin32 : public GNUmakeTokenPool {
-   virtual void WaitForTokenAvailability(HANDLE ioport);
-   virtual bool TokenIsAvailable(ULONG_PTR key);
- 
--  virtual const char *GetEnv(const char *name);
--  virtual bool ParseAuth(const char *jobserver);
++#include <windows.h>
++
++#include <ctype.h>
++#include <stdlib.h>
++#include <string.h>
++
++#include "util.h"
++
++// TokenPool implementation for GNU make jobserver - Win32 implementation
++// (https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html)
++struct GNUmakeTokenPoolWin32 : public GNUmakeTokenPool {
++  GNUmakeTokenPoolWin32();
++  virtual ~GNUmakeTokenPoolWin32();
++
++  virtual void WaitForTokenAvailability(HANDLE ioport);
++  virtual bool TokenIsAvailable(ULONG_PTR key);
++
 +  virtual const char* GetEnv(const char* name);
 +  virtual bool ParseAuth(const char* jobserver);
-   virtual bool AcquireToken();
-   virtual bool ReturnToken();
- 
-@@ -98,19 +99,19 @@ GNUmakeTokenPoolWin32::~GNUmakeTokenPoolWin32() {
-   }
- }
- 
--const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) {
++  virtual bool AcquireToken();
++  virtual bool ReturnToken();
++
++ private:
++  // Semaphore for GNU make jobserver protocol
++  HANDLE semaphore_jobserver_;
++  // Semaphore Child -> Parent
++  // - child releases it before entering wait on jobserver semaphore
++  // - parent blocks on it to know when child enters wait
++  HANDLE semaphore_enter_wait_;
++  // Semaphore Parent -> Child
++  // - parent releases it to allow child to restart loop
++  // - child blocks on it to know when to restart loop
++  HANDLE semaphore_restart_;
++  // set to false if child should exit loop and terminate thread
++  bool running_;
++  // child thread
++  HANDLE child_;
++  // I/O completion port from SubprocessSet
++  HANDLE ioport_;
++
++
++  DWORD SemaphoreThread();
++  void ReleaseSemaphore(HANDLE semaphore);
++  void WaitForObject(HANDLE object);
++  static DWORD WINAPI SemaphoreThreadWrapper(LPVOID param);
++  static void NoopAPCFunc(ULONG_PTR param);
++};
++
++GNUmakeTokenPoolWin32::GNUmakeTokenPoolWin32() : semaphore_jobserver_(NULL),
++                                                 semaphore_enter_wait_(NULL),
++                                                 semaphore_restart_(NULL),
++                                                 running_(false),
++                                                 child_(NULL),
++                                                 ioport_(NULL) {
++}
++
++GNUmakeTokenPoolWin32::~GNUmakeTokenPoolWin32() {
++  Clear();
++  CloseHandle(semaphore_jobserver_);
++  semaphore_jobserver_ = NULL;
++
++  if (child_) {
++    // tell child thread to exit
++    running_ = false;
++    ReleaseSemaphore(semaphore_restart_);
++
++    // wait for child thread to exit
++    WaitForObject(child_);
++    CloseHandle(child_);
++    child_ = NULL;
++  }
++
++  if (semaphore_restart_) {
++    CloseHandle(semaphore_restart_);
++    semaphore_restart_ = NULL;
++  }
++
++  if (semaphore_enter_wait_) {
++    CloseHandle(semaphore_enter_wait_);
++    semaphore_enter_wait_ = NULL;
++  }
++}
++
 +const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) {
-   // getenv() does not work correctly together with tokenpool_tests.cc
-   static char buffer[MAX_PATH + 1];
--  if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0)
++  // getenv() does not work correctly together with tokenpool_tests.cc
++  static char buffer[MAX_PATH + 1];
 +  if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0)
-     return NULL;
--  return(buffer);
++    return NULL;
 +  return buffer;
- }
- 
--bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) {
++}
++
 +bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) {
-   // match "--jobserver-auth=gmake_semaphore_<INTEGER>..."
--  const char *start = strchr(jobserver, '=');
++  // match "--jobserver-auth=gmake_semaphore_<INTEGER>..."
 +  const char* start = strchr(jobserver, '=');
-   if (start) {
--    const char *end = start;
++  if (start) {
 +    const char* end = start;
-     unsigned int len;
-     char c, *auth;
- 
-@@ -119,14 +120,15 @@ bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) {
-         break;
-     len = end - start; // includes string terminator in count
- 
--    if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) {
++    unsigned int len;
++    char c, *auth;
++
++    while ((c = *++end) != '\0')
++      if (!(isalnum(c) || (c == '_')))
++        break;
++    len = end - start; // includes string terminator in count
++
 +    if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) {
-       strncpy(auth, start + 1, len - 1);
-       auth[len - 1] = '\0';
- 
--      if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */
--                                                FALSE,                /* Child processes DON'T inherit */
--                                                auth                  /* Semaphore name */
--                                                )) != NULL) {
++      strncpy(auth, start + 1, len - 1);
++      auth[len - 1] = '\0';
++
 +      if ((semaphore_jobserver_ =
 +           OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */
 +                         FALSE,                /* Child processes DON'T inherit */
 +                         auth                  /* Semaphore name */
 +                        )) != NULL) {
-         free(auth);
-         return true;
-       }
-@@ -171,7 +173,7 @@ DWORD GNUmakeTokenPoolWin32::SemaphoreThread() {
- }
- 
- DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) {
--  GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param;
++        free(auth);
++        return true;
++      }
++
++      free(auth);
++    }
++  }
++
++  return false;
++}
++
++bool GNUmakeTokenPoolWin32::AcquireToken() {
++  return WaitForSingleObject(semaphore_jobserver_, 0) == WAIT_OBJECT_0;
++}
++
++bool GNUmakeTokenPoolWin32::ReturnToken() {
++  ReleaseSemaphore(semaphore_jobserver_);
++  return true;
++}
++
++DWORD GNUmakeTokenPoolWin32::SemaphoreThread() {
++  while (running_) {
++    // indicate to parent that we are entering wait
++    ReleaseSemaphore(semaphore_enter_wait_);
++
++    // alertable wait forever on token semaphore
++    if (WaitForSingleObjectEx(semaphore_jobserver_, INFINITE, TRUE) == WAIT_OBJECT_0) {
++      // release token again for AcquireToken()
++      ReleaseSemaphore(semaphore_jobserver_);
++
++      // indicate to parent on ioport that a token might be available
++      if (!PostQueuedCompletionStatus(ioport_, 0, (ULONG_PTR) this, NULL))
++        Win32Fatal("PostQueuedCompletionStatus");
++    }
++
++    // wait for parent to allow loop restart
++    WaitForObject(semaphore_restart_);
++    // semaphore is now in nonsignaled state again for next run...
++  }
++
++  return 0;
++}
++
++DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) {
 +  GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param;
-   return This->SemaphoreThread();
- }
- 
-@@ -216,7 +218,7 @@ void GNUmakeTokenPoolWin32::WaitForTokenAvailability(HANDLE ioport) {
- 
- bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) {
-   // alert child thread to break wait on token semaphore
--  QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL);
-+  QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL);
- 
-   // return true when GetQueuedCompletionStatus() returned our key
-   return key == (ULONG_PTR) this;
-@@ -232,6 +234,6 @@ void GNUmakeTokenPoolWin32::WaitForObject(HANDLE object) {
-     Win32Fatal("WaitForSingleObject");
- }
- 
--struct TokenPool *TokenPool::Get() {
-+TokenPool* TokenPool::Get() {
-   return new GNUmakeTokenPoolWin32;
- }
-diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc
-index 92ff611721..60e0552924 100644
---- a/src/tokenpool-gnu-make.cc
-+++ b/src/tokenpool-gnu-make.cc
-@@ -31,36 +31,37 @@ GNUmakeTokenPool::~GNUmakeTokenPool() {
- bool GNUmakeTokenPool::Setup(bool ignore,
-                              bool verbose,
-                              double& max_load_average) {
--  const char *value = GetEnv("MAKEFLAGS");
--  if (value) {
--    // GNU make <= 4.1
--    const char *jobserver = strstr(value, "--jobserver-fds=");
-+  const char* value = GetEnv("MAKEFLAGS");
-+  if (!value)
-+    return false;
++  return This->SemaphoreThread();
++}
 +
-+  // GNU make <= 4.1
-+  const char* jobserver = strstr(value, "--jobserver-fds=");
-+  if (!jobserver)
-     // GNU make => 4.2
--    if (!jobserver)
--      jobserver = strstr(value, "--jobserver-auth=");
--    if (jobserver) {
--      LinePrinter printer;
--
--      if (ignore) {
--        printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
--      } else {
--        if (ParseAuth(jobserver)) {
--          const char *l_arg = strstr(value, " -l");
--          int load_limit = -1;
--
--          if (verbose) {
--            printer.PrintOnNewLine("ninja: using GNU make jobserver.\n");
--          }
--
--          // translate GNU make -lN to ninja -lN
--          if (l_arg &&
--              (sscanf(l_arg + 3, "%d ", &load_limit) == 1) &&
--              (load_limit > 0)) {
--            max_load_average = load_limit;
--          }
--
--          return true;
-+    jobserver = strstr(value, "--jobserver-auth=");
-+  if (jobserver) {
-+    LinePrinter printer;
++void GNUmakeTokenPoolWin32::NoopAPCFunc(ULONG_PTR param) {
++}
 +
-+    if (ignore) {
-+      printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
-+    } else {
-+      if (ParseAuth(jobserver)) {
-+        const char* l_arg = strstr(value, " -l");
-+        int load_limit = -1;
++void GNUmakeTokenPoolWin32::WaitForTokenAvailability(HANDLE ioport) {
++  if (child_ == NULL) {
++    // first invocation
++    //
++    // subprocess-win32.cc uses I/O completion port (IOCP) which can't be
++    // used as a waitable object. Therefore we can't use WaitMultipleObjects()
++    // to wait on the IOCP and the token semaphore at the same time. Create
++    // a child thread that waits on the semaphore and posts an I/O completion
++    ioport_ = ioport;
 +
-+        if (verbose) {
-+          printer.PrintOnNewLine("ninja: using GNU make jobserver.\n");
-+        }
++    // create both semaphores in nonsignaled state
++    if ((semaphore_enter_wait_ = CreateSemaphore(NULL, 0, 1, NULL))
++        == NULL)
++      Win32Fatal("CreateSemaphore/enter_wait");
++    if ((semaphore_restart_ = CreateSemaphore(NULL, 0, 1, NULL))
++        == NULL)
++      Win32Fatal("CreateSemaphore/restart");
 +
-+        // translate GNU make -lN to ninja -lN
-+        if (l_arg &&
-+            (sscanf(l_arg + 3, "%d ", &load_limit) == 1) &&
-+            (load_limit > 0)) {
-+          max_load_average = load_limit;
-         }
++    // start child thread
++    running_ = true;
++    if ((child_ = CreateThread(NULL, 0, &SemaphoreThreadWrapper, this, 0, NULL))
++        == NULL)
++      Win32Fatal("CreateThread");
 +
-+        return true;
-       }
-     }
-   }
-@@ -76,10 +77,10 @@ bool GNUmakeTokenPool::Acquire() {
-     // token acquired
-     available_++;
-     return true;
--  } else {
--    // no token available
--    return false;
-   }
++  } else {
++    // all further invocations - allow child thread to loop
++    ReleaseSemaphore(semaphore_restart_);
++  }
 +
-+  // no token available
-+  return false;
- }
- 
- void GNUmakeTokenPool::Reserve() {
-diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h
-index d3852088e2..c94cca5e2d 100644
---- a/src/tokenpool-gnu-make.h
++  // wait for child thread to enter wait
++  WaitForObject(semaphore_enter_wait_);
++  // semaphore is now in nonsignaled state again for next run...
++
++  // now SubprocessSet::DoWork() can enter GetQueuedCompletionStatus()...
++}
++
++bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) {
++  // alert child thread to break wait on token semaphore
++  QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL);
++
++  // return true when GetQueuedCompletionStatus() returned our key
++  return key == (ULONG_PTR) this;
++}
++
++void GNUmakeTokenPoolWin32::ReleaseSemaphore(HANDLE semaphore) {
++  if (!::ReleaseSemaphore(semaphore, 1, NULL))
++    Win32Fatal("ReleaseSemaphore");
++}
++
++void GNUmakeTokenPoolWin32::WaitForObject(HANDLE object) {
++  if (WaitForSingleObject(object, INFINITE) != WAIT_OBJECT_0)
++    Win32Fatal("WaitForSingleObject");
++}
++
++TokenPool* TokenPool::Get() {
++  return new GNUmakeTokenPoolWin32;
++}
+--- /dev/null
 +++ b/src/tokenpool-gnu-make.h
-@@ -17,7 +17,7 @@
- // interface to GNU make token pool
- struct GNUmakeTokenPool : public TokenPool {
-   GNUmakeTokenPool();
--  virtual ~GNUmakeTokenPool();
+@@ -0,0 +1,40 @@
++// Copyright 2016-2018 Google Inc. All Rights Reserved.
++//
++// Licensed under the Apache License, Version 2.0 (the "License");
++// you may not use this file except in compliance with the License.
++// You may obtain a copy of the License at
++//
++//     http://www.apache.org/licenses/LICENSE-2.0
++//
++// Unless required by applicable law or agreed to in writing, software
++// distributed under the License is distributed on an "AS IS" BASIS,
++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++// See the License for the specific language governing permissions and
++// limitations under the License.
++
++#include "tokenpool.h"
++
++// interface to GNU make token pool
++struct GNUmakeTokenPool : public TokenPool {
++  GNUmakeTokenPool();
 +  ~GNUmakeTokenPool();
- 
-   // token pool implementation
-   virtual bool Acquire();
-@@ -27,8 +27,8 @@ struct GNUmakeTokenPool : public TokenPool {
-   virtual bool Setup(bool ignore, bool verbose, double& max_load_average);
- 
-   // platform specific implementation
--  virtual const char *GetEnv(const char *name) = 0;
--  virtual bool ParseAuth(const char *jobserver) = 0;
++
++  // token pool implementation
++  virtual bool Acquire();
++  virtual void Reserve();
++  virtual void Release();
++  virtual void Clear();
++  virtual bool Setup(bool ignore, bool verbose, double& max_load_average);
++
++  // platform specific implementation
 +  virtual const char* GetEnv(const char* name) = 0;
 +  virtual bool ParseAuth(const char* jobserver) = 0;
-   virtual bool AcquireToken() = 0;
-   virtual bool ReturnToken() = 0;
- 
-diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc
-deleted file mode 100644
-index 613d16882d..0000000000
---- a/src/tokenpool-none.cc
-+++ /dev/null
-@@ -1,22 +0,0 @@
--// Copyright 2016-2018 Google Inc. All Rights Reserved.
--//
--// Licensed under the Apache License, Version 2.0 (the "License");
--// you may not use this file except in compliance with the License.
--// You may obtain a copy of the License at
--//
--//     http://www.apache.org/licenses/LICENSE-2.0
--//
--// Unless required by applicable law or agreed to in writing, software
--// distributed under the License is distributed on an "AS IS" BASIS,
--// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--// See the License for the specific language governing permissions and
--// limitations under the License.
--
--#include "tokenpool.h"
--
--#include <stdlib.h>
--
--// No-op TokenPool implementation
--struct TokenPool *TokenPool::Get() {
--  return NULL;
--}
-diff --git a/src/tokenpool.h b/src/tokenpool.h
-index 1be8e1d5ce..931c22754d 100644
---- a/src/tokenpool.h
-+++ b/src/tokenpool.h
-@@ -38,5 +38,5 @@ struct TokenPool {
- #endif
- 
-   // returns NULL if token pool is not available
--  static struct TokenPool *Get();
-+  static TokenPool* Get();
- };
-diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc
-index 8d4fd7d33a..8d3061cb30 100644
---- a/src/tokenpool_test.cc
-+++ b/src/tokenpool_test.cc
-@@ -43,10 +43,10 @@ const double kLoadAverageDefault = -1.23456789;
- 
- struct TokenPoolTest : public testing::Test {
-   double load_avg_;
--  TokenPool *tokens_;
-+  TokenPool* tokens_;
-   char buf_[1024];
- #ifdef _WIN32
--  const char *semaphore_name_;
-+  const char* semaphore_name_;
-   HANDLE semaphore_;
- #else
-   int fds_[2];
-@@ -65,7 +65,7 @@ struct TokenPoolTest : public testing::Test {
-       ASSERT_TRUE(false);
-   }
- 
--  void CreatePool(const char *format, bool ignore_jobserver = false) {
-+  void CreatePool(const char* format, bool ignore_jobserver = false) {
-     if (format) {
-       sprintf(buf_, format,
- #ifdef _WIN32
-@@ -209,7 +209,7 @@ TEST_F(TokenPoolTest, TwoTokens) {
-   tokens_->Release();
-   EXPECT_TRUE(tokens_->Acquire());
- 
--  // release implict token - must return 2nd token back to jobserver
-+  // release implicit token - must return 2nd token back to jobserver
-   tokens_->Release();
-   EXPECT_TRUE(tokens_->Acquire());
- 
++  virtual bool AcquireToken() = 0;
++  virtual bool ReturnToken() = 0;
++
++ private:
++  int available_;
++  int used_;
++
++  void Return();
++};
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -112,6 +112,7 @@ add_library(libninja OBJECT
+ 	src/state.cc
+ 	src/status.cc
+ 	src/string_piece_util.cc
++	src/tokenpool-gnu-make.cc
+ 	src/util.cc
+ 	src/version.cc
+ )
+@@ -123,9 +124,14 @@ if(WIN32)
+ 		src/msvc_helper_main-win32.cc
+ 		src/getopt.c
+ 		src/minidump-win32.cc
++		src/tokenpool-gnu-make-win32.cc
+ 	)
+ else()
+ 	target_sources(libninja PRIVATE src/subprocess-posix.cc)
++	target_sources(libninja PRIVATE
++		src/subprocess-posix.cc
++		src/tokenpool-gnu-make-posix.cc
++	)
+ 	if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
+ 		target_sources(libninja PRIVATE src/getopt.c)
+ 	endif()
+@@ -204,6 +210,7 @@ if(BUILD_TESTING)
+     src/string_piece_util_test.cc
+     src/subprocess_test.cc
+     src/test.cc
++    src/tokenpool_test.cc
+     src/util_test.cc
+   )
+   if(WIN32)




More information about the lede-commits mailing list