| # Copyright (c) 2009, Google Inc. All rights reserved. |
| # Copyright (c) 2009 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import csv |
| import ctypes |
| import errno |
| import logging |
| import multiprocessing |
| import os |
| import signal |
| import subprocess |
| import sys |
| import threading |
| import time |
| |
| import six |
| |
| _log = logging.getLogger(__name__) |
| |
| |
| class ScriptError(Exception): |
| def __init__(self, |
| message=None, |
| script_args=None, |
| exit_code=None, |
| output=None, |
| cwd=None, |
| output_limit=500): |
| shortened_output = output |
| if output and output_limit and len(output) > output_limit: |
| shortened_output = 'Last %s characters of output:\n%s' % ( |
| output_limit, output[-output_limit:]) |
| |
| if not message: |
| message = 'Failed to run "%s"' % repr(script_args) |
| if exit_code: |
| message += ' exit_code: %d' % exit_code |
| if cwd: |
| message += ' cwd: %s' % cwd |
| |
| if shortened_output: |
| message += '\n\noutput: %s' % shortened_output |
| |
| Exception.__init__(self, message) |
| self.script_args = script_args # 'args' is already used by Exception |
| self.exit_code = exit_code |
| self.output = output |
| self.cwd = cwd |
| |
| def message_with_output(self): |
| return six.text_type(self) |
| |
| def command_name(self): |
| command_path = self.script_args |
| if isinstance(command_path, list): |
| command_path = command_path[0] |
| return os.path.basename(command_path) |
| |
| |
| class Executive(object): |
| PIPE = subprocess.PIPE |
| STDOUT = subprocess.STDOUT |
| DEVNULL = open(os.devnull, 'wb') |
| |
| def __init__(self, error_output_limit=500): |
| """Args: |
| error_output_limit: The maximum length of output included in the |
| message of ScriptError when run_command sees a non-zero exit |
| code. None means no limit. |
| """ |
| self.error_output_limit = error_output_limit |
| |
| def _should_close_fds(self): |
| # We need to pass close_fds=True to work around Python bug #2320 |
| # (otherwise we can hang when we kill DumpRenderTree when we are running |
| # multiple threads). See http://bugs.python.org/issue2320 . |
| # Note that close_fds isn't supported on Windows, but this bug only |
| # shows up on Mac and Linux. |
| return sys.platform != 'win32' |
| |
| def cpu_count(self): |
| return multiprocessing.cpu_count() |
| |
| def kill_process(self, pid, kill_tree=True): |
| """Attempts to kill the given pid. |
| |
| if kill_tree is True, the whole process group will be killed. |
| |
| Will fail silently if pid does not exist. |
| """ |
| if sys.platform == 'win32': |
| # Workaround for race condition that occurs when the browser is |
| # killed as it's launching a process. This sometimes leaves a child |
| # process that is in a suspended state. |
| # Follow the Win32 API naming style. pylint: disable=invalid-name |
| OpenProcess = ctypes.windll.kernel32.OpenProcess |
| CloseHandle = ctypes.windll.kernel32.CloseHandle |
| NtSuspendProcess = ctypes.windll.ntdll.NtSuspendProcess |
| PROCESS_ALL_ACCESS = 0x1F0FFF |
| process_handle = OpenProcess(PROCESS_ALL_ACCESS, False, pid) |
| if process_handle != 0: |
| NtSuspendProcess(process_handle) |
| CloseHandle(process_handle) |
| |
| command = ['taskkill.exe', '/f'] |
| if kill_tree: |
| command.append('/t') |
| command += ['/pid', pid] |
| # taskkill will exit 128 if the process is not found. We should log. |
| self.run_command(command, error_handler=self.log_error) |
| return |
| |
| try: |
| if kill_tree: |
| os.killpg(os.getpgid(pid), signal.SIGKILL) |
| else: |
| os.kill(pid, signal.SIGKILL) |
| # At this point if no exception has been raised, the kill has |
| # succeeded, so we can safely use a blocking wait. |
| os.waitpid(pid, 0) |
| except OSError as error: |
| if error.errno == errno.ESRCH: |
| _log.debug("PID %s does not exist.", pid) |
| return |
| if error.errno == errno.ECHILD: |
| # Can't wait on a non-child process, but the kill worked. |
| return |
| if error.errno == errno.EPERM and \ |
| kill_tree and sys.platform == 'darwin': |
| # Calling killpg on a process group whose leader is defunct |
| # causes a permission error on macOS, in which case we try to |
| # collect the defunct process. |
| if os.waitpid(pid, os.WNOHANG) == (0, 0): |
| return |
| raise |
| |
| def _win32_check_running_pid(self, pid): |
| class PROCESSENTRY32(ctypes.Structure): |
| _fields_ = [('dwSize', ctypes.c_ulong), ('cntUsage', |
| ctypes.c_ulong), |
| ('th32ProcessID', ctypes.c_ulong), |
| ('th32DefaultHeapID', ctypes.POINTER(ctypes.c_ulong)), |
| ('th32ModuleID', ctypes.c_ulong), |
| ('cntThreads', ctypes.c_ulong), |
| ('th32ParentProcessID', ctypes.c_ulong), |
| ('pcPriClassBase', ctypes.c_ulong), |
| ('dwFlags', ctypes.c_ulong), |
| ('szExeFile', ctypes.c_char * 260)] |
| |
| # Follow the Win32 API naming style. pylint: disable=invalid-name |
| CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot |
| Process32First = ctypes.windll.kernel32.Process32First |
| Process32Next = ctypes.windll.kernel32.Process32Next |
| CloseHandle = ctypes.windll.kernel32.CloseHandle |
| TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number |
| hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) |
| # pylint does not understand ctypes. pylint: disable=attribute-defined-outside-init |
| pe32 = PROCESSENTRY32() |
| pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) |
| result = False |
| if not Process32First(hProcessSnap, ctypes.byref(pe32)): |
| _log.debug('Failed getting first process.') |
| CloseHandle(hProcessSnap) |
| return result |
| while True: |
| if pe32.th32ProcessID == pid: |
| result = True |
| break |
| if not Process32Next(hProcessSnap, ctypes.byref(pe32)): |
| break |
| CloseHandle(hProcessSnap) |
| return result |
| |
| def check_running_pid(self, pid): |
| """Return True if pid is alive, otherwise return False.""" |
| _log.debug('Checking whether pid %d is alive.', pid) |
| if sys.platform == 'win32': |
| return self._win32_check_running_pid(pid) |
| |
| try: |
| os.kill(pid, 0) |
| return True |
| except OSError: |
| return False |
| |
| def _running_processes(self): |
| processes = [] |
| if sys.platform == 'win32': |
| tasklist_process = self.popen(['tasklist', '/fo', 'csv'], |
| stdout=self.PIPE, |
| stderr=self.PIPE) |
| stdout, _ = tasklist_process.communicate() |
| stdout_reader = csv.reader(stdout.splitlines()) |
| for line in stdout_reader: |
| processes.append([column for column in line]) |
| else: |
| ps_process = self.popen(['ps', '-eo', 'pid,comm'], |
| stdout=self.PIPE, |
| stderr=self.PIPE) |
| stdout, _ = ps_process.communicate() |
| for line in stdout.splitlines(): |
| # In some cases the line can contain one or more |
| # leading white-spaces, so strip it before split. |
| pid, process_name = line.strip().split(' ', 1) |
| processes.append([process_name, pid]) |
| return processes |
| |
| def running_pids(self, process_name_filter=None): |
| if not process_name_filter: |
| process_name_filter = lambda process_name: True |
| |
| running_pids = [] |
| for line in self._running_processes(): |
| try: |
| process_name = line[0] |
| pid = line[1] |
| if process_name_filter(process_name): |
| running_pids.append(int(pid)) |
| except (ValueError, IndexError): |
| pass |
| |
| return sorted(running_pids) |
| |
| def wait_limited(self, |
| pid, |
| limit_in_seconds=None, |
| check_frequency_in_seconds=None): |
| seconds_left = limit_in_seconds or 10 |
| sleep_length = check_frequency_in_seconds or 1 |
| while seconds_left > 0 and self.check_running_pid(pid): |
| seconds_left -= sleep_length |
| time.sleep(sleep_length) |
| |
| def interrupt(self, pid): |
| interrupt_signal = signal.SIGINT |
| # Note: The python docs seem to suggest that on Windows, we may want to use |
| # signal.CTRL_C_EVENT (http://docs.python.org/2/library/signal.html), but |
| # it appears that signal.SIGINT also appears to work on Windows. |
| try: |
| os.kill(pid, interrupt_signal) |
| except OSError: |
| # Silently ignore when the pid doesn't exist. |
| # It's impossible for callers to avoid race conditions with process shutdown. |
| pass |
| |
| # Error handlers do not need to be static methods once all callers are |
| # updated to use an Executive object. |
| |
| @staticmethod |
| def default_error_handler(error): |
| raise error |
| |
| @staticmethod |
| def ignore_error(error): |
| pass |
| |
| @staticmethod |
| def log_error(error): |
| _log.debug(error) |
| |
| def _compute_stdin(self, user_input): |
| """Returns (stdin, string_to_communicate)""" |
| # FIXME: We should be returning /dev/null for stdin |
| # or closing stdin after process creation to prevent |
| # child processes from getting input from the user. |
| if not user_input: |
| return (None, None) |
| if hasattr(user_input, 'read'): # Check if the user_input is a file. |
| # Assume the file is in the right encoding. |
| return (user_input, None) |
| |
| # Popen in Python 2.5 and before does not automatically encode unicode objects. |
| # http://bugs.python.org/issue5290 |
| # See https://bugs.webkit.org/show_bug.cgi?id=37528 |
| # for an example of a regression caused by passing a unicode string directly. |
| # FIXME: We may need to encode differently on different platforms. |
| if isinstance(user_input, six.text_type): |
| user_input = user_input.encode(self._child_process_encoding()) |
| return (self.PIPE, user_input) |
| |
| def command_for_printing(self, args): |
| """Returns a print-ready string representing command args. |
| The string should be copy/paste ready for execution in a shell. |
| """ |
| args = self._stringify_args(args) |
| escaped_args = [] |
| for arg in args: |
| if isinstance(arg, six.text_type): |
| # Escape any non-ascii characters for easy copy/paste |
| arg = arg.encode('unicode_escape') |
| # FIXME: Do we need to fix quotes here? |
| escaped_args.append(arg) |
| return ' '.join(escaped_args) |
| |
| def run_command( |
| self, |
| args, |
| cwd=None, |
| env=None, |
| input=None, # pylint: disable=redefined-builtin |
| timeout_seconds=None, |
| error_handler=None, |
| return_exit_code=False, |
| return_stderr=True, |
| ignore_stderr=False, |
| decode_output=True, |
| debug_logging=True): |
| """Popen wrapper for convenience and to work around python bugs. |
| |
| By default, run_command will expect a zero exit code and will return the |
| program output in that case, or throw a ScriptError if the program has a |
| non-zero exit code. This behavior can be changed by setting the |
| appropriate input parameters. |
| |
| Args: |
| args: the program arguments. Passed to Popen. |
| cwd: the current working directory for the program. Passed to Popen. |
| env: the environment for the program. Passed to Popen. |
| input: input to give to the program on stdin. Accepts either a file |
| handler (will be passed directly) or a string (will be passed |
| via a pipe). |
| timeout_seconds: maximum time in seconds to wait for the program to |
| terminate; on a timeout the process will be killed |
| error_handler: a custom error handler called with a ScriptError when |
| the program fails. The default handler raises the error. |
| return_exit_code: instead of returning the program output, return |
| the exit code. Setting this makes non-zero exit codes non-fatal |
| (the error_handler will not be called). |
| return_stderr: if True, include stderr in the returned output. If |
| False, stderr will be printed to the console unless ignore_stderr |
| is also True. |
| ignore_stderr: squash stderr so it doesn't appear in the console. |
| decode_output: whether to decode the program output. |
| debug_logging: whether to log details about program execution. |
| """ |
| assert isinstance(args, list) or isinstance(args, tuple) |
| start_time = time.time() |
| |
| assert not (return_stderr and ignore_stderr) |
| stdin, string_to_communicate = self._compute_stdin(input) |
| stderr = self.STDOUT if return_stderr else ( |
| self.DEVNULL if ignore_stderr else None) |
| |
| process = self.popen( |
| args, |
| stdin=stdin, |
| stdout=self.PIPE, |
| stderr=stderr, |
| cwd=cwd, |
| env=env, |
| close_fds=self._should_close_fds()) |
| |
| def on_command_timeout(): |
| _log.error('Error: Command timed out after %s seconds', |
| timeout_seconds) |
| process.kill() |
| |
| if timeout_seconds: |
| timer = threading.Timer(timeout_seconds, on_command_timeout) |
| timer.start() |
| |
| output = process.communicate(string_to_communicate)[0] |
| |
| # run_command automatically decodes to unicode() unless explicitly told not to. |
| if decode_output: |
| output = output.decode( |
| self._child_process_encoding(), errors='replace') |
| |
| # wait() is not threadsafe and can throw OSError due to: |
| # http://bugs.python.org/issue1731717 |
| exit_code = process.wait() |
| |
| if timeout_seconds: |
| timer.cancel() |
| |
| if debug_logging: |
| _log.debug('"%s" took %.2fs', self.command_for_printing(args), |
| time.time() - start_time) |
| |
| if return_exit_code: |
| return exit_code |
| |
| if exit_code: |
| script_error = ScriptError( |
| script_args=args, |
| exit_code=exit_code, |
| output=output, |
| cwd=cwd, |
| output_limit=self.error_output_limit) |
| (error_handler or self.default_error_handler)(script_error) |
| return output |
| |
| def _child_process_encoding(self): |
| # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW |
| # to launch subprocesses, so we have to encode arguments using the |
| # current code page. |
| if sys.platform == 'win32' and sys.version < '3': |
| return 'mbcs' |
| # All other platforms use UTF-8. |
| # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands |
| # which will expect arguments to be encoded using the current code |
| # page. |
| return 'utf-8' |
| |
| def _should_encode_child_process_arguments(self): |
| # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW |
| # to launch subprocesses, so we have to encode arguments using the |
| # current code page. |
| if sys.platform == 'win32' and sys.version < '3': |
| return True |
| |
| # On other (POSIX) platforms, we need to encode arguments if the system |
| # does not use UTF-8 encoding. Otherwise, subprocess.Popen will raise |
| # TypeError. Note that macOS always uses UTF-8, while on UNIX it |
| # depends on user locale (LC_CTYPE) and sys.getfilesystemencoding() may |
| # fail and return None. |
| return (sys.getfilesystemencoding() or '').lower() != 'utf-8' |
| |
| def _encode_argument_if_needed(self, argument): |
| if not self._should_encode_child_process_arguments(): |
| return argument |
| return argument.encode(self._child_process_encoding()) |
| |
| def _stringify_args(self, args): |
| # Popen will throw an exception if args are non-strings (like int()) |
| string_args = map(six.text_type, args) |
| # The Windows implementation of Popen cannot handle unicode strings. :( |
| return map(self._encode_argument_if_needed, string_args) |
| |
| def popen(self, args, **kwargs): |
| assert not kwargs.get('shell') |
| string_args = self._stringify_args(args) |
| |
| # os.setpgid is required, otherwise, kill_process function will kill |
| # the parent processes unexpectedly. |
| if sys.platform != 'win32': |
| kwargs['preexec_fn'] = lambda: os.setpgid(0, 0) |
| return subprocess.Popen(string_args, **kwargs) |
| |
| def call(self, args, **kwargs): |
| return subprocess.call(self._stringify_args(args), **kwargs) |
| |
| def run_in_parallel(self, command_lines_and_cwds, processes=None): |
| """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples.""" |
| assert len(command_lines_and_cwds) |
| return self.map(_run_command_thunk, command_lines_and_cwds, processes) |
| |
| def map(self, thunk, arglist, processes=None): |
| if sys.platform == 'win32' or len(arglist) == 1: |
| return map(thunk, arglist) |
| pool = multiprocessing.Pool( |
| processes=(processes or multiprocessing.cpu_count())) |
| try: |
| return pool.map(thunk, arglist) |
| finally: |
| pool.close() |
| pool.join() |
| |
| |
| def _run_command_thunk(cmd_line_and_cwd): |
| # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool. |
| (cmd_line, cwd) = cmd_line_and_cwd |
| proc = subprocess.Popen( |
| cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| stdout, stderr = proc.communicate() |
| return (proc.returncode, stdout, stderr) |
