| #!/usr/bin/env python |
| # |
| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this |
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| ########################################################################## |
| # |
| # This is a collection of helper tools to get stuff done in NSS. |
| # |
| |
| import sys |
| import argparse |
| import fnmatch |
| import subprocess |
| import os |
| import platform |
| import tempfile |
| |
| from hashlib import sha256 |
| |
| DEVNULL = open(os.devnull, 'wb') |
| cwd = os.path.dirname(os.path.abspath(__file__)) |
| |
| def run_tests(test, cycles="standard", env={}, silent=False): |
| domsuf = os.getenv('DOMSUF', "localdomain") |
| host = os.getenv('HOST', "localhost") |
| env = env.copy() |
| env.update({ |
| "NSS_TESTS": test, |
| "NSS_CYCLES": cycles, |
| "DOMSUF": domsuf, |
| "HOST": host |
| }) |
| os_env = os.environ |
| os_env.update(env) |
| command = cwd + "/tests/all.sh" |
| stdout = stderr = DEVNULL if silent else None |
| subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr) |
| |
| class cfAction(argparse.Action): |
| docker_command = ["docker"] |
| restorecon = None |
| |
| def __call__(self, parser, args, values, option_string=None): |
| if not args.noroot: |
| self.setDockerCommand() |
| |
| if values: |
| files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values] |
| else: |
| files = self.modifiedFiles() |
| files = [os.path.join('/home/worker/nss', x) for x in files] |
| |
| # First check if we can run docker. |
| try: |
| with open(os.devnull, "w") as f: |
| subprocess.check_call( |
| self.docker_command + ["images"], stdout=f) |
| except: |
| print("Please install docker and start the docker daemon.") |
| sys.exit(1) |
| |
| docker_image = 'clang-format-service:latest' |
| cf_docker_folder = cwd + "/automation/clang-format" |
| |
| # Build the image if necessary. |
| if self.filesChanged(cf_docker_folder): |
| self.buildImage(docker_image, cf_docker_folder) |
| |
| # Check if we have the docker image. |
| try: |
| command = self.docker_command + [ |
| "image", "inspect", "clang-format-service:latest" |
| ] |
| with open(os.devnull, "w") as f: |
| subprocess.check_call(command, stdout=f) |
| except: |
| print("I have to build the docker image first.") |
| self.buildImage(docker_image, cf_docker_folder) |
| |
| command = self.docker_command + [ |
| 'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image |
| ] |
| # The clang format script returns 1 if something's to do. We don't |
| # care. |
| subprocess.call(command + files) |
| if self.restorecon is not None: |
| subprocess.call([self.restorecon, '-R', cwd]) |
| |
| def filesChanged(self, path): |
| hash = sha256() |
| for dirname, dirnames, files in os.walk(path): |
| for file in files: |
| with open(os.path.join(dirname, file), "rb") as f: |
| hash.update(f.read()) |
| chk_file = cwd + "/.chk" |
| old_chk = "" |
| new_chk = hash.hexdigest() |
| if os.path.exists(chk_file): |
| with open(chk_file) as f: |
| old_chk = f.readline() |
| if old_chk != new_chk: |
| with open(chk_file, "w+") as f: |
| f.write(new_chk) |
| return True |
| return False |
| |
| def buildImage(self, docker_image, cf_docker_folder): |
| command = self.docker_command + [ |
| "build", "-t", docker_image, cf_docker_folder |
| ] |
| subprocess.check_call(command) |
| return |
| |
| def setDockerCommand(self): |
| if platform.system() == "Linux": |
| from distutils.spawn import find_executable |
| self.restorecon = find_executable('restorecon') |
| self.docker_command = ["sudo"] + self.docker_command |
| |
| def modifiedFiles(self): |
| files = [] |
| if os.path.exists(os.path.join(cwd, '.hg')): |
| st = subprocess.Popen(['hg', 'status', '-m', '-a'], |
| cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True) |
| for line in iter(st.stdout.readline, ''): |
| files += [line[2:].rstrip()] |
| elif os.path.exists(os.path.join(cwd, '.git')): |
| st = subprocess.Popen(['git', 'status', '--porcelain'], |
| cwd=cwd, stdout=subprocess.PIPE) |
| for line in iter(st.stdout.readline, ''): |
| if line[1] == 'M' or line[1] != 'D' and \ |
| (line[0] == 'M' or line[0] == 'A' or |
| line[0] == 'C' or line[0] == 'U'): |
| files += [line[3:].rstrip()] |
| elif line[0] == 'R': |
| files += [line[line.index(' -> ', beg=4) + 4:]] |
| else: |
| print('Warning: neither mercurial nor git detected!') |
| |
| def isFormatted(x): |
| return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h' |
| return [x for x in files if isFormatted(x)] |
| |
| |
| class buildAction(argparse.Action): |
| |
| def __call__(self, parser, args, values, option_string=None): |
| subprocess.check_call([cwd + "/build.sh"] + values) |
| |
| |
| class testAction(argparse.Action): |
| |
| def __call__(self, parser, args, values, option_string=None): |
| run_tests(values) |
| |
| |
| class covAction(argparse.Action): |
| |
| def runSslGtests(self, outdir): |
| env = { |
| "GTESTFILTER": "*", # Prevent parallel test runs. |
| "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir |
| } |
| |
| run_tests("ssl_gtests", env=env, silent=True) |
| |
| def findSanCovFile(self, outdir): |
| for file in os.listdir(outdir): |
| if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'): |
| return os.path.join(outdir, file) |
| |
| return None |
| |
| def __call__(self, parser, args, values, option_string=None): |
| outdir = args.outdir |
| print("Output directory: " + outdir) |
| |
| print("\nBuild with coverage sanitizers...\n") |
| sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp" |
| subprocess.check_call([ |
| os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", |
| "--sancov=" + sancov_args |
| ]) |
| |
| print("\nRun ssl_gtests to get a coverage report...") |
| self.runSslGtests(outdir) |
| print("Done.") |
| |
| sancov_file = self.findSanCovFile(outdir) |
| if not sancov_file: |
| print("Couldn't find .sancov file.") |
| sys.exit(1) |
| |
| symcov_file = os.path.join(outdir, "ssl_gtest.symcov") |
| out = open(symcov_file, 'wb') |
| subprocess.check_call([ |
| "sancov", |
| "-blacklist=" + os.path.join(cwd, ".sancov-blacklist"), |
| "-symbolize", sancov_file, |
| os.path.join(cwd, "../dist/Debug/bin/ssl_gtest") |
| ], stdout=out) |
| out.close() |
| |
| print("\nCoverage report: " + symcov_file) |
| |
| |
| class commandsAction(argparse.Action): |
| commands = [] |
| |
| def __call__(self, parser, args, values, option_string=None): |
| for c in commandsAction.commands: |
| print(c) |
| |
| |
| def parse_arguments(): |
| parser = argparse.ArgumentParser( |
| description='NSS helper script. ' + |
| 'Make sure to separate sub-command arguments with --.') |
| subparsers = parser.add_subparsers() |
| |
| parser_build = subparsers.add_parser( |
| 'build', help='All arguments are passed to build.sh') |
| parser_build.add_argument( |
| 'build_args', nargs='*', help="build arguments", action=buildAction) |
| |
| parser_cf = subparsers.add_parser( |
| 'clang-format', |
| help=""" |
| Run clang-format. |
| |
| By default this runs against any files that you have modified. If |
| there are no modified files, it checks everything. |
| """) |
| parser_cf.add_argument( |
| '--noroot', |
| help='On linux, suppress the use of \'sudo\' for running docker.', |
| action='store_true') |
| parser_cf.add_argument( |
| '<file/dir>', |
| nargs='*', |
| help="Specify files or directories to run clang-format on", |
| action=cfAction) |
| |
| parser_test = subparsers.add_parser( |
| 'tests', help='Run tests through tests/all.sh.') |
| tests = [ |
| "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips", |
| "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec", |
| "gtests", "ssl_gtests", "bogo", "interop", "policy" |
| ] |
| parser_test.add_argument( |
| 'test', choices=tests, help="Available tests", action=testAction) |
| |
| parser_cov = subparsers.add_parser( |
| 'coverage', help='Generate coverage report') |
| cov_modules = ["ssl_gtests"] |
| parser_cov.add_argument( |
| '--outdir', help='Output directory for coverage report data.', |
| default=tempfile.mkdtemp()) |
| parser_cov.add_argument( |
| 'module', choices=cov_modules, help="Available coverage modules", |
| action=covAction) |
| |
| parser_commands = subparsers.add_parser( |
| 'mach-commands', |
| help="list commands") |
| parser_commands.add_argument( |
| 'mach-commands', |
| nargs='*', |
| action=commandsAction) |
| |
| commandsAction.commands = [c for c in subparsers.choices] |
| return parser.parse_args() |
| |
| |
| def main(): |
| parse_arguments() |
| |
| |
| if __name__ == '__main__': |
| main() |