cq/scripts/slave/step/clang_tidy_step.py - manifest_repos/continuous-tests - Git at Google

 """Build step class for formatting C++ files with clang-tidy-diff."""

 from __future__ import absolute_import
 import logging
 import os
 from collections import defaultdict
 import yaml

 from helpers import commit_message_utils
 from helpers import diff_utils
 from helpers import git_utils
 from slave import base_step

 CLANG_FORMAT_DIFF_SCRIPT = 'clang-tidy-diff.py'
 GIT_DIFF_COMMAND = ['git', 'diff', '-U0', 'HEAD^']

 # TODO(mingcl): point the go link to g3doc and add instructions/explanations
 # there
 CLANG_TIDY_MSG = (
     'ClangTidy have some suggestions. This will not block submission. See '
     'go/eureka-clang-tidy')


 # Given file name and character offset, return a tuple consisting line number
 # and character position in the line
 def char_offset_to_line_number(filename, char_offset):
   try:
     with open(filename) as f:
       content = f.read(char_offset)
       line_number = content.count('\n') + 1
       line_start_position = content.rfind('\n')
       if line_start_position == -1:
         char_position = char_offset
       else:
         char_position = char_offset - (line_start_position + 1)
       return (line_number, char_position)
   except (OSError, IOError) as _:
     logging.info('Cannot open {} in repo. '.format(filename) +
                  'It might be a generated file. ' +
                  'Returning arbitrary line number and offset ' +
                  'because Gerrit will ignore this comment anyway.')
     return (1, 0)


 # Given file, offset and length, convert it to CommentRange object that Gerrit
 # API accepts
 def to_comment_range_object(base_dir, file_path, offset, length):
   (start_line, start_character) = char_offset_to_line_number(
       os.path.join(base_dir, file_path), offset)
   (end_line, end_character) = char_offset_to_line_number(
       os.path.join(base_dir, file_path), offset + length)
   return {
       'start_line': int(start_line),
       'start_character': int(start_character),
       'end_line': int(end_line),
       'end_character': int(end_character)
   }


 # Given git repo root and file_path from the repo, normalize and convert
 # file_path to the relative path from repo_root.
 # This is needed because pathes outputed by clang-tidy-diff script might
 # contain redundant double dots
 def path_relative_to_repo_root(repo_root, file_path):
   return os.path.relpath(
       os.path.normpath(os.path.join(repo_root, file_path)), repo_root)


 def suggestion_to_gerrit_comments(suggestions_str, base_dir):
   r""" Convert a YAML clang_tidy output to gerrit comment format

   Comments are formatted according to:
     https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#comment-input

   Args:
     suggestions_str: YAML string exported by clang tidy
     base_dir: Absolution path of the directory which the clang tidy command is
       run in
   """
   if not suggestions_str:
     return {}

   diagnostics = yaml.safe_load(suggestions_str).get('Diagnostics', [])

   if not diagnostics:
     return {}

   comments = defaultdict(list)

   for diagnostic in diagnostics:
     diagnostic_name = diagnostic['DiagnosticName']
     diagnostic_message = diagnostic['DiagnosticMessage']
     diagnostic_level = diagnostic['Level']
     # Don't report errors. They will be caught by other builders anyway.
     if diagnostic_level == 'Error':
       continue
     comments[path_relative_to_repo_root(
         base_dir, diagnostic_message['FilePath'])].append({
             'message':
                 '[non-blocking] ClangTidy: [{}] {}: {}'.format(
                     diagnostic_name, diagnostic_level,
                     diagnostic_message['Message']),
             'range':
                 to_comment_range_object(base_dir,
                                         diagnostic_message['FilePath'],
                                         diagnostic_message['FileOffset'], 1)
         })
     for replacement in diagnostic_message['Replacements']:
       if replacement['Length'] > 0 and replacement['ReplacementText']:
         replacement_suggestion_message = 'Replace this with `' + replacement[
             'ReplacementText'] + '`.'
       elif replacement['Length'] > 0:
         replacement_suggestion_message = 'Remove this.'
       elif replacement['ReplacementText']:
         replacement_suggestion_message = 'Insert `' + replacement[
             'ReplacementText'] + '` here.'
         # Avoid zero length comment length to accurately indicate the location
         # of insertion
         replacement['Length'] = 1
       else:
         # It makes no sense to suggest to replace empty string with empty
         # string. Leave the suggestion message empty
         replacement_suggestion_message = ''
       comments[path_relative_to_repo_root(
           base_dir, replacement['FilePath'])].append({
               'message':
                   '[non-blocking] ClangTidy: [{}] {}\n{}'.format(
                       diagnostic_name, diagnostic_message['Message'],
                       replacement_suggestion_message),
               'range':
                   to_comment_range_object(base_dir, replacement['FilePath'],
                                           replacement['Offset'],
                                           replacement['Length'])
           })
     # If Notes exists, also convert them to comments
     for note in diagnostic.get('Notes', []):
       comments[path_relative_to_repo_root(base_dir, note['FilePath'])].append({
           'message':
               '[non-blocking] ClangTidy: [{}] {}'.format(
                   diagnostic_name, note['Message']),
           'range':
               to_comment_range_object(base_dir, note['FilePath'],
                                       note['FileOffset'], 1)
       })
   return comments


 class ClangTidyStep(base_step.BaseStep):
   """Build step class for clang-tidying C++ files."""

   def __init__(self, **kwargs):
     """Creates a ClangTidyStep instance.

     Note that this step uses clang-tidy-diff, so that only the changed
     parts of a file are checked.

     Args:
       **kwargs: Any additional args to pass to BaseStep.
     """
     base_step.BaseStep.__init__(self, name='clang-tidy', **kwargs)

   def _filter_comments(self, comments):
     for filename in list(comments):
       if self.allow_listed(filename) and not self.deny_listed(filename):
         continue
       del comments[filename]
     return comments

   def run(self):
     """Runs clang-tidy-diff on changed files for the current commit.

     Sets a build property "review" with the output.

     Returns:
       True iff there were no suggestions.
     """
     # Do not run on cherry-picks; errors are ignored for upstream consistency
     if git_utils.is_cherry_pick(self, self.directory):
       logging.info('clang_tidy check skipped because this is a cherry-pick.')
       return True

     # Do not run if the build looks like a revert.
     commit_message = git_utils.commit_message(self, self.directory)
     if commit_message_utils.is_revert_commit_message(commit_message):
       logging.info('looks like a revert, allowing it through.')
       return True

     # Ensure there is prebuilt clang-tidy binary available in
     # chromium/src/third_party/llvm-build/Release+Asserts/bin/
     # It won't be there if this step has never been run before.
     returncode, _, _ = self.exec_subprocess([
         'python3',
         os.path.join(
             self.get_project_path('chromium/src'),
             'tools/clang/scripts/update.py'), '--package', 'clang-tidy'
     ])
     if returncode != 0:
       return False

     output_file = 'clang_tidy_fixes.yaml'

     diff_command = (GIT_DIFF_COMMAND, {'cwd': self.directory})
     clang_tidy_command = ([
         'clang-tidy-diff.py',
         '-clang-tidy-binary',
         os.path.join(self._getcwd(), self.get_project_path('chromium/src'),
                      'third_party/llvm-build/Release+Asserts/bin/clang-tidy'),
         '-p1',
         '-export-fixes',
         output_file,
     ], {
         'cwd': self.directory
     })

     returncode, stdout, _ = self.exec_subprocess_chain(
         [diff_command, clang_tidy_command])
     if returncode != 0:
       return False

     clang_tidy_fixes = ''
     try:
       with open(os.path.join(self.directory, output_file)) as file:
         clang_tidy_fixes = file.read()
       logging.info('yaml outputed by clang-tidy:\n%s', clang_tidy_fixes)
     except (OSError, IOError) as _:
       logging.info('clang tidy suggestion file not exist')

     gerrit_comments = suggestion_to_gerrit_comments(clang_tidy_fixes,
                                                     self.directory)

     gerrit_comments = self._filter_comments(gerrit_comments)

     if gerrit_comments == {}:
       logging.info('clang_tidy has no comments')
       return True

     self.add_review({'message': CLANG_TIDY_MSG, 'comments': gerrit_comments})

     return False
	"""Build step class for formatting C++ files with clang-tidy-diff."""

	from __future__ import absolute_import
	import logging
	import os
	from collections import defaultdict
	import yaml

	from helpers import commit_message_utils
	from helpers import diff_utils
	from helpers import git_utils
	from slave import base_step

	CLANG_FORMAT_DIFF_SCRIPT = 'clang-tidy-diff.py'
	GIT_DIFF_COMMAND = ['git', 'diff', '-U0', 'HEAD^']

	# TODO(mingcl): point the go link to g3doc and add instructions/explanations
	# there
	CLANG_TIDY_MSG = (
	'ClangTidy have some suggestions. This will not block submission. See '
	'go/eureka-clang-tidy')


	# Given file name and character offset, return a tuple consisting line number
	# and character position in the line
	def char_offset_to_line_number(filename, char_offset):
	try:
	with open(filename) as f:
	content = f.read(char_offset)
	line_number = content.count('\n') + 1
	line_start_position = content.rfind('\n')
	if line_start_position == -1:
	char_position = char_offset
	else:
	char_position = char_offset - (line_start_position + 1)
	return (line_number, char_position)
	except (OSError, IOError) as _:
	logging.info('Cannot open {} in repo. '.format(filename) +
	'It might be a generated file. ' +
	'Returning arbitrary line number and offset ' +
	'because Gerrit will ignore this comment anyway.')
	return (1, 0)


	# Given file, offset and length, convert it to CommentRange object that Gerrit
	# API accepts
	def to_comment_range_object(base_dir, file_path, offset, length):
	(start_line, start_character) = char_offset_to_line_number(
	os.path.join(base_dir, file_path), offset)
	(end_line, end_character) = char_offset_to_line_number(
	os.path.join(base_dir, file_path), offset + length)
	return {
	'start_line': int(start_line),
	'start_character': int(start_character),
	'end_line': int(end_line),
	'end_character': int(end_character)
	}


	# Given git repo root and file_path from the repo, normalize and convert
	# file_path to the relative path from repo_root.
	# This is needed because pathes outputed by clang-tidy-diff script might
	# contain redundant double dots
	def path_relative_to_repo_root(repo_root, file_path):
	return os.path.relpath(
	os.path.normpath(os.path.join(repo_root, file_path)), repo_root)


	def suggestion_to_gerrit_comments(suggestions_str, base_dir):
	r""" Convert a YAML clang_tidy output to gerrit comment format

	Comments are formatted according to:
	https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#comment-input

	Args:
	suggestions_str: YAML string exported by clang tidy
	base_dir: Absolution path of the directory which the clang tidy command is
	run in
	"""
	if not suggestions_str:
	return {}

	diagnostics = yaml.safe_load(suggestions_str).get('Diagnostics', [])

	if not diagnostics:
	return {}

	comments = defaultdict(list)

	for diagnostic in diagnostics:
	diagnostic_name = diagnostic['DiagnosticName']
	diagnostic_message = diagnostic['DiagnosticMessage']
	diagnostic_level = diagnostic['Level']
	# Don't report errors. They will be caught by other builders anyway.
	if diagnostic_level == 'Error':
	continue
	comments[path_relative_to_repo_root(
	base_dir, diagnostic_message['FilePath'])].append({
	'message':
	'[non-blocking] ClangTidy: [{}] {}: {}'.format(
	diagnostic_name, diagnostic_level,
	diagnostic_message['Message']),
	'range':
	to_comment_range_object(base_dir,
	diagnostic_message['FilePath'],
	diagnostic_message['FileOffset'], 1)
	})
	for replacement in diagnostic_message['Replacements']:
	if replacement['Length'] > 0 and replacement['ReplacementText']:
	replacement_suggestion_message = 'Replace this with `' + replacement[
	'ReplacementText'] + '`.'
	elif replacement['Length'] > 0:
	replacement_suggestion_message = 'Remove this.'
	elif replacement['ReplacementText']:
	replacement_suggestion_message = 'Insert `' + replacement[
	'ReplacementText'] + '` here.'
	# Avoid zero length comment length to accurately indicate the location
	# of insertion
	replacement['Length'] = 1
	else:
	# It makes no sense to suggest to replace empty string with empty
	# string. Leave the suggestion message empty
	replacement_suggestion_message = ''
	comments[path_relative_to_repo_root(
	base_dir, replacement['FilePath'])].append({
	'message':
	'[non-blocking] ClangTidy: [{}] {}\n{}'.format(
	diagnostic_name, diagnostic_message['Message'],
	replacement_suggestion_message),
	'range':
	to_comment_range_object(base_dir, replacement['FilePath'],
	replacement['Offset'],
	replacement['Length'])
	})
	# If Notes exists, also convert them to comments
	for note in diagnostic.get('Notes', []):
	comments[path_relative_to_repo_root(base_dir, note['FilePath'])].append({
	'message':
	'[non-blocking] ClangTidy: [{}] {}'.format(
	diagnostic_name, note['Message']),
	'range':
	to_comment_range_object(base_dir, note['FilePath'],
	note['FileOffset'], 1)
	})
	return comments


	class ClangTidyStep(base_step.BaseStep):
	"""Build step class for clang-tidying C++ files."""

	def __init__(self, **kwargs):
	"""Creates a ClangTidyStep instance.

	Note that this step uses clang-tidy-diff, so that only the changed
	parts of a file are checked.

	Args:
	**kwargs: Any additional args to pass to BaseStep.
	"""
	base_step.BaseStep.__init__(self, name='clang-tidy', **kwargs)

	def _filter_comments(self, comments):
	for filename in list(comments):
	if self.allow_listed(filename) and not self.deny_listed(filename):
	continue
	del comments[filename]
	return comments

	def run(self):
	"""Runs clang-tidy-diff on changed files for the current commit.

	Sets a build property "review" with the output.

	Returns:
	True iff there were no suggestions.
	"""
	# Do not run on cherry-picks; errors are ignored for upstream consistency
	if git_utils.is_cherry_pick(self, self.directory):
	logging.info('clang_tidy check skipped because this is a cherry-pick.')
	return True

	# Do not run if the build looks like a revert.
	commit_message = git_utils.commit_message(self, self.directory)
	if commit_message_utils.is_revert_commit_message(commit_message):
	logging.info('looks like a revert, allowing it through.')
	return True

	# Ensure there is prebuilt clang-tidy binary available in
	# chromium/src/third_party/llvm-build/Release+Asserts/bin/
	# It won't be there if this step has never been run before.
	returncode, _, _ = self.exec_subprocess([
	'python3',
	os.path.join(
	self.get_project_path('chromium/src'),
	'tools/clang/scripts/update.py'), '--package', 'clang-tidy'
	])
	if returncode != 0:
	return False

	output_file = 'clang_tidy_fixes.yaml'

	diff_command = (GIT_DIFF_COMMAND, {'cwd': self.directory})
	clang_tidy_command = ([
	'clang-tidy-diff.py',
	'-clang-tidy-binary',
	os.path.join(self._getcwd(), self.get_project_path('chromium/src'),
	'third_party/llvm-build/Release+Asserts/bin/clang-tidy'),
	'-p1',
	'-export-fixes',
	output_file,
	], {
	'cwd': self.directory
	})

	returncode, stdout, _ = self.exec_subprocess_chain(
	[diff_command, clang_tidy_command])
	if returncode != 0:
	return False

	clang_tidy_fixes = ''
	try:
	with open(os.path.join(self.directory, output_file)) as file:
	clang_tidy_fixes = file.read()
	logging.info('yaml outputed by clang-tidy:\n%s', clang_tidy_fixes)
	except (OSError, IOError) as _:
	logging.info('clang tidy suggestion file not exist')

	gerrit_comments = suggestion_to_gerrit_comments(clang_tidy_fixes,
	self.directory)

	gerrit_comments = self._filter_comments(gerrit_comments)

	if gerrit_comments == {}:
	logging.info('clang_tidy has no comments')
	return True

	self.add_review({'message': CLANG_TIDY_MSG, 'comments': gerrit_comments})

	return False