blob: 7feeee0f76f52d0a62d1fa68581214bce98d4961 [file] [log] [blame] [edit]
#!/usr/bin/env perl
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
##===----------------------------------------------------------------------===##
#
# A script designed to wrap a build so that all calls to gcc are intercepted
# and piped to the static analyzer.
#
##===----------------------------------------------------------------------===##
use strict;
use warnings;
use FindBin qw($RealBin);
use Digest::MD5;
use File::Basename;
use File::Find;
use Term::ANSIColor;
use Term::ANSIColor qw(:constants);
use Cwd qw/ getcwd abs_path /;
use Sys::Hostname;
use Data::Dumper;
my $Verbose = 0; # Verbose output from this script.
my $Prog = "post-process";
my $BuildName;
my $BuildDate;
my $TERM = $ENV{'TERM'};
my $UseColor = (defined $TERM and $TERM =~ 'xterm-.*color' and -t STDOUT
and defined $ENV{'SCAN_BUILD_COLOR'});
my $UserName = HtmlEscape(getpwuid($<) || 'unknown');
my $HostName = HtmlEscape(hostname() || 'unknown');
my $CurrentDir = HtmlEscape(getcwd());
my $CurrentDirSuffix = basename($CurrentDir);
my @PluginsToLoad;
my $HtmlTitle = "Clang static analysis report generated by post-process script";
my $Date = localtime();
my $HtmlDir;
my $HtmlDirSpecified = 0;
my $AnalyzerStats = 0;
my $KeepEmpty = 0;
my @filesFound;
my $baseDir;
sub FileWanted {
my $baseDirRegEx = quotemeta $baseDir;
my $file = $File::Find::name;
if ($file =~ /report-.*\.html$/) {
my $relative_file = $file;
$relative_file =~ s/$baseDirRegEx//g;
push @filesFound, $relative_file;
}
}
##----------------------------------------------------------------------------##
# Diagnostics
##----------------------------------------------------------------------------##
sub Diag {
if ($UseColor) {
print BOLD, MAGENTA "$Prog: @_";
print RESET;
}
else {
print "$Prog: @_";
}
}
sub ErrorDiag {
if ($UseColor) {
print STDERR BOLD, RED "$Prog: ";
print STDERR RESET, RED @_;
print STDERR RESET;
} else {
print STDERR "$Prog: @_";
}
}
sub DiagCrashes {
my $Dir = shift;
Diag ("The analyzer encountered problems on some source files.\n");
Diag ("Preprocessed versions of these sources were deposited in '$Dir/failures'.\n");
Diag ("Please consider submitting a bug report using these files:\n");
Diag (" http://clang-analyzer.llvm.org/filing_bugs.html\n")
}
sub DieDiag {
if ($UseColor) {
print STDERR BOLD, RED "$Prog: ";
print STDERR RESET, RED @_;
print STDERR RESET;
}
else {
print STDERR "$Prog: ", @_;
}
exit 1;
}
##----------------------------------------------------------------------------##
# CopyFiles - Copy resource files to target directory.
##----------------------------------------------------------------------------##
sub CopyFiles {
my $Dir = shift;
my $JS = Cwd::realpath("$RealBin/../lib/static-analyzer/sorttable.js");
DieDiag("Cannot find 'sorttable.js'.\n")
if (! -r $JS);
system ("cp", $JS, "$Dir");
DieDiag("Could not copy 'sorttable.js' to '$Dir'.\n")
if (! -r "$Dir/sorttable.js");
my $CSS = Cwd::realpath("$RealBin/../lib/static-analyzer/scanview.css");
DieDiag("Cannot find 'scanview.css'.\n")
if (! -r $CSS);
system ("cp", $CSS, "$Dir");
DieDiag("Could not copy 'scanview.css' to '$Dir'.\n")
if (! -r $CSS);
}
##----------------------------------------------------------------------------##
# UpdatePrefix - Compute the common prefix of files.
##----------------------------------------------------------------------------##
my $Prefix;
sub UpdatePrefix {
my $x = shift;
print "\nUpdating prefix of $x";
my $y = basename($x);
$x =~ s/\Q$y\E$//;
if (!defined $Prefix) {
$Prefix = $x;
return;
}
chop $Prefix while (!($x =~ /^\Q$Prefix/));
}
sub GetPrefix {
return $Prefix;
}
##----------------------------------------------------------------------------##
# UpdateInFilePath - Update the path in the report file.
##----------------------------------------------------------------------------##
sub UpdateInFilePath {
my $fname = shift;
my $regex = shift;
my $newtext = shift;
open (RIN, $fname) or die "cannot open $fname";
open (ROUT, ">", "$fname.tmp") or die "cannot open $fname.tmp";
while (<RIN>) {
s/$regex/$newtext/;
print ROUT $_;
}
close (ROUT);
close (RIN);
system("mv", "$fname.tmp", $fname);
}
##----------------------------------------------------------------------------##
# ComputeDigest - Compute a digest of the specified file.
##----------------------------------------------------------------------------##
sub ComputeDigest {
my $FName = shift;
DieDiag("Cannot read $FName to compute Digest.\n") if (! -r $FName);
# Use Digest::MD5. We don't have to be cryptographically secure. We're
# just looking for duplicate files that come from a non-malicious source.
# We use Digest::MD5 because it is a standard Perl module that should
# come bundled on most systems.
open(FILE, $FName) or DieDiag("Cannot open $FName when computing Digest.\n");
binmode FILE;
my $Result = Digest::MD5->new->addfile(*FILE)->hexdigest;
close(FILE);
# Return the digest.
return $Result;
}
my $printedPerc = 0;
sub ShowProgress {
my $numFilesScanned = shift;
my $numFilesFound = shift;
my $perc = int(($numFilesScanned / $numFilesFound)*100);
my $pperc = $perc % 10;
if (($pperc == 0) and ($printedPerc != $perc)) {
print "..($perc%)";
if ($perc == 100) {
print "\n";
}
$printedPerc = $perc;
}
}
##----------------------------------------------------------------------------##
# ScanFile - Scan a report file for various identifying attributes.
##----------------------------------------------------------------------------##
# Sometimes a source file is scanned more than once, and thus produces
# multiple error reports. We use a cache to solve this problem.
my %AlreadyScanned;
sub ScanFile {
my $Index = shift;
my $Dir = shift;
my $FName = shift;
my $Stats = shift;
# Compute a digest for the report file. Determine if we have already
# scanned a file that looks just like it.
my $digest = ComputeDigest("$Dir/$FName");
if (defined $AlreadyScanned{$digest}) {
# Redundant file. Remove it.
system ("rm", "-f", "$Dir/$FName");
return;
}
$AlreadyScanned{$digest} = 1;
# At this point the report file is not world readable. Make it happen.
system ("chmod", "644", "$Dir/$FName");
# Scan the report file for tags.
open(IN, "$Dir/$FName") or DieDiag("Cannot open '$Dir/$FName'\n");
my $BugType = "";
my $BugFile = "";
my $BugCategory = "";
my $BugDescription = "";
my $BugPathLength = 1;
my $BugLine = 0;
if ($Verbose) {
Diag("Analyzing '$Dir/$FName'\n");
}
while (<IN>) {
last if (/<!-- BUGMETAEND -->/);
if (/<!-- BUGTYPE (.*) -->$/) {
$BugType = $1;
}
elsif (/<!-- BUGFILE (.*) -->$/) {
$BugFile = $1;
#$BugFile = abs_path($1);
#UpdatePrefix($BugFile);
#print "\nUpdating prefix of $BugFile";
}
elsif (/<!-- BUGPATHLENGTH (.*) -->$/) {
$BugPathLength = $1;
}
elsif (/<!-- BUGLINE (.*) -->$/) {
$BugLine = $1;
}
elsif (/<!-- BUGCATEGORY (.*) -->$/) {
$BugCategory = $1;
}
elsif (/<!-- BUGDESC (.*) -->$/) {
$BugDescription = $1;
}
}
close(IN);
if (!defined $BugCategory) {
$BugCategory = "Other";
}
# Don't add internal statistics to the bug reports
if ($BugCategory =~ /statistics/i) {
AddStatLine($BugDescription, $Stats, $BugFile);
return;
}
push @$Index,[ $FName, $BugCategory, $BugType, $BugFile, $BugLine,
$BugPathLength ];
}
##----------------------------------------------------------------------------##
# Diagnostics
##----------------------------------------------------------------------------##
sub Postprocess {
my $Dir = shift;
my $AnalyzerStats = shift;
my $KeepEmpty = shift;
die "No directory specified." if (!defined $Dir);
if (! -d $Dir) {
Diag("No bugs found.\n");
return 0;
}
$baseDir = $Dir . "/";
find({ wanted => \&FileWanted, follow => 0}, $Dir);
my $numFilesFound = scalar(@filesFound);
if ($numFilesFound == 0 and ! -e "$Dir/failures") {
if (! $KeepEmpty) {
Diag("Removing directory '$Dir' because it contains no reports.\n");
system ("rm", "-fR", $Dir);
}
Diag("No bugs found.\n");
return 0;
}
# Scan each report file and build an index.
my @Index;
my @Stats;
my $numFilesScanned = 1;
foreach my $file (@filesFound) {
ScanFile(\@Index, $Dir, $file, \@Stats);
if (!$Verbose) {
ShowProgress($numFilesScanned, $numFilesFound);
}
++$numFilesScanned;
}
# Scan the failures directory and use the information in the .info files
# to update the common prefix directory.
my @failures;
my @attributes_ignored;
if (-d "$Dir/failures") {
opendir(DIR, "$Dir/failures");
@failures = grep { /[.]info.txt$/ && !/attribute_ignored/; } readdir(DIR);
closedir(DIR);
opendir(DIR, "$Dir/failures");
@attributes_ignored = grep { /^attribute_ignored/; } readdir(DIR);
closedir(DIR);
foreach my $file (@failures) {
open IN, "$Dir/failures/$file" or DieDiag("cannot open $file\n");
my $Path = <IN>;
if (defined $Path) { UpdatePrefix($Path); }
close IN;
}
}
# Generate an index.html file.
my $FName = "$Dir/index.html";
open(OUT, ">", $FName) or DieDiag("Cannot create file '$FName'\n");
# Print out the header.
print OUT <<ENDTEXT;
<html>
<head>
<title>${HtmlTitle}</title>
<link type="text/css" rel="stylesheet" href="scanview.css"/>
<script src="sorttable.js"></script>
<script language='javascript' type="text/javascript">
function SetDisplay(RowClass, DisplayVal)
{
var Rows = document.getElementsByTagName("tr");
for ( var i = 0 ; i < Rows.length; ++i ) {
if (Rows[i].className == RowClass) {
Rows[i].style.display = DisplayVal;
}
}
}
function CopyCheckedStateToCheckButtons(SummaryCheckButton) {
var Inputs = document.getElementsByTagName("input");
for ( var i = 0 ; i < Inputs.length; ++i ) {
if (Inputs[i].type == "checkbox") {
if(Inputs[i] != SummaryCheckButton) {
Inputs[i].checked = SummaryCheckButton.checked;
Inputs[i].onclick();
}
}
}
}
function returnObjById( id ) {
if (document.getElementById)
var returnVar = document.getElementById(id);
else if (document.all)
var returnVar = document.all[id];
else if (document.layers)
var returnVar = document.layers[id];
return returnVar;
}
var NumUnchecked = 0;
function ToggleDisplay(CheckButton, ClassName) {
if (CheckButton.checked) {
SetDisplay(ClassName, "");
if (--NumUnchecked == 0) {
returnObjById("AllBugsCheck").checked = true;
}
}
else {
SetDisplay(ClassName, "none");
NumUnchecked++;
returnObjById("AllBugsCheck").checked = false;
}
}
</script>
<!-- SUMMARYENDHEAD -->
</head>
<body>
<h1>${HtmlTitle}</h1>
<table>
<tr><th>User:</th><td>${UserName}\@${HostName}</td></tr>
<tr><th>Working Directory:</th><td>${CurrentDir}</td></tr>
<tr><th>Date:</th><td>${Date}</td></tr>
ENDTEXT
print OUT "<tr><th>Version:</th><td>${BuildName} (${BuildDate})</td></tr>\n"
if (defined($BuildName) && defined($BuildDate));
print OUT <<ENDTEXT;
</table>
ENDTEXT
if (scalar(@filesFound)) {
# Print out the summary table.
my %Totals;
for my $row ( @Index ) {
my $bug_type = ($row->[2]);
my $bug_category = ($row->[1]);
my $key = "$bug_category:$bug_type";
if (!defined $Totals{$key}) { $Totals{$key} = [1,$bug_category,$bug_type]; }
else { $Totals{$key}->[0]++; }
}
print OUT "<h2>Bug Summary</h2>";
if (defined $BuildName) {
print OUT "\n<p>Results in this analysis run are based on analyzer build <b>$BuildName</b>.</p>\n"
}
my $TotalBugs = scalar(@Index);
print OUT <<ENDTEXT;
<table>
<thead><tr><td>Bug Type</td><td>Quantity</td><td class="sorttable_nosort">Display?</td></tr></thead>
<tr style="font-weight:bold"><td class="SUMM_DESC">All Bugs</td><td class="Q">$TotalBugs</td><td><center><input type="checkbox" id="AllBugsCheck" onClick="CopyCheckedStateToCheckButtons(this);" checked/></center></td></tr>
ENDTEXT
my $last_category;
for my $key (
sort {
my $x = $Totals{$a};
my $y = $Totals{$b};
my $res = $x->[1] cmp $y->[1];
$res = $x->[2] cmp $y->[2] if ($res == 0);
$res
} keys %Totals )
{
my $val = $Totals{$key};
my $category = $val->[1];
if (!defined $last_category or $last_category ne $category) {
$last_category = $category;
print OUT "<tr><th>$category</th><th colspan=2></th></tr>\n";
}
my $x = lc $key;
$x =~ s/[ ,'":\/()]+/_/g;
print OUT "<tr><td class=\"SUMM_DESC\">";
print OUT $val->[2];
print OUT "</td><td class=\"Q\">";
print OUT $val->[0];
print OUT "</td><td><center><input type=\"checkbox\" onClick=\"ToggleDisplay(this,'bt_$x');\" checked/></center></td></tr>\n";
}
# Print out the table of errors.
print OUT <<ENDTEXT;
</table>
<h2>Reports</h2>
<table class="sortable" style="table-layout:automatic">
<thead><tr>
<td>Bug Group</td>
<td class="sorttable_sorted">Bug Type<span id="sorttable_sortfwdind">&nbsp;&#x25BE;</span></td>
<td>File</td>
<td class="Q">Line</td>
<td class="Q">Path Length</td>
<td class="sorttable_nosort"></td>
<!-- REPORTBUGCOL -->
</tr></thead>
<tbody>
ENDTEXT
my $prefix = GetPrefix();
my $regex;
my $InFileRegex;
my $InFilePrefix = "File:</td><td>";
if (defined $prefix) {
$regex = qr/^\Q$prefix\E/is;
$InFileRegex = qr/\Q$InFilePrefix$prefix\E/is;
}
for my $row ( sort { $a->[2] cmp $b->[2] } @Index ) {
my $x = "$row->[1]:$row->[2]";
$x = lc $x;
$x =~ s/[ ,'":\/()]+/_/g;
my $ReportFile = $row->[0];
print OUT "<tr class=\"bt_$x\">";
print OUT "<td class=\"DESC\">";
print OUT $row->[1];
print OUT "</td>";
print OUT "<td class=\"DESC\">";
print OUT $row->[2];
print OUT "</td>";
# Update the file prefix.
my $fname = $row->[3];
if (defined $regex) {
$fname =~ s/$regex//;
UpdateInFilePath("$Dir/$ReportFile", $InFileRegex, $InFilePrefix)
}
print OUT "<td>";
my @fname = split /\//,$fname;
if ($#fname > 0) {
while ($#fname >= 0) {
my $x = shift @fname;
print OUT $x;
if ($#fname >= 0) {
print OUT "<span class=\"W\"> </span>/";
}
}
}
else {
print OUT $fname;
}
print OUT "</td>";
# Print out the quantities.
for my $j ( 4 .. 5 ) {
print OUT "<td class=\"Q\">$row->[$j]</td>";
}
# Print the rest of the columns.
for (my $j = 6; $j <= $#{$row}; ++$j) {
print OUT "<td>$row->[$j]</td>"
}
# Emit the "View" link.
print OUT "<td><a href=\"$ReportFile#EndPath\">View Report</a></td>";
# Emit REPORTBUG markers.
print OUT "\n<!-- REPORTBUG id=\"$ReportFile\" -->\n";
# End the row.
print OUT "</tr>\n";
}
print OUT "</tbody>\n</table>\n\n";
}
if (scalar (@failures) || scalar(@attributes_ignored)) {
print OUT "<h2>Analyzer Failures</h2>\n";
if (scalar @attributes_ignored) {
print OUT "The analyzer's parser ignored the following attributes:<p>\n";
print OUT "<table>\n";
print OUT "<thead><tr><td>Attribute</td><td>Source File</td><td>Preprocessed File</td><td>STDERR Output</td></tr></thead>\n";
foreach my $file (sort @attributes_ignored) {
die "cannot demangle attribute name\n" if (! ($file =~ /^attribute_ignored_(.+).txt/));
my $attribute = $1;
# Open the attribute file to get the first file that failed.
next if (!open (ATTR, "$Dir/failures/$file"));
my $ppfile = <ATTR>;
chomp $ppfile;
close ATTR;
next if (! -e "$Dir/failures/$ppfile");
# Open the info file and get the name of the source file.
open (INFO, "$Dir/failures/$ppfile.info.txt") or
die "Cannot open $Dir/failures/$ppfile.info.txt\n";
my $srcfile = <INFO>;
chomp $srcfile;
close (INFO);
# Print the information in the table.
my $prefix = GetPrefix();
if (defined $prefix) { $srcfile =~ s/^\Q$prefix//; }
print OUT "<tr><td>$attribute</td><td>$srcfile</td><td><a href=\"failures/$ppfile\">$ppfile</a></td><td><a href=\"failures/$ppfile.stderr.txt\">$ppfile.stderr.txt</a></td></tr>\n";
my $ppfile_clang = $ppfile;
$ppfile_clang =~ s/[.](.+)$/.clang.$1/;
print OUT " <!-- REPORTPROBLEM src=\"$srcfile\" file=\"failures/$ppfile\" clangfile=\"failures/$ppfile_clang\" stderr=\"failures/$ppfile.stderr.txt\" info=\"failures/$ppfile.info.txt\" -->\n";
}
print OUT "</table>\n";
}
if (scalar @failures) {
print OUT "<p>The analyzer had problems processing the following files:</p>\n";
print OUT "<table>\n";
print OUT "<thead><tr><td>Problem</td><td>Source File</td><td>Preprocessed File</td><td>STDERR Output</td></tr></thead>\n";
foreach my $file (sort @failures) {
$file =~ /(.+).info.txt$/;
# Get the preprocessed file.
my $ppfile = $1;
# Open the info file and get the name of the source file.
open (INFO, "$Dir/failures/$file") or
die "Cannot open $Dir/failures/$file\n";
my $srcfile = <INFO>;
chomp $srcfile;
my $problem = <INFO>;
chomp $problem;
close (INFO);
# Print the information in the table.
my $prefix = GetPrefix();
if (defined $prefix) { $srcfile =~ s/^\Q$prefix//; }
print OUT "<tr><td>$problem</td><td>$srcfile</td><td><a href=\"failures/$ppfile\">$ppfile</a></td><td><a href=\"failures/$ppfile.stderr.txt\">$ppfile.stderr.txt</a></td></tr>\n";
my $ppfile_clang = $ppfile;
$ppfile_clang =~ s/[.](.+)$/.clang.$1/;
print OUT " <!-- REPORTPROBLEM src=\"$srcfile\" file=\"failures/$ppfile\" clangfile=\"failures/$ppfile_clang\" stderr=\"failures/$ppfile.stderr.txt\" info=\"failures/$ppfile.info.txt\" -->\n";
}
print OUT "</table>\n";
}
print OUT "<p>Please consider submitting preprocessed files as <a href=\"http://clang-analyzer.llvm.org/filing_bugs.html\">bug reports</a>. <!-- REPORTCRASHES --> </p>\n";
}
print OUT "</body></html>\n";
close(OUT);
CopyFiles($Dir);
# Print statistics
print CalcStats(\@Stats) if $AnalyzerStats;
my $Num = scalar(@Index);
Diag("$Num bugs found.\n");
if ($Num > 0 && -r "$Dir/index.html") {
Diag("Open '$Dir/index.html' to examine summarized report.\n");
}
DiagCrashes($Dir) if (scalar @failures || scalar @attributes_ignored);
return $Num;
}
##----------------------------------------------------------------------------##
# HtmlEscape - HTML entity encode characters that are special in HTML
##----------------------------------------------------------------------------##
sub HtmlEscape {
# copy argument to new variable so we don't clobber the original
my $arg = shift || '';
my $tmp = $arg;
$tmp =~ s/&/&amp;/g;
$tmp =~ s/</&lt;/g;
$tmp =~ s/>/&gt;/g;
return $tmp;
}
##----------------------------------------------------------------------------##
# DisplayHelp - Utility function to display all help options.
##----------------------------------------------------------------------------##
sub DisplayHelp {
print <<ENDTEXT;
USAGE: $Prog [options] --report-dir <report dir> [build options]
ENDTEXT
if (defined $BuildName) {
print "ANALYZER BUILD: $BuildName ($BuildDate)\n\n";
}
print <<ENDTEXT;
OPTIONS:
--report-dir <report dir>
Specifies the directory with static analyzer report <html> files. If this
option is not specified the program exits.
-h
--help
Display this message.
--html-title [title]
--html-title=[title]
Specify the title used on generated HTML pages. If not specified, a default
title will be used.
--keep-empty
Don't remove the build results directory even if no issues were reported.
--verbose
Verbose output.
ENDTEXT
}
##----------------------------------------------------------------------------##
# Process command-line arguments.
##----------------------------------------------------------------------------##
my $RequestDisplayHelp = 0;
my $ForceDisplayHelp = 0;
if (!@ARGV) {
$ForceDisplayHelp = 1;
}
while (@ARGV) {
# Scan for options we recognize.
my $arg = $ARGV[0];
if ($arg eq "-h" or $arg eq "--help") {
$RequestDisplayHelp = 1;
shift @ARGV;
next;
}
if ($arg eq "--verbose") {
$Verbose = 1;
shift @ARGV;
next;
}
if ($arg eq "--report-dir") {
shift @ARGV;
if (!@ARGV) {
DieDiag("'--report-dir' option requires a target directory name.\n");
}
# Construct an absolute path. Uses the current working directory
# as a base if the original path was not absolute.
$HtmlDir = abs_path(shift @ARGV);
$HtmlDirSpecified = 1;
next;
}
if ($arg =~ /^--html-title(=(.+))?$/) {
shift @ARGV;
if (!defined $2 || $2 eq '') {
if (!@ARGV) {
DieDiag("'--html-title' option requires a string.\n");
}
$HtmlTitle = shift @ARGV;
} else {
$HtmlTitle = $2;
}
next;
}
if ($arg eq "--keep-empty") {
shift @ARGV;
$KeepEmpty = 1;
next;
}
DieDiag("unrecognized option '$arg'\n");
last;
}
if(!$RequestDisplayHelp && $HtmlDirSpecified) {
if (-d $HtmlDir) {
if (! -r $HtmlDir) {
DieDiag("directory '$HtmlDir' exists but is not readable.\n");
}
} else {
DieDiag("report directory does not exist.\n");
$ForceDisplayHelp = 1;
}
} else {
$ForceDisplayHelp = 1;
}
if ($ForceDisplayHelp || $RequestDisplayHelp) {
DisplayHelp();
exit $ForceDisplayHelp;
}
Diag("Will read the reports from '$HtmlDir'\n");
if (!$KeepEmpty) {
Diag("Will remove the report directory if it contains no reports.\n");
}
Diag("Scanning started...\n");
my $NumBugs = Postprocess($HtmlDir, $AnalyzerStats, $KeepEmpty);
exit $NumBugs;