| /* |
| * Copyright (c) 2016, The OpenThread Authors. |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * 3. Neither the name of the copyright holder nor the |
| * names of its contributors may be used to endorse or promote products |
| * derived from this software without specific prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| * POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| using System; |
| using System.Collections.Generic; |
| using System.Diagnostics; |
| using System.IO; |
| using System.Net; |
| using System.Text; |
| using System.Threading; |
| using System.Threading.Tasks; |
| |
| namespace otTestRunner |
| { |
| class Program |
| { |
| struct TestResults |
| { |
| public bool Pass; |
| public List<string> Output; |
| public string Error; |
| } |
| |
| static string EscapeJson(string data) |
| { |
| return data.Replace("\\", "\\\\").Replace("\"", "\\\""); |
| } |
| |
| /// <summary> |
| /// Executes an exe with the given args and captures the output |
| /// </summary> |
| static async Task<TestResults> ExecuteAsync(string name, string args = null, int timeoutMilliseconds = -1, int instanceIndex = -1) |
| { |
| ProcessStartInfo startInfo = new ProcessStartInfo(); |
| startInfo.WindowStyle = ProcessWindowStyle.Hidden; |
| startInfo.CreateNoWindow = true; |
| startInfo.UseShellExecute = false; |
| startInfo.RedirectStandardError = true; |
| startInfo.RedirectStandardOutput = true; |
| startInfo.FileName = name; |
| startInfo.Arguments = args; |
| |
| startInfo.EnvironmentVariables["NODE_TYPE"] = "win-sim"; |
| |
| if (instanceIndex != -1) |
| { |
| startInfo.EnvironmentVariables["INSTANCE"] = instanceIndex.ToString(); |
| } |
| |
| TestResults Results = new TestResults(); |
| Results.Pass = false; |
| Results.Output = new List<string>(); |
| Results.Error = null; |
| var Errors = new List<string>(); |
| |
| Results.Output.Add(string.Format("> set NODE_TYPE=win-sim")); |
| Results.Output.Add(string.Format("> {0} {1}", name, args)); |
| Results.Output.Add("----------------------------------------------------------------------"); |
| |
| try |
| { |
| // Execute process |
| using (Process process = Process.Start(startInfo)) |
| { |
| process.OutputDataReceived += |
| (object sender, DataReceivedEventArgs e) => { |
| if (e.Data != null && e.Data.Length > 0) |
| lock (Results.Output) { Results.Output.Add(e.Data); } |
| }; |
| process.ErrorDataReceived += |
| (object sender, DataReceivedEventArgs e) => { |
| if (e.Data != null && e.Data.Length > 0) |
| lock (Results.Output) { Errors.Add(e.Data); } |
| }; |
| |
| process.BeginErrorReadLine(); |
| process.BeginOutputReadLine(); |
| |
| #if DEBUG |
| Console.WriteLine("Starting {0} {1}", name, args); |
| #endif |
| |
| // Wait for process to complete |
| await Task.Run( |
| () => { |
| if (timeoutMilliseconds == -1) |
| process.WaitForExit(); |
| else if (!process.WaitForExit(timeoutMilliseconds)) |
| { |
| process.Kill(); |
| Results.Output.Add(string.Format("Killed {0} on execution timeout!", name)); |
| } |
| }); |
| |
| // Wait a bit for any output to collect |
| await Task.Delay(1000); |
| |
| process.CancelOutputRead(); |
| process.CancelErrorRead(); |
| |
| Results.Output.AddRange(Errors); |
| Results.Pass = Errors.Count > 0 && Errors[Errors.Count - 1] == "OK"; |
| |
| if (!Results.Pass) Results.Error = EscapeJson(string.Join("\\r\\n", Errors)); |
| |
| // Make sure the process is killed |
| try { process.Kill(); } catch (Exception) { } |
| |
| #if DEBUG |
| Console.WriteLine("Completed {0} {1}", name, args); |
| #endif |
| } |
| } |
| catch (Exception e) |
| { |
| Results.Output.Add("Encountered exception: " + e.Message); |
| Results.Output.Add(e.StackTrace); |
| } |
| |
| return Results; |
| } |
| |
| static bool VerboseOutput = false; |
| static int RetiresOnFailure = 0; |
| static bool AppVeyorMode = false; |
| static string AppVeyorApiUrl = null; |
| static string ResultsFolder = "Results_" + DateTime.Now.ToString("yyyyMMdd_HH.mm.ss"); |
| |
| static void UploadAppVeyorTestResult(string name, bool passed, long durationMS, string error = null) |
| { |
| if (AppVeyorApiUrl == null) return; |
| |
| string jsonData = null; |
| |
| try |
| { |
| var request = (HttpWebRequest)WebRequest.Create(Path.Combine(AppVeyorApiUrl, "api/tests")); |
| |
| jsonData = |
| string.Format( |
| "{{" + |
| "\"testName\": \"{0}\", " + |
| "\"testFramework\": \"MSTest\", " + |
| "\"fileName\": \"{0}.py\", " + |
| "\"outcome\": \"{1}\", " + |
| "\"durationMilliseconds\": \"{2}\", " + |
| "\"ErrorMessage\": \"{3}\"" + |
| "}}", |
| name, |
| passed ? "Passed" : "Failed", |
| durationMS, |
| error == null ? "" : error |
| ); |
| |
| var data = Encoding.UTF8.GetBytes(jsonData); |
| |
| request.Method = "POST"; |
| request.ContentType = "application/json"; |
| request.ContentLength = data.Length; |
| |
| using (var stream = request.GetRequestStream()) |
| { |
| stream.Write(data, 0, data.Length); |
| } |
| |
| var response = (HttpWebResponse)request.GetResponse(); |
| var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd(); |
| } |
| catch (Exception e) |
| { |
| Console.WriteLine("Encountered exception for http post:"); |
| Console.WriteLine(e.Message); |
| Console.WriteLine(e.StackTrace); |
| |
| Console.WriteLine("Json content:"); |
| Console.WriteLine(jsonData); |
| } |
| } |
| |
| enum TestResult |
| { |
| Fail, |
| Pass, |
| PassWithRetry |
| } |
| |
| /// <summary> |
| /// Runs a python test file and returns success/failure |
| /// </summary> |
| static async Task<TestResult> RunTest(string file, int index) |
| { |
| string pythonPath = "python.exe"; |
| if (AppVeyorMode) |
| { |
| if (Environment.GetEnvironmentVariable("Platform").ToLower() == "x64") |
| { |
| pythonPath = @"c:\python35-x64\python.exe"; |
| } |
| else |
| { |
| pythonPath = @"c:\python35\python.exe"; |
| } |
| } |
| |
| int tries = 0; |
| Stopwatch Timer; |
| TestResults Results; |
| |
| do |
| { |
| Timer = new Stopwatch(); |
| Timer.Start(); |
| |
| Results = await ExecuteAsync(pythonPath, file, 30 * 60 * 1000, index); |
| |
| Timer.Stop(); |
| |
| } while (++tries < RetiresOnFailure + 1 && Results.Pass == false); |
| |
| if (VerboseOutput) |
| { |
| lock (ResultsFolder) |
| { |
| foreach (var line in Results.Output) |
| Console.WriteLine(line); |
| } |
| } |
| |
| UploadAppVeyorTestResult(Path.GetFileNameWithoutExtension(file), Results.Pass, Timer.ElapsedMilliseconds, Results.Error); |
| |
| // Write the output to a file |
| var filePrefix = Results.Pass ? "P_" : "F_"; |
| var outputFilePath = Path.Combine(ResultsFolder, filePrefix + Path.GetFileNameWithoutExtension(file) + ".txt"); |
| try { |
| File.WriteAllLines(outputFilePath, Results.Output); |
| } catch (Exception e) { |
| Console.WriteLine("Exception while trying to write {0}:\n{1}!", outputFilePath, e.Message); |
| } |
| |
| return Results.Pass ? |
| (tries == 1 ? TestResult.Pass : TestResult.PassWithRetry) : |
| TestResult.Fail; |
| } |
| |
| /// <summary> |
| /// Runs all the tests as indicated by input arguments |
| /// </summary> |
| static void Main(string[] args) |
| { |
| if (args.Length < 2) |
| { |
| Console.WriteLine("Usage: otTestRunner.exe [path] [search pattern] (parallel:n) (verbose)"); |
| return; |
| } |
| |
| var files = Directory.GetFiles(args[0], args[1]); |
| if (files.Length == 0) |
| { |
| Console.WriteLine("No tests found with that path & pattern!"); |
| return; |
| } |
| |
| var NumberOfTestsToRunInParallel = 1; |
| for (var i = 2; i < args.Length; i++) |
| { |
| if (args[i].StartsWith("parallel:")) |
| NumberOfTestsToRunInParallel = int.Parse(args[i].Substring(9)); |
| else if (args[i].StartsWith("retry:")) |
| RetiresOnFailure = int.Parse(args[i].Substring(6)); |
| else if (args[i].StartsWith("verbose")) |
| VerboseOutput = true; |
| else if (args[i].StartsWith("appveyor")) |
| { |
| AppVeyorMode = true; |
| AppVeyorApiUrl = Environment.GetEnvironmentVariable("APPVEYOR_API_URL"); |
| //Console.WriteLine("AppVeyorApiUrl = {0}", AppVeyorApiUrl); |
| } |
| } |
| |
| var CurNumTestsRunning = 0; |
| var ReadyToRunEvent = new ManualResetEvent(true); |
| |
| var TestPassCount = 0; |
| Stopwatch Timer = new Stopwatch(); |
| |
| Directory.CreateDirectory(ResultsFolder); |
| Console.WriteLine("Test results saved: .\\{0}", ResultsFolder); |
| |
| Console.WriteLine("Running {0} tests, {1} at a time:", files.Length, NumberOfTestsToRunInParallel); |
| /*for (var i = 0; i < files.Length; i++) |
| Console.WriteLine(Path.GetFileName(files[i]));*/ |
| Console.WriteLine(""); |
| |
| Timer.Start(); |
| for (var i = 0; i < files.Length; i++) |
| { |
| // Wait for the event to be set, if not already |
| ReadyToRunEvent.WaitOne(); |
| |
| if (i != 0) |
| { |
| // Wait a bit to stagger the starts |
| Task.Delay(1000).Wait(); |
| } |
| |
| lock (ResultsFolder) |
| { |
| if (++CurNumTestsRunning == NumberOfTestsToRunInParallel) |
| ReadyToRunEvent.Reset(); |
| } |
| |
| var index = i; |
| var fileName = files[i]; |
| |
| // Start the test, but don't wait for it |
| Task.Run( |
| async () => |
| { |
| var result = await RunTest(fileName, index); |
| lock (ResultsFolder) |
| { |
| var PrevColor = Console.ForegroundColor; |
| if (result != TestResult.Fail) |
| { |
| if (result == TestResult.Pass) |
| { |
| Console.ForegroundColor = ConsoleColor.Green; |
| } |
| else |
| { |
| Console.ForegroundColor = ConsoleColor.Yellow; |
| } |
| Console.Write("PASS"); |
| TestPassCount++; |
| } |
| else |
| { |
| Console.ForegroundColor = ConsoleColor.Red; |
| Console.Write("FAIL"); |
| } |
| Console.ForegroundColor = PrevColor; |
| Console.WriteLine(": {0}", Path.GetFileNameWithoutExtension(fileName)); |
| |
| CurNumTestsRunning--; |
| ReadyToRunEvent.Set(); |
| } |
| }); |
| } |
| |
| // Wait for all the tests to complete |
| while (CurNumTestsRunning != 0) |
| ReadyToRunEvent.WaitOne(); |
| |
| Timer.Stop(); |
| TimeSpan ts = Timer.Elapsed; |
| string elapsedTime = |
| String.Format("{0:00}:{1:00}:{2:00}.{3:00}", |
| ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); |
| |
| Console.WriteLine("{0} tests run in {1}", files.Length, elapsedTime); |
| Console.WriteLine("{0} passed and {1} failed", TestPassCount, files.Length - TestPassCount); |
| |
| if (!AppVeyorMode) |
| Environment.ExitCode = files.Length == TestPassCount ? 0 : 1; |
| } |
| } |
| } |
