From e585aa3131cc6a55ac2e6b8b9f7c571c83499e6d Mon Sep 17 00:00:00 2001 From: Nicolas Saillant Date: Wed, 5 Apr 2023 11:34:10 +0200 Subject: [PATCH] Adding a timeout column to the testsuite --- Maintenance/test_handling/create_testresult_page | 15 ++++++++++++--- Maintenance/test_handling/testresult.css | 6 ++++++ Testsuite/test/parse-ctest-dashboard-xml.py | 9 +++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Maintenance/test_handling/create_testresult_page b/Maintenance/test_handling/create_testresult_page index 6a3632481b7..1744b4e575d 100755 --- a/Maintenance/test_handling/create_testresult_page +++ b/Maintenance/test_handling/create_testresult_page @@ -151,7 +151,7 @@ sub collect_results_of_platform($) # Create an anonymous hash that hashes packages to their result. my $platform_results = {}; my $test_result="results_${platform}.txt"; - my ($yeahs, $nays, $warnings,$third_party_warnings,$reqs) = (0,0,0,0,0); + my ($yeahs, $nays, $warnings,$third_party_warnings,$timeout,$reqs) = (0,0,0,0,0,0); my $resulttext; open(TESTRESULT, $test_result) or return $platform_results; while () { @@ -169,7 +169,10 @@ sub collect_results_of_platform($) } elsif ($2 eq 'n' or $2 eq 'N') { $resulttext = 'n'; ++$nays; - } elsif ($2 eq 'r') { + } elsif ($2 eq 'o' or $2 eq 'O') { + $resulttext = 'o'; + ++$timeout; + } elsif ($2 eq 'r') { $resulttext = 'r'; ++$reqs; } else { @@ -183,6 +186,7 @@ sub collect_results_of_platform($) $platform_results->{"n"} = $nays; $platform_results->{"w"} = $warnings; $platform_results->{"t"} = $third_party_warnings; + $platform_results->{"o"} = $timeout; $platform_results->{"r"} = $reqs; return $platform_results; } @@ -260,6 +264,8 @@ EOF print OUTPUT ' class="warning"'; } elsif ($resulttext eq 't') { print OUTPUT ' class="third_party_warning"'; + } elsif ($resulttext eq 'o') { + print OUTPUT ' class="timeout"'; } elsif ($resulttext eq 'n') { print OUTPUT ' class="error"'; } elsif ($resulttext eq 'r') { @@ -461,6 +467,7 @@ sub print_platform_descriptions() y t w +o n r CMake @@ -500,6 +507,7 @@ EOF my $county = $testresults[$platform_num]->{"y"}; my $countt = $testresults[$platform_num]->{"t"}; my $countw = $testresults[$platform_num]->{"w"}; + my $counto = $testresults[$platform_num]->{"o"}; my $countn = $testresults[$platform_num]->{"n"}; my $countr = $testresults[$platform_num]->{"r"}; @@ -519,6 +527,7 @@ EOF print OUTPUT "$county\n"; print OUTPUT "$countt\n"; print OUTPUT "$countw\n"; + print OUTPUT "$counto\n"; print OUTPUT "$countn\n"; print OUTPUT "$countr\n"; $index = 8; @@ -611,7 +620,7 @@ sub print_little_header(){ jump to results

The results of the tests are presented in a table -('y' = success, 'w' = warning, 't' = third party warning 'n' = failure, 'r' = a requirement is not found), +('y' = success, 'w' = warning, 't' = third party warning, 'o' = timeout, 'n' = failure, 'r' = a requirement is not found), and the error + compiler output from each test can be retrieved by clicking on it.

N.B. The detection of warnings is not exact. diff --git a/Maintenance/test_handling/testresult.css b/Maintenance/test_handling/testresult.css index c44cba095ce..b48d35ee171 100644 --- a/Maintenance/test_handling/testresult.css +++ b/Maintenance/test_handling/testresult.css @@ -28,6 +28,7 @@ TD.os64bits {font-style:italic} TD.ok {background-color: rgb(44%,88%,44%)} TD.warning {background-color: rgb(100%,100%,50%)} TD.third_party_warning {background-color: rgb(75%,100%,50%)} +TD.timeout {background-color: rgb(100%,75%,25%)} TD.error {background-color: rgb(100%,50%,50%)} TD.na {background-color: white;} TD.requirements { background-color: rgb(65%,65%,100%) } @@ -35,6 +36,7 @@ TD.requirements { background-color: rgb(65%,65%,100%) } TH.ok {background-color: rgb(44%,88%,44%)} TH.warning {background-color: rgb(100%,100%,50%)} TH.third_party_warning {background-color: rgb(75%,100%,50%)} +TH.timeout {background-color: rgb(100%,75%,25%)} TH.error {background-color: rgb(100%,50%,50%)} TH.requirements { background-color: rgb(65%,65%,100%) } @@ -50,6 +52,10 @@ TD.third_party_warning A {font-size:large; text-decoration: none} TD.third_party_warning A:link {color: rgb(0%,0%,100%)} TD.third_party_warning A:visited {color: rgb(80%,80%,100%)} +TD.timeout A {font-size:large; text-decoration: none} +TD.timeout A:link {color: rgb(0%,0%,100%)} +TD.timeout A:visited {color: rgb(80%,80%,100%)} + TD.error A {font-size: large; text-decoration: none} TD.error A:link {color: rgb(0%,0%,100%)} TD.error A:visited {color: rgb(80%,0%,100%)} diff --git a/Testsuite/test/parse-ctest-dashboard-xml.py b/Testsuite/test/parse-ctest-dashboard-xml.py index 0df042fa1d5..748ada8a48c 100644 --- a/Testsuite/test/parse-ctest-dashboard-xml.py +++ b/Testsuite/test/parse-ctest-dashboard-xml.py @@ -83,13 +83,16 @@ with open_file_create_dir(result_file_name.format(dir=os.getcwd(), tester=tester_name, platform=platform_name), 'a+') as results: for label, tests in tests_per_label.items(): - counts={"n": 0, "w": 0, "t": 0} + counts={"n": 0, "w": 0, "t": 0, "o": 0} result_for_label='y' with open_file_create_dir("{}/error.txt".format(label), 'w') as error: for t in tests: print(" {result} {name} in {time} s : {value} ".format(result = "successful " if (t['Status'] == 'passed') else "ERROR: ", name = t['Name'], value = t['ExitValue'] if(t['ExitValue'] != "") else "SUCCESS" , time = t['ExecutionTime']), file=error) if t['Status'] != 'passed': - counts["n"]+=1 + if t['ExitValue'] == "Timeout": + counts["o"]+=1 + else: + counts["n"]+=1 elif t['Output'] != None and w_det.search(t['Output']): entries = re.split("\n+", t['Output']) for entry in entries: @@ -108,6 +111,8 @@ with open_file_create_dir(result_file_name.format(dir=os.getcwd(), if counts["n"] > 0: result_for_label='n' + elif counts["o"] > 0: + result_for_label='o' elif counts["w"] > 0: result_for_label='w' elif counts["t"] > 0: