fuzz: use qemu_get_exec_dir
[qemu.git] / scripts / tap-driver.pl
1 #! /usr/bin/env perl
2 # Copyright (C) 2011-2013 Free Software Foundation, Inc.
3 # Copyright (C) 2018 Red Hat, Inc.
4 #
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2, or (at your option)
8 # any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <https://www.gnu.org/licenses/>.
17
18 # As a special exception to the GNU General Public License, if you
19 # distribute this file as part of a program that contains a
20 # configuration script generated by Autoconf, you may include it under
21 # the same distribution terms that you use for the rest of that program.
22
23 # ---------------------------------- #
24 # Imports, static data, and setup. #
25 # ---------------------------------- #
26
27 use warnings FATAL => 'all';
28 use strict;
29 use Getopt::Long ();
30 use TAP::Parser;
31 use Term::ANSIColor qw(:constants);
32
33 my $ME = "tap-driver.pl";
34 my $VERSION = "2018-11-30";
35
36 my $USAGE = <<'END';
37 Usage:
38 tap-driver [--test-name=TEST] [--color={always|never|auto}]
39 [--verbose] [--show-failures-only]
40 END
41
42 my $HELP = "$ME: TAP-aware test driver for QEMU testsuite harness." .
43 "\n" . $USAGE;
44
45 # It's important that NO_PLAN evaluates "false" as a boolean.
46 use constant NO_PLAN => 0;
47 use constant EARLY_PLAN => 1;
48 use constant LATE_PLAN => 2;
49
50 use constant DIAG_STRING => "#";
51
52 # ------------------- #
53 # Global variables. #
54 # ------------------- #
55
56 my $testno = 0; # Number of test results seen so far.
57 my $bailed_out = 0; # Whether a "Bail out!" directive has been seen.
58 my $failed = 0; # Final exit code
59
60 # Whether the TAP plan has been seen or not, and if yes, which kind
61 # it is ("early" is seen before any test result, "late" otherwise).
62 my $plan_seen = NO_PLAN;
63
64 # ----------------- #
65 # Option parsing. #
66 # ----------------- #
67
68 my %cfg = (
69 "color" => 0,
70 "verbose" => 0,
71 "show-failures-only" => 0,
72 );
73
74 my $color = "auto";
75 my $test_name = undef;
76
77 # Perl's Getopt::Long allows options to take optional arguments after a space.
78 # Prevent --color by itself from consuming other arguments
79 foreach (@ARGV) {
80 if ($_ eq "--color" || $_ eq "-color") {
81 $_ = "--color=$color";
82 }
83 }
84
85 Getopt::Long::GetOptions
86 (
87 'help' => sub { print $HELP; exit 0; },
88 'version' => sub { print "$ME $VERSION\n"; exit 0; },
89 'test-name=s' => \$test_name,
90 'color=s' => \$color,
91 'show-failures-only' => sub { $cfg{"show-failures-only"} = 1; },
92 'verbose' => sub { $cfg{"verbose"} = 1; },
93 ) or exit 1;
94
95 if ($color =~ /^always$/i) {
96 $cfg{'color'} = 1;
97 } elsif ($color =~ /^never$/i) {
98 $cfg{'color'} = 0;
99 } elsif ($color =~ /^auto$/i) {
100 $cfg{'color'} = (-t STDOUT);
101 } else {
102 die "Invalid color mode: $color\n";
103 }
104
105 # ------------- #
106 # Prototypes. #
107 # ------------- #
108
109 sub colored ($$);
110 sub decorate_result ($);
111 sub extract_tap_comment ($);
112 sub handle_tap_bailout ($);
113 sub handle_tap_plan ($);
114 sub handle_tap_result ($);
115 sub is_null_string ($);
116 sub main ();
117 sub report ($;$);
118 sub stringify_result_obj ($);
119 sub testsuite_error ($);
120
121 # -------------- #
122 # Subroutines. #
123 # -------------- #
124
125 # If the given string is undefined or empty, return true, otherwise
126 # return false. This function is useful to avoid pitfalls like:
127 # if ($message) { print "$message\n"; }
128 # which wouldn't print anything if $message is the literal "0".
129 sub is_null_string ($)
130 {
131 my $str = shift;
132 return ! (defined $str and length $str);
133 }
134
135 sub stringify_result_obj ($)
136 {
137 my $result_obj = shift;
138 if ($result_obj->is_unplanned || $result_obj->number != $testno)
139 {
140 return "ERROR";
141 }
142 elsif ($plan_seen == LATE_PLAN)
143 {
144 return "ERROR";
145 }
146 elsif (!$result_obj->directive)
147 {
148 return $result_obj->is_ok ? "PASS" : "FAIL";
149 }
150 elsif ($result_obj->has_todo)
151 {
152 return $result_obj->is_actual_ok ? "XPASS" : "XFAIL";
153 }
154 elsif ($result_obj->has_skip)
155 {
156 return $result_obj->is_ok ? "SKIP" : "FAIL";
157 }
158 die "$ME: INTERNAL ERROR"; # NOTREACHED
159 }
160
161 sub colored ($$)
162 {
163 my ($color_string, $text) = @_;
164 return $color_string . $text . RESET;
165 }
166
167 sub decorate_result ($)
168 {
169 my $result = shift;
170 return $result unless $cfg{"color"};
171 my %color_for_result =
172 (
173 "ERROR" => BOLD.MAGENTA,
174 "PASS" => GREEN,
175 "XPASS" => BOLD.YELLOW,
176 "FAIL" => BOLD.RED,
177 "XFAIL" => YELLOW,
178 "SKIP" => BLUE,
179 );
180 if (my $color = $color_for_result{$result})
181 {
182 return colored ($color, $result);
183 }
184 else
185 {
186 return $result; # Don't colorize unknown stuff.
187 }
188 }
189
190 sub report ($;$)
191 {
192 my ($msg, $result, $explanation) = (undef, @_);
193 if ($result =~ /^(?:X?(?:PASS|FAIL)|SKIP|ERROR)/)
194 {
195 # Output on console might be colorized.
196 $msg = decorate_result($result);
197 if ($result =~ /^(?:PASS|XFAIL|SKIP)/)
198 {
199 return if $cfg{"show-failures-only"};
200 }
201 else
202 {
203 $failed = 1;
204 }
205 }
206 elsif ($result eq "#")
207 {
208 $msg = " ";
209 }
210 else
211 {
212 die "$ME: INTERNAL ERROR"; # NOTREACHED
213 }
214 $msg .= " $explanation" if defined $explanation;
215 print $msg . "\n";
216 }
217
218 sub testsuite_error ($)
219 {
220 report "ERROR", "$test_name - $_[0]";
221 }
222
223 sub handle_tap_result ($)
224 {
225 $testno++;
226 my $result_obj = shift;
227
228 my $test_result = stringify_result_obj $result_obj;
229 my $string = $result_obj->number;
230
231 my $description = $result_obj->description;
232 $string .= " $test_name" unless is_null_string $test_name;
233 $string .= " $description" unless is_null_string $description;
234
235 if ($plan_seen == LATE_PLAN)
236 {
237 $string .= " # AFTER LATE PLAN";
238 }
239 elsif ($result_obj->is_unplanned)
240 {
241 $string .= " # UNPLANNED";
242 }
243 elsif ($result_obj->number != $testno)
244 {
245 $string .= " # OUT-OF-ORDER (expecting $testno)";
246 }
247 elsif (my $directive = $result_obj->directive)
248 {
249 $string .= " # $directive";
250 my $explanation = $result_obj->explanation;
251 $string .= " $explanation"
252 unless is_null_string $explanation;
253 }
254
255 report $test_result, $string;
256 }
257
258 sub handle_tap_plan ($)
259 {
260 my $plan = shift;
261 if ($plan_seen)
262 {
263 # Error, only one plan per stream is acceptable.
264 testsuite_error "multiple test plans";
265 return;
266 }
267 # The TAP plan can come before or after *all* the TAP results; we speak
268 # respectively of an "early" or a "late" plan. If we see the plan line
269 # after at least one TAP result has been seen, assume we have a late
270 # plan; in this case, any further test result seen after the plan will
271 # be flagged as an error.
272 $plan_seen = ($testno >= 1 ? LATE_PLAN : EARLY_PLAN);
273 # If $testno > 0, we have an error ("too many tests run") that will be
274 # automatically dealt with later, so don't worry about it here. If
275 # $plan_seen is true, we have an error due to a repeated plan, and that
276 # has already been dealt with above. Otherwise, we have a valid "plan
277 # with SKIP" specification, and should report it as a particular kind
278 # of SKIP result.
279 if ($plan->directive && $testno == 0)
280 {
281 my $explanation = is_null_string ($plan->explanation) ?
282 undef : "- " . $plan->explanation;
283 report "SKIP", $explanation;
284 }
285 }
286
287 sub handle_tap_bailout ($)
288 {
289 my ($bailout, $msg) = ($_[0], "Bail out!");
290 $bailed_out = 1;
291 $msg .= " " . $bailout->explanation
292 unless is_null_string $bailout->explanation;
293 testsuite_error $msg;
294 }
295
296 sub extract_tap_comment ($)
297 {
298 my $line = shift;
299 if (index ($line, DIAG_STRING) == 0)
300 {
301 # Strip leading `DIAG_STRING' from `$line'.
302 $line = substr ($line, length (DIAG_STRING));
303 # And strip any leading and trailing whitespace left.
304 $line =~ s/(?:^\s*|\s*$)//g;
305 # Return what is left (if any).
306 return $line;
307 }
308 return "";
309 }
310
311 sub main ()
312 {
313 my $iterator = TAP::Parser::Iterator::Stream->new(\*STDIN);
314 my $parser = TAP::Parser->new ({iterator => $iterator });
315
316 STDOUT->autoflush(1);
317 while (defined (my $cur = $parser->next))
318 {
319 # Parsing of TAP input should stop after a "Bail out!" directive.
320 next if $bailed_out;
321
322 if ($cur->is_plan)
323 {
324 handle_tap_plan ($cur);
325 }
326 elsif ($cur->is_test)
327 {
328 handle_tap_result ($cur);
329 }
330 elsif ($cur->is_bailout)
331 {
332 handle_tap_bailout ($cur);
333 }
334 elsif ($cfg{"verbose"})
335 {
336 my $comment = extract_tap_comment ($cur->raw);
337 report "#", "$comment" if length $comment;
338 }
339 }
340 # A "Bail out!" directive should cause us to ignore any following TAP
341 # error.
342 if (!$bailed_out)
343 {
344 if (!$plan_seen)
345 {
346 testsuite_error "missing test plan";
347 }
348 elsif ($parser->tests_planned != $parser->tests_run)
349 {
350 my ($planned, $run) = ($parser->tests_planned, $parser->tests_run);
351 my $bad_amount = $run > $planned ? "many" : "few";
352 testsuite_error (sprintf "too %s tests run (expected %d, got %d)",
353 $bad_amount, $planned, $run);
354 }
355 }
356 }
357
358 # ----------- #
359 # Main code. #
360 # ----------- #
361
362 main;
363 exit($failed);
364
365 # Local Variables:
366 # perl-indent-level: 2
367 # perl-continued-statement-offset: 2
368 # perl-continued-brace-offset: 0
369 # perl-brace-offset: 0
370 # perl-brace-imaginary-offset: 0
371 # perl-label-offset: -2
372 # cperl-indent-level: 2
373 # cperl-brace-offset: 0
374 # cperl-continued-brace-offset: 0
375 # cperl-label-offset: -2
376 # cperl-extra-newline-before-brace: t
377 # cperl-merge-trailing-else: nil
378 # cperl-continued-statement-offset: 2
379 # End: