qapi: enable use of g_autoptr with QAPI types
[qemu.git] / scripts / qapi / parser.py
1 # -*- coding: utf-8 -*-
2 #
3 # QAPI schema parser
4 #
5 # Copyright IBM, Corp. 2011
6 # Copyright (c) 2013-2019 Red Hat Inc.
7 #
8 # Authors:
9 # Anthony Liguori <aliguori@us.ibm.com>
10 # Markus Armbruster <armbru@redhat.com>
11 # Marc-André Lureau <marcandre.lureau@redhat.com>
12 # Kevin Wolf <kwolf@redhat.com>
13 #
14 # This work is licensed under the terms of the GNU GPL, version 2.
15 # See the COPYING file in the top-level directory.
16
17 import os
18 import re
19 from collections import OrderedDict
20
21 from qapi.error import QAPIParseError, QAPISemError
22 from qapi.source import QAPISourceInfo
23
24
25 class QAPISchemaParser:
26
27 def __init__(self, fname, previously_included=None, incl_info=None):
28 previously_included = previously_included or set()
29 previously_included.add(os.path.abspath(fname))
30
31 try:
32 fp = open(fname, 'r', encoding='utf-8')
33 self.src = fp.read()
34 except IOError as e:
35 raise QAPISemError(incl_info or QAPISourceInfo(None, None, None),
36 "can't read %s file '%s': %s"
37 % ("include" if incl_info else "schema",
38 fname,
39 e.strerror))
40
41 if self.src == '' or self.src[-1] != '\n':
42 self.src += '\n'
43 self.cursor = 0
44 self.info = QAPISourceInfo(fname, 1, incl_info)
45 self.line_pos = 0
46 self.exprs = []
47 self.docs = []
48 self.accept()
49 cur_doc = None
50
51 while self.tok is not None:
52 info = self.info
53 if self.tok == '#':
54 self.reject_expr_doc(cur_doc)
55 cur_doc = self.get_doc(info)
56 self.docs.append(cur_doc)
57 continue
58
59 expr = self.get_expr(False)
60 if 'include' in expr:
61 self.reject_expr_doc(cur_doc)
62 if len(expr) != 1:
63 raise QAPISemError(info, "invalid 'include' directive")
64 include = expr['include']
65 if not isinstance(include, str):
66 raise QAPISemError(info,
67 "value of 'include' must be a string")
68 incl_fname = os.path.join(os.path.dirname(fname),
69 include)
70 self.exprs.append({'expr': {'include': incl_fname},
71 'info': info})
72 exprs_include = self._include(include, info, incl_fname,
73 previously_included)
74 if exprs_include:
75 self.exprs.extend(exprs_include.exprs)
76 self.docs.extend(exprs_include.docs)
77 elif "pragma" in expr:
78 self.reject_expr_doc(cur_doc)
79 if len(expr) != 1:
80 raise QAPISemError(info, "invalid 'pragma' directive")
81 pragma = expr['pragma']
82 if not isinstance(pragma, dict):
83 raise QAPISemError(
84 info, "value of 'pragma' must be an object")
85 for name, value in pragma.items():
86 self._pragma(name, value, info)
87 else:
88 expr_elem = {'expr': expr,
89 'info': info}
90 if cur_doc:
91 if not cur_doc.symbol:
92 raise QAPISemError(
93 cur_doc.info, "definition documentation required")
94 expr_elem['doc'] = cur_doc
95 self.exprs.append(expr_elem)
96 cur_doc = None
97 self.reject_expr_doc(cur_doc)
98
99 @staticmethod
100 def reject_expr_doc(doc):
101 if doc and doc.symbol:
102 raise QAPISemError(
103 doc.info,
104 "documentation for '%s' is not followed by the definition"
105 % doc.symbol)
106
107 def _include(self, include, info, incl_fname, previously_included):
108 incl_abs_fname = os.path.abspath(incl_fname)
109 # catch inclusion cycle
110 inf = info
111 while inf:
112 if incl_abs_fname == os.path.abspath(inf.fname):
113 raise QAPISemError(info, "inclusion loop for %s" % include)
114 inf = inf.parent
115
116 # skip multiple include of the same file
117 if incl_abs_fname in previously_included:
118 return None
119
120 return QAPISchemaParser(incl_fname, previously_included, info)
121
122 def _pragma(self, name, value, info):
123 if name == 'doc-required':
124 if not isinstance(value, bool):
125 raise QAPISemError(info,
126 "pragma 'doc-required' must be boolean")
127 info.pragma.doc_required = value
128 elif name == 'returns-whitelist':
129 if (not isinstance(value, list)
130 or any([not isinstance(elt, str) for elt in value])):
131 raise QAPISemError(
132 info,
133 "pragma returns-whitelist must be a list of strings")
134 info.pragma.returns_whitelist = value
135 elif name == 'name-case-whitelist':
136 if (not isinstance(value, list)
137 or any([not isinstance(elt, str) for elt in value])):
138 raise QAPISemError(
139 info,
140 "pragma name-case-whitelist must be a list of strings")
141 info.pragma.name_case_whitelist = value
142 else:
143 raise QAPISemError(info, "unknown pragma '%s'" % name)
144
145 def accept(self, skip_comment=True):
146 while True:
147 self.tok = self.src[self.cursor]
148 self.pos = self.cursor
149 self.cursor += 1
150 self.val = None
151
152 if self.tok == '#':
153 if self.src[self.cursor] == '#':
154 # Start of doc comment
155 skip_comment = False
156 self.cursor = self.src.find('\n', self.cursor)
157 if not skip_comment:
158 self.val = self.src[self.pos:self.cursor]
159 return
160 elif self.tok in '{}:,[]':
161 return
162 elif self.tok == "'":
163 # Note: we accept only printable ASCII
164 string = ''
165 esc = False
166 while True:
167 ch = self.src[self.cursor]
168 self.cursor += 1
169 if ch == '\n':
170 raise QAPIParseError(self, "missing terminating \"'\"")
171 if esc:
172 # Note: we recognize only \\ because we have
173 # no use for funny characters in strings
174 if ch != '\\':
175 raise QAPIParseError(self,
176 "unknown escape \\%s" % ch)
177 esc = False
178 elif ch == '\\':
179 esc = True
180 continue
181 elif ch == "'":
182 self.val = string
183 return
184 if ord(ch) < 32 or ord(ch) >= 127:
185 raise QAPIParseError(
186 self, "funny character in string")
187 string += ch
188 elif self.src.startswith('true', self.pos):
189 self.val = True
190 self.cursor += 3
191 return
192 elif self.src.startswith('false', self.pos):
193 self.val = False
194 self.cursor += 4
195 return
196 elif self.tok == '\n':
197 if self.cursor == len(self.src):
198 self.tok = None
199 return
200 self.info = self.info.next_line()
201 self.line_pos = self.cursor
202 elif not self.tok.isspace():
203 # Show up to next structural, whitespace or quote
204 # character
205 match = re.match('[^[\\]{}:,\\s\'"]+',
206 self.src[self.cursor-1:])
207 raise QAPIParseError(self, "stray '%s'" % match.group(0))
208
209 def get_members(self):
210 expr = OrderedDict()
211 if self.tok == '}':
212 self.accept()
213 return expr
214 if self.tok != "'":
215 raise QAPIParseError(self, "expected string or '}'")
216 while True:
217 key = self.val
218 self.accept()
219 if self.tok != ':':
220 raise QAPIParseError(self, "expected ':'")
221 self.accept()
222 if key in expr:
223 raise QAPIParseError(self, "duplicate key '%s'" % key)
224 expr[key] = self.get_expr(True)
225 if self.tok == '}':
226 self.accept()
227 return expr
228 if self.tok != ',':
229 raise QAPIParseError(self, "expected ',' or '}'")
230 self.accept()
231 if self.tok != "'":
232 raise QAPIParseError(self, "expected string")
233
234 def get_values(self):
235 expr = []
236 if self.tok == ']':
237 self.accept()
238 return expr
239 if self.tok not in "{['tfn":
240 raise QAPIParseError(
241 self, "expected '{', '[', ']', string, boolean or 'null'")
242 while True:
243 expr.append(self.get_expr(True))
244 if self.tok == ']':
245 self.accept()
246 return expr
247 if self.tok != ',':
248 raise QAPIParseError(self, "expected ',' or ']'")
249 self.accept()
250
251 def get_expr(self, nested):
252 if self.tok != '{' and not nested:
253 raise QAPIParseError(self, "expected '{'")
254 if self.tok == '{':
255 self.accept()
256 expr = self.get_members()
257 elif self.tok == '[':
258 self.accept()
259 expr = self.get_values()
260 elif self.tok in "'tfn":
261 expr = self.val
262 self.accept()
263 else:
264 raise QAPIParseError(
265 self, "expected '{', '[', string, boolean or 'null'")
266 return expr
267
268 def get_doc(self, info):
269 if self.val != '##':
270 raise QAPIParseError(
271 self, "junk after '##' at start of documentation comment")
272
273 doc = QAPIDoc(self, info)
274 self.accept(False)
275 while self.tok == '#':
276 if self.val.startswith('##'):
277 # End of doc comment
278 if self.val != '##':
279 raise QAPIParseError(
280 self,
281 "junk after '##' at end of documentation comment")
282 doc.end_comment()
283 self.accept()
284 return doc
285 doc.append(self.val)
286 self.accept(False)
287
288 raise QAPIParseError(self, "documentation comment must end with '##'")
289
290
291 class QAPIDoc:
292 """
293 A documentation comment block, either definition or free-form
294
295 Definition documentation blocks consist of
296
297 * a body section: one line naming the definition, followed by an
298 overview (any number of lines)
299
300 * argument sections: a description of each argument (for commands
301 and events) or member (for structs, unions and alternates)
302
303 * features sections: a description of each feature flag
304
305 * additional (non-argument) sections, possibly tagged
306
307 Free-form documentation blocks consist only of a body section.
308 """
309
310 class Section:
311 def __init__(self, name=None):
312 # optional section name (argument/member or section name)
313 self.name = name
314 # the list of lines for this section
315 self.text = ''
316
317 def append(self, line):
318 self.text += line.rstrip() + '\n'
319
320 class ArgSection(Section):
321 def __init__(self, name):
322 super().__init__(name)
323 self.member = None
324
325 def connect(self, member):
326 self.member = member
327
328 def __init__(self, parser, info):
329 # self._parser is used to report errors with QAPIParseError. The
330 # resulting error position depends on the state of the parser.
331 # It happens to be the beginning of the comment. More or less
332 # servicable, but action at a distance.
333 self._parser = parser
334 self.info = info
335 self.symbol = None
336 self.body = QAPIDoc.Section()
337 # dict mapping parameter name to ArgSection
338 self.args = OrderedDict()
339 self.features = OrderedDict()
340 # a list of Section
341 self.sections = []
342 # the current section
343 self._section = self.body
344 self._append_line = self._append_body_line
345
346 def has_section(self, name):
347 """Return True if we have a section with this name."""
348 for i in self.sections:
349 if i.name == name:
350 return True
351 return False
352
353 def append(self, line):
354 """
355 Parse a comment line and add it to the documentation.
356
357 The way that the line is dealt with depends on which part of
358 the documentation we're parsing right now:
359 * The body section: ._append_line is ._append_body_line
360 * An argument section: ._append_line is ._append_args_line
361 * A features section: ._append_line is ._append_features_line
362 * An additional section: ._append_line is ._append_various_line
363 """
364 line = line[1:]
365 if not line:
366 self._append_freeform(line)
367 return
368
369 if line[0] != ' ':
370 raise QAPIParseError(self._parser, "missing space after #")
371 line = line[1:]
372 self._append_line(line)
373
374 def end_comment(self):
375 self._end_section()
376
377 @staticmethod
378 def _is_section_tag(name):
379 return name in ('Returns:', 'Since:',
380 # those are often singular or plural
381 'Note:', 'Notes:',
382 'Example:', 'Examples:',
383 'TODO:')
384
385 def _append_body_line(self, line):
386 """
387 Process a line of documentation text in the body section.
388
389 If this a symbol line and it is the section's first line, this
390 is a definition documentation block for that symbol.
391
392 If it's a definition documentation block, another symbol line
393 begins the argument section for the argument named by it, and
394 a section tag begins an additional section. Start that
395 section and append the line to it.
396
397 Else, append the line to the current section.
398 """
399 name = line.split(' ', 1)[0]
400 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't
401 # recognized, and get silently treated as ordinary text
402 if not self.symbol and not self.body.text and line.startswith('@'):
403 if not line.endswith(':'):
404 raise QAPIParseError(self._parser, "line should end with ':'")
405 self.symbol = line[1:-1]
406 # FIXME invalid names other than the empty string aren't flagged
407 if not self.symbol:
408 raise QAPIParseError(self._parser, "invalid name")
409 elif self.symbol:
410 # This is a definition documentation block
411 if name.startswith('@') and name.endswith(':'):
412 self._append_line = self._append_args_line
413 self._append_args_line(line)
414 elif line == 'Features:':
415 self._append_line = self._append_features_line
416 elif self._is_section_tag(name):
417 self._append_line = self._append_various_line
418 self._append_various_line(line)
419 else:
420 self._append_freeform(line.strip())
421 else:
422 # This is a free-form documentation block
423 self._append_freeform(line.strip())
424
425 def _append_args_line(self, line):
426 """
427 Process a line of documentation text in an argument section.
428
429 A symbol line begins the next argument section, a section tag
430 section or a non-indented line after a blank line begins an
431 additional section. Start that section and append the line to
432 it.
433
434 Else, append the line to the current section.
435
436 """
437 name = line.split(' ', 1)[0]
438
439 if name.startswith('@') and name.endswith(':'):
440 line = line[len(name)+1:]
441 self._start_args_section(name[1:-1])
442 elif self._is_section_tag(name):
443 self._append_line = self._append_various_line
444 self._append_various_line(line)
445 return
446 elif (self._section.text.endswith('\n\n')
447 and line and not line[0].isspace()):
448 if line == 'Features:':
449 self._append_line = self._append_features_line
450 else:
451 self._start_section()
452 self._append_line = self._append_various_line
453 self._append_various_line(line)
454 return
455
456 self._append_freeform(line.strip())
457
458 def _append_features_line(self, line):
459 name = line.split(' ', 1)[0]
460
461 if name.startswith('@') and name.endswith(':'):
462 line = line[len(name)+1:]
463 self._start_features_section(name[1:-1])
464 elif self._is_section_tag(name):
465 self._append_line = self._append_various_line
466 self._append_various_line(line)
467 return
468 elif (self._section.text.endswith('\n\n')
469 and line and not line[0].isspace()):
470 self._start_section()
471 self._append_line = self._append_various_line
472 self._append_various_line(line)
473 return
474
475 self._append_freeform(line.strip())
476
477 def _append_various_line(self, line):
478 """
479 Process a line of documentation text in an additional section.
480
481 A symbol line is an error.
482
483 A section tag begins an additional section. Start that
484 section and append the line to it.
485
486 Else, append the line to the current section.
487 """
488 name = line.split(' ', 1)[0]
489
490 if name.startswith('@') and name.endswith(':'):
491 raise QAPIParseError(self._parser,
492 "'%s' can't follow '%s' section"
493 % (name, self.sections[0].name))
494 if self._is_section_tag(name):
495 line = line[len(name)+1:]
496 self._start_section(name[:-1])
497
498 if (not self._section.name or
499 not self._section.name.startswith('Example')):
500 line = line.strip()
501
502 self._append_freeform(line)
503
504 def _start_symbol_section(self, symbols_dict, name):
505 # FIXME invalid names other than the empty string aren't flagged
506 if not name:
507 raise QAPIParseError(self._parser, "invalid parameter name")
508 if name in symbols_dict:
509 raise QAPIParseError(self._parser,
510 "'%s' parameter name duplicated" % name)
511 assert not self.sections
512 self._end_section()
513 self._section = QAPIDoc.ArgSection(name)
514 symbols_dict[name] = self._section
515
516 def _start_args_section(self, name):
517 self._start_symbol_section(self.args, name)
518
519 def _start_features_section(self, name):
520 self._start_symbol_section(self.features, name)
521
522 def _start_section(self, name=None):
523 if name in ('Returns', 'Since') and self.has_section(name):
524 raise QAPIParseError(self._parser,
525 "duplicated '%s' section" % name)
526 self._end_section()
527 self._section = QAPIDoc.Section(name)
528 self.sections.append(self._section)
529
530 def _end_section(self):
531 if self._section:
532 text = self._section.text = self._section.text.strip()
533 if self._section.name and (not text or text.isspace()):
534 raise QAPIParseError(
535 self._parser,
536 "empty doc section '%s'" % self._section.name)
537 self._section = None
538
539 def _append_freeform(self, line):
540 match = re.match(r'(@\S+:)', line)
541 if match:
542 raise QAPIParseError(self._parser,
543 "'%s' not allowed in free-form documentation"
544 % match.group(1))
545 self._section.append(line)
546
547 def connect_member(self, member):
548 if member.name not in self.args:
549 # Undocumented TODO outlaw
550 self.args[member.name] = QAPIDoc.ArgSection(member.name)
551 self.args[member.name].connect(member)
552
553 def connect_feature(self, feature):
554 if feature.name not in self.features:
555 raise QAPISemError(feature.info,
556 "feature '%s' lacks documentation"
557 % feature.name)
558 self.features[feature.name].connect(feature)
559
560 def check_expr(self, expr):
561 if self.has_section('Returns') and 'command' not in expr:
562 raise QAPISemError(self.info,
563 "'Returns:' is only valid for commands")
564
565 def check(self):
566
567 def check_args_section(args, info, what):
568 bogus = [name for name, section in args.items()
569 if not section.member]
570 if bogus:
571 raise QAPISemError(
572 self.info,
573 "documented member%s '%s' %s not exist"
574 % ("s" if len(bogus) > 1 else "",
575 "', '".join(bogus),
576 "do" if len(bogus) > 1 else "does"))
577
578 check_args_section(self.args, self.info, 'members')
579 check_args_section(self.features, self.info, 'features')