mirror of
https://github.com/python/cpython.git
synced 2026-10-28 10:25:05 +08:00
gh-153195: Allow splitting the PEG grammar into several files (#159917)
Add an @include meta to pegen: the rules of an included grammar file are appended to the including file's rules. The C generator emits the rules of each included file into a separate C file next to the main output. Use it to move the invalid_* rules from Grammar/python.gram into Grammar/invalid.gram, which generates Parser/parser_invalid.c.
This commit is contained in:
Lysandros Nikolaou authored and GitHub committed
1 parent
e03def8c23
commit
e548b472f3
29 files changed
+15066
-12354
No files matched your search
@@ -102,6 +102,7 @@ Modules/_testinternalcapi/test_cases.c.h generated
|
||||
Modules/_testinternalcapi/test_targets.h generated
|
||||
PC/python3dll.c generated
|
||||
Parser/parser.c generated
|
||||
Parser/parser_invalid.c generated
|
||||
Parser/token.c generated
|
||||
Programs/test_frozenmain.h generated
|
||||
Python/Python-ast.c generated
|
||||
|
||||
@@ -302,6 +302,7 @@ Python/optimizer_symbols.c @markshannon @tomasr8 @Fidget-Spinner
|
||||
|
||||
# Parser, Lexer, and Grammar
|
||||
Grammar/python.gram @pablogsal @lysnikolaou
|
||||
Grammar/invalid.gram @pablogsal @lysnikolaou
|
||||
Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
|
||||
Lib/test/test_tokenize.py @pablogsal @lysnikolaou
|
||||
Lib/tokenize.py @pablogsal @lysnikolaou
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
# PEG grammar for Python: rules for invalid syntax
|
||||
#
|
||||
# The rule names starting with "invalid_" here are used for specialized syntax errors
|
||||
# - These rules are NOT used in the first pass of the parser.
|
||||
# - Only if the first pass fails to parse, a second pass including the invalid
|
||||
# rules will be executed.
|
||||
# - If the parser fails in the second phase with a generic syntax error, the
|
||||
# location of the generic failure of the first pass will be used (this avoids
|
||||
# reporting incorrect locations due to the invalid rules).
|
||||
# - The order of the alternatives involving invalid rules matter
|
||||
# (like any rule in PEG).
|
||||
|
||||
invalid_arguments:
|
||||
| ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "iterable argument unpacking follows keyword argument unpacking") }
|
||||
| a=expression b=for_if_clauses ',' [args | expression for_if_clauses] {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") }
|
||||
| a=NAME b='=' expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")}
|
||||
| (args ',')? a=NAME b='=' &(',' | ')') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected argument value expression")}
|
||||
| a=args b=for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a, b) }
|
||||
| args ',' a=expression b=for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") }
|
||||
| a=args ',' args { _PyPegen_arguments_parsing_error(p, a) }
|
||||
invalid_kwarg:
|
||||
| a='True' b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to True") }
|
||||
| a='False' b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to False") }
|
||||
| a='None' b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to None") }
|
||||
| a=NAME b='=' expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")}
|
||||
| !(NAME '=') a=expression b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b, "expression cannot contain assignment, perhaps you meant \"==\"?") }
|
||||
| a='**' expression '=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to keyword argument unpacking") }
|
||||
|
||||
# IMPORTANT: Note that the "_without_invalid" suffix causes the rule to not call invalid rules under it
|
||||
expression_without_invalid[expr_ty]:
|
||||
| a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) }
|
||||
| disjunction
|
||||
| lambdef
|
||||
invalid_legacy_expression:
|
||||
| a=NAME !'(' b=star_expressions {
|
||||
_PyPegen_check_legacy_stmt(p, a) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b,
|
||||
"Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL}
|
||||
|
||||
invalid_type_param:
|
||||
| '*' a=NAME colon=':' e=expression {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
|
||||
? "cannot use constraints with TypeVarTuple"
|
||||
: "cannot use bound with TypeVarTuple")
|
||||
}
|
||||
| '**' a=NAME colon=':' e=expression {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
|
||||
? "cannot use constraints with ParamSpec"
|
||||
: "cannot use bound with ParamSpec")
|
||||
}
|
||||
|
||||
invalid_expression:
|
||||
| STRING a=(!STRING expression_without_invalid)+ STRING {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE( PyPegen_first_item(a, expr_ty), PyPegen_last_item(a, expr_ty),
|
||||
"invalid syntax. Is this intended to be part of the string?") }
|
||||
# !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf"
|
||||
# Soft keywords need to also be ignored because they can be parsed as NAME NAME
|
||||
| !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid {
|
||||
_PyPegen_raise_error_for_missing_comma(p, a, b) }
|
||||
| a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") }
|
||||
| a=disjunction 'if' b=disjunction 'else' !expression {
|
||||
RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("expected expression after 'else', but statement is given") }
|
||||
| a[stmt_ty]=(pass_stmt|break_stmt|continue_stmt) 'if' b=disjunction 'else' c=simple_stmt {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION (a, "expected expression before 'if', but statement is given") }
|
||||
| a='lambda' [lambda_params] b=':' &FSTRING_MIDDLE {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") }
|
||||
| a='lambda' [lambda_params] b=':' &TSTRING_MIDDLE {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "t-string: lambda expressions are not allowed without parentheses") }
|
||||
|
||||
invalid_if_expression:
|
||||
| disjunction 'if' b=disjunction 'else' a='*' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot unpack only part of a conditional expression") }
|
||||
| disjunction 'if' b=disjunction 'else' a='**' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking on only part of a conditional expression") }
|
||||
|
||||
invalid_named_expression(memo):
|
||||
| a=expression ':=' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) }
|
||||
| a=NAME '=' b=bitwise_or !('='|':=') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?") }
|
||||
| !(list|tuple|genexp|'True'|'None'|'False') a=bitwise_or b='=' bitwise_or !('='|':=') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot assign to %s here. Maybe you meant '==' instead of '='?",
|
||||
_PyPegen_get_expr_name(a)) }
|
||||
|
||||
invalid_assignment:
|
||||
| a=invalid_ann_assign_target ':' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a,
|
||||
"only single target (not %s) can be annotated",
|
||||
_PyPegen_get_expr_name(a)
|
||||
)}
|
||||
| a=star_named_expression ',' star_named_expressions* ':' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not tuple) can be annotated") }
|
||||
| a=expression ':' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "illegal target for annotation") }
|
||||
| (star_targets '=')* a=star_expressions '=' {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) }
|
||||
| (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") }
|
||||
| a=star_expressions augassign annotated_rhs {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a,
|
||||
"'%s' is an illegal expression for augmented assignment",
|
||||
_PyPegen_get_expr_name(a)
|
||||
)}
|
||||
invalid_ann_assign_target[expr_ty]:
|
||||
| list
|
||||
| tuple
|
||||
| '(' a=invalid_ann_assign_target ')' { a }
|
||||
invalid_raise_stmt:
|
||||
| a='raise' b='from' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget an expression between 'raise' and 'from'?") }
|
||||
| 'raise' expression a='from' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "did you forget an expression after 'from'?") }
|
||||
invalid_del_stmt:
|
||||
| 'del' a=star_expressions {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) }
|
||||
invalid_assert_stmt:
|
||||
| 'assert' a=expression '=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot assign to %s here. Maybe you meant '==' instead of '='?",
|
||||
_PyPegen_get_expr_name(a)) }
|
||||
| 'assert' expression ',' a=expression '=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot assign to %s here. Maybe you meant '==' instead of '='?",
|
||||
_PyPegen_get_expr_name(a)) }
|
||||
| 'assert' a=expression ':=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot use named expression without parentheses here") }
|
||||
| 'assert' expression ',' a=expression ':=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot use named expression without parentheses here") }
|
||||
invalid_block:
|
||||
| NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") }
|
||||
invalid_comprehension:
|
||||
| '[' a='**' b=expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in list comprehension") }
|
||||
| '(' a='**' b=expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in generator expression") }
|
||||
| ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, expr_ty),
|
||||
"did you forget parentheses around the comprehension target?") }
|
||||
| ('[' | '{') a=star_named_expression b=',' for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget parentheses around the comprehension target?") }
|
||||
invalid_parameters:
|
||||
| a="/" ',' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") }
|
||||
| (slash_no_default | slash_with_default) param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ may appear only once") }
|
||||
| slash_no_default? param_no_default* invalid_parameters_helper a=param_no_default {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameter without a default follows parameter with a default") }
|
||||
| param_no_default* a='(' param_no_default+ ','? b=')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Function parameters cannot be parenthesized") }
|
||||
| (slash_no_default | slash_with_default)? param_maybe_default* '*' (',' | param_no_default) param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ must be ahead of *") }
|
||||
| param_maybe_default+ '/' a='*' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expected comma between / and *") }
|
||||
invalid_default:
|
||||
| a='=' &(')'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expected default value expression") }
|
||||
invalid_star_etc:
|
||||
| a='*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "named parameters must follow bare *") }
|
||||
| '*' ',' TYPE_COMMENT { RAISE_SYNTAX_ERROR("bare * has associated type comment") }
|
||||
| '*' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional parameter cannot have default value") }
|
||||
| '*' (param_no_default | ',') param_maybe_default* a='*' (param_no_default | ',') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* may appear only once") }
|
||||
invalid_kwds:
|
||||
| '**' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword parameter cannot have default value") }
|
||||
| '**' param ',' a=param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
| '**' param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
invalid_parameters_helper: # This is only there to avoid type errors
|
||||
| a=slash_with_default { _PyPegen_singleton_seq(p, a) }
|
||||
| param_with_default+
|
||||
invalid_lambda_parameters:
|
||||
| a="/" ',' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") }
|
||||
| (lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ may appear only once") }
|
||||
| lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper a=lambda_param_no_default {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameter without a default follows parameter with a default") }
|
||||
| lambda_param_no_default* a='(' ','.lambda_param+ ','? b=')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Lambda expression parameters cannot be parenthesized") }
|
||||
| (lambda_slash_no_default | lambda_slash_with_default)? lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ must be ahead of *") }
|
||||
| lambda_param_maybe_default+ '/' a='*' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expected comma between / and *") }
|
||||
invalid_lambda_parameters_helper:
|
||||
| a=lambda_slash_with_default { _PyPegen_singleton_seq(p, a) }
|
||||
| lambda_param_with_default+
|
||||
invalid_lambda_star_etc:
|
||||
| '*' (':' | ',' (':' | '**')) { RAISE_SYNTAX_ERROR("named parameters must follow bare *") }
|
||||
| '*' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional parameter cannot have default value") }
|
||||
| '*' (lambda_param_no_default | ',') lambda_param_maybe_default* a='*' (lambda_param_no_default | ',') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* may appear only once") }
|
||||
invalid_lambda_kwds:
|
||||
| '**' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword parameter cannot have default value") }
|
||||
| '**' lambda_param ',' a=lambda_param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
| '**' lambda_param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
invalid_double_type_comments:
|
||||
| TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT {
|
||||
RAISE_SYNTAX_ERROR("Cannot have two type comments on def") }
|
||||
invalid_with_item:
|
||||
| expression 'as' a=expression &(',' | ')' | ':') {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) }
|
||||
|
||||
invalid_for_if_clause:
|
||||
| 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' {
|
||||
RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") }
|
||||
|
||||
invalid_for_target:
|
||||
| 'async'? 'for' a=star_expressions {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) }
|
||||
|
||||
invalid_group:
|
||||
| '(' a=starred_expression ')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use starred expression here") }
|
||||
| '(' a='**' expression ')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use double starred expression here") }
|
||||
invalid_import:
|
||||
| a='import' ','.dotted_name+ 'from' dotted_name {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") }
|
||||
| 'import' token=NEWLINE {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") }
|
||||
invalid_dotted_as_name:
|
||||
| dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a,
|
||||
"cannot use %s as import target", _PyPegen_get_expr_name(a)) }
|
||||
invalid_import_from_as_name:
|
||||
| NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a,
|
||||
"cannot use %s as import target", _PyPegen_get_expr_name(a)) }
|
||||
|
||||
invalid_import_from:
|
||||
| 'from' ('.' | '...')* dotted_name a="lazy" 'import' import_from_targets {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a,
|
||||
"use 'lazy from ... ' instead of 'from ... lazy import'") }
|
||||
|
||||
invalid_import_from_targets:
|
||||
| import_from_as_names ',' NEWLINE {
|
||||
RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") }
|
||||
| token=NEWLINE {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") }
|
||||
|
||||
invalid_with_stmt:
|
||||
| ['async'] 'with' ','.(expression ['as' star_target])+ trailing=',' ':' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(trailing, "the last 'with' item has a trailing comma") }
|
||||
| ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
invalid_with_stmt_indent:
|
||||
| ['async'] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }
|
||||
| ['async'] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }
|
||||
|
||||
invalid_try_stmt:
|
||||
| a='try' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'try' statement on line %d", a->lineno) }
|
||||
| 'try' ':' block !('except' | 'finally') { RAISE_SYNTAX_ERROR("expected 'except' or 'finally' block") }
|
||||
| 'try' ':' block* except_block+ a='except' b='*' expression ['as' NAME] ':' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot have both 'except' and 'except*' on the same 'try'") }
|
||||
| 'try' ':' block* except_star_block+ a='except' [expression ['as' NAME]] ':' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot have both 'except' and 'except*' on the same 'try'") }
|
||||
invalid_except_stmt:
|
||||
| 'except' a=expression ',' expressions 'as' NAME ':' {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") }
|
||||
| a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| 'except' expression 'as' a=expression ':' block {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use except statement with %s", _PyPegen_get_expr_name(a)) }
|
||||
invalid_except_star_stmt:
|
||||
| 'except' '*' a=expression ',' expressions 'as' NAME ':' {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") }
|
||||
| a='except' '*' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='except' '*' (NEWLINE | ':') { RAISE_SYNTAX_ERROR("expected one or more exception types") }
|
||||
| 'except' '*' expression 'as' a=expression ':' block {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use except* statement with %s", _PyPegen_get_expr_name(a)) }
|
||||
invalid_finally_stmt:
|
||||
| a='finally' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'finally' statement on line %d", a->lineno) }
|
||||
invalid_except_stmt_indent:
|
||||
| a='except' expression ['as' NAME ] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) }
|
||||
| a='except' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) }
|
||||
invalid_except_star_stmt_indent:
|
||||
| a='except' '*' expression ['as' NAME ] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'except*' statement on line %d", a->lineno) }
|
||||
invalid_match_stmt:
|
||||
| "match" subject_expr NEWLINE { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) }
|
||||
| a="match" subject=subject_expr ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'match' statement on line %d", a->lineno) }
|
||||
| a="case" patterns guard? b=':' block {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"case statement must be inside match statement") }
|
||||
invalid_case_block:
|
||||
| "case" patterns guard? NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a="case" patterns guard? ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'case' statement on line %d", a->lineno) }
|
||||
invalid_as_pattern:
|
||||
| or_pattern 'as' a="_" { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use '_' as a target") }
|
||||
| or_pattern 'as' a=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use %s as pattern target", _PyPegen_get_expr_name(a)) }
|
||||
invalid_class_pattern:
|
||||
| name_or_attr '(' a=invalid_class_argument_pattern { RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
PyPegen_first_item(a, pattern_ty),
|
||||
PyPegen_last_item(a, pattern_ty),
|
||||
"positional patterns follow keyword patterns") }
|
||||
invalid_mapping_pattern:
|
||||
| '{' (items_pattern ',')? rest=double_star_pattern ',' items_pattern ','? '}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
rest,
|
||||
"double star pattern must be the last (right-most) subpattern in the mapping pattern") }
|
||||
invalid_class_argument_pattern[asdl_pattern_seq*]:
|
||||
| [positional_patterns ','] keyword_patterns ',' a=positional_patterns { a }
|
||||
invalid_if_stmt:
|
||||
| 'if' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='if' a=named_expression ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'if' statement on line %d", a->lineno) }
|
||||
invalid_elif_stmt:
|
||||
| 'elif' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='elif' named_expression ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'elif' statement on line %d", a->lineno) }
|
||||
invalid_else_stmt:
|
||||
| a='else' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'else' statement on line %d", a->lineno) }
|
||||
| 'else' ':' block 'elif' { RAISE_SYNTAX_ERROR("'elif' block follows an 'else' block")}
|
||||
invalid_while_stmt:
|
||||
| 'while' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='while' named_expression ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'while' statement on line %d", a->lineno) }
|
||||
invalid_for_stmt:
|
||||
| ['async'] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| ['async'] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'for' statement on line %d", a->lineno) }
|
||||
invalid_def_raw:
|
||||
| ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) }
|
||||
| ['async'] 'def' NAME [type_params] &&'(' [params] ')' ['->' expression] &&':' [func_type_comment] block
|
||||
invalid_class_def_raw:
|
||||
| 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) }
|
||||
|
||||
invalid_double_starred_kvpairs:
|
||||
| invalid_kvpair_unpacking [',']
|
||||
| ','.double_starred_kvpair+ ',' (invalid_kvpair | invalid_kvpair_unpacking)
|
||||
| expression a=':' &('}'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
|
||||
invalid_kvpair_unpacking:
|
||||
| a='**' b=if_expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid double starred expression. Did you forget to wrap the conditional expression in parentheses?") }
|
||||
| a='*' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary key") }
|
||||
| a='**' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary key") }
|
||||
| expression ':' a='*' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary value") }
|
||||
| expression ':' a='**' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary value") }
|
||||
invalid_kvpair:
|
||||
| a=expression !(':') {
|
||||
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") }
|
||||
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") }
|
||||
| expression ':' a='**' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use dict unpacking in a dictionary value") }
|
||||
| expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
|
||||
invalid_starred_expression_unpacking:
|
||||
| a='*' b=if_expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid starred expression. Did you forget to wrap the conditional expression in parentheses?") }
|
||||
| a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") }
|
||||
invalid_starred_expression_unpacking_sequence:
|
||||
| a='**' bitwise_or {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking here") }
|
||||
| invalid_starred_expression_unpacking
|
||||
invalid_starred_expression:
|
||||
| '*' { RAISE_SYNTAX_ERROR("Invalid star expression") }
|
||||
invalid_fstring_replacement_field:
|
||||
| '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") }
|
||||
| '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") }
|
||||
| '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before ':'") }
|
||||
| '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '}'") }
|
||||
| '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'") }
|
||||
| '{' annotated_rhs !('=' | '!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '=', or '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '=' !('!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '='? invalid_fstring_conversion_character
|
||||
| '{' annotated_rhs '='? ['!' NAME] !(':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting ':' or '}'") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}', or format specs") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}'") }
|
||||
|
||||
invalid_fstring_conversion_character:
|
||||
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: missing conversion character") }
|
||||
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: invalid conversion character") }
|
||||
|
||||
invalid_tstring_replacement_field:
|
||||
| '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '='") }
|
||||
| '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '!'") }
|
||||
| '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before ':'") }
|
||||
| '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '}'") }
|
||||
| '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting a valid expression after '{'") }
|
||||
| '{' annotated_rhs !('=' | '!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '=', or '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '=' !('!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '='? invalid_tstring_conversion_character
|
||||
| '{' annotated_rhs '='? ['!' NAME] !(':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting ':' or '}'") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '}', or format specs") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '}'") }
|
||||
|
||||
invalid_tstring_conversion_character:
|
||||
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
|
||||
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
|
||||
|
||||
invalid_string_tstring_concat:
|
||||
| a=(fstring|string)+ b[expr_ty]=tstring {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
|
||||
| a=tstring+ b[expr_ty]=(fstring|string) {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
|
||||
|
||||
invalid_arithmetic:
|
||||
| sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
|
||||
invalid_factor:
|
||||
| ('+' | '-' | '~') a='not' b=factor { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
|
||||
|
||||
invalid_type_params:
|
||||
| '[' token=']' {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(
|
||||
token,
|
||||
"Type parameter list cannot be empty")}
|
||||
|
||||
invalid_bitwise_and:
|
||||
| a=bitwise_and b='&' c='&' {
|
||||
_PyPegen_tokens_are_adjacent(b, c)
|
||||
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'and' or '&' instead of '&&'?")
|
||||
: NULL
|
||||
}
|
||||
|
||||
invalid_bitwise_or:
|
||||
| a=bitwise_or b='|' c='|' {
|
||||
_PyPegen_tokens_are_adjacent(b, c)
|
||||
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'or' or '|' instead of '||'?")
|
||||
: NULL
|
||||
}
|
||||
|
||||
invalid_noteq:
|
||||
| a='<' b='>' {
|
||||
_PyPegen_tokens_are_adjacent(a, b)
|
||||
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '!=' instead of '<>'?")
|
||||
: NULL
|
||||
}
|
||||
+2
-470
@@ -25,6 +25,8 @@ _PyPegen_parse(Parser *p)
|
||||
}
|
||||
'''
|
||||
|
||||
@include "invalid.gram"
|
||||
|
||||
# ========================= START OF THE GRAMMAR =========================
|
||||
|
||||
# General grammatical elements and rules:
|
||||
@@ -32,15 +34,6 @@ _PyPegen_parse(Parser *p)
|
||||
# * Strings with double quotes (") denote SOFT KEYWORDS
|
||||
# * Strings with single quotes (') denote KEYWORDS
|
||||
# * Upper case names (NAME) denote tokens in the Grammar/Tokens file
|
||||
# * Rule names starting with "invalid_" are used for specialized syntax errors
|
||||
# - These rules are NOT used in the first pass of the parser.
|
||||
# - Only if the first pass fails to parse, a second pass including the invalid
|
||||
# rules will be executed.
|
||||
# - If the parser fails in the second phase with a generic syntax error, the
|
||||
# location of the generic failure of the first pass will be used (this avoids
|
||||
# reporting incorrect locations due to the invalid rules).
|
||||
# - The order of the alternatives involving invalid rules matter
|
||||
# (like any rule in PEG).
|
||||
#
|
||||
# Grammar Syntax (see PEP 617 for more information):
|
||||
#
|
||||
@@ -1162,464 +1155,3 @@ func_type_comment[Token*]:
|
||||
| TYPE_COMMENT
|
||||
|
||||
# ========================= END OF THE GRAMMAR ===========================
|
||||
|
||||
|
||||
|
||||
# ========================= START OF INVALID RULES =======================
|
||||
|
||||
# From here on, there are rules for invalid syntax with specialised error messages
|
||||
invalid_arguments:
|
||||
| ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "iterable argument unpacking follows keyword argument unpacking") }
|
||||
| a=expression b=for_if_clauses ',' [args | expression for_if_clauses] {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") }
|
||||
| a=NAME b='=' expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")}
|
||||
| (args ',')? a=NAME b='=' &(',' | ')') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected argument value expression")}
|
||||
| a=args b=for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a, b) }
|
||||
| args ',' a=expression b=for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") }
|
||||
| a=args ',' args { _PyPegen_arguments_parsing_error(p, a) }
|
||||
invalid_kwarg:
|
||||
| a='True' b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to True") }
|
||||
| a='False' b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to False") }
|
||||
| a='None' b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to None") }
|
||||
| a=NAME b='=' expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")}
|
||||
| !(NAME '=') a=expression b='=' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b, "expression cannot contain assignment, perhaps you meant \"==\"?") }
|
||||
| a='**' expression '=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to keyword argument unpacking") }
|
||||
|
||||
# IMPORTANT: Note that the "_without_invalid" suffix causes the rule to not call invalid rules under it
|
||||
expression_without_invalid[expr_ty]:
|
||||
| a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) }
|
||||
| disjunction
|
||||
| lambdef
|
||||
invalid_legacy_expression:
|
||||
| a=NAME !'(' b=star_expressions {
|
||||
_PyPegen_check_legacy_stmt(p, a) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b,
|
||||
"Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL}
|
||||
|
||||
invalid_type_param:
|
||||
| '*' a=NAME colon=':' e=expression {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
|
||||
? "cannot use constraints with TypeVarTuple"
|
||||
: "cannot use bound with TypeVarTuple")
|
||||
}
|
||||
| '**' a=NAME colon=':' e=expression {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
|
||||
? "cannot use constraints with ParamSpec"
|
||||
: "cannot use bound with ParamSpec")
|
||||
}
|
||||
|
||||
invalid_expression:
|
||||
| STRING a=(!STRING expression_without_invalid)+ STRING {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE( PyPegen_first_item(a, expr_ty), PyPegen_last_item(a, expr_ty),
|
||||
"invalid syntax. Is this intended to be part of the string?") }
|
||||
# !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf"
|
||||
# Soft keywords need to also be ignored because they can be parsed as NAME NAME
|
||||
| !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid {
|
||||
_PyPegen_raise_error_for_missing_comma(p, a, b) }
|
||||
| a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") }
|
||||
| a=disjunction 'if' b=disjunction 'else' !expression {
|
||||
RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("expected expression after 'else', but statement is given") }
|
||||
| a[stmt_ty]=(pass_stmt|break_stmt|continue_stmt) 'if' b=disjunction 'else' c=simple_stmt {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION (a, "expected expression before 'if', but statement is given") }
|
||||
| a='lambda' [lambda_params] b=':' &FSTRING_MIDDLE {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") }
|
||||
| a='lambda' [lambda_params] b=':' &TSTRING_MIDDLE {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "t-string: lambda expressions are not allowed without parentheses") }
|
||||
|
||||
invalid_if_expression:
|
||||
| disjunction 'if' b=disjunction 'else' a='*' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot unpack only part of a conditional expression") }
|
||||
| disjunction 'if' b=disjunction 'else' a='**' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking on only part of a conditional expression") }
|
||||
|
||||
invalid_named_expression(memo):
|
||||
| a=expression ':=' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) }
|
||||
| a=NAME '=' b=bitwise_or !('='|':=') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?") }
|
||||
| !(list|tuple|genexp|'True'|'None'|'False') a=bitwise_or b='=' bitwise_or !('='|':=') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot assign to %s here. Maybe you meant '==' instead of '='?",
|
||||
_PyPegen_get_expr_name(a)) }
|
||||
|
||||
invalid_assignment:
|
||||
| a=invalid_ann_assign_target ':' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a,
|
||||
"only single target (not %s) can be annotated",
|
||||
_PyPegen_get_expr_name(a)
|
||||
)}
|
||||
| a=star_named_expression ',' star_named_expressions* ':' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not tuple) can be annotated") }
|
||||
| a=expression ':' expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "illegal target for annotation") }
|
||||
| (star_targets '=')* a=star_expressions '=' {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) }
|
||||
| (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") }
|
||||
| a=star_expressions augassign annotated_rhs {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a,
|
||||
"'%s' is an illegal expression for augmented assignment",
|
||||
_PyPegen_get_expr_name(a)
|
||||
)}
|
||||
invalid_ann_assign_target[expr_ty]:
|
||||
| list
|
||||
| tuple
|
||||
| '(' a=invalid_ann_assign_target ')' { a }
|
||||
invalid_raise_stmt:
|
||||
| a='raise' b='from' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget an expression between 'raise' and 'from'?") }
|
||||
| 'raise' expression a='from' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "did you forget an expression after 'from'?") }
|
||||
invalid_del_stmt:
|
||||
| 'del' a=star_expressions {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) }
|
||||
invalid_assert_stmt:
|
||||
| 'assert' a=expression '=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot assign to %s here. Maybe you meant '==' instead of '='?",
|
||||
_PyPegen_get_expr_name(a)) }
|
||||
| 'assert' expression ',' a=expression '=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot assign to %s here. Maybe you meant '==' instead of '='?",
|
||||
_PyPegen_get_expr_name(a)) }
|
||||
| 'assert' a=expression ':=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot use named expression without parentheses here") }
|
||||
| 'assert' expression ',' a=expression ':=' b=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"cannot use named expression without parentheses here") }
|
||||
invalid_block:
|
||||
| NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") }
|
||||
invalid_comprehension:
|
||||
| '[' a='**' b=expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in list comprehension") }
|
||||
| '(' a='**' b=expression for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in generator expression") }
|
||||
| ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, expr_ty),
|
||||
"did you forget parentheses around the comprehension target?") }
|
||||
| ('[' | '{') a=star_named_expression b=',' for_if_clauses {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget parentheses around the comprehension target?") }
|
||||
invalid_parameters:
|
||||
| a="/" ',' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") }
|
||||
| (slash_no_default | slash_with_default) param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ may appear only once") }
|
||||
| slash_no_default? param_no_default* invalid_parameters_helper a=param_no_default {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameter without a default follows parameter with a default") }
|
||||
| param_no_default* a='(' param_no_default+ ','? b=')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Function parameters cannot be parenthesized") }
|
||||
| (slash_no_default | slash_with_default)? param_maybe_default* '*' (',' | param_no_default) param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ must be ahead of *") }
|
||||
| param_maybe_default+ '/' a='*' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expected comma between / and *") }
|
||||
invalid_default:
|
||||
| a='=' &(')'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expected default value expression") }
|
||||
invalid_star_etc:
|
||||
| a='*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "named parameters must follow bare *") }
|
||||
| '*' ',' TYPE_COMMENT { RAISE_SYNTAX_ERROR("bare * has associated type comment") }
|
||||
| '*' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional parameter cannot have default value") }
|
||||
| '*' (param_no_default | ',') param_maybe_default* a='*' (param_no_default | ',') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* may appear only once") }
|
||||
invalid_kwds:
|
||||
| '**' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword parameter cannot have default value") }
|
||||
| '**' param ',' a=param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
| '**' param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
invalid_parameters_helper: # This is only there to avoid type errors
|
||||
| a=slash_with_default { _PyPegen_singleton_seq(p, a) }
|
||||
| param_with_default+
|
||||
invalid_lambda_parameters:
|
||||
| a="/" ',' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") }
|
||||
| (lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ may appear only once") }
|
||||
| lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper a=lambda_param_no_default {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameter without a default follows parameter with a default") }
|
||||
| lambda_param_no_default* a='(' ','.lambda_param+ ','? b=')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Lambda expression parameters cannot be parenthesized") }
|
||||
| (lambda_slash_no_default | lambda_slash_with_default)? lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* a='/' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ must be ahead of *") }
|
||||
| lambda_param_maybe_default+ '/' a='*' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expected comma between / and *") }
|
||||
invalid_lambda_parameters_helper:
|
||||
| a=lambda_slash_with_default { _PyPegen_singleton_seq(p, a) }
|
||||
| lambda_param_with_default+
|
||||
invalid_lambda_star_etc:
|
||||
| '*' (':' | ',' (':' | '**')) { RAISE_SYNTAX_ERROR("named parameters must follow bare *") }
|
||||
| '*' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional parameter cannot have default value") }
|
||||
| '*' (lambda_param_no_default | ',') lambda_param_maybe_default* a='*' (lambda_param_no_default | ',') {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* may appear only once") }
|
||||
invalid_lambda_kwds:
|
||||
| '**' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword parameter cannot have default value") }
|
||||
| '**' lambda_param ',' a=lambda_param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
| '**' lambda_param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") }
|
||||
invalid_double_type_comments:
|
||||
| TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT {
|
||||
RAISE_SYNTAX_ERROR("Cannot have two type comments on def") }
|
||||
invalid_with_item:
|
||||
| expression 'as' a=expression &(',' | ')' | ':') {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) }
|
||||
|
||||
invalid_for_if_clause:
|
||||
| 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' {
|
||||
RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") }
|
||||
|
||||
invalid_for_target:
|
||||
| 'async'? 'for' a=star_expressions {
|
||||
RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) }
|
||||
|
||||
invalid_group:
|
||||
| '(' a=starred_expression ')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use starred expression here") }
|
||||
| '(' a='**' expression ')' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use double starred expression here") }
|
||||
invalid_import:
|
||||
| a='import' ','.dotted_name+ 'from' dotted_name {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") }
|
||||
| 'import' token=NEWLINE {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") }
|
||||
invalid_dotted_as_name:
|
||||
| dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a,
|
||||
"cannot use %s as import target", _PyPegen_get_expr_name(a)) }
|
||||
invalid_import_from_as_name:
|
||||
| NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a,
|
||||
"cannot use %s as import target", _PyPegen_get_expr_name(a)) }
|
||||
|
||||
invalid_import_from:
|
||||
| 'from' ('.' | '...')* dotted_name a="lazy" 'import' import_from_targets {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a,
|
||||
"use 'lazy from ... ' instead of 'from ... lazy import'") }
|
||||
|
||||
invalid_import_from_targets:
|
||||
| import_from_as_names ',' NEWLINE {
|
||||
RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") }
|
||||
| token=NEWLINE {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") }
|
||||
|
||||
invalid_with_stmt:
|
||||
| ['async'] 'with' ','.(expression ['as' star_target])+ trailing=',' ':' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(trailing, "the last 'with' item has a trailing comma") }
|
||||
| ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
invalid_with_stmt_indent:
|
||||
| ['async'] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }
|
||||
| ['async'] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }
|
||||
|
||||
invalid_try_stmt:
|
||||
| a='try' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'try' statement on line %d", a->lineno) }
|
||||
| 'try' ':' block !('except' | 'finally') { RAISE_SYNTAX_ERROR("expected 'except' or 'finally' block") }
|
||||
| 'try' ':' block* except_block+ a='except' b='*' expression ['as' NAME] ':' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot have both 'except' and 'except*' on the same 'try'") }
|
||||
| 'try' ':' block* except_star_block+ a='except' [expression ['as' NAME]] ':' {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot have both 'except' and 'except*' on the same 'try'") }
|
||||
invalid_except_stmt:
|
||||
| 'except' a=expression ',' expressions 'as' NAME ':' {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") }
|
||||
| a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| 'except' expression 'as' a=expression ':' block {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use except statement with %s", _PyPegen_get_expr_name(a)) }
|
||||
invalid_except_star_stmt:
|
||||
| 'except' '*' a=expression ',' expressions 'as' NAME ':' {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") }
|
||||
| a='except' '*' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='except' '*' (NEWLINE | ':') { RAISE_SYNTAX_ERROR("expected one or more exception types") }
|
||||
| 'except' '*' expression 'as' a=expression ':' block {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use except* statement with %s", _PyPegen_get_expr_name(a)) }
|
||||
invalid_finally_stmt:
|
||||
| a='finally' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'finally' statement on line %d", a->lineno) }
|
||||
invalid_except_stmt_indent:
|
||||
| a='except' expression ['as' NAME ] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) }
|
||||
| a='except' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) }
|
||||
invalid_except_star_stmt_indent:
|
||||
| a='except' '*' expression ['as' NAME ] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'except*' statement on line %d", a->lineno) }
|
||||
invalid_match_stmt:
|
||||
| "match" subject_expr NEWLINE { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) }
|
||||
| a="match" subject=subject_expr ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'match' statement on line %d", a->lineno) }
|
||||
| a="case" patterns guard? b=':' block {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
a, b,
|
||||
"case statement must be inside match statement") }
|
||||
invalid_case_block:
|
||||
| "case" patterns guard? NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a="case" patterns guard? ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'case' statement on line %d", a->lineno) }
|
||||
invalid_as_pattern:
|
||||
| or_pattern 'as' a="_" { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use '_' as a target") }
|
||||
| or_pattern 'as' a=expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
a, "cannot use %s as pattern target", _PyPegen_get_expr_name(a)) }
|
||||
invalid_class_pattern:
|
||||
| name_or_attr '(' a=invalid_class_argument_pattern { RAISE_SYNTAX_ERROR_KNOWN_RANGE(
|
||||
PyPegen_first_item(a, pattern_ty),
|
||||
PyPegen_last_item(a, pattern_ty),
|
||||
"positional patterns follow keyword patterns") }
|
||||
invalid_mapping_pattern:
|
||||
| '{' (items_pattern ',')? rest=double_star_pattern ',' items_pattern ','? '}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
rest,
|
||||
"double star pattern must be the last (right-most) subpattern in the mapping pattern") }
|
||||
invalid_class_argument_pattern[asdl_pattern_seq*]:
|
||||
| [positional_patterns ','] keyword_patterns ',' a=positional_patterns { a }
|
||||
invalid_if_stmt:
|
||||
| 'if' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='if' a=named_expression ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'if' statement on line %d", a->lineno) }
|
||||
invalid_elif_stmt:
|
||||
| 'elif' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='elif' named_expression ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'elif' statement on line %d", a->lineno) }
|
||||
invalid_else_stmt:
|
||||
| a='else' ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'else' statement on line %d", a->lineno) }
|
||||
| 'else' ':' block 'elif' { RAISE_SYNTAX_ERROR("'elif' block follows an 'else' block")}
|
||||
invalid_while_stmt:
|
||||
| 'while' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='while' named_expression ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'while' statement on line %d", a->lineno) }
|
||||
invalid_for_stmt:
|
||||
| ['async'] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| ['async'] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after 'for' statement on line %d", a->lineno) }
|
||||
invalid_def_raw:
|
||||
| ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) }
|
||||
| ['async'] 'def' NAME [type_params] &&'(' [params] ')' ['->' expression] &&':' [func_type_comment] block
|
||||
invalid_class_def_raw:
|
||||
| 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
|
||||
| a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT {
|
||||
RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) }
|
||||
|
||||
invalid_double_starred_kvpairs:
|
||||
| invalid_kvpair_unpacking [',']
|
||||
| ','.double_starred_kvpair+ ',' (invalid_kvpair | invalid_kvpair_unpacking)
|
||||
| expression a=':' &('}'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
|
||||
invalid_kvpair_unpacking:
|
||||
| a='**' b=if_expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid double starred expression. Did you forget to wrap the conditional expression in parentheses?") }
|
||||
| a='*' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary key") }
|
||||
| a='**' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary key") }
|
||||
| expression ':' a='*' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary value") }
|
||||
| expression ':' a='**' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary value") }
|
||||
invalid_kvpair:
|
||||
| a=expression !(':') {
|
||||
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") }
|
||||
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") }
|
||||
| expression ':' a='**' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use dict unpacking in a dictionary value") }
|
||||
| expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
|
||||
invalid_starred_expression_unpacking:
|
||||
| a='*' b=if_expression {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid starred expression. Did you forget to wrap the conditional expression in parentheses?") }
|
||||
| a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") }
|
||||
invalid_starred_expression_unpacking_sequence:
|
||||
| a='**' bitwise_or {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking here") }
|
||||
| invalid_starred_expression_unpacking
|
||||
invalid_starred_expression:
|
||||
| '*' { RAISE_SYNTAX_ERROR("Invalid star expression") }
|
||||
invalid_fstring_replacement_field:
|
||||
| '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") }
|
||||
| '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") }
|
||||
| '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before ':'") }
|
||||
| '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '}'") }
|
||||
| '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'") }
|
||||
| '{' annotated_rhs !('=' | '!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '=', or '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '=' !('!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '='? invalid_fstring_conversion_character
|
||||
| '{' annotated_rhs '='? ['!' NAME] !(':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting ':' or '}'") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}', or format specs") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}'") }
|
||||
|
||||
invalid_fstring_conversion_character:
|
||||
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: missing conversion character") }
|
||||
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: invalid conversion character") }
|
||||
|
||||
invalid_tstring_replacement_field:
|
||||
| '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '='") }
|
||||
| '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '!'") }
|
||||
| '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before ':'") }
|
||||
| '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '}'") }
|
||||
| '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting a valid expression after '{'") }
|
||||
| '{' annotated_rhs !('=' | '!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '=', or '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '=' !('!' | ':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '!', or ':', or '}'") }
|
||||
| '{' annotated_rhs '='? invalid_tstring_conversion_character
|
||||
| '{' annotated_rhs '='? ['!' NAME] !(':' | '}') {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting ':' or '}'") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '}', or format specs") }
|
||||
| '{' annotated_rhs '='? ['!' NAME] !'}' {
|
||||
PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '}'") }
|
||||
|
||||
invalid_tstring_conversion_character:
|
||||
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
|
||||
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
|
||||
|
||||
invalid_string_tstring_concat:
|
||||
| a=(fstring|string)+ b[expr_ty]=tstring {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
|
||||
| a=tstring+ b[expr_ty]=(fstring|string) {
|
||||
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
|
||||
|
||||
invalid_arithmetic:
|
||||
| sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
|
||||
invalid_factor:
|
||||
| ('+' | '-' | '~') a='not' b=factor { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
|
||||
|
||||
invalid_type_params:
|
||||
| '[' token=']' {
|
||||
RAISE_SYNTAX_ERROR_STARTING_FROM(
|
||||
token,
|
||||
"Type parameter list cannot be empty")}
|
||||
|
||||
invalid_bitwise_and:
|
||||
| a=bitwise_and b='&' c='&' {
|
||||
_PyPegen_tokens_are_adjacent(b, c)
|
||||
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'and' or '&' instead of '&&'?")
|
||||
: NULL
|
||||
}
|
||||
|
||||
invalid_bitwise_or:
|
||||
| a=bitwise_or b='|' c='|' {
|
||||
_PyPegen_tokens_are_adjacent(b, c)
|
||||
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'or' or '|' instead of '||'?")
|
||||
: NULL
|
||||
}
|
||||
|
||||
invalid_noteq:
|
||||
| a='<' b='>' {
|
||||
_PyPegen_tokens_are_adjacent(a, b)
|
||||
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '!=' instead of '<>'?")
|
||||
: NULL
|
||||
}
|
||||
@@ -13,10 +13,13 @@ Below is a checklist of things that may need to change.
|
||||
## Checklist
|
||||
|
||||
* [`Grammar/python.gram`](../Grammar/python.gram): The grammar definition,
|
||||
with actions that build AST nodes.
|
||||
After changing it, run ``make regen-pegen`` (or ``build.bat --regen`` on Windows),
|
||||
to regenerate [`Parser/parser.c`](../Parser/parser.c).
|
||||
(This runs Python's parser generator, [`Tools/peg_generator`](../Tools/peg_generator)).
|
||||
with actions that build AST nodes. The rules for specialized syntax errors
|
||||
(`invalid_*` rules) are in [`Grammar/invalid.gram`](../Grammar/invalid.gram),
|
||||
included by `python.gram`. After changing either, run ``make regen-pegen``
|
||||
(or ``build.bat --regen`` on Windows), to regenerate
|
||||
[`Parser/parser.c`](../Parser/parser.c) and
|
||||
[`Parser/parser_invalid.c`](../Parser/parser_invalid.c). (This runs Python's
|
||||
parser generator, [`Tools/peg_generator`](../Tools/peg_generator)).
|
||||
|
||||
* [`Grammar/Tokens`](../Grammar/Tokens) is a place for adding new token types. After
|
||||
changing it, run ``make regen-token`` to regenerate
|
||||
|
||||
@@ -498,6 +498,42 @@ Pegen has some special grammatical elements and rules:
|
||||
reporting incorrect locations due to the invalid rules).
|
||||
- The order of the alternatives involving invalid rules matter
|
||||
(like any rule in PEG).
|
||||
- These rules live in [`Grammar/invalid.gram`](../Grammar/invalid.gram),
|
||||
which is included from [`Grammar/python.gram`](../Grammar/python.gram).
|
||||
|
||||
A grammar can be split into several files with the `@include` meta, which
|
||||
takes a file name relative to the including file:
|
||||
|
||||
```
|
||||
@include "invalid.gram"
|
||||
```
|
||||
|
||||
The rules of each included file are appended, in order, to the rules of the
|
||||
including file, so the parser is the same as if the files were concatenated.
|
||||
Included files may themselves use `@include`, but may not define other metas
|
||||
(such as `@trailer`), and a rule can only be defined in one of the files.
|
||||
Includes are resolved by `pegen.build.build_parser()`, which every pegen
|
||||
command uses to load a grammar file. A grammar parsed some other way must be
|
||||
passed to `pegen.build.resolve_includes()` before generating a parser from it;
|
||||
otherwise the generator refuses it.
|
||||
|
||||
The C generator emits the rules of each included file into a separate C file,
|
||||
with the included file's name appended: for `invalid.gram` that is
|
||||
[`Parser/parser_invalid.c`](../Parser/parser_invalid.c). An included file in a
|
||||
subdirectory of the root grammar file's directory gets its C file in the same
|
||||
subdirectory of the output's directory (`sub/extra.gram` gives
|
||||
`Parser/sub/parser_extra.c`); included files outside that directory are
|
||||
rejected. This keeps the generated files smaller, and lets them be compiled
|
||||
separately. The keyword tables and the `@trailer` code are only in the main C
|
||||
file, while the `@header` and `@subheader` code is copied into every C file,
|
||||
so it must not define non-`static` functions or variables (they would be defined
|
||||
more than once). Rules that are called from another C file, including rules of
|
||||
included files that the trailer or a C action calls, can't be `static`; the
|
||||
generated code gives them a `_PyPegen_` prefix with a `#define`, since CPython
|
||||
requires that prefix on its non-static symbols, and marks them
|
||||
`Py_LOCAL_SYMBOL` so that they aren't exported from the shared library. Helper
|
||||
rules (for groups, repetitions and so on)
|
||||
are never shared between files: each C file gets its own `static` copy.
|
||||
|
||||
Tokenization
|
||||
------------
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
import io
|
||||
import textwrap
|
||||
import unittest
|
||||
from pathlib import PurePath
|
||||
from pathlib import Path, PurePath
|
||||
from unittest import mock
|
||||
|
||||
from test import test_tools
|
||||
from test.support import os_helper
|
||||
|
||||
test_tools.skip_if_missing("peg_generator")
|
||||
with test_tools.imports_under_tool("peg_generator"):
|
||||
from pegen import grammar as grammar_module
|
||||
from pegen.build import build_parser, included_output_files
|
||||
from pegen.c_generator import CParserGenerator
|
||||
from pegen.c_generator_file import CParserEmitter
|
||||
from pegen.grammar import NamedItem, RuleKind
|
||||
from pegen.grammar import GrammarError, NamedItem, RuleKind
|
||||
from pegen.grammar_parser import GeneratedParser as GrammarParser
|
||||
from pegen.testutil import ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, parse_string
|
||||
|
||||
@@ -287,5 +290,144 @@ class TestCGenerator(unittest.TestCase):
|
||||
self.assertEqual(generator.soft_keywords, {"soft"})
|
||||
|
||||
|
||||
class TestIncludedCFiles(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp_dir = self.enterContext(os_helper.temp_dir())
|
||||
|
||||
def write(self, name, source):
|
||||
path = Path(self.tmp_dir, name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(textwrap.dedent(source))
|
||||
return path
|
||||
|
||||
def generate(self, grammar_file):
|
||||
grammar, _, _ = build_parser(grammar_file)
|
||||
output = io.StringIO()
|
||||
included = {source: io.StringIO() for source in {
|
||||
rule.source for rule in grammar.rules.values() if rule.source is not None
|
||||
}}
|
||||
generator = CParserGenerator(
|
||||
grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, output,
|
||||
included_files=included,
|
||||
)
|
||||
generator.generate(grammar_file)
|
||||
return output.getvalue(), {
|
||||
source.name: file.getvalue() for source, file in included.items()
|
||||
}
|
||||
|
||||
def test_included_rules_go_to_their_own_file(self):
|
||||
main = self.write("main.gram", """\
|
||||
@include "extra.gram"
|
||||
start: item+ NEWLINE
|
||||
item: NAME ('+' | '-') | invalid_item
|
||||
""")
|
||||
self.write("extra.gram", """\
|
||||
invalid_item: NUMBER ('+' | '-') item
|
||||
""")
|
||||
root, included = self.generate(main)
|
||||
extra = included["extra.gram"]
|
||||
|
||||
self.assertTrue(root.startswith("// @generated by pegen from main.gram\n"))
|
||||
self.assertTrue(extra.startswith("// @generated by pegen from extra.gram\n"))
|
||||
# Each rule is defined in the C file of its grammar file, and only
|
||||
# the root file has the keyword tables and the trailer.
|
||||
self.assertIn("\nstart_rule(Parser *p)\n{", root)
|
||||
self.assertNotIn("\ninvalid_item_rule(Parser *p)\n{", root)
|
||||
self.assertIn("\ninvalid_item_rule(Parser *p)\n{", extra)
|
||||
self.assertNotIn("\nitem_rule(Parser *p)\n{", extra)
|
||||
self.assertIn("reserved_keywords", root)
|
||||
self.assertNotIn("reserved_keywords", extra)
|
||||
self.assertIn("_PyPegen_parse", root)
|
||||
self.assertNotIn("_PyPegen_parse", extra)
|
||||
# Rule type ids stay unique across the files.
|
||||
self.assertIn("#define start_type 1000\n", root)
|
||||
self.assertIn("#define item_type 1001\n", root)
|
||||
self.assertIn("#define invalid_item_type 1002\n", extra)
|
||||
# Rules called across files are not static and get a prefixed
|
||||
# linker name; the others stay static.
|
||||
for output in root, extra:
|
||||
self.assertIn("#define item_rule _PyPegen_item_rule\n", output)
|
||||
self.assertIn("#define invalid_item_rule _PyPegen_invalid_item_rule\n", output)
|
||||
self.assertNotIn("_PyPegen_start_rule", output)
|
||||
self.assertIn("static asdl_seq *_loop1_", root)
|
||||
self.assertIn("static void *\nstart_rule(Parser *p)", root)
|
||||
self.assertIn("\nPy_LOCAL_SYMBOL void *\nitem_rule(Parser *p)", root)
|
||||
self.assertIn("\nPy_LOCAL_SYMBOL void *\ninvalid_item_rule(Parser *p)", extra)
|
||||
# A helper for an identical subexpression is not shared between
|
||||
# files: each one has its own static copy.
|
||||
self.assertIn("static void *_tmp_", root)
|
||||
self.assertIn("static void *_tmp_", extra)
|
||||
|
||||
def test_trailer_can_call_rules_of_included_files(self):
|
||||
# The default trailer calls start_rule().
|
||||
main = self.write("main.gram", """\
|
||||
@include "rules.gram"
|
||||
item: NAME
|
||||
""")
|
||||
self.write("rules.gram", "start: item+ NEWLINE\n")
|
||||
root, included = self.generate(main)
|
||||
rules = included["rules.gram"]
|
||||
self.assertIn("return start_rule(p);", root)
|
||||
for output in root, rules:
|
||||
self.assertIn("#define start_rule _PyPegen_start_rule\n", output)
|
||||
self.assertIn("\nPy_LOCAL_SYMBOL void *start_rule(Parser *p);\n", output)
|
||||
self.assertIn("\nPy_LOCAL_SYMBOL void *\nstart_rule(Parser *p)\n{", rules)
|
||||
|
||||
def test_actions_can_call_rules_of_included_files(self):
|
||||
main = self.write("main.gram", """\
|
||||
@include "extra.gram"
|
||||
start: NAME NEWLINE { extra_rule(p) }
|
||||
""")
|
||||
self.write("extra.gram", "extra: NAME\n")
|
||||
root, included = self.generate(main)
|
||||
extra = included["extra.gram"]
|
||||
for output in root, extra:
|
||||
self.assertIn("#define extra_rule _PyPegen_extra_rule\n", output)
|
||||
self.assertIn("\nPy_LOCAL_SYMBOL void *extra_rule(Parser *p);\n", output)
|
||||
self.assertIn("\nPy_LOCAL_SYMBOL void *\nextra_rule(Parser *p)\n{", extra)
|
||||
|
||||
def test_without_includes_all_rules_are_static(self):
|
||||
main = self.write("main.gram", """\
|
||||
start: item NEWLINE
|
||||
item: NAME
|
||||
""")
|
||||
root, included = self.generate(main)
|
||||
self.assertEqual(included, {})
|
||||
self.assertNotIn("Rules shared", root)
|
||||
self.assertNotIn("_PyPegen_item_rule", root)
|
||||
self.assertIn("static void *\nitem_rule(Parser *p)", root)
|
||||
|
||||
def test_included_output_files(self):
|
||||
main = self.write("main.gram", """\
|
||||
@include "extra.gram"
|
||||
@include "sub/more.rules.gram"
|
||||
@include "a/same.gram"
|
||||
@include "b/same.gram"
|
||||
start: extra more one two NEWLINE
|
||||
""")
|
||||
self.write("extra.gram", "extra: NAME\n")
|
||||
self.write("sub/more.rules.gram", "more: NAME\n")
|
||||
self.write("a/same.gram", "one: NAME\n")
|
||||
self.write("b/same.gram", "two: NAME\n")
|
||||
grammar, _, _ = build_parser(main)
|
||||
out = Path("out", "parser.c.new")
|
||||
self.assertEqual(included_output_files(grammar, out), {
|
||||
Path("extra.gram"): Path("out", "parser_extra.c.new"),
|
||||
Path("sub", "more.rules.gram"): Path("out", "sub", "parser_more.rules.c.new"),
|
||||
Path("a", "same.gram"): Path("out", "a", "parser_same.c.new"),
|
||||
Path("b", "same.gram"): Path("out", "b", "parser_same.c.new"),
|
||||
})
|
||||
|
||||
def test_included_output_files_outside_output_directory(self):
|
||||
main = self.write("grammar/main.gram", """\
|
||||
@include "../outside.gram"
|
||||
start: outside NEWLINE
|
||||
""")
|
||||
self.write("outside.gram", "outside: NAME\n")
|
||||
grammar, _, _ = build_parser(main)
|
||||
with self.assertRaisesRegex(GrammarError, "would be outside the output directory"):
|
||||
included_output_files(grammar, Path("parser.c"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -18,6 +18,7 @@ if support.check_cflags_pgo():
|
||||
|
||||
test_tools.skip_if_missing("peg_generator")
|
||||
with test_tools.imports_under_tool("peg_generator"):
|
||||
from pegen.build import build_parser
|
||||
from pegen.grammar_parser import GeneratedParser as GrammarParser
|
||||
from pegen.testutil import (
|
||||
parse_string,
|
||||
@@ -139,6 +140,9 @@ class TestCParser(unittest.TestCase):
|
||||
|
||||
def run_test(self, grammar_source, test_source):
|
||||
self.build_extension(grammar_source)
|
||||
self.run_extension_test(test_source)
|
||||
|
||||
def run_extension_test(self, test_source):
|
||||
test_source = textwrap.indent(textwrap.dedent(test_source), 8 * " ")
|
||||
assert_python_ok(
|
||||
"-c",
|
||||
@@ -674,6 +678,36 @@ class TestCParser(unittest.TestCase):
|
||||
"""
|
||||
self.run_test(grammar_source, test_source)
|
||||
|
||||
def test_included_grammar_file(self) -> None:
|
||||
# The rules of an included grammar file are compiled from a separate
|
||||
# C file, in the same subdirectory as the grammar file; rules call
|
||||
# each other across the two files.
|
||||
grammar_dir = self.enterContext(os_helper.temp_dir())
|
||||
with open(os.path.join(grammar_dir, "main.gram"), "w") as f:
|
||||
f.write(textwrap.dedent("""\
|
||||
@include "errors/invalid.gram"
|
||||
start: stmt+ NEWLINE? ENDMARKER
|
||||
stmt: invalid_stmt | NAME '=' expr
|
||||
expr: NAME | NUMBER
|
||||
"""))
|
||||
os.mkdir(os.path.join(grammar_dir, "errors"))
|
||||
with open(os.path.join(grammar_dir, "errors", "invalid.gram"), "w") as f:
|
||||
f.write(textwrap.dedent("""\
|
||||
invalid_stmt: expr '==' expr { RAISE_SYNTAX_ERROR("use '=' to assign") }
|
||||
"""))
|
||||
grammar, _, _ = build_parser(Path(grammar_dir, "main.gram"))
|
||||
generate_parser_c_extension(grammar, Path('.'), library_dir=self.library_dir)
|
||||
self.assertTrue(os.path.exists(os.path.join("errors", "parse_invalid.c")))
|
||||
self.run_extension_test("""
|
||||
self.check_input_strings_for_grammar(
|
||||
valid_cases=["a = b", "a = 1"],
|
||||
invalid_cases=["a =", "a == "],
|
||||
)
|
||||
with self.assertRaises(SyntaxError) as e:
|
||||
parse.parse_string("a == b", mode=0)
|
||||
self.assertIn("use '=' to assign", str(e.exception))
|
||||
""")
|
||||
|
||||
def test_forced(self) -> None:
|
||||
grammar_source = """
|
||||
start: NAME &&':' | NAME
|
||||
|
||||
@@ -3,16 +3,18 @@ import difflib
|
||||
import io
|
||||
import textwrap
|
||||
import unittest
|
||||
from pathlib import PurePath
|
||||
from pathlib import Path, PurePath
|
||||
|
||||
from test import test_tools
|
||||
from test.support import os_helper
|
||||
from typing import Dict, Any
|
||||
from tokenize import TokenInfo, NAME, NEWLINE, NUMBER, OP
|
||||
|
||||
test_tools.skip_if_missing("peg_generator")
|
||||
with test_tools.imports_under_tool("peg_generator"):
|
||||
from pegen.build import build_parser
|
||||
from pegen.grammar_parser import GeneratedParser as GrammarParser
|
||||
from pegen.testutil import parse_string, generate_parser, make_parser
|
||||
from pegen.testutil import parse_string, generate_parser, make_parser, run_parser
|
||||
from pegen.grammar import GrammarVisitor, GrammarError, Grammar, RuleKind
|
||||
from pegen.grammar_visualizer import ASTGrammarPrinter
|
||||
from pegen.parser import Parser
|
||||
@@ -1204,3 +1206,120 @@ class TestGrammarVisualizer(unittest.TestCase):
|
||||
"no_flags_rule should not have memo")
|
||||
self.assertEqual(no_flags_rule.flags, frozenset(),
|
||||
f"no_flags_rule flags should be the empty set, got {no_flags_rule.flags}")
|
||||
|
||||
|
||||
class TestGrammarIncludes(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp_dir = self.enterContext(os_helper.temp_dir())
|
||||
|
||||
def write(self, name: str, source: str) -> Path:
|
||||
path = Path(self.tmp_dir, name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(textwrap.dedent(source))
|
||||
return path
|
||||
|
||||
def test_included_rules_are_appended_in_order(self) -> None:
|
||||
main = self.write("main.gram", """\
|
||||
@subheader "# subheader"
|
||||
@include "first.gram"
|
||||
@include "sub/second.gram"
|
||||
start: a b c d NEWLINE
|
||||
a: 'a'
|
||||
""")
|
||||
self.write("first.gram", """\
|
||||
@include "third.gram"
|
||||
b: 'b'
|
||||
""")
|
||||
self.write("third.gram", "c: 'c'\n")
|
||||
self.write("sub/second.gram", "d: 'd'\n")
|
||||
grammar, _, _ = build_parser(main)
|
||||
self.assertEqual(list(grammar.rules), ["start", "a", "b", "c", "d"])
|
||||
self.assertEqual(grammar.metas, {"subheader": "# subheader"})
|
||||
self.assertEqual(grammar.includes, [])
|
||||
parser_class = generate_parser(grammar)
|
||||
self.assertIsNotNone(parse_string("a b c d\n", parser_class))
|
||||
|
||||
def test_repeated_rule_in_included_file(self) -> None:
|
||||
main = self.write("main.gram", """\
|
||||
@include "other.gram"
|
||||
start: a NEWLINE
|
||||
a: 'a'
|
||||
""")
|
||||
other = self.write("other.gram", "a: 'b'\n")
|
||||
with self.assertRaises(GrammarError) as cm:
|
||||
build_parser(main)
|
||||
self.assertEqual(
|
||||
str(cm.exception),
|
||||
f"Repeated rule 'a' in '{other.resolve()}' (first defined in '{main}')",
|
||||
)
|
||||
|
||||
def test_circular_include(self) -> None:
|
||||
main = self.write("main.gram", """\
|
||||
@include "other.gram"
|
||||
start: NEWLINE
|
||||
""")
|
||||
self.write("other.gram", """\
|
||||
@include "main.gram"
|
||||
a: 'a'
|
||||
""")
|
||||
with self.assertRaisesRegex(GrammarError, "Circular @include"):
|
||||
build_parser(main)
|
||||
|
||||
def test_file_included_twice(self) -> None:
|
||||
main = self.write("main.gram", """\
|
||||
@include "a.gram"
|
||||
@include "b.gram"
|
||||
start: a b NEWLINE
|
||||
""")
|
||||
a = self.write("a.gram", """\
|
||||
@include "common.gram"
|
||||
a: common
|
||||
""")
|
||||
b = self.write("b.gram", """\
|
||||
@include "common.gram"
|
||||
b: common
|
||||
""")
|
||||
common = self.write("common.gram", "common: NAME\n")
|
||||
with self.assertRaises(GrammarError) as cm:
|
||||
build_parser(main)
|
||||
self.assertEqual(
|
||||
str(cm.exception),
|
||||
f"'{common.resolve()}' is included more than once: "
|
||||
f"from '{a.resolve()}' and from '{b.resolve()}'",
|
||||
)
|
||||
|
||||
def test_current_source_context_restores_previous_source(self) -> None:
|
||||
grammar = parse_string("start: NAME\n", GrammarParser)
|
||||
generator = PythonParserGenerator(grammar, io.StringIO())
|
||||
outer, inner = Path("outer.gram"), Path("inner.gram")
|
||||
with generator.current_source_context(outer):
|
||||
with self.assertRaises(ValueError):
|
||||
with generator.current_source_context(inner):
|
||||
self.assertEqual(generator.current_source, inner)
|
||||
raise ValueError
|
||||
self.assertEqual(generator.current_source, outer)
|
||||
self.assertIsNone(generator.current_source)
|
||||
|
||||
def test_included_file_cannot_define_metas(self) -> None:
|
||||
main = self.write("main.gram", """\
|
||||
@include "other.gram"
|
||||
start: NEWLINE
|
||||
""")
|
||||
self.write("other.gram", """\
|
||||
@trailer "# trailer"
|
||||
a: 'a'
|
||||
""")
|
||||
with self.assertRaisesRegex(GrammarError, "cannot define metas: @trailer"):
|
||||
build_parser(main)
|
||||
|
||||
def test_unresolved_include(self) -> None:
|
||||
# Running GrammarParser directly leaves the includes unresolved.
|
||||
grammar_source = textwrap.dedent("""
|
||||
@include "other.gram"
|
||||
start: NEWLINE
|
||||
""")
|
||||
grammar = run_parser(io.StringIO(grammar_source), GrammarParser)
|
||||
self.assertEqual(grammar.includes, ["other.gram"])
|
||||
self.assertEqual(grammar.metas, {})
|
||||
with self.assertRaisesRegex(GrammarError, "Unresolved @include of 'other.gram'"):
|
||||
generate_parser(grammar)
|
||||
@@ -398,6 +398,7 @@ PEGEN_OBJS= \
|
||||
Parser/pegen_errors.o \
|
||||
Parser/action_helpers.o \
|
||||
Parser/parser.o \
|
||||
Parser/parser_invalid.o \
|
||||
Parser/string_parser.o \
|
||||
Parser/peg_api.o
|
||||
|
||||
@@ -1987,6 +1988,7 @@ regen-pegen:
|
||||
$(srcdir)/Grammar/Tokens \
|
||||
-o $(srcdir)/Parser/parser.c.new
|
||||
$(UPDATE_FILE) --create $(srcdir)/Parser/parser.c $(srcdir)/Parser/parser.c.new
|
||||
$(UPDATE_FILE) --create $(srcdir)/Parser/parser_invalid.c $(srcdir)/Parser/parser_invalid.c.new
|
||||
|
||||
.PHONY: regen-ast
|
||||
regen-ast:
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
The PEG parser generator now supports splitting a grammar into several files
|
||||
with the ``@include`` meta, and generates a separate C file for the rules of
|
||||
each included file. The rules for specialized syntax errors moved from
|
||||
:file:`Grammar/python.gram` to :file:`Grammar/invalid.gram`, which generates
|
||||
the new :file:`Parser/parser_invalid.c`.
|
||||
@@ -175,6 +175,7 @@
|
||||
<ClCompile Include="..\Objects\weakrefobject.c" />
|
||||
<ClCompile Include="..\Parser\myreadline.c" />
|
||||
<ClCompile Include="..\Parser\parser.c" />
|
||||
<ClCompile Include="..\Parser\parser_invalid.c" />
|
||||
<ClCompile Include="..\Parser\peg_api.c" />
|
||||
<ClCompile Include="..\Parser\pegen.c" />
|
||||
<ClCompile Include="..\Parser\pegen_errors.c" />
|
||||
|
||||
@@ -334,6 +334,9 @@
|
||||
<ClCompile Include="..\Parser\parser.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\Parser\parser_invalid.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\Python\parking_lot.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
|
||||
@@ -607,6 +607,7 @@
|
||||
<ClCompile Include="..\Parser\pegen_errors.c" />
|
||||
<ClCompile Include="..\Parser\action_helpers.c" />
|
||||
<ClCompile Include="..\Parser\parser.c" />
|
||||
<ClCompile Include="..\Parser\parser_invalid.c" />
|
||||
<ClCompile Include="..\Parser\string_parser.c" />
|
||||
<ClCompile Include="..\Parser\peg_api.c" />
|
||||
<ClCompile Include="..\PC\invalid_parameter_handler.c" />
|
||||
|
||||
@@ -1715,6 +1715,9 @@
|
||||
<ClCompile Include="..\Parser\parser.c">
|
||||
<Filter>Parser</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\Parser\parser_invalid.c">
|
||||
<Filter>Parser</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\Parser\pegen.c">
|
||||
<Filter>Parser</Filter>
|
||||
</ClCompile>
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup>
|
||||
<_PegenSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
|
||||
<_PegenOutputs Include="$(PySourcePath)Parser\parser.c" />
|
||||
<_PegenSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\invalid.gram;$(PySourcePath)Grammar\Tokens" />
|
||||
<_PegenOutputs Include="$(PySourcePath)Parser\parser.c;$(PySourcePath)Parser\parser_invalid.c" />
|
||||
<_ASTSources Include="$(PySourcePath)Parser\Python.asdl" />
|
||||
<_ASTOutputs Include="$(PySourcePath)Include\internal\pycore_ast.h">
|
||||
<Argument>-H</Argument>
|
||||
@@ -26,7 +26,7 @@
|
||||
<_TokenOutputs Include="$(PySourcePath)Lib\token.py">
|
||||
<Format>py</Format>
|
||||
</_TokenOutputs>
|
||||
<_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
|
||||
<_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\invalid.gram;$(PySourcePath)Grammar\Tokens" />
|
||||
<_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
|
||||
<!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: -->
|
||||
<_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(PySourcePath)PC\pyconfig.h;$(PySourcePath)Tools\jit\**"/>
|
||||
|
||||
Generated
+515
-11822
File diff suppressed because it is too large.
Load diff
Generated
+13389
File diff suppressed because it is too large.
Load diff
@@ -1,2 +1,3 @@
|
||||
peg_extension/parse.c
|
||||
peg_extension/parse_invalid.c
|
||||
venv/
|
||||
@@ -12,15 +12,16 @@ MYPY ?= ./venv/bin/mypy
|
||||
BLACK ?= ./venv/bin/black
|
||||
|
||||
GRAMMAR = ../../Grammar/python.gram
|
||||
INCLUDED_GRAMMARS = ../../Grammar/invalid.gram
|
||||
TOKENS = ../../Grammar/Tokens
|
||||
|
||||
build: peg_extension/parse.c
|
||||
|
||||
peg_extension/parse.c: $(GRAMMAR) $(TOKENS) pegen/*.py peg_extension/peg_extension.c ../../Parser/pegen.c ../../Parser/pegen_errors.c ../../Parser/string_parser.c ../../Parser/action_helpers.c ../../Parser/*.h pegen/grammar_parser.py
|
||||
peg_extension/parse.c: $(GRAMMAR) $(INCLUDED_GRAMMARS) $(TOKENS) pegen/*.py peg_extension/peg_extension.c ../../Parser/pegen.c ../../Parser/pegen_errors.c ../../Parser/string_parser.c ../../Parser/action_helpers.c ../../Parser/*.h pegen/grammar_parser.py
|
||||
$(VENVPYTHON) -m pegen -q c $(GRAMMAR) $(TOKENS) -o peg_extension/parse.c --compile-extension
|
||||
|
||||
clean:
|
||||
-rm -f peg_extension/*.o peg_extension/*.so peg_extension/parse.c
|
||||
-rm -f peg_extension/*.o peg_extension/*.so peg_extension/parse.c peg_extension/parse_invalid.c
|
||||
-rm -rf $(VENVDIR)
|
||||
|
||||
regen-metaparser: pegen/metagrammar.gram pegen/*.py
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import contextlib
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
@@ -6,10 +7,11 @@ import sys
|
||||
import sysconfig
|
||||
import tempfile
|
||||
import tokenize
|
||||
from collections.abc import Sequence
|
||||
from typing import IO, Any
|
||||
|
||||
from pegen.c_generator import CParserGenerator
|
||||
from pegen.grammar import Grammar
|
||||
from pegen.grammar import Grammar, GrammarError
|
||||
from pegen.grammar_parser import GeneratedParser as GrammarParser
|
||||
from pegen.parser import Parser
|
||||
from pegen.parser_generator import ParserGenerator
|
||||
@@ -76,6 +78,7 @@ def compile_c_extension(
|
||||
keep_asserts: bool = True,
|
||||
disable_optimization: bool = False,
|
||||
library_dir: str | None = None,
|
||||
extra_generated_sources: Sequence[pathlib.Path] = (),
|
||||
) -> pathlib.Path:
|
||||
"""Compile the generated source for a parser generator into an extension module.
|
||||
|
||||
@@ -90,6 +93,9 @@ def compile_c_extension(
|
||||
If *library_dir* is provided, that path will be used as the directory for a
|
||||
static library of the common parser sources (this is useful in case you are
|
||||
creating multiple extensions).
|
||||
|
||||
*extra_generated_sources* are the C files generated for the included
|
||||
grammar files (see included_output_files()).
|
||||
"""
|
||||
import setuptools.command.build_ext
|
||||
import setuptools.logging
|
||||
@@ -154,7 +160,7 @@ def compile_c_extension(
|
||||
# HACK: Our output directory for free-threaded builds has moved, and so
|
||||
# tests running in-tree require our sys.executable directory for libs
|
||||
library_dirs.append(pathlib.Path(sys.executable).parent)
|
||||
sources = [generated_source_path]
|
||||
sources = [generated_source_path, *extra_generated_sources]
|
||||
extension = Extension(
|
||||
extension_name,
|
||||
sources=sources,
|
||||
@@ -245,8 +251,8 @@ def compile_c_extension(
|
||||
return pathlib.Path(ext_path)
|
||||
|
||||
|
||||
def build_parser(
|
||||
grammar_file: pathlib.Path, verbose_tokenizer: bool = False, verbose_parser: bool = False
|
||||
def _parse_grammar_file(
|
||||
grammar_file: pathlib.Path, verbose_tokenizer: bool, verbose_parser: bool
|
||||
) -> tuple[Grammar, Parser, Tokenizer]:
|
||||
with open(grammar_file) as file:
|
||||
tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer)
|
||||
@@ -259,6 +265,90 @@ def build_parser(
|
||||
return grammar, parser, tokenizer
|
||||
|
||||
|
||||
def _include_rules(
|
||||
root_grammar: Grammar,
|
||||
root_dir: pathlib.Path,
|
||||
includes: list[str],
|
||||
including_file: pathlib.Path,
|
||||
rule_files: dict[str, pathlib.Path],
|
||||
include_stack: list[pathlib.Path],
|
||||
included_from: dict[pathlib.Path, pathlib.Path],
|
||||
verbose_tokenizer: bool,
|
||||
verbose_parser: bool,
|
||||
) -> None:
|
||||
# Append the rules of each included file (and, depth-first, of the files
|
||||
# it includes) to *root_grammar*, in order.
|
||||
for include in includes:
|
||||
path = (including_file.parent / include).resolve()
|
||||
if path in include_stack:
|
||||
raise GrammarError(f"Circular @include of '{path!s}' in '{including_file!s}'")
|
||||
if path in included_from:
|
||||
raise GrammarError(
|
||||
f"'{path}' is included more than once: "
|
||||
f"from '{included_from[path]}' and from '{including_file}'"
|
||||
)
|
||||
included_from[path] = including_file
|
||||
new_grammar, _, _ = _parse_grammar_file(path, verbose_tokenizer, verbose_parser)
|
||||
if new_grammar.metas:
|
||||
names = ", ".join(f"@{name}" for name in new_grammar.metas)
|
||||
raise GrammarError(f"Included grammar '{path!s}' cannot define metas: {names}")
|
||||
for name, rule in new_grammar.rules.items():
|
||||
if name in root_grammar.rules:
|
||||
raise GrammarError(
|
||||
f"Repeated rule {name!r} in '{path!s}' "
|
||||
f"(first defined in '{rule_files[name]!s}')"
|
||||
)
|
||||
rule.source = pathlib.Path(os.path.relpath(path, root_dir))
|
||||
root_grammar.rules[name] = rule
|
||||
rule_files[name] = path
|
||||
_include_rules(
|
||||
root_grammar,
|
||||
root_dir,
|
||||
new_grammar.includes,
|
||||
path,
|
||||
rule_files,
|
||||
include_stack + [path],
|
||||
included_from,
|
||||
verbose_tokenizer,
|
||||
verbose_parser,
|
||||
)
|
||||
|
||||
|
||||
def build_parser(
|
||||
grammar_file: pathlib.Path, verbose_tokenizer: bool = False, verbose_parser: bool = False
|
||||
) -> tuple[Grammar, Parser, Tokenizer]:
|
||||
grammar, parser, tokenizer = _parse_grammar_file(
|
||||
grammar_file, verbose_tokenizer, verbose_parser
|
||||
)
|
||||
resolve_includes(grammar, grammar_file, verbose_tokenizer, verbose_parser)
|
||||
return grammar, parser, tokenizer
|
||||
|
||||
|
||||
def resolve_includes(
|
||||
grammar: Grammar,
|
||||
grammar_file: pathlib.Path,
|
||||
verbose_tokenizer: bool = False,
|
||||
verbose_parser: bool = False,
|
||||
) -> None:
|
||||
"""Add the rules of the files that *grammar* includes, and clear its includes.
|
||||
|
||||
*grammar_file* is the file *grammar* was parsed from: its @include paths
|
||||
are relative to that file's directory.
|
||||
"""
|
||||
_include_rules(
|
||||
grammar,
|
||||
grammar_file.resolve().parent,
|
||||
grammar.includes,
|
||||
grammar_file,
|
||||
dict.fromkeys(grammar.rules, grammar_file),
|
||||
[grammar_file.resolve()],
|
||||
{},
|
||||
verbose_tokenizer,
|
||||
verbose_parser,
|
||||
)
|
||||
grammar.includes = []
|
||||
|
||||
|
||||
def generate_token_definitions(tokens: IO[str]) -> TokenDefinitions:
|
||||
all_tokens = {}
|
||||
exact_tokens = {}
|
||||
@@ -288,6 +378,34 @@ def generate_token_definitions(tokens: IO[str]) -> TokenDefinitions:
|
||||
return all_tokens, exact_tokens, non_exact_tokens
|
||||
|
||||
|
||||
def included_output_files(
|
||||
grammar: Grammar, output_file: pathlib.Path
|
||||
) -> dict[pathlib.Path, pathlib.Path]:
|
||||
stem, dot, rest = output_file.name.partition(".")
|
||||
outputs: dict[pathlib.Path, pathlib.Path] = {}
|
||||
for rule in grammar.rules.values():
|
||||
if rule.source is None or rule.source in outputs:
|
||||
continue
|
||||
if rule.source.is_absolute() or ".." in rule.source.parts:
|
||||
raise GrammarError(
|
||||
f"The C file for '{rule.source}' would be outside the output directory; "
|
||||
"included grammar files must be in the root grammar file's directory "
|
||||
"or its subdirectories"
|
||||
)
|
||||
path = (
|
||||
output_file.parent
|
||||
/ rule.source.parent
|
||||
/ f"{stem}_{rule.source.stem}{dot}{rest}"
|
||||
)
|
||||
if path in outputs.values():
|
||||
raise GrammarError(
|
||||
f"The C file for '{rule.source}' would be '{path}', "
|
||||
"which is already used by another grammar file"
|
||||
)
|
||||
outputs[rule.source] = path
|
||||
return outputs
|
||||
|
||||
|
||||
def build_c_generator(
|
||||
grammar: Grammar,
|
||||
grammar_file: pathlib.Path,
|
||||
@@ -300,9 +418,23 @@ def build_c_generator(
|
||||
) -> ParserGenerator:
|
||||
with open(tokens_file) as tok_file:
|
||||
all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file)
|
||||
with open(output_file, "w") as file:
|
||||
included_outputs = included_output_files(grammar, output_file)
|
||||
with contextlib.ExitStack() as stack:
|
||||
file: IO[str] = stack.enter_context(open(output_file, "w"))
|
||||
for path in included_outputs.values():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
included_files: dict[pathlib.Path, IO[str]] = {
|
||||
source: stack.enter_context(open(path, "w"))
|
||||
for source, path in included_outputs.items()
|
||||
}
|
||||
gen: ParserGenerator = CParserGenerator(
|
||||
grammar, all_tokens, exact_tok, non_exact_tok, file, skip_actions=skip_actions
|
||||
grammar,
|
||||
all_tokens,
|
||||
exact_tok,
|
||||
non_exact_tok,
|
||||
file,
|
||||
skip_actions=skip_actions,
|
||||
included_files=included_files,
|
||||
)
|
||||
gen.generate(grammar_file)
|
||||
|
||||
@@ -313,6 +445,7 @@ def build_c_generator(
|
||||
build_dir=build_dir,
|
||||
verbose=verbose_c_extension,
|
||||
keep_asserts=keep_asserts_in_extension,
|
||||
extra_generated_sources=list(included_outputs.values()),
|
||||
)
|
||||
return gen
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Prepare a complete C parser description and emit it."""
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
from typing import IO
|
||||
|
||||
from pegen import grammar
|
||||
@@ -67,8 +68,10 @@ class CParserGenerator(ParserGenerator):
|
||||
file: IO[str] | None,
|
||||
debug: bool = False,
|
||||
skip_actions: bool = False,
|
||||
included_files: dict[pathlib.Path, IO[str]] | None = None,
|
||||
):
|
||||
super().__init__(grammar, set(tokens.values()), file)
|
||||
self.included_files = included_files or {}
|
||||
self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor(
|
||||
self, exact_tokens, non_exact_tokens
|
||||
)
|
||||
@@ -77,14 +80,25 @@ class CParserGenerator(ParserGenerator):
|
||||
self.skip_actions = skip_actions
|
||||
|
||||
def generate(self, filename: pathlib.PurePath) -> None:
|
||||
sources = {rule.source for rule in self.rules.values() if rule.source is not None}
|
||||
if missing := sources - self.included_files.keys():
|
||||
raise grammar.GrammarError(
|
||||
"No output file for the rules of "
|
||||
+ ", ".join(f"'{source}'" for source in sorted(missing))
|
||||
)
|
||||
parser = self.prepare(filename)
|
||||
CParserEmitter(parser, self.file).emit()
|
||||
for source, file in self.included_files.items():
|
||||
CParserEmitter(parser, file, source).emit()
|
||||
|
||||
def prepare(self, filename: pathlib.PurePath) -> CParser:
|
||||
self.collect_rules()
|
||||
lowerer = self.callmakervisitor.make_lowerer()
|
||||
shared = self._shared_rules()
|
||||
rules = tuple(
|
||||
lowerer.prepare_rule(rule, skip_actions=self.skip_actions)
|
||||
lowerer.prepare_rule(
|
||||
rule, skip_actions=self.skip_actions, shared=rule.name in shared
|
||||
)
|
||||
for rule in self.all_rules.values()
|
||||
)
|
||||
headers = []
|
||||
@@ -108,6 +122,29 @@ class CParserGenerator(ParserGenerator):
|
||||
super().collect_rules()
|
||||
self._collected = True
|
||||
|
||||
def _shared_rules(self) -> set[str]:
|
||||
# Rules called from a rule (or its actions) whose code goes in
|
||||
# another C file.
|
||||
shared = set()
|
||||
for rule in self.all_rules.values():
|
||||
collector = _NameCollector()
|
||||
collector.visit(rule)
|
||||
for name in collector.names:
|
||||
callee = self.all_rules.get(name)
|
||||
if callee is None or callee.source == rule.source:
|
||||
continue
|
||||
# Artificial rules are only reused within one grammar file.
|
||||
assert name in self.rules, f"{name} is used outside its grammar file"
|
||||
shared.add(name)
|
||||
# The trailer is C code in the root file that calls rules by their
|
||||
# function name (the default one calls start_rule()).
|
||||
trailer = self.grammar.metas.get("trailer", EXTENSION_SUFFIX) or ""
|
||||
for name in _RULE_CALL.findall(trailer):
|
||||
callee = self.rules.get(name)
|
||||
if callee is not None and callee.source is not None:
|
||||
shared.add(name)
|
||||
return shared
|
||||
|
||||
def _prepare_keywords(self) -> tuple[tuple[tuple[str, int], ...], ...]:
|
||||
if not self.keywords:
|
||||
return ()
|
||||
@@ -132,3 +169,20 @@ class CParserGenerator(ParserGenerator):
|
||||
if trailer := self.grammar.metas.get("trailer", EXTENSION_SUFFIX):
|
||||
return trailer.rstrip("\n") % dict(mode=mode, modulename=modulename)
|
||||
return None
|
||||
|
||||
|
||||
# A call to a rule's C function, as written in actions and the trailer.
|
||||
_RULE_CALL = re.compile(r"\b(\w+)_rule\b")
|
||||
|
||||
|
||||
class _NameCollector(grammar.GrammarVisitor):
|
||||
def __init__(self) -> None:
|
||||
self.names: set[str] = set()
|
||||
|
||||
def visit_NameLeaf(self, node: grammar.NameLeaf) -> None:
|
||||
self.names.add(node.value)
|
||||
|
||||
def visit_Alt(self, node: grammar.Alt) -> None:
|
||||
if node.action:
|
||||
self.names.update(_RULE_CALL.findall(node.action))
|
||||
self.generic_visit(node)
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Discover calls and helper rules, then prepare rules from resolved calls."""
|
||||
|
||||
import ast
|
||||
import pathlib
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
@@ -115,7 +116,7 @@ class CCallMakerVisitor(GrammarVisitor):
|
||||
self._keywords = parser_generator.keywords
|
||||
self._exact_tokens = exact_tokens
|
||||
self._non_exact_tokens = non_exact_tokens
|
||||
self._helper_cache: dict[tuple[type, str], str] = {}
|
||||
self._helper_cache: dict[tuple[pathlib.Path | None, type, str], str] = {}
|
||||
self._calls: dict[NamedItem, tuple[Item, FunctionCall]] = {}
|
||||
|
||||
def visit(self, node: Any, *args: Any, **kwargs: Any) -> FunctionCall:
|
||||
@@ -326,7 +327,9 @@ class CCallMakerVisitor(GrammarVisitor):
|
||||
|
||||
def _resolve_artificial_rule(self, node: _HelperNode) -> CRuleSignature:
|
||||
# Preserve helper reuse and numbering from the fixed-point traversal.
|
||||
key = (type(node), str(node))
|
||||
# Helpers are only reused within one grammar file, so that each
|
||||
# generated C file has its own (static) copy.
|
||||
key = (self._registry.current_source, type(node), str(node))
|
||||
if (name := self._helper_cache.get(key)) is None:
|
||||
match node:
|
||||
case Rhs():
|
||||
@@ -406,7 +409,9 @@ class CCallLowerer:
|
||||
i = j
|
||||
self._prefixes[rule.name] = tuple(prefixes)
|
||||
|
||||
def prepare_rule(self, rule: Rule, *, skip_actions: bool = False) -> CRule:
|
||||
def prepare_rule(
|
||||
self, rule: Rule, *, skip_actions: bool = False, shared: bool = False
|
||||
) -> CRule:
|
||||
if (signature := self._signatures.get(rule.name)) is None:
|
||||
raise RuntimeError(f"Rule {rule.name!r} was not discovered")
|
||||
rhs = rule.flatten()
|
||||
@@ -424,6 +429,8 @@ class CCallLowerer:
|
||||
memoize="memo" in rule.flags and not rule.left_recursive,
|
||||
disable_invalid_rules=rule.name.endswith("without_invalid"),
|
||||
prefixes=self._prefixes.get(rule.name, ()),
|
||||
source=rule.source,
|
||||
shared=shared,
|
||||
)
|
||||
|
||||
def prepare_alt(
|
||||
|
||||
@@ -1,29 +1,45 @@
|
||||
"""Emit a complete C parser from an immutable plan and local output state."""
|
||||
|
||||
import pathlib
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import IO
|
||||
|
||||
from pegen.c_generator_model import CParser
|
||||
from pegen.c_generator_model import CParser, CRule
|
||||
from pegen.c_generator_rules import CRuleEmitter
|
||||
|
||||
SHARED_RULE_PREFIX = "_PyPegen_"
|
||||
|
||||
|
||||
class CParserEmitter:
|
||||
def __init__(self, parser: CParser, file: IO[str] | None):
|
||||
def __init__(self, parser: CParser, file: IO[str] | None, source: pathlib.Path | None = None):
|
||||
self._parser = parser
|
||||
self._file = file
|
||||
self._source = source
|
||||
self._level = 0
|
||||
|
||||
def emit(self) -> None:
|
||||
self._emit_headers()
|
||||
self._emit_keywords()
|
||||
if self._source is None:
|
||||
self._emit_keywords()
|
||||
self._emit_declarations()
|
||||
self._emit_rules()
|
||||
if self._parser.trailer is not None:
|
||||
if self._source is None and self._parser.trailer is not None:
|
||||
self.print(self._parser.trailer)
|
||||
|
||||
def _own_rules(self) -> Iterator[tuple[int, CRule]]:
|
||||
# Rule type ids number all rules of the parser, so that they stay
|
||||
# unique across its C files.
|
||||
for rule_id, rule in enumerate(self._parser.rules, 1000):
|
||||
if rule.source == self._source:
|
||||
yield rule_id, rule
|
||||
|
||||
def _emit_headers(self) -> None:
|
||||
self.print(f"// @generated by pegen from {self._parser.source_name}")
|
||||
if self._source is None:
|
||||
source_name = self._parser.source_name
|
||||
else:
|
||||
source_name = self._source.name
|
||||
self.print(f"// @generated by pegen from {source_name}")
|
||||
for header in self._parser.headers:
|
||||
self.print(header)
|
||||
|
||||
@@ -51,16 +67,25 @@ class CParserEmitter:
|
||||
self.print("};")
|
||||
|
||||
def _emit_declarations(self) -> None:
|
||||
for rule_id, rule in enumerate(self._parser.rules, 1000):
|
||||
for rule_id, rule in self._own_rules():
|
||||
comment = " // Left-recursive" if rule.left_recursive else ""
|
||||
self.print(f"#define {rule.signature.name}_type {rule_id}{comment}")
|
||||
self.print()
|
||||
for rule in self._parser.rules:
|
||||
self.print(rule.signature.declaration())
|
||||
shared_rules = [rule for rule in self._parser.rules if rule.shared]
|
||||
if shared_rules:
|
||||
self.print("// Rules shared between the C files of this parser")
|
||||
for rule in shared_rules:
|
||||
name = f"{rule.signature.name}_rule"
|
||||
self.print(f"#define {name} {SHARED_RULE_PREFIX}{name}")
|
||||
self.print(f"Py_LOCAL_SYMBOL {rule.signature.declaration(static=False)}")
|
||||
self.print()
|
||||
for _, rule in self._own_rules():
|
||||
if not rule.shared:
|
||||
self.print(rule.signature.declaration())
|
||||
self.print()
|
||||
|
||||
def _emit_rules(self) -> None:
|
||||
for rule in self._parser.rules:
|
||||
for _, rule in self._own_rules():
|
||||
self.print()
|
||||
if rule.left_recursive:
|
||||
self.print("// Left-recursive")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Immutable C parser descriptions shared by lowering and emission."""
|
||||
|
||||
import pathlib
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
|
||||
@@ -64,9 +65,10 @@ class CRuleSignature:
|
||||
def c_return_type(self) -> str:
|
||||
return self.return_type or "void *"
|
||||
|
||||
def declaration(self) -> str:
|
||||
def declaration(self, *, static: bool = True) -> str:
|
||||
separator = " " if self.kind is RuleKind.NORMAL and self.return_type else ""
|
||||
return f"static {self.c_return_type}{separator}{self.name}_rule(Parser *p);"
|
||||
storage = "static " if static else ""
|
||||
return f"{storage}{self.c_return_type}{separator}{self.name}_rule(Parser *p);"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -123,6 +125,8 @@ class CRule:
|
||||
memoize: bool
|
||||
disable_invalid_rules: bool
|
||||
prefixes: tuple[CPrefix, ...] = ()
|
||||
source: pathlib.Path | None = None
|
||||
shared: bool = False
|
||||
|
||||
@property
|
||||
def uses_locations(self) -> bool:
|
||||
|
||||
@@ -97,7 +97,7 @@ class CRuleEmitter:
|
||||
self._print(f"// {line}")
|
||||
if rule.left_recursive and rule.leader:
|
||||
self._print(f"static {result_type} {signature.name}_raw(Parser *);")
|
||||
self._print(f"static {result_type}")
|
||||
self._print(f"Py_LOCAL_SYMBOL {result_type}" if rule.shared else f"static {result_type}")
|
||||
self._print(f"{signature.name}_rule(Parser *p)")
|
||||
if rule.left_recursive and rule.leader:
|
||||
self._emit_left_recursive_wrapper()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
from collections.abc import Iterable, Iterator, Set
|
||||
from enum import Enum, auto
|
||||
from typing import Any
|
||||
@@ -35,7 +36,16 @@ class Grammar:
|
||||
raise GrammarError(f"Repeated rule {rule.name!r}")
|
||||
all_rules[rule.name] = rule
|
||||
self.rules = all_rules
|
||||
self.metas = dict(metas)
|
||||
# "@include" can appear more than once, so it is kept apart from the
|
||||
# other metas.
|
||||
metas = list(metas)
|
||||
self.includes: list[str] = []
|
||||
for name, value in metas:
|
||||
if name == "include":
|
||||
if value is None:
|
||||
raise GrammarError("@include requires a file name")
|
||||
self.includes.append(value)
|
||||
self.metas = {name: value for name, value in metas if name != "include"}
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "\n".join(str(rule) for name, rule in self.rules.items())
|
||||
@@ -74,12 +84,16 @@ class Rule:
|
||||
flags: frozenset[str] | None = None,
|
||||
*,
|
||||
kind: RuleKind = RuleKind.NORMAL,
|
||||
source: pathlib.Path | None = None,
|
||||
):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.rhs = rhs
|
||||
self.flags = flags or frozenset()
|
||||
self.kind = kind
|
||||
# The included grammar file that defines this rule, relative to the
|
||||
# root grammar file's directory, or None for the root grammar file itself.
|
||||
self.source: pathlib.Path | None = source
|
||||
self.left_recursive = False
|
||||
self.leader = False
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import pathlib
|
||||
import re
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Generator
|
||||
from typing import IO
|
||||
|
||||
from pegen.grammar import (
|
||||
@@ -112,7 +112,17 @@ class ParserGenerator:
|
||||
self.all_rules: dict[str, Rule] = self.rules.copy() # Rules + temporal rules
|
||||
self._local_variable_stack: list[list[str]] = []
|
||||
|
||||
# Source of the rule being visited; artificial rules created for it
|
||||
# belong to the same grammar file.
|
||||
self.current_source: pathlib.Path | None = None
|
||||
|
||||
def _validate_grammar(self) -> None:
|
||||
if self.grammar.includes:
|
||||
raise GrammarError(
|
||||
f"Unresolved @include of {', '.join(map(repr, self.grammar.includes))}; "
|
||||
"load the grammar with pegen.build.build_parser() or call "
|
||||
"pegen.build.resolve_includes() on it"
|
||||
)
|
||||
self.validate_rule_names()
|
||||
if "trailer" not in self.grammar.metas and "start" not in self.rules:
|
||||
raise GrammarError("Grammar without a trailer must have a 'start' rule")
|
||||
@@ -126,7 +136,7 @@ class ParserGenerator:
|
||||
raise GrammarError(f"Rule names cannot start with underscore: '{rule}'")
|
||||
|
||||
@contextlib.contextmanager
|
||||
def local_variable_context(self) -> Iterator[None]:
|
||||
def local_variable_context(self) -> Generator[None]:
|
||||
self._local_variable_stack.append([])
|
||||
yield
|
||||
self._local_variable_stack.pop()
|
||||
@@ -140,13 +150,22 @@ class ParserGenerator:
|
||||
raise NotImplementedError
|
||||
|
||||
@contextlib.contextmanager
|
||||
def indent(self) -> Iterator[None]:
|
||||
def indent(self) -> Generator[None]:
|
||||
self.level += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.level -= 1
|
||||
|
||||
@contextlib.contextmanager
|
||||
def current_source_context(self, current_source: pathlib.Path | None) -> Generator[None]:
|
||||
previous = self.current_source
|
||||
self.current_source = current_source
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.current_source = previous
|
||||
|
||||
def print(self, *args: object) -> None:
|
||||
if not args:
|
||||
print(file=self.file)
|
||||
@@ -172,7 +191,9 @@ class ParserGenerator:
|
||||
break
|
||||
done = set(self.all_rules)
|
||||
for rulename in todo:
|
||||
rule_collector.visit(self.all_rules[rulename])
|
||||
rule = self.all_rules[rulename]
|
||||
with self.current_source_context(rule.source):
|
||||
rule_collector.visit(rule)
|
||||
|
||||
def keyword_type(self) -> int:
|
||||
self.keyword_counter += 1
|
||||
@@ -181,7 +202,7 @@ class ParserGenerator:
|
||||
def artificial_rule_from_rhs(self, rhs: Rhs) -> str:
|
||||
self.counter += 1
|
||||
name = f"_tmp_{self.counter}" # TODO: Pick a nicer name.
|
||||
self.all_rules[name] = Rule(name, None, rhs)
|
||||
self._add_artificial_rule(name, rhs)
|
||||
return name
|
||||
|
||||
def artificial_rule_from_repeat(self, node: Plain, is_repeat1: bool) -> str:
|
||||
@@ -193,9 +214,7 @@ class ParserGenerator:
|
||||
prefix = "_loop0_"
|
||||
kind = RuleKind.LOOP0
|
||||
name = f"{prefix}{self.counter}"
|
||||
self.all_rules[name] = Rule(
|
||||
name, None, Rhs([Alt([NamedItem(None, node)])]), kind=kind
|
||||
)
|
||||
self._add_artificial_rule(name, Rhs([Alt([NamedItem(None, node)])]), kind)
|
||||
return name
|
||||
|
||||
def artificial_rule_from_gather(self, node: Gather) -> str:
|
||||
@@ -205,25 +224,18 @@ class ParserGenerator:
|
||||
[NamedItem(None, node.separator), NamedItem("elem", node.node)],
|
||||
action="elem",
|
||||
)
|
||||
self.all_rules[extra_function_name] = Rule(
|
||||
extra_function_name,
|
||||
None,
|
||||
Rhs([extra_function_alt]),
|
||||
kind=RuleKind.LOOP0,
|
||||
)
|
||||
self._add_artificial_rule(extra_function_name, Rhs([extra_function_alt]), RuleKind.LOOP0)
|
||||
self.counter += 1
|
||||
name = f"_gather_{self.counter}"
|
||||
alt = Alt(
|
||||
[NamedItem("elem", node.node), NamedItem("seq", NameLeaf(extra_function_name))],
|
||||
)
|
||||
self.all_rules[name] = Rule(
|
||||
name,
|
||||
None,
|
||||
Rhs([alt]),
|
||||
kind=RuleKind.GATHER,
|
||||
)
|
||||
self._add_artificial_rule(name, Rhs([alt]), RuleKind.GATHER)
|
||||
return name
|
||||
|
||||
def _add_artificial_rule(self, name: str, rhs: Rhs, kind: RuleKind = RuleKind.NORMAL) -> None:
|
||||
self.all_rules[name] = Rule(name, None, rhs, kind=kind, source=self.current_source)
|
||||
|
||||
def dedupe(self, name: str) -> str:
|
||||
origname = name
|
||||
counter = 0
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
@@ -8,7 +9,11 @@ import tokenize
|
||||
from pathlib import Path, PurePath
|
||||
from typing import IO, Any, Final, cast
|
||||
|
||||
from pegen.build import compile_c_extension
|
||||
from pegen.build import (
|
||||
compile_c_extension,
|
||||
included_output_files,
|
||||
resolve_includes,
|
||||
)
|
||||
from pegen.c_generator import CParserGenerator
|
||||
from pegen.grammar import Grammar
|
||||
from pegen.grammar_parser import GeneratedParser as GrammarParser
|
||||
@@ -46,13 +51,22 @@ def run_parser(file: IO[bytes], parser_class: type[Parser], *, verbose: bool = F
|
||||
|
||||
|
||||
def parse_string(
|
||||
source: str, parser_class: type[Parser], *, dedent: bool = True, verbose: bool = False
|
||||
source: str,
|
||||
parser_class: type[Parser],
|
||||
*,
|
||||
dedent: bool = True,
|
||||
verbose: bool = False,
|
||||
base_dir: Path | None = None,
|
||||
) -> Any:
|
||||
# Run the parser on a string.
|
||||
# Run the parser on a string. A grammar's @include paths are relative
|
||||
# to *base_dir* (the current directory by default).
|
||||
if dedent:
|
||||
source = textwrap.dedent(source)
|
||||
file = io.StringIO(source)
|
||||
return run_parser(file, parser_class, verbose=verbose) # type: ignore[arg-type] # typeshed issue #3515
|
||||
result = run_parser(file, parser_class, verbose=verbose) # type: ignore[arg-type] # typeshed issue #3515
|
||||
if isinstance(result, Grammar):
|
||||
resolve_includes(result, (base_dir or Path.cwd()) / "<string>")
|
||||
return result
|
||||
|
||||
|
||||
def make_parser(source: str) -> type[Parser]:
|
||||
@@ -99,9 +113,23 @@ def generate_parser_c_extension(
|
||||
# context.
|
||||
assert not os.listdir(path)
|
||||
source = path / "parse.c"
|
||||
with open(source, "w", encoding="utf-8") as file:
|
||||
included_outputs = included_output_files(grammar, source)
|
||||
with contextlib.ExitStack() as stack:
|
||||
file = stack.enter_context(open(source, "w", encoding="utf-8"))
|
||||
for output in included_outputs.values():
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
included_files: dict[Path, IO[str]] = {
|
||||
include: stack.enter_context(open(output, "w", encoding="utf-8"))
|
||||
for include, output in included_outputs.items()
|
||||
}
|
||||
genr = CParserGenerator(
|
||||
grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, file, debug=debug
|
||||
grammar,
|
||||
ALL_TOKENS,
|
||||
EXACT_TOKENS,
|
||||
NON_EXACT_TOKENS,
|
||||
file,
|
||||
debug=debug,
|
||||
included_files=included_files,
|
||||
)
|
||||
genr.generate(PurePath("parse.c"))
|
||||
compile_c_extension(
|
||||
@@ -110,6 +138,7 @@ def generate_parser_c_extension(
|
||||
# Significant test_peg_generator speedups
|
||||
disable_optimization=True,
|
||||
library_dir=library_dir,
|
||||
extra_generated_sources=list(included_outputs.values()),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user