mirrors/zig
0
0 Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2026-10-28 10:45:48 +08:00
Code Activity

Merge pull request 'Make std.zig.Ast.parse() and grammar.peg finally agree' (#36094) from ifreund/zig:parser-oracle into master

Browse Source 
Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36094
Reviewed-by: Andrew Kelley <andrew@ziglang.org>
This commit is contained in:
Andrew Kelley committed 2026-07-08 21:46:59 +02:00
2 parents de7fdfecd8 b2c3c33eef
commit cf8120f58f
26 files changed
+8686 -3767
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

build.zig
+29 -2
View File
@@ -719,6 +719,30 @@ pub fn build(b: *std.Build) !void {
check_mingw_run.addDirectoryArg(b.path("lib/libc/mingw"));
check_mingw_step.dependOn(&check_mingw_run.step);
{
const gen_oracle_exe = b.addExecutable(.{
.name = "gen_parser_oracle",
.root_module = b.createModule(.{
.root_source_file = b.path("tools/gen_parser_oracle.zig"),
.target = b.graph.host,
}),
});
const gen_oracle_step = b.step("gen-parser-oracle", "Regenerate lib/std/zig/parser_generated_oracle.zig from doc/langref/grammar.peg");
const gen_oracle_run = b.addRunArtifact(gen_oracle_exe);
gen_oracle_run.addFileArg(b.path("doc/langref/grammar.peg"));
gen_oracle_run.addFileArg(b.path("lib/std/zig/parser_generated_oracle.zig"));
gen_oracle_step.dependOn(&gen_oracle_run.step);
const check_oracle_step = b.step("check-parser-oracle", "Check if doc/langref/grammar.peg was modified without regenerating the oracle");
const check_oracle_run = b.addRunArtifact(gen_oracle_exe);
check_oracle_run.addFileArg(b.path("doc/langref/grammar.peg"));
check_oracle_run.addFileArg(b.path("lib/std/zig/parser_generated_oracle.zig"));
check_oracle_run.addArg("--check");
check_oracle_step.dependOn(&check_oracle_run.step);
test_step.dependOn(check_oracle_step);
}
const test_incremental_step = b.step("test-incremental", "Run the incremental compilation test cases");
try tests.addIncrementalTests(b, test_incremental_step, test_filters);
if (!skip_test_incremental) test_step.dependOn(test_incremental_step);
@@ -1569,8 +1593,9 @@ fn generateLangRef(b: *std.Build) !std.Build.LazyPath {
var it = dir.iterateAssumeFirstIteration();
while (it.next(io) catch @panic("failed to read dir")) |entry| {
if (std.mem.startsWith(u8, entry.name, ".") or entry.kind != .file)
continue;
if (entry.kind != .file) continue;
if (std.mem.startsWith(u8, entry.name, ".")) continue;
if (!std.mem.endsWith(u8, entry.name, ".zig")) continue;
const out_basename = b.fmt("{s}.out", .{std.fs.path.stem(entry.name)});
const cmd = b.addRunArtifact(doctest_exe);
@@ -1603,6 +1628,8 @@ fn generateLangRef(b: *std.Build) !std.Build.LazyPath {
const docgen_cmd = b.addRunArtifact(docgen_exe);
docgen_cmd.addArgs(&.{"--code-dir"});
docgen_cmd.addDirectoryArg(wf.getDirectory());
docgen_cmd.addArgs(&.{"--grammar"});
docgen_cmd.addFileArg(b.path("doc/langref/grammar.peg"));
docgen_cmd.addFileArg(b.path("doc/langref.html.in"));
return docgen_cmd.addOutputFileArg("langref.html");
doc/langref.html.in
+1 -591
View File
@@ -7798,597 +7798,7 @@ fn readU32Be() u32 {}
{#header_open|Appendix#}
{#header_open|Grammar#}
{#syntax_block|peg|grammar.peg#}
Root <- skip ContainerMembers eof
# *** Top level ***
ContainerMembers <- container_doc_comment? ContainerDeclaration* (ContainerField COMMA)* (ContainerField / ContainerDeclaration*)
ContainerDeclaration <- TestDecl / ComptimeDecl / doc_comment? KEYWORD_pub? Decl
TestDecl <- KEYWORD_test (STRINGLITERALSINGLE / IDENTIFIER)? Block
ComptimeDecl <- KEYWORD_comptime Block
Decl
<- (KEYWORD_export / KEYWORD_inline / KEYWORD_noinline)? FnProto (SEMICOLON / Block)
/ KEYWORD_extern STRINGLITERALSINGLE? FnProto SEMICOLON
/ (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? GlobalVarDecl
FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? AddrSpace? LinkSection? CallConv? EXCLAMATIONMARK? TypeExpr !ExprSuffix
VarDeclProto <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? AddrSpace? LinkSection?
GlobalVarDecl <- VarDeclProto (EQUAL Expr)? SEMICOLON
ContainerField <- doc_comment? (KEYWORD_comptime / !KEYWORD_comptime) !KEYWORD_fn (IDENTIFIER COLON / !(IDENTIFIER COLON))? TypeExpr ByteAlign? (EQUAL Expr)?
# *** Block Level ***
BlockStatement
<- Statement
/ KEYWORD_defer BlockExprStatement
/ KEYWORD_errdefer BlockExprStatement
/ !ExprStatement (KEYWORD_comptime !BlockExpr)? VarAssignStatement
Statement
<- ExprStatement
/ KEYWORD_suspend BlockExprStatement
/ !ExprStatement (KEYWORD_comptime !BlockExpr)? AssignExpr SEMICOLON
ExprStatement
<- IfStatement
/ LabeledStatement
/ KEYWORD_nosuspend BlockExprStatement
/ KEYWORD_comptime BlockExpr
IfStatement
<- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )?
/ IfPrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
LabeledStatement <- BlockLabel? (Block / LoopStatement / SwitchExpr)
LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement)
ForStatement
<- ForPrefix BlockExpr ( KEYWORD_else Statement / !KEYWORD_else )
/ ForPrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Statement )
WhileStatement
<- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )?
/ WhilePrefix !BlockExpr AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
BlockExprStatement
<- BlockExpr
/ !BlockExpr AssignExpr SEMICOLON
BlockExpr <- BlockLabel? Block
# An assignment or a destructure whose LHS are all lvalue expressions or variable declarations.
VarAssignStatement <- (VarDeclProto / Expr) (COMMA (VarDeclProto / Expr))* EQUAL Expr SEMICOLON
# *** Expression Level ***
# An assignment or a destructure whose LHS are all lvalue expressions.
AssignExpr <- Expr (AssignOp Expr / (COMMA Expr)+ EQUAL Expr)?
SingleAssignExpr <- Expr (AssignOp Expr)?
Expr <- BoolOrExpr
BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)*
BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)*
CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)?
BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)*
BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)*
AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)*
MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)*
PrefixExpr <- PrefixOp* PrimaryExpr
PrimaryExpr
<- AsmExpr
/ IfExpr
/ KEYWORD_break (BreakLabel / !BreakLabel) (Expr !ExprSuffix / !SinglePtrTypeStart)
/ KEYWORD_comptime Expr !ExprSuffix
/ KEYWORD_nosuspend Expr !ExprSuffix
/ KEYWORD_continue (BreakLabel / !BreakLabel) (Expr !ExprSuffix / !SinglePtrTypeStart)
/ KEYWORD_resume Expr !ExprSuffix
/ KEYWORD_return (Expr !ExprSuffix / !SinglePtrTypeStart)
/ BlockLabel? LoopExpr
/ Block
/ CurlySuffixExpr
IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? !ExprSuffix
Block <- LBRACE BlockStatement* RBRACE
LoopExpr <- KEYWORD_inline? (ForExpr / WhileExpr)
ForExpr <- ForPrefix Expr (KEYWORD_else Expr / !KEYWORD_else) !ExprSuffix
WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? !ExprSuffix
CurlySuffixExpr <- TypeExpr InitList?
InitList
<- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
/ LBRACE Expr (COMMA Expr)* COMMA? RBRACE
/ LBRACE RBRACE
TypeExpr <- PrefixTypeOp* ErrorUnionExpr
ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)?
SuffixExpr
<- PrimaryTypeExpr (SuffixOp / FnCallArguments)*
PrimaryTypeExpr
<- BUILTINIDENTIFIER FnCallArguments
/ CHAR_LITERAL
/ ContainerDecl
/ DOT IDENTIFIER
/ DOT InitList
/ ErrorSetDecl
/ FLOAT
/ FnProto
/ GroupedExpr
/ LabeledTypeExpr
/ IDENTIFIER !(COLON LabelableExpr)
/ IfTypeExpr
/ INTEGER
/ KEYWORD_comptime TypeExpr !ExprSuffix
/ KEYWORD_error DOT IDENTIFIER
/ KEYWORD_anyframe
/ KEYWORD_unreachable
/ STRINGLITERAL
ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
GroupedExpr <- LPAREN Expr RPAREN
IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? !ExprSuffix
LabeledTypeExpr
<- BlockLabel Block
/ BlockLabel? LoopTypeExpr
/ BlockLabel? SwitchExpr
LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr)
ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr / !KEYWORD_else) !ExprSuffix
WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? !ExprSuffix
SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE
# *** Assembly ***
AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN
AsmOutput <- COLON AsmOutputList AsmInput?
AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN
AsmInput <- COLON AsmInputList AsmClobbers?
AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN Expr RPAREN
AsmClobbers <- COLON Expr
# *** Helper grammar ***
BreakLabel <- COLON IDENTIFIER
BlockLabel <- IDENTIFIER COLON
FieldInit <- DOT IDENTIFIER EQUAL Expr
WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN
LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN
AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN
# Fn specific
CallConv <- KEYWORD_callconv LPAREN Expr RPAREN
ParamDecl <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime / !KEYWORD_comptime) (IDENTIFIER COLON / !(IDENTIFIER COLON)) ParamType
ParamType
<- KEYWORD_anytype
/ TypeExpr
# Control flow prefixes
IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
ForPrefix <- KEYWORD_for LPAREN ForArgumentsList RPAREN PtrListPayload
# Payloads
Payload <- PIPE IDENTIFIER PIPE
PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE
PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
PtrListPayload <- PIPE ASTERISK? IDENTIFIER (COMMA ASTERISK? IDENTIFIER)* COMMA? PIPE
# Switch specific
SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? SingleAssignExpr
SwitchCase
<- SwitchItem (COMMA SwitchItem)* COMMA?
/ KEYWORD_else
SwitchItem <- Expr (DOT3 Expr)?
# For specific
ForArgumentsList <- ForItem (COMMA ForItem)* COMMA?
ForItem <- Expr (DOT2 Expr?)?
# Operators
AssignOp
<- ASTERISKEQUAL
/ ASTERISKPIPEEQUAL
/ SLASHEQUAL
/ PERCENTEQUAL
/ PLUSEQUAL
/ PLUSPIPEEQUAL
/ MINUSEQUAL
/ MINUSPIPEEQUAL
/ LARROW2EQUAL
/ LARROW2PIPEEQUAL
/ RARROW2EQUAL
/ AMPERSANDEQUAL
/ CARETEQUAL
/ PIPEEQUAL
/ ASTERISKPERCENTEQUAL
/ PLUSPERCENTEQUAL
/ MINUSPERCENTEQUAL
/ EQUAL
CompareOp
<- EQUALEQUAL
/ EXCLAMATIONMARKEQUAL
/ LARROW
/ RARROW
/ LARROWEQUAL
/ RARROWEQUAL
BitwiseOp
<- AMPERSAND
/ CARET
/ PIPE
/ KEYWORD_orelse
/ KEYWORD_catch Payload?
BitShiftOp
<- LARROW2
/ RARROW2
/ LARROW2PIPE
AdditionOp
<- PLUS
/ MINUS
/ PLUS2
/ PLUSPERCENT
/ MINUSPERCENT
/ PLUSPIPE
/ MINUSPIPE
MultiplyOp
<- PIPE2
/ ASTERISK
/ SLASH
/ PERCENT
/ ASTERISKPERCENT
/ ASTERISKPIPE
PrefixOp
<- EXCLAMATIONMARK
/ MINUS
/ TILDE
/ MINUSPERCENT
/ AMPERSAND
/ KEYWORD_try
PrefixTypeOp
<- QUESTIONMARK
/ KEYWORD_anyframe MINUSRARROW
/ (ManyPtrTypeStart / SliceTypeStart) KEYWORD_allowzero? ByteAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile?
/ SinglePtrTypeStart KEYWORD_allowzero? BitAlign? AddrSpace? KEYWORD_const? KEYWORD_volatile?
/ ArrayTypeStart
SuffixOp
<- LBRACKET Expr (DOT2 Expr? (COLON Expr)?)? RBRACKET
/ DOT IDENTIFIER
/ DOTASTERISK
/ DOTQUESTIONMARK
FnCallArguments <- LPAREN ExprList RPAREN
ExprSuffix
<- KEYWORD_or
/ KEYWORD_and
/ CompareOp
/ BitwiseOp
/ BitShiftOp
/ AdditionOp
/ MultiplyOp
/ EXCLAMATIONMARK
/ SuffixOp
/ FnCallArguments
LabelableExpr
<- Block
/ SwitchExpr
/ LoopExpr
# Ptr specific
SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET
SinglePtrTypeStart <- ASTERISK
ManyPtrTypeStart <- LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
ArrayTypeStart <- LBRACKET Expr !ASTERISK (COLON Expr)? RBRACKET
# ContainerDecl specific
ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
ContainerDeclType
<- KEYWORD_struct (LPAREN Expr RPAREN)?
/ KEYWORD_opaque
/ KEYWORD_enum (LPAREN Expr RPAREN)?
/ KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / !KEYWORD_enum Expr) RPAREN)?
# Alignment
ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
BitAlign <- KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN
# Lists
IdentifierList <- (doc_comment? IDENTIFIER COMMA)* (doc_comment? IDENTIFIER)?
SwitchProngList <- (SwitchProng COMMA)* SwitchProng?
AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem?
AsmInputList <- (AsmInputItem COMMA)* AsmInputItem?
ParamDeclList <- (ParamDecl COMMA)* (ParamDecl / DOT3 COMMA?)?
ExprList <- (Expr COMMA)* Expr?
# *** Tokens ***
eof <- !.
bin <- [01]
bin_ <- '_'? bin
oct <- [0-7]
oct_ <- '_'? oct
hex <- [0-9a-fA-F]
hex_ <- '_'? hex
dec <- [0-9]
dec_ <- '_'? dec
bin_int <- bin bin_*
oct_int <- oct oct_*
dec_int <- dec dec_*
hex_int <- hex hex_*
ox80_oxBF <- [\200-\277]
oxF4 <- '\364'
ox80_ox8F <- [\200-\217]
oxF1_oxF3 <- [\361-\363]
oxF0 <- '\360'
ox90_0xBF <- [\220-\277]
oxEE_oxEF <- [\356-\357]
oxED <- '\355'
ox80_ox9F <- [\200-\237]
oxE1_oxEC <- [\341-\354]
oxE0 <- '\340'
oxA0_oxBF <- [\240-\277]
oxC2_oxDF <- [\302-\337]
# From https://lemire.me/blog/2018/05/09/how-quickly-can-you-check-that-a-string-is-valid-unicode-utf-8/
# First Byte Second Byte Third Byte Fourth Byte
# [0x00,0x7F]
# [0xC2,0xDF] [0x80,0xBF]
# 0xE0 [0xA0,0xBF] [0x80,0xBF]
# [0xE1,0xEC] [0x80,0xBF] [0x80,0xBF]
# 0xED [0x80,0x9F] [0x80,0xBF]
# [0xEE,0xEF] [0x80,0xBF] [0x80,0xBF]
# 0xF0 [0x90,0xBF] [0x80,0xBF] [0x80,0xBF]
# [0xF1,0xF3] [0x80,0xBF] [0x80,0xBF] [0x80,0xBF]
# 0xF4 [0x80,0x8F] [0x80,0xBF] [0x80,0xBF]
multibyte_utf8 <-
oxF4 ox80_ox8F ox80_oxBF ox80_oxBF
/ oxF1_oxF3 ox80_oxBF ox80_oxBF ox80_oxBF
/ oxF0 ox90_0xBF ox80_oxBF ox80_oxBF
/ oxEE_oxEF ox80_oxBF ox80_oxBF
/ oxED ox80_ox9F ox80_oxBF
/ oxE1_oxEC ox80_oxBF ox80_oxBF
/ oxE0 oxA0_oxBF ox80_oxBF
/ oxC2_oxDF ox80_oxBF
non_control_ascii <- [\040-\176]
non_control_utf8 <- [\040-\377]
char_escape
<- "\\x" hex hex
/ "\\u{" hex+ "}"
/ "\\" [nr\\t'"]
char_char
<- multibyte_utf8
/ char_escape
/ ![\\'\n] non_control_ascii
string_char
<- multibyte_utf8
/ char_escape
/ ![\\"\n] non_control_ascii
container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+
doc_comment <- ('///' non_control_utf8* [ \n]* skip)+
line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8*
line_string <- '\\\\' non_control_utf8* [ \n]*
skip <- ([ \n] / line_comment)*
CHAR_LITERAL <- ['] char_char ['] skip
FLOAT
<- '0x' hex_int '.' hex_int ([pP] [-+]? dec_int)? skip
/ dec_int '.' dec_int ([eE] [-+]? dec_int)? skip
/ '0x' hex_int [pP] [-+]? dec_int skip
/ dec_int [eE] [-+]? dec_int skip
INTEGER
<- '0b' bin_int skip
/ '0o' oct_int skip
/ '0x' hex_int skip
/ dec_int skip
STRINGLITERALSINGLE <- ["] string_char* ["] skip
STRINGLITERAL
<- STRINGLITERALSINGLE
/ (line_string skip)+
IDENTIFIER
<- !keyword [A-Za-z_] [A-Za-z0-9_]* skip
/ '@' STRINGLITERALSINGLE
BUILTINIDENTIFIER <- '@'[A-Za-z_][A-Za-z0-9_]* skip
AMPERSAND <- '&' ![=] skip
AMPERSANDEQUAL <- '&=' skip
ASTERISK <- '*' ![%=|] skip
ASTERISKEQUAL <- '*=' skip
ASTERISKPERCENT <- '*%' ![=] skip
ASTERISKPERCENTEQUAL <- '*%=' skip
ASTERISKPIPE <- '*|' ![=] skip
ASTERISKPIPEEQUAL <- '*|=' skip
CARET <- '^' ![=] skip
CARETEQUAL <- '^=' skip
COLON <- ':' skip
COMMA <- ',' skip
DOT <- '.' ![*.?] skip
DOT2 <- '..' ![.] skip
DOT3 <- '...' skip
DOTASTERISK <- '.*' skip
DOTQUESTIONMARK <- '.?' skip
EQUAL <- '=' ![>=] skip
EQUALEQUAL <- '==' skip
EQUALRARROW <- '=>' skip
EXCLAMATIONMARK <- '!' ![=] skip
EXCLAMATIONMARKEQUAL <- '!=' skip
LARROW <- '<' ![<=] skip
LARROW2 <- '<<' ![=|] skip
LARROW2EQUAL <- '<<=' skip
LARROW2PIPE <- '<<|' ![=] skip
LARROW2PIPEEQUAL <- '<<|=' skip
LARROWEQUAL <- '<=' skip
LBRACE <- '{' skip
LBRACKET <- '[' skip
LPAREN <- '(' skip
MINUS <- '-' ![%=>|] skip
MINUSEQUAL <- '-=' skip
MINUSPERCENT <- '-%' ![=] skip
MINUSPERCENTEQUAL <- '-%=' skip
MINUSPIPE <- '-|' ![=] skip
MINUSPIPEEQUAL <- '-|=' skip
MINUSRARROW <- '->' skip
PERCENT <- '%' ![=] skip
PERCENTEQUAL <- '%=' skip
PIPE <- '|' ![|=] skip
PIPE2 <- '||' skip
PIPEEQUAL <- '|=' skip
PLUS <- '+' ![%+=|] skip
PLUS2 <- '++' skip
PLUSEQUAL <- '+=' skip
PLUSPERCENT <- '+%' ![=] skip
PLUSPERCENTEQUAL <- '+%=' skip
PLUSPIPE <- '+|' ![=] skip
PLUSPIPEEQUAL <- '+|=' skip
LETTERC <- 'c' skip
QUESTIONMARK <- '?' skip
RARROW <- '>' ![>=] skip
RARROW2 <- '>>' ![=] skip
RARROW2EQUAL <- '>>=' skip
RARROWEQUAL <- '>=' skip
RBRACE <- '}' skip
RBRACKET <- ']' skip
RPAREN <- ')' skip
SEMICOLON <- ';' skip
SLASH <- '/' ![=] skip
SLASHEQUAL <- '/=' skip
TILDE <- '~' skip
end_of_word <- ![a-zA-Z0-9_] skip
KEYWORD_addrspace <- 'addrspace' end_of_word
KEYWORD_align <- 'align' end_of_word
KEYWORD_allowzero <- 'allowzero' end_of_word
KEYWORD_and <- 'and' end_of_word
KEYWORD_anyframe <- 'anyframe' end_of_word
KEYWORD_anytype <- 'anytype' end_of_word
KEYWORD_asm <- 'asm' end_of_word
KEYWORD_break <- 'break' end_of_word
KEYWORD_callconv <- 'callconv' end_of_word
KEYWORD_catch <- 'catch' end_of_word
KEYWORD_comptime <- 'comptime' end_of_word
KEYWORD_const <- 'const' end_of_word
KEYWORD_continue <- 'continue' end_of_word
KEYWORD_defer <- 'defer' end_of_word
KEYWORD_else <- 'else' end_of_word
KEYWORD_enum <- 'enum' end_of_word
KEYWORD_errdefer <- 'errdefer' end_of_word
KEYWORD_error <- 'error' end_of_word
KEYWORD_export <- 'export' end_of_word
KEYWORD_extern <- 'extern' end_of_word
KEYWORD_fn <- 'fn' end_of_word
KEYWORD_for <- 'for' end_of_word
KEYWORD_if <- 'if' end_of_word
KEYWORD_inline <- 'inline' end_of_word
KEYWORD_noalias <- 'noalias' end_of_word
KEYWORD_nosuspend <- 'nosuspend' end_of_word
KEYWORD_noinline <- 'noinline' end_of_word
KEYWORD_opaque <- 'opaque' end_of_word
KEYWORD_or <- 'or' end_of_word
KEYWORD_orelse <- 'orelse' end_of_word
KEYWORD_packed <- 'packed' end_of_word
KEYWORD_pub <- 'pub' end_of_word
KEYWORD_resume <- 'resume' end_of_word
KEYWORD_return <- 'return' end_of_word
KEYWORD_linksection <- 'linksection' end_of_word
KEYWORD_struct <- 'struct' end_of_word
KEYWORD_suspend <- 'suspend' end_of_word
KEYWORD_switch <- 'switch' end_of_word
KEYWORD_test <- 'test' end_of_word
KEYWORD_threadlocal <- 'threadlocal' end_of_word
KEYWORD_try <- 'try' end_of_word
KEYWORD_union <- 'union' end_of_word
KEYWORD_unreachable <- 'unreachable' end_of_word
KEYWORD_var <- 'var' end_of_word
KEYWORD_volatile <- 'volatile' end_of_word
KEYWORD_while <- 'while' end_of_word
keyword <- KEYWORD_addrspace / KEYWORD_align / KEYWORD_allowzero / KEYWORD_and
/ KEYWORD_anyframe / KEYWORD_anytype / KEYWORD_asm
/ KEYWORD_break / KEYWORD_callconv / KEYWORD_catch
/ KEYWORD_comptime / KEYWORD_const / KEYWORD_continue / KEYWORD_defer
/ KEYWORD_else / KEYWORD_enum / KEYWORD_errdefer / KEYWORD_error / KEYWORD_export
/ KEYWORD_extern / KEYWORD_fn / KEYWORD_for / KEYWORD_if
/ KEYWORD_inline / KEYWORD_noalias / KEYWORD_nosuspend / KEYWORD_noinline
/ KEYWORD_opaque / KEYWORD_or / KEYWORD_orelse / KEYWORD_packed
/ KEYWORD_pub / KEYWORD_resume / KEYWORD_return / KEYWORD_linksection
/ KEYWORD_struct / KEYWORD_suspend / KEYWORD_switch / KEYWORD_test
/ KEYWORD_threadlocal / KEYWORD_try / KEYWORD_union / KEYWORD_unreachable
/ KEYWORD_var / KEYWORD_volatile / KEYWORD_while
{#end_syntax_block#}
{#grammar#}
{#header_close#}
{#header_open|Zen#}
<ul>
doc/langref/grammar.peg
+701
View File
@@ -0,0 +1,701 @@
# This grammar is intended to be LL(k) at the token level.
# In other words, whenever the parser is required to make a choice it must be
# able to choose "correctly" with constant token lookahead. This means that the
# handwritten recursive descent parser for this grammar can be implemented with
# worst-case linear run time rather than worst-case exponential runtime.
#
# To ensure this property is upheld, care must be taken to avoid requiring
# unbounded token lookahead in the grammar. To illustrate this, consider the
# following examples requiring unbounded lookahead:
#
# This is rather subtle as it looks fine at first glance:
#
# SingleAssignExpr <- Expr (AssignOp Expr)?
#
# Consider the case where the AssignOp token matches but Expr after it does
# not match. The parser would attempt to parse the Expr after the AssignOp,
# fail, then backtrack and succeed the SingleAssignExpr rule matching only
# the first Expr. This is an example of unbounded lookahead. The rule must
# be rewritten so that if the AssignOp token is not followed by an Expr the
# whole SingleAssignExpr rule fails to match:
#
# SingleAssignExpr <- Expr (AssignOp Expr / !AssignOp)
#
# The pattern is similar for e.g. the * operator. The following rule requires
# unbounded lookahead:
#
# BoolOrExpr <- BoolAndExpr (OrOp BoolAndExpr)*
#
# and can be fixed to require only bounded lookahead by adding a !OrOp:
#
# BoolOrExpr <- BoolAndExpr (OrOp BoolAndExpr)* !OrOp
#
# Note also that the negative lookahead added with the ! operator must always be
# for a constant maximum number of tokens.
#
# See this paper for analysis of how LL(k) grammars relate to PEGs:
# https://arxiv.org/abs/1304.3177
Root <- ContainerMembers skip eof
# *** Top level ***
ContainerMembers <- container_doc_comment? ContainerDecl* ContainerMembersRest
ContainerMembersRest
<- !ContainerDeclPrefix ContainerField (COMMA ContainerMembersRest)?
/ ContainerDecl*
ContainerDecl <- TestDecl / ComptimeDecl / doc_comment? KEYWORD_pub? Decl
ContainerDeclPrefix
<- KEYWORD_test
/ KEYWORD_comptime LBRACE
/ doc_comment? KEYWORD_pub? DeclPrefix
TestDecl <- KEYWORD_test (STRINGLITERALSINGLE / IDENTIFIER)? Block
ComptimeDecl <- KEYWORD_comptime Block
Decl
<- (KEYWORD_export / KEYWORD_inline / KEYWORD_noinline)? FnProto (SEMICOLON / Block)
/ KEYWORD_extern STRINGLITERALSINGLE? FnProto SEMICOLON
/ (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? GlobalVarDecl
DeclPrefix
<- (KEYWORD_export / KEYWORD_inline / KEYWORD_noinline)? KEYWORD_fn
/ KEYWORD_extern STRINGLITERALSINGLE? KEYWORD_fn
/ (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? (KEYWORD_const / KEYWORD_var)
FnProto <- KEYWORD_fn IDENTIFIER? ParamDeclList ByteAlign? AddrSpace? LinkSection? CallConv? EXCLAMATIONMARK? TypeExpr
VarDeclProto <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? AddrSpace? LinkSection?
GlobalVarDecl <- VarDeclProto (EQUAL Expr)? SEMICOLON
ContainerField <- doc_comment? KEYWORD_comptime? (IDENTIFIER COLON)? TypeExpr ByteAlign? (EQUAL Expr)?
# *** Block Level ***
BlockStatement
<- Statement
/ KEYWORD_defer BlockExprStatement
/ KEYWORD_errdefer BlockExprStatement
/ !StatementPrefix KEYWORD_comptime? VarAssignStatement
Statement
<- IfStatement
/ LabeledStatement
/ KEYWORD_nosuspend BlockExprStatement
/ KEYWORD_comptime BlockExpr
/ KEYWORD_suspend BlockExprStatement
/ !StatementPrefix KEYWORD_comptime? AssignExpr SEMICOLON
StatementPrefix
<- KEYWORD_if
/ BlockLabel? (LBRACE / KEYWORD_inline? (KEYWORD_for / KEYWORD_while) / KEYWORD_switch)
/ KEYWORD_nosuspend
/ KEYWORD_comptime BlockExprPrefix
IfStatement
<- IfPrefix BlockExpr (KEYWORD_else Payload? Statement / !KEYWORD_else)
/ IfPrefix !BlockExprPrefix AssignExpr (SEMICOLON / KEYWORD_else Payload? Statement)
LabeledStatement <- BlockLabel? (Block / LoopStatement / SwitchExpr)
LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement)
ForStatement
<- ForPrefix BlockExpr (KEYWORD_else Statement / !KEYWORD_else)
/ ForPrefix !BlockExprPrefix AssignExpr (SEMICOLON / KEYWORD_else Statement)
WhileStatement
<- WhilePrefix BlockExpr (KEYWORD_else Payload? Statement / !KEYWORD_else)
/ WhilePrefix !BlockExprPrefix AssignExpr (SEMICOLON / KEYWORD_else Payload? Statement)
BlockExprStatement
<- BlockExpr
/ !BlockExprPrefix AssignExpr SEMICOLON
BlockExprPrefix <- BlockLabel? LBRACE
BlockExpr <- BlockLabel? Block
# An assignment or a destructure whose LHS are all lvalue expressions or variable declarations.
VarAssignStatement <- (VarDeclProto / Expr) (COMMA (VarDeclProto / Expr))* EQUAL Expr SEMICOLON
# *** Expression Level ***
# An assignment or a destructure whose LHS are all lvalue expressions.
AssignExpr <- Expr (AssignOp Expr / (COMMA Expr)+ EQUAL Expr / !AssignOp !COMMA)
SingleAssignExpr <- Expr (AssignOp Expr / !AssignOp)
Expr <- BoolOrExpr
BoolOrExpr <- BoolAndExpr (OrOp BoolAndExpr)* !OrOp
BoolAndExpr <- CompareExpr (AndOp CompareExpr)* !AndOp
CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)? !CompareOp
BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)* !BitwiseOp
BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)* !BitShiftOp
AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)* !AdditionOp
MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)* !MultiplyOp
PrefixExpr <- PrefixOp* !PrefixOp PrimaryExpr
PrimaryExpr
<- AsmExpr
/ IfExpr
/ KEYWORD_break BreakLabel? (Expr / !ExprPrefix)
/ KEYWORD_comptime Expr
/ KEYWORD_nosuspend Expr
/ KEYWORD_continue BreakLabel? (Expr / !ExprPrefix)
/ KEYWORD_resume Expr
/ KEYWORD_return (Expr / !ExprPrefix)
/ BlockLabel? LoopExpr
/ Block
/ CurlySuffixExpr
IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr / !KEYWORD_else)
Block <- LBRACE BlockStatement* RBRACE
LoopExpr <- KEYWORD_inline? (ForExpr / WhileExpr)
ForExpr <- ForPrefix Expr (KEYWORD_else Expr / !KEYWORD_else)
WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)?
CurlySuffixExpr <- TypeExpr (InitList / !LBRACE)
InitList
<- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
/ LBRACE Expr (COMMA Expr)* COMMA? RBRACE
/ LBRACE RBRACE
TypeExpr <- PrefixTypeOp* !PrefixTypeOpPrefix ErrorUnionExpr
ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr / !EXCLAMATIONMARK)
SuffixExpr <- PrimaryTypeExpr SuffixOp* !SuffixOpPrefix
PrimaryTypeExpr
<- BUILTINIDENTIFIER FnCallArguments
/ CHAR_LITERAL
/ ContainerType
/ DOT IDENTIFIER
/ DOT InitList
/ ErrorSetDecl
/ FnProto
/ GroupedExpr
/ LabeledTypeExpr
/ IDENTIFIER
/ IfTypeExpr
/ KEYWORD_comptime TypeExpr
/ KEYWORD_error DOT IDENTIFIER
/ KEYWORD_anyframe
/ KEYWORD_unreachable
/ NUMBERLITERAL
/ STRINGLITERAL
ContainerType <- (KEYWORD_extern / KEYWORD_packed)? ContainerTypeAuto
ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
GroupedExpr <- LPAREN Expr RPAREN
IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr / !KEYWORD_else)
LabeledTypeExpr
<- BlockLabel Block
/ BlockLabel? LoopTypeExpr
/ BlockLabel? SwitchExpr
LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr)
ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr / !KEYWORD_else)
WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr / !KEYWORD_else)
SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE
# *** Assembly ***
AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN
AsmOutput <- COLON AsmOutputList AsmInput?
AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN
AsmInput <- COLON AsmInputList AsmClobbers?
AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERALSINGLE LPAREN Expr RPAREN
AsmClobbers <- COLON Expr
# *** Helper grammar ***
BreakLabel <- COLON IDENTIFIER
BlockLabel <- IDENTIFIER COLON
FieldInit <- DOT IDENTIFIER EQUAL Expr
WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN
LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN
AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN
# Fn specific
CallConv <- KEYWORD_callconv LPAREN Expr RPAREN
ParamDecl <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime?) (IDENTIFIER COLON)? ParamType
ParamType
<- KEYWORD_anytype
/ TypeExpr
# Control flow prefixes
IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
ForPrefix <- KEYWORD_for LPAREN ForArgumentsList RPAREN PtrListPayload
# Payloads
Payload <- PIPE IDENTIFIER PIPE
PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE
PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
PtrListPayload <- PIPE ASTERISK? IDENTIFIER (COMMA ASTERISK? IDENTIFIER)* COMMA? PIPE
# Switch specific
SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? SingleAssignExpr
SwitchCase
<- SwitchItem (COMMA SwitchItem)* COMMA?
/ KEYWORD_else
SwitchItem <- Expr (DOT3 Expr)?
# For specific
ForArgumentsList <- ForItem (COMMA ForItem)* COMMA?
ForItem <- Expr (DOT2 (Expr / !ExprPrefix) / !DOT2)
# Operators
AssignOp
<- ASTERISKEQUAL
/ ASTERISKPIPEEQUAL
/ SLASHEQUAL
/ PERCENTEQUAL
/ PLUSEQUAL
/ PLUSPIPEEQUAL
/ MINUSEQUAL
/ MINUSPIPEEQUAL
/ LARROW2EQUAL
/ LARROW2PIPEEQUAL
/ RARROW2EQUAL
/ AMPERSANDEQUAL
/ CARETEQUAL
/ PIPEEQUAL
/ ASTERISKPERCENTEQUAL
/ PLUSPERCENTEQUAL
/ MINUSPERCENTEQUAL
/ EQUAL
OrOp <- pre_op_white KEYWORD_or post_op_white
/ !pre_op_white KEYWORD_or !post_op_white
AndOp <- pre_op_white KEYWORD_and post_op_white
/ !pre_op_white KEYWORD_and !post_op_white
CompareOp <- pre_op_white CompareOpTok post_op_white
/ !pre_op_white CompareOpTok !post_op_white
CompareOpTok
<- EQUALEQUAL
/ EXCLAMATIONMARKEQUAL
/ LARROW
/ RARROW
/ LARROWEQUAL
/ RARROWEQUAL
BitwiseOp <- pre_op_white BitwiseOpTok post_op_white
/ !pre_op_white BitwiseOpTok !post_op_white
/ pre_op_white KEYWORD_catch post_op_white Payload?
/ !pre_op_white KEYWORD_catch !post_op_white Payload?
BitwiseOpTok
<- AMPERSAND ![&]
/ CARET
/ PIPE
/ KEYWORD_orelse
BitShiftOp <- pre_op_white BitShiftOpTok post_op_white
/ !pre_op_white BitShiftOpTok !post_op_white
BitShiftOpTok
<- LARROW2
/ RARROW2
/ LARROW2PIPE
AdditionOp <- pre_op_white AdditionOpTok post_op_white
/ !pre_op_white AdditionOpTok !post_op_white
AdditionOpTok
<- PLUS
/ MINUS
/ PLUS2
/ PLUSPERCENT
/ MINUSPERCENT
/ PLUSPIPE
/ MINUSPIPE
MultiplyOp <- pre_op_white MultiplyOpTok post_op_white
/ !pre_op_white MultiplyOpTok !post_op_white
MultiplyOpTok
<- PIPE2
/ ASTERISK
/ SLASH
/ PERCENT
/ ASTERISKPERCENT
/ ASTERISKPIPE
PrefixOp
<- EXCLAMATIONMARK
/ MINUS
/ TILDE
/ MINUSPERCENT
/ AMPERSAND
/ KEYWORD_try
PrefixTypeOp
<- QUESTIONMARK
/ KEYWORD_anyframe MINUSRARROW
/ (ManyPtrTypeStart / SliceTypeStart) PtrMods
/ SinglePtrTypeStart SinglePtrMods
/ ArrayTypeStart
# Forbid more than one align or addrspace pointer modifier since these
# modifiers contain sub-expressions. This allows for a simpler AST data layout.
# Allow duplicate single-token modifiers (allowzero/const/volatile) in the grammar
# to avoid the combinatorial explosion of grammar rules necessary to forbid duplicates
# while permitting arbitrary order. A compile error for duplicates single-token modifiers
# is emitted during "AstGen" after parsing is complete.
PtrMods
<- PtrMod* !KEYWORD_addrspace ByteAlign? PtrMod* AddrSpace? PtrMod*
/ PtrMod* !KEYWORD_align AddrSpace? PtrMod* ByteAlign? PtrMod*
SinglePtrMods
<- PtrMod* !KEYWORD_addrspace BitAlign? PtrMod* AddrSpace? PtrMod*
/ PtrMod* !KEYWORD_align AddrSpace? PtrMod* BitAlign? PtrMod*
PtrMod
<- KEYWORD_allowzero
/ KEYWORD_const
/ KEYWORD_volatile
PrefixTypeOpPrefix
<- QUESTIONMARK
/ KEYWORD_anyframe MINUSRARROW
/ LBRACKET
/ ASTERISK
SuffixOp
<- LBRACKET Expr (DOT2 (Expr / !ExprPrefix) (COLON Expr)?)? RBRACKET
/ DOT IDENTIFIER
/ DOTASTERISK
/ DOT QUESTIONMARK
/ FnCallArguments
SuffixOpPrefix
<- LBRACKET
/ DOT IDENTIFIER
/ DOTASTERISK
/ DOT QUESTIONMARK
/ LPAREN
FnCallArguments <- LPAREN ExprList RPAREN
# Ptr specific
SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET
SinglePtrTypeStart <- ASTERISK
ManyPtrTypeStart <- LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
ArrayTypeStart <- LBRACKET !ASTERISK Expr (COLON Expr)? RBRACKET
# ContainerType specific
ContainerTypeAuto <- ContainerTypeKind LBRACE ContainerMembers RBRACE
ContainerTypeKind
<- KEYWORD_struct (LPAREN Expr RPAREN)?
/ KEYWORD_opaque
/ KEYWORD_enum (LPAREN Expr RPAREN)?
/ KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)?
# Alignment
ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
BitAlign <- KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN
# Lists
IdentifierList <- (doc_comment? IDENTIFIER (COMMA IdentifierList)?)?
SwitchProngList <- (SwitchProng (COMMA SwitchProngList)?)?
AsmOutputList <- (AsmOutputItem (COMMA AsmOutputList)?)?
AsmInputList <- (AsmInputItem (COMMA AsmInputList)?)?
ParamDeclList <- LPAREN ParamDeclListRest
ParamDeclListRest
<- RPAREN
/ DOT3 COMMA? RPAREN
/ ParamDecl (COMMA ParamDeclListRest / RPAREN)
ExprList <- Expr (COMMA ExprList)? / !ExprPrefix
ExprPrefix <- ASTERISK
# *** Tokens ***
# https://en.wikipedia.org/wiki/Byte_order_mark
byte_order_mark <- '\357\273\277'
# Unfortunately, there is not a standard way to match the start of the file in PEG.
# This rule definition is compatible with the original peg(1) tool but is not
# portable. It is however trivial to implement an equivalent rule in hand written
# parsers and other PEG tooling should have similar mechanisms.
sof <- &{ (yy->__pos == 0) } byte_order_mark?
eof <- !.
ox80_oxBF <- [\200-\277]
oxF4 <- '\364'
ox80_ox8F <- [\200-\217]
oxF1_oxF3 <- [\361-\363]
oxF0 <- '\360'
ox90_0xBF <- [\220-\277]
oxEE_oxEF <- [\356-\357]
oxED <- '\355'
ox80_ox9F <- [\200-\237]
oxE1_oxEC <- [\341-\354]
oxE0 <- '\340'
oxA0_oxBF <- [\240-\277]
oxC2_oxDF <- [\302-\337]
# From https://lemire.me/blog/2018/05/09/how-quickly-can-you-check-that-a-string-is-valid-unicode-utf-8/
# First Byte Second Byte Third Byte Fourth Byte
# [0x00,0x7F]
# [0xC2,0xDF] [0x80,0xBF]
# 0xE0 [0xA0,0xBF] [0x80,0xBF]
# [0xE1,0xEC] [0x80,0xBF] [0x80,0xBF]
# 0xED [0x80,0x9F] [0x80,0xBF]
# [0xEE,0xEF] [0x80,0xBF] [0x80,0xBF]
# 0xF0 [0x90,0xBF] [0x80,0xBF] [0x80,0xBF]
# [0xF1,0xF3] [0x80,0xBF] [0x80,0xBF] [0x80,0xBF]
# 0xF4 [0x80,0x8F] [0x80,0xBF] [0x80,0xBF]
multibyte_utf8 <-
oxF4 ox80_ox8F ox80_oxBF ox80_oxBF
/ oxF1_oxF3 ox80_oxBF ox80_oxBF ox80_oxBF
/ oxF0 ox90_0xBF ox80_oxBF ox80_oxBF
/ oxEE_oxEF ox80_oxBF ox80_oxBF
/ oxED ox80_ox9F ox80_oxBF
/ oxE1_oxEC ox80_oxBF ox80_oxBF
/ oxE0 oxA0_oxBF ox80_oxBF
/ oxC2_oxDF ox80_oxBF
# Exclude \177 which is DEL
non_control_ascii <- [\040-\176]
non_control_utf8 <- [\040-\176\200-\377]
# XXX: the Zig tokenizer doesn't yet perform UTF-8 validation
# When the tokenizer is fixed, switch back to these definitions
# that forbid invalid UTF-8:
#
# char_char
# <- multibyte_utf8
# / "\\\\"
# / "\\'"
# / !['] non_control_ascii
# string_char
# <- multibyte_utf8
# / '\\\\'
# / '\\"'
# / !["] non_control_ascii
char_char
<- "\\\\"
/ "\\'"
/ !['] non_control_utf8
string_char
<- '\\\\'
/ '\\"'
/ !["] non_control_utf8
container_doc_comment <- (skip '//!' non_control_utf8* newline)+
# We forbid same-line doc comments to disambiguate the mapping to e.g. struct
# fields for documentation generation tooling.
doc_comment <- (sof / skip_require_newline) (skip '///' non_control_utf8* newline)+
line_comment
<- '//' ![!/] non_control_utf8* newline
/ '////' non_control_utf8* newline
line_string <- '\\\\' non_control_utf8* newline
# This uses a positive lookahead rather than consuming input to make e.g.
# the newline terminating a multiline string literal or doc comment visible
# to the pre_op_white non-terminal.
newline <- &("\n" / "\r\n" / eof)
skip <- sof? ([ \n\t\r] / line_comment)*
skip_require_newline <- [ \t\r]* ([\n] / line_comment) skip
pre_op_white <- ([ \n\t\r] / line_comment)+
post_op_white <- [ \n\t\r] skip
CHAR_LITERAL <- skip ['] char_char* [']
digit <- [_0-9A-DF-OQ-Za-df-oq-z]
digit_int <- digit / [eEpP]
digit_float <- digit / [eEpP] [-+]?
NUMBERLITERAL
<- skip [0-9] digit_int* '.' digit_float+
/ skip [0-9] digit_float*
string <- ["] string_char* ["]
STRINGLITERALSINGLE <- skip string
STRINGLITERAL
<- skip string
/ (skip line_string)+
IDENTIFIER
<- skip !keyword [A-Za-z_] [A-Za-z0-9_]*
/ skip '@' string
BUILTINIDENTIFIER <- skip '@'[A-Za-z_][A-Za-z0-9_]*
AMPERSAND <- skip '&' ![=]
AMPERSANDEQUAL <- skip '&='
ASTERISK <- skip '*' ![%=|]
ASTERISKEQUAL <- skip '*='
ASTERISKPERCENT <- skip '*%' ![=]
ASTERISKPERCENTEQUAL <- skip '*%='
ASTERISKPIPE <- skip '*|' ![=]
ASTERISKPIPEEQUAL <- skip '*|='
CARET <- skip '^' ![=]
CARETEQUAL <- skip '^='
COLON <- skip ':'
COMMA <- skip ','
DOT <- skip '.' ![*.]
DOT2 <- skip '..' ![.]
DOT3 <- skip '...'
DOTASTERISK <- skip '.*'
EQUAL <- skip '=' ![>=]
EQUALEQUAL <- skip '=='
EQUALRARROW <- skip '=>'
EXCLAMATIONMARK <- skip '!' ![=]
EXCLAMATIONMARKEQUAL <- skip '!='
LARROW <- skip '<' ![<=]
LARROW2 <- skip '<<' ![=|]
LARROW2EQUAL <- skip '<<='
LARROW2PIPE <- skip '<<|' ![=]
LARROW2PIPEEQUAL <- skip '<<|='
LARROWEQUAL <- skip '<='
LBRACE <- skip '{'
LBRACKET <- skip '['
LPAREN <- skip '('
MINUS <- skip '-' ![%=>|]
MINUSEQUAL <- skip '-='
MINUSPERCENT <- skip '-%' ![=]
MINUSPERCENTEQUAL <- skip '-%='
MINUSPIPE <- skip '-|' ![=]
MINUSPIPEEQUAL <- skip '-|='
MINUSRARROW <- skip '->'
PERCENT <- skip '%' ![=]
PERCENTEQUAL <- skip '%='
PIPE <- skip '|' ![|=]
PIPE2 <- skip '||'
PIPEEQUAL <- skip '|='
PLUS <- skip '+' ![%+=|]
PLUS2 <- skip '++'
PLUSEQUAL <- skip '+='
PLUSPERCENT <- skip '+%' ![=]
PLUSPERCENTEQUAL <- skip '+%='
PLUSPIPE <- skip '+|' ![=]
PLUSPIPEEQUAL <- skip '+|='
LETTERC <- skip 'c'
QUESTIONMARK <- skip '?'
RARROW <- skip '>' ![>=]
RARROW2 <- skip '>>' ![=]
RARROW2EQUAL <- skip '>>='
RARROWEQUAL <- skip '>='
RBRACE <- skip '}'
RBRACKET <- skip ']'
RPAREN <- skip ')'
SEMICOLON <- skip ';'
SLASH <- skip '/' ![=/]
SLASHEQUAL <- skip '/='
TILDE <- skip '~'
end_of_word <- ![a-zA-Z0-9_]
KEYWORD_addrspace <- skip 'addrspace' end_of_word
KEYWORD_align <- skip 'align' end_of_word
KEYWORD_allowzero <- skip 'allowzero' end_of_word
KEYWORD_and <- skip 'and' end_of_word
KEYWORD_anyframe <- skip 'anyframe' end_of_word
KEYWORD_anytype <- skip 'anytype' end_of_word
KEYWORD_asm <- skip 'asm' end_of_word
KEYWORD_break <- skip 'break' end_of_word
KEYWORD_callconv <- skip 'callconv' end_of_word
KEYWORD_catch <- skip 'catch' end_of_word
KEYWORD_comptime <- skip 'comptime' end_of_word
KEYWORD_const <- skip 'const' end_of_word
KEYWORD_continue <- skip 'continue' end_of_word
KEYWORD_defer <- skip 'defer' end_of_word
KEYWORD_else <- skip 'else' end_of_word
KEYWORD_enum <- skip 'enum' end_of_word
KEYWORD_errdefer <- skip 'errdefer' end_of_word
KEYWORD_error <- skip 'error' end_of_word
KEYWORD_export <- skip 'export' end_of_word
KEYWORD_extern <- skip 'extern' end_of_word
KEYWORD_fn <- skip 'fn' end_of_word
KEYWORD_for <- skip 'for' end_of_word
KEYWORD_if <- skip 'if' end_of_word
KEYWORD_inline <- skip 'inline' end_of_word
KEYWORD_noalias <- skip 'noalias' end_of_word
KEYWORD_nosuspend <- skip 'nosuspend' end_of_word
KEYWORD_noinline <- skip 'noinline' end_of_word
KEYWORD_opaque <- skip 'opaque' end_of_word
KEYWORD_or <- skip 'or' end_of_word
KEYWORD_orelse <- skip 'orelse' end_of_word
KEYWORD_packed <- skip 'packed' end_of_word
KEYWORD_pub <- skip 'pub' end_of_word
KEYWORD_resume <- skip 'resume' end_of_word
KEYWORD_return <- skip 'return' end_of_word
KEYWORD_linksection <- skip 'linksection' end_of_word
KEYWORD_struct <- skip 'struct' end_of_word
KEYWORD_suspend <- skip 'suspend' end_of_word
KEYWORD_switch <- skip 'switch' end_of_word
KEYWORD_test <- skip 'test' end_of_word
KEYWORD_threadlocal <- skip 'threadlocal' end_of_word
KEYWORD_try <- skip 'try' end_of_word
KEYWORD_union <- skip 'union' end_of_word
KEYWORD_unreachable <- skip 'unreachable' end_of_word
KEYWORD_var <- skip 'var' end_of_word
KEYWORD_volatile <- skip 'volatile' end_of_word
KEYWORD_while <- skip 'while' end_of_word
keyword <- KEYWORD_addrspace / KEYWORD_align / KEYWORD_allowzero / KEYWORD_and
/ KEYWORD_anyframe / KEYWORD_anytype / KEYWORD_asm
/ KEYWORD_break / KEYWORD_callconv / KEYWORD_catch
/ KEYWORD_comptime / KEYWORD_const / KEYWORD_continue / KEYWORD_defer
/ KEYWORD_else / KEYWORD_enum / KEYWORD_errdefer / KEYWORD_error / KEYWORD_export
/ KEYWORD_extern / KEYWORD_fn / KEYWORD_for / KEYWORD_if
/ KEYWORD_inline / KEYWORD_noalias / KEYWORD_nosuspend / KEYWORD_noinline
/ KEYWORD_opaque / KEYWORD_or / KEYWORD_orelse / KEYWORD_packed
/ KEYWORD_pub / KEYWORD_resume / KEYWORD_return / KEYWORD_linksection
/ KEYWORD_struct / KEYWORD_suspend / KEYWORD_switch / KEYWORD_test
/ KEYWORD_threadlocal / KEYWORD_try / KEYWORD_union / KEYWORD_unreachable
/ KEYWORD_var / KEYWORD_volatile / KEYWORD_while
lib/compiler/Maker.zig
+1 -1
View File
@@ -3536,7 +3536,7 @@ fn loadManifest(
else => |e| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }),
};
};
var ast = try std.zig.Ast.parse(gpa, manifest_bytes, .zon);
var ast = try std.zig.Ast.parse(gpa, manifest_bytes, .zon, .{});
errdefer ast.deinit(gpa);
if (ast.errors.len > 0) {
lib/compiler/Maker/Package/Manifest.zig
+4 -4
View File
@@ -597,7 +597,7 @@ pub fn load(
0,
);
ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon, .{});
if (ast.errors.len > 0) {
const file_path = try manifest_path.joinString(arena, "");
@@ -636,7 +636,7 @@ test "basic" {
\\}
;
var ast = try Ast.parse(gpa, example, .zon);
var ast = try Ast.parse(gpa, example, .zon, .{});
defer ast.deinit(gpa);
try testing.expect(ast.errors.len == 0);
@@ -682,7 +682,7 @@ test "minimum_zig_version" {
\\}
;
var ast = try Ast.parse(gpa, example, .zon);
var ast = try Ast.parse(gpa, example, .zon, .{});
defer ast.deinit(gpa);
try testing.expect(ast.errors.len == 0);
@@ -717,7 +717,7 @@ test "minimum_zig_version - invalid version" {
\\}
;
var ast = try Ast.parse(gpa, example, .zon);
var ast = try Ast.parse(gpa, example, .zon, .{});
defer ast.deinit(gpa);
try testing.expect(ast.errors.len == 0);
lib/compiler/reduce.zig
+2 -2
View File
@@ -188,7 +188,7 @@ pub fn main(init: std.process.Init) !void {
try astgen_input.writer.writeAll(rendered.written());
try astgen_input.writer.writeByte(0);
const source_with_null = astgen_input.written()[0..(astgen_input.written().len - 1) :0];
var astgen_tree = try Ast.parse(gpa, source_with_null, .zig);
var astgen_tree = try Ast.parse(gpa, source_with_null, .zig, .{});
defer astgen_tree.deinit(gpa);
if (astgen_tree.errors.len != 0) {
@panic("syntax errors occurred");
@@ -407,7 +407,7 @@ fn parse(gpa: Allocator, io: Io, file_path: []const u8) !Ast {
};
errdefer gpa.free(source_code);
var tree = try Ast.parse(gpa, source_code, .zig);
var tree = try Ast.parse(gpa, source_code, .zig, .{});
errdefer tree.deinit(gpa);
if (tree.errors.len != 0) {
lib/docs/wasm/Walk.zig
+3 -3
View File
@@ -428,7 +428,7 @@ fn parse(file_name: []const u8, source: []u8) Oom!Ast {
break :s source[0 .. source.len - 1 :0];
};
var ast = try Ast.parse(gpa, adjusted_source, .zig);
var ast = try Ast.parse(gpa, adjusted_source, .zig, .{});
if (ast.errors.len > 0) {
defer ast.deinit(gpa);
@@ -446,7 +446,7 @@ fn parse(file_name: []const u8, source: []u8) Oom!Ast {
file_name, err_loc.line + 1, err_loc.column + 1, rendered_err.written(),
});
}
return Ast.parse(gpa, "", .zig);
return Ast.parse(gpa, "", .zig, .{});
}
return ast;
}
@@ -1085,7 +1085,7 @@ pub fn isPrimitiveNonType(name: []const u8) bool {
//
// // example test command:
// // zig test --dep input.zig -Mroot=src/Walk.zig -Minput.zig=/home/andy/dev/zig/lib/std/fs/File/zig
// var ast = try Ast.parse(gpa, @embedFile("input.zig"), .zig);
// var ast = try Ast.parse(gpa, @embedFile("input.zig"), .zig, .{});
// defer ast.deinit(gpa);
//
// var w: Walk = .{
lib/std/zig.zig
+3 -2
View File
@@ -29,7 +29,6 @@ pub const primitives = @import("zig/primitives.zig");
pub const isPrimitive = primitives.isPrimitive;
pub const Ast = @import("zig/Ast.zig");
pub const AstGen = @import("zig/AstGen.zig");
pub const AstSmith = @import("zig/AstSmith.zig");
pub const Zir = @import("zig/Zir.zig");
pub const Zoir = @import("zig/Zoir.zig");
pub const ZonGen = @import("zig/ZonGen.zig");
@@ -43,6 +42,8 @@ pub const PkgConfig = @import("zig/PkgConfig.zig");
pub const target = @import("zig/target.zig");
pub const llvm = @import("zig/llvm.zig");
pub const parser_generated_oracle = @import("zig/parser_generated_oracle.zig");
// Character literal parsing
pub const ParsedCharLiteral = string_literal.ParsedCharLiteral;
pub const parseCharLiteral = string_literal.parseCharLiteral;
@@ -1863,7 +1864,6 @@ fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8
test {
_ = Ast;
_ = AstRlAnnotate;
_ = AstSmith;
_ = BuiltinFn;
_ = Client;
_ = ErrorBundle;
@@ -1879,4 +1879,5 @@ test {
_ = target;
_ = c_translation;
_ = llvm;
_ = @import("zig/parser_fuzz.zig");
}
lib/std/zig/Ast.zig
+27 -25
View File
@@ -139,10 +139,13 @@ pub fn deinit(tree: *Ast, gpa: Allocator) void {
}
pub const Mode = enum { zig, zon };
pub const ParseOptions = struct {
recover: bool = true,
};
/// Result should be freed with tree.deinit() when there are
/// no more references to any of the tokens or nodes.
pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!Ast {
pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode, options: ParseOptions) Allocator.Error!Ast {
var tokens = Ast.TokenList{};
defer tokens.deinit(gpa);
@@ -162,7 +165,7 @@ pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!A
var tokens_slice = tokens.toOwnedSlice();
errdefer tokens_slice.deinit(gpa);
return parseTokens(gpa, source, tokens_slice, mode);
return parseTokens(gpa, source, tokens_slice, mode, options);
}
pub fn parseTokens(
@@ -170,6 +173,7 @@ pub fn parseTokens(
source: [:0]const u8,
tokens: Ast.TokenList.Slice,
mode: Mode,
options: ParseOptions,
) Allocator.Error!Ast {
var parser: Parse = .{
.source = source,
@@ -180,6 +184,7 @@ pub fn parseTokens(
.extra_data = .empty,
.scratch = .empty,
.tok_i = 0,
.recover = options.recover,
};
defer parser.errors.deinit(gpa);
defer parser.nodes.deinit(gpa);
@@ -465,15 +470,6 @@ pub fn renderError(tree: Ast, parse_error: Error, w: *Writer) Writer.Error!void
.extra_align_qualifier => {
return w.writeAll("extra align qualifier");
},
.extra_allowzero_qualifier => {
return w.writeAll("extra allowzero qualifier");
},
.extra_const_qualifier => {
return w.writeAll("extra const qualifier");
},
.extra_volatile_qualifier => {
return w.writeAll("extra volatile qualifier");
},
.ptr_mod_on_array_child_type => {
return w.print("pointer modifier '{s}' not allowed on array child type", .{
tree.tokenTag(parse_error.token).symbol(),
@@ -559,12 +555,6 @@ pub fn renderError(tree: Ast, parse_error: Error, w: *Writer) Writer.Error!void
.var_const_decl => {
return w.writeAll("use 'var' or 'const' to declare variable");
},
.extra_for_capture => {
return w.writeAll("extra capture in for loop");
},
.for_input_not_captured => {
return w.writeAll("for input is not captured");
},
.invalid_byte => {
const tok_slice = tree.source[tree.tokens.items(.start)[parse_error.token]..];
@@ -2116,6 +2106,7 @@ fn fullPtrTypeComponents(tree: Ast, info: full.PtrType.Components) full.PtrType
.allowzero_token = null,
.const_token = null,
.volatile_token = null,
.duplicate_token = null,
.ast = info,
};
// We need to be careful that we don't iterate over any sub-expressions
@@ -2131,9 +2122,24 @@ fn fullPtrTypeComponents(tree: Ast, info: full.PtrType.Components) full.PtrType
const end = tree.firstToken(info.child_type);
while (i < end) : (i += 1) {
switch (tree.tokenTag(i)) {
.keyword_allowzero => result.allowzero_token = i,
.keyword_const => result.const_token = i,
.keyword_volatile => result.volatile_token = i,
.keyword_allowzero => {
if (result.allowzero_token != null) {
result.duplicate_token = i;
}
result.allowzero_token = i;
},
.keyword_const => {
if (result.const_token != null) {
result.duplicate_token = i;
}
result.const_token = i;
},
.keyword_volatile => {
if (result.volatile_token != null) {
result.duplicate_token = i;
}
result.volatile_token = i;
},
.keyword_align => {
if (info.bit_range_end.unwrap()) |bit_range_end| {
assert(info.bit_range_start != .none);
@@ -2730,6 +2736,7 @@ pub const full = struct {
allowzero_token: ?TokenIndex,
const_token: ?TokenIndex,
volatile_token: ?TokenIndex,
duplicate_token: ?TokenIndex,
ast: Components,
pub const Components = struct {
@@ -2863,9 +2870,6 @@ pub const Error = struct {
extern_fn_body,
extra_addrspace_qualifier,
extra_align_qualifier,
extra_allowzero_qualifier,
extra_const_qualifier,
extra_volatile_qualifier,
ptr_mod_on_array_child_type,
invalid_bit_range,
same_line_doc_comment,
@@ -2889,8 +2893,6 @@ pub const Error = struct {
expected_var_const,
wrong_equal_var_decl,
var_const_decl,
extra_for_capture,
for_input_not_captured,
zig_style_container,
previous_field,
lib/std/zig/AstGen.zig
+10
View File
@@ -3746,6 +3746,9 @@ fn ptrType(
if (ptr_info.size == .c and ptr_info.allowzero_token != null) {
return gz.astgen.failTok(ptr_info.allowzero_token.?, "C pointers always allow address zero", .{});
}
if (ptr_info.duplicate_token) |duplicate| {
return gz.astgen.failTok(duplicate, "Extra pointer qualifier", .{});
}
const source_offset = gz.astgen.source_offset;
const source_line = gz.astgen.source_line;
@@ -6709,6 +6712,9 @@ fn forExpr(
for (for_full.ast.inputs, indexables, lens) |input, *indexable_ref, *len_refs| {
const capture_is_ref = tree.tokenTag(capture_token) == .asterisk;
const ident_tok = capture_token + @intFromBool(capture_is_ref);
if (tree.tokenTag(ident_tok) != .identifier) {
return astgen.failNode(input, "for input is not captured", .{});
}
const is_discard = mem.eql(u8, tree.tokenSlice(ident_tok), "_");
if (is_discard and capture_is_ref) {
@@ -6751,6 +6757,10 @@ fn forExpr(
len_refs.* = .{ indexable, .none };
}
}
// There may or may not be a trailing comma after the final capture
if (tree.tokenTag(capture_token) != .pipe and tree.tokenTag(capture_token - 1) != .pipe) {
return astgen.failTok(capture_token, "extra capture in for loop", .{});
}
}
if (!any_len_checks) {
lib/std/zig/AstSmith.zig
-2595
View File
File diff suppressed because it is too large. Load diff
lib/std/zig/Parse.zig
+272 -256
View File
File diff suppressed because it is too large. Load diff
lib/std/zig/parser_fuzz.zig
+243
View File
@@ -0,0 +1,243 @@
const std = @import("../std.zig");
const Allocator = std.mem.Allocator;
const Smith = std.testing.Smith;
const oracle = @import("parser_generated_oracle.zig");
test "fuzz std.zig.Ast.parse() against generated oracle" {
try std.testing.fuzz({}, fuzzAgainstOracle, .{});
}
fn fuzzAgainstOracle(_: void, smith: *Smith) !void {
var buffer: [1 << 14]u8 = undefined;
const len = smith.slice(buffer[0 .. buffer.len - 1]);
buffer[len] = 0;
const source = buffer[0..len :0];
checkAgainstOracle(source) catch |err| switch (err) {
error.MaxDepth => return error.SkipZigTest,
else => |e| return e,
};
}
test "max depth" {
try std.testing.expectError(error.MaxDepth, checkAgainstOracle("((((("));
_ = checkAgainstOracle("((((") catch |err| switch (err) {
error.MaxDepth => try std.testing.expect(false),
else => |e| return e,
};
}
// Found using AFL++
test "operator whitespace" {
try checkAgainstOracle(
\\test {
\\ _!= 0;
\\}
);
try checkAgainstOracle(
\\test{{\\
\\*0;}}
);
}
// Found using AFL++
test "doc comment or division operator" {
try checkAgainstOracle("0=0///\n0");
}
// Found using AFL++
test "double ampersand" {
try checkAgainstOracle("0=0&&0"); // error
try checkAgainstOracle("test{&&0;}"); // ok
}
// Found using AFL++
test "newline required before doc comment not at start of file" {
try checkAgainstOracle("0,///\n0"); // error
try checkAgainstOracle("///\n0"); // ok
try checkAgainstOracle(" ///\n0"); // ok
try checkAgainstOracle("\n///\n0"); // ok
try checkAgainstOracle("///"); // error
try checkAgainstOracle("///\n//!");
try checkAgainstOracle("///\ntest {}");
try checkAgainstOracle("///\ncomptime 0");
}
// Found using AFL++
test "extra capture in for loop" {
try checkAgainstOracle("for(0)|t,r|0");
}
// Found using AFL++
test "expression nesting" {
try checkAgainstOracle("test{*comptime 0 == 0;}");
}
// Found using AFL++
test "comptime fn" {
try checkAgainstOracle("comptime fn()0");
}
// Found using AFL++
test "return asterisk" {
try checkAgainstOracle("test{return*!0;}");
}
// Found using AFL++
test "fn container field" {
try checkAgainstOracle("fn()0");
}
// Found using AFL++
test "at newline string" {
try checkAgainstOracle(
\\@
\\""
);
}
// Found using AFL++
test "string lit" {
try checkAgainstOracle(
\\"\\"
);
}
// Found using AFL++
test "dot question" {
try checkAgainstOracle("0. ?");
}
// Found using AFL++
test "volatile const" {
try checkAgainstOracle("*volatile\nconst\n0");
}
// Found using AFL++
test "addrspace align" {
try checkAgainstOracle("*addrspace(0) align(0) 0");
try checkAgainstOracle("*align(0) addrspace(0) 0");
try checkAgainstOracle("[*]addrspace(0) align(0) 0");
try checkAgainstOracle("[*]align(0) addrspace(0) 0");
}
// Found using AFL++
test "catch capture whitespace" {
try checkAgainstOracle("test{0 catch |h|0;}");
}
// Found using AFL++
test "byte order mark" {
// https://en.wikipedia.org/wiki/Byte_order_mark
try checkAgainstOracle("\xef\xbb\xbf");
try checkAgainstOracle("\xef\xbb\xbf///\n0");
}
// Found using AFL++
test "nosuspend multi assign" {
try checkAgainstOracle("test{nosuspend*0,var _=0;}");
}
// Found using AFL++
test "bin op at end of file" {
try checkAgainstOracle(
\\test {
\\ _ = 00
\\|
);
}
// Found using AFL++
test "resume block chained compare ops" {
try checkAgainstOracle("test{resume{0 > 0;} > 0 > 0;}");
}
// Found using AFL++
test "for multiassign" {
try checkAgainstOracle("test{for(0)|t|0,const w=0;}");
}
fn checkAgainstOracle(source: [:0]const u8) !void {
var fba_buf: [1 << 18]u8 = undefined;
var fba: std.heap.FixedBufferAllocator = .init(&fba_buf);
const expected = try oracle.parse(source);
// It is important to disable recovery for fuzz testing.
// Consider the case where there is a parse error right at the beginning of the file,
// followed by a valid declaration with a million nested parens. The oracle will not
// skip this input due to the max depth being exceeded since the oracle hits a parse
// error right away and does no recovery. However, std.zig.Ast.parse() does recovery
// by default and will hit a stack overflow rather than returning after the parser error.
// Stack overflows are not interesting and we do not want the fuzzer to be able to find them.
const ast = try std.zig.Ast.parse(fba.allocator(), source, .zig, .{ .recover = false });
errdefer logBadSource(source, ast);
try std.testing.expectEqual(expected, ast.errors.len == 0);
}
fn logBadSource(source: []const u8, ast: std.zig.Ast) void {
@disableInstrumentation();
var buf: [256]u8 = undefined;
const ls = std.debug.lockStderr(&buf);
defer std.debug.unlockStderr();
logBadSourceInner(source, ls.terminal(), ast) catch {};
}
fn logBadSourceInner(source: []const u8, t: std.Io.Terminal, ast: std.zig.Ast) std.Io.Writer.Error!void {
@disableInstrumentation();
try logSourceInner(source, t);
const w = t.writer;
try w.writeAll("=== Parse Errors ===\n");
for (ast.errors) |err| {
const loc = ast.tokenLocation(0, err.token);
try w.print("{}:{}: ", .{ loc.line + 1, loc.column + 1 });
try ast.renderError(err, w);
try w.writeByte('\n');
}
}
pub fn logSource(source: []const u8) void {
@disableInstrumentation();
var buf: [256]u8 = undefined;
const ls = std.debug.lockStderr(&buf);
defer std.debug.unlockStderr();
logSourceInner(source, ls.terminal()) catch {};
}
fn logSourceInner(source: []const u8, t: std.Io.Terminal) std.Io.Writer.Error!void {
@disableInstrumentation();
const w = t.writer;
t.setColor(.dim) catch {};
try w.writeAll("=== Source ===\n");
t.setColor(.reset) catch {};
var line: usize = 1;
try w.print("{: >5} ", .{line});
for (source) |c| switch (c) {
' '...0x7e => try w.writeByte(c),
'\n' => {
line += 1;
try w.print("\n{: >5} ", .{line});
},
'\r' => {
t.setColor(.cyan) catch {};
try w.writeAll("\\r");
t.setColor(.reset) catch {};
},
'\t' => {
t.setColor(.cyan) catch {};
try w.writeAll("\\t");
t.setColor(.reset) catch {};
},
else => {
t.setColor(.cyan) catch {};
try w.print("\\x{x:0>2}", .{c});
t.setColor(.reset) catch {};
},
};
try w.writeByte('\n');
}
lib/std/zig/parser_generated_oracle.zig
+6552
View File
File diff suppressed because it is too large. Load diff
lib/std/zig/parser_test.zig
+7 -269
View File
@@ -6976,6 +6976,7 @@ test "recovery: missing comma" {
, &[_]Error{
.expected_comma_after_switch_prong,
.expected_comma_after_switch_prong,
.mismatched_binary_op_whitespace,
.expected_expr,
});
}
@@ -6992,10 +6993,10 @@ test "recovery: non-associative operators" {
test "recovery: extra qualifier" {
try testError(
\\const a: *const const u8;
\\const a: *align(4) align(8) u8;
\\test ""
, &[_]Error{
.extra_const_qualifier,
.extra_align_qualifier,
.expected_block,
});
}
@@ -7230,7 +7231,7 @@ test "ampersand" {
test "Ast: pointer types with subexprs containing qualifiers" {
var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
const allocator = fixed_allocator.allocator();
var tree = try std.zig.Ast.parse(allocator, "**addrspace(*align(1)T)T", .zon);
var tree = try std.zig.Ast.parse(allocator, "**addrspace(*align(1)T)T", .zon, .{});
defer tree.deinit(allocator);
const regular_ptr_node = tree.nodeData(.root).node;
@@ -7252,7 +7253,7 @@ fn testParse(io: Io, source: [:0]const u8, allocator: Allocator, anything_change
defer io.unlockStderr();
const writer = &stderr.file_writer.interface;
var tree = try std.zig.Ast.parse(allocator, source, .zig);
var tree = try std.zig.Ast.parse(allocator, source, .zig, .{});
defer tree.deinit(allocator);
for (tree.errors) |parse_error| {
@@ -7312,7 +7313,7 @@ fn testCanonical(source: [:0]const u8) !void {
const Error = std.zig.Ast.Error.Tag;
fn testError(source: [:0]const u8, expected_errors: []const Error) !void {
var tree = try std.zig.Ast.parse(std.testing.allocator, source, .zig);
var tree = try std.zig.Ast.parse(std.testing.allocator, source, .zig, .{});
defer tree.deinit(std.testing.allocator);
std.testing.expectEqual(expected_errors.len, tree.errors.len) catch |err| {
@@ -7332,268 +7333,5 @@ fn fuzzTestOneParse(_: void, smith: *std.testing.Smith) !void {
const mode = smith.value(std.zig.Ast.Mode);
var tokens: std.zig.TokenSmith = .gen(smith);
var fba: std.heap.FixedBufferAllocator = .init(&fixed_buffer_mem);
_ = std.zig.Ast.parseTokens(fba.allocator(), tokens.source(), tokens.list(), mode) catch return;
}
test "zig fmt: fuzz" {
try std.testing.fuzz({}, fuzzRender, .{});
}
fn isRewritable(source: []const u8, tokens: std.zig.Ast.TokenList.Slice) !bool {
@disableInstrumentation();
// Byte-order marker is stripped
var maybe_rewritable = std.mem.startsWith(u8, source, "\xEF\xBB\xBF");
// The above variable can not yet be replaced by returns since error.SkipZigTest still needs to
// be checked for.
for (0.., tokens.items(.tag), tokens.items(.start)) |i, tag, start| switch (tag) {
// Extra colons can be removed
.keyword_asm,
// Qualifiers can be reordered
// keyword_const is intentionally excluded since it is used in other contexts and
// having only one qualifier will never lead to reordering.
.keyword_addrspace,
.keyword_align,
.keyword_allowzero,
.keyword_callconv,
.keyword_linksection,
.keyword_volatile,
=> maybe_rewritable = true,
.builtin,
// Pointer casts can be reordered
=> for ([_][]const u8{
"ptrCast",
"alignCast",
"addrSpaceCast",
"constCast",
"volatileCast",
}) |id| {
if (std.mem.startsWith(u8, source[start + 1 ..], id)) {
maybe_rewritable = true;
}
},
// Quoted identifiers can be unquoted
.identifier => if (source[start] == '@') {
maybe_rewritable = true;
},
else => {},
// #23754
.container_doc_comment,
=> if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{.l_brace})) {
return error.SkipZigTest; // Can cause I.B.
},
// #24507
.keyword_inline,
.keyword_for,
.keyword_while,
.l_brace,
=> if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{ .identifier, .colon })) {
return error.SkipZigTest; // Can cause I.B. due to double rendering of zig fmt on/off
},
};
return maybe_rewritable;
}
/// Checks equivelence of non-whitespace characters.
/// If there are commas in `source`, then it is checked they are also present
/// in `rendered`. Extra commas in `rendered` are ignored.
fn isRewritten(source: [:0]const u8, rendered: [:0]const u8) bool {
@disableInstrumentation();
var i: usize = 0;
for (source[0 .. source.len + 1]) |c| switch (c) {
' ', '\r', '\t', '\n' => {},
else => while (true) {
defer i += 1;
switch (rendered[i]) {
' ', '\n' => {},
',' => if (c == ',') break,
else => |r| if (c != r) return false else break,
}
},
};
std.debug.assert(i >= rendered.len);
return false;
}
/// Checks that no line ends in whitespace
fn checkBetweenTokens(src: []const u8, fmt_on: *bool) error{
TrailingLineWhitespace,
DoubleEmptyLine,
}!void {
@disableInstrumentation();
var pos: usize = 0;
while (true) {
const nl_pos = std.mem.indexOfScalarPos(u8, src, pos, '\n');
var check_trailing = fmt_on.*;
const line = src[pos .. nl_pos orelse src.len];
if (std.mem.indexOfScalar(u8, line, '/')) |comment_start| {
const comment_content = line[comment_start..][2..];
const trimmed_comment = std.mem.trim(u8, comment_content, &std.ascii.whitespace);
if (std.mem.eql(u8, trimmed_comment, "zig fmt: off")) {
fmt_on.* = false;
} else if (std.mem.eql(u8, trimmed_comment, "zig fmt: on")) {
fmt_on.* = true;
check_trailing = true;
}
}
pos = nl_pos orelse break;
if (check_trailing and pos != 0) switch (src[pos - 1]) {
' ', '\t', '\r' => return error.TrailingLineWhitespace,
'\n' => if (pos != 1 and src[pos - 2] == '\n') return error.DoubleEmptyLine,
else => {},
};
pos += 1;
}
}
/// Ignores extre `.comma` tokens in `rendered`
fn reparseTokens(
fba: Allocator,
rendered: [:0]const u8,
expected_tags: [:.eof]const Token.Tag,
) error{
OutOfMemory,
SameLineMultilineStringLiteral,
TrailingLineWhitespace,
DoubleEmptyLine,
}!struct {
toks: std.zig.Ast.TokenList,
rewritten: bool,
} {
@disableInstrumentation();
var rewritten = false;
var tokens: std.zig.Ast.TokenList = .{};
var last_token_end: usize = 0;
var fmt_on = true;
try tokens.ensureTotalCapacity(fba, expected_tags.len + 2); // 1 for EOF and 1 for maybe a comma
var tokenizer: std.zig.Tokenizer = .init(rendered);
var i: usize = 0;
while (true) {
const tok = tokenizer.next();
try tokens.append(fba, .{
.tag = tok.tag,
.start = @intCast(tok.loc.start),
});
const between = rendered[last_token_end..tok.loc.start];
last_token_end = tok.loc.end;
try checkBetweenTokens(between, &fmt_on);
if (tok.tag == .multiline_string_literal_line and fmt_on) blk: {
if (tokens.len == 1)
break :blk; // first token
if (std.mem.indexOfScalar(u8, between, '\n') == null)
return error.SameLineMultilineStringLiteral;
}
if (tok.tag == expected_tags[i]) {
if (tok.tag == .eof)
break;
i += 1;
} else if (tok.tag != .comma or !fmt_on) {
rewritten = true;
}
}
std.debug.assert(i == expected_tags.len);
try checkBetweenTokens(rendered[last_token_end..], &fmt_on);
return .{ .toks = tokens, .rewritten = rewritten };
}
fn fuzzRender(_: void, smith: *std.testing.Smith) !void {
@disableInstrumentation();
var ast_smith: std.zig.AstSmith = .init(smith);
try ast_smith.generateSource();
var fba_ctx = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem);
var opt_rendered: ?[]const u8 = null;
fuzzRenderInner(&ast_smith, fba_ctx.allocator(), &opt_rendered) catch |e| switch (e) {
error.SkipZigTest, error.OutOfMemory, error.WriteFailed => return error.SkipZigTest,
else => |failure| {
ast_smith.logSource();
if (opt_rendered) |rendered| {
logRenderedSource(rendered);
}
return failure;
},
};
}
fn fuzzRenderInner(ast_smith: *std.zig.AstSmith, fba: Allocator, opt_rendered: *?[]const u8) !void {
@disableInstrumentation();
const source = ast_smith.source();
const src_rewritable = try isRewritable(source, ast_smith.tokens());
const src_tree = try std.zig.Ast.parseTokens(fba, source, ast_smith.tokens(), .zig);
std.debug.assert(src_tree.errors.len == 0);
for (src_tree.nodes.items(.tag)) |tag| switch (tag) {
// #24507 (`switch(x) { inline for (a) |a| a => {} }` to
// `switch(x) { { inline for (a) |a| a => {} }` since
// AST determines inline case token as one before the case expression's first)
.switch_case_inline, .switch_case_inline_one => return error.SkipZigTest,
else => {},
};
var rendered_w: std.Io.Writer.Allocating = .init(fba);
try rendered_w.ensureUnusedCapacity(source.len + source.len / 2);
try src_tree.render(fba, &rendered_w.writer, .{});
// `toOwnedSliceSentinel` is not used since it reallocates the entire
// list to save space which is useless for fixed buffer allocators.
try rendered_w.writer.writeByte(0);
const rendered = rendered_w.written()[0 .. rendered_w.written().len - 1 :0];
opt_rendered.* = rendered;
// First check that the non-whitespace characters match. This ensures that
// identifier names, numbers, comments, et cetera are preserved.
if (!src_rewritable and isRewritten(source, rendered))
return error.Rewritten;
// Next check that the tokens are the same since whitespace removal can change the tokens
const src_tags = ast_smith.tokens().items(.tag);
const rendered_toks = try reparseTokens(fba, rendered, src_tags[0 .. src_tags.len - 1 :.eof]);
if (!src_rewritable and rendered_toks.rewritten)
return error.Rewritten;
// Rerender the tree to check idempotency and that new commas
// and whitespace changes did not create an AST error.
const rendered_tree = try std.zig.Ast.parseTokens(fba, rendered, rendered_toks.toks.slice(), .zig);
if (rendered_tree.errors.len != 0)
return error.Rewritten;
var rerendered_w: std.Io.Writer.Allocating = .init(fba);
try rerendered_w.ensureUnusedCapacity(source.len);
try rendered_tree.render(fba, &rerendered_w.writer, .{});
try std.testing.expectEqualStrings(rendered, rerendered_w.written());
}
fn logRenderedSource(source: []const u8) void {
var buf: [256]u8 = undefined;
const ls = std.debug.lockStderr(&buf);
defer std.debug.unlockStderr();
logRenderedSourceInner(source, ls.terminal()) catch {};
}
fn logRenderedSourceInner(source: []const u8, t: std.Io.Terminal) std.Io.Writer.Error!void {
const w = t.writer;
t.setColor(.dim) catch {};
try w.writeAll("=== Rendered Source ===\n");
t.setColor(.reset) catch {};
for (0.., source) |i, c| switch (c) {
' '...0x7e => try w.writeByte(c),
'\n' => {
if (i != 0 and source[i - 1] == ' ') {
try w.writeAll("⏎");
}
try w.writeByte('\n');
},
else => {
t.setColor(.cyan) catch {};
try w.print("\\x{x:0>2}", .{c});
t.setColor(.reset) catch {};
},
};
try w.writeAll("␃\n");
_ = std.zig.Ast.parseTokens(fba.allocator(), tokens.source(), tokens.list(), mode, .{ .recover = false }) catch return;
}
lib/std/zig/perf_test.zig
+1 -1
View File
@@ -32,6 +32,6 @@ pub fn main() !void {
fn testOnce() usize {
var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem);
const allocator = fixed_buf_alloc.allocator();
_ = std.zig.Ast.parse(allocator, source, .zig) catch @panic("parse failure");
_ = std.zig.Ast.parse(allocator, source, .zig, .{}) catch @panic("parse failure");
return fixed_buf_alloc.end_index;
}
lib/std/zon/parse.zig
+3 -3
View File
@@ -294,7 +294,7 @@ pub fn fromSliceAlloc(
) error{ OutOfMemory, ParseZon }!T {
if (diag) |s| s.assertEmpty();
var ast = try std.zig.Ast.parse(gpa, source, .zon);
var ast = try std.zig.Ast.parse(gpa, source, .zon, .{});
defer if (diag == null) ast.deinit(gpa);
if (diag) |s| s.ast = ast;
@@ -2140,7 +2140,7 @@ test "std.zon string literal" {
// Passing string literal to a array
{
{
var ast = try std.zig.Ast.parse(gpa, "\"abcd\"", .zon);
var ast = try std.zig.Ast.parse(gpa, "\"abcd\"", .zon, .{});
defer ast.deinit(gpa);
var zoir = try ZonGen.generate(gpa, ast, .{ .parse_str_lits = false });
defer zoir.deinit(gpa);
@@ -3534,7 +3534,7 @@ test "std.zon no alloc" {
const Nested = struct { u8, u8, struct { u8, u8 } };
var ast = try std.zig.Ast.parse(gpa, ".{ 1, 2, .{ 3, 4 } }", .zon);
var ast = try std.zig.Ast.parse(gpa, ".{ 1, 2, .{ 3, 4 } }", .zon, .{});
defer ast.deinit(gpa);
var zoir = try ZonGen.generate(gpa, ast, .{ .parse_str_lits = false });
src/Builtin.zig
+1 -1
View File
@@ -296,7 +296,7 @@ pub fn populateFile(opts: @This(), gpa: Allocator, file: *File) Allocator.Error!
log.debug("parsing and generating 'builtin.zig'", .{});
file.tree = try std.zig.Ast.parse(gpa, file.source.?, .zig);
file.tree = try std.zig.Ast.parse(gpa, file.source.?, .zig, .{});
assert(file.tree.?.errors.len == 0); // builtin.zig must parse
file.zir = try AstGen.generate(gpa, file.tree.?);
src/Zcu.zig
+1 -1
View File
@@ -1145,7 +1145,7 @@ pub const File = struct {
if (file.tree) |*tree| return tree;
const source = try file.getSource(zcu);
file.tree = try .parse(zcu.gpa, source, file.getMode());
file.tree = try .parse(zcu.gpa, source, file.getMode(), .{});
return &file.tree.?;
}
src/Zcu/PerThread.zig
+1 -1
View File
@@ -642,7 +642,7 @@ pub fn updateFile(
var timer = comp.startTimer();
// Any potential AST errors are converted to ZIR errors when we run AstGen/ZonGen.
file.tree = try Ast.parse(gpa, source, file.getMode());
file.tree = try Ast.parse(gpa, source, file.getMode(), .{});
if (timer.finish(io)) |ns_parse| {
comp.mutex.lockUncancelable(io);
defer comp.mutex.unlock(io);
src/fmt.zig
+2 -2
View File
@@ -117,7 +117,7 @@ pub fn run(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8) !
};
defer gpa.free(source_code);
var tree = std.zig.Ast.parse(gpa, source_code, if (force_zon) .zon else .zig) catch |err| {
var tree = std.zig.Ast.parse(gpa, source_code, if (force_zon) .zon else .zig, .{}) catch |err| {
fatal("error parsing stdin: {}", .{err});
};
defer tree.deinit(gpa);
@@ -312,7 +312,7 @@ fn fmtPathFile(
break :mode .zig;
};
var tree = try std.zig.Ast.parse(gpa, source_code, mode);
var tree = try std.zig.Ast.parse(gpa, source_code, mode, .{});
defer tree.deinit(gpa);
if (tree.errors.len != 0) {
src/main.zig
+3 -3
View File
@@ -5528,7 +5528,7 @@ fn cmdAstCheck(arena: Allocator, io: Io, args: []const []const u8, environ_map:
break :mode .zig;
};
const tree = try Ast.parse(arena, source, mode);
const tree = try Ast.parse(arena, source, mode, .{});
var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer);
const stdout_bw = &stdout_writer.interface;
@@ -5698,7 +5698,7 @@ fn cmdChangelist(arena: Allocator, io: Io, args: []const []const u8, environ_map
fatal("unable to read new source file {q}: {t}", .{ new_source_path, err });
};
const old_tree = try Ast.parse(arena, old_source, .zig);
const old_tree = try Ast.parse(arena, old_source, .zig, .{});
const old_zir = try AstGen.generate(arena, old_tree);
if (old_zir.loweringFailed()) {
@@ -5710,7 +5710,7 @@ fn cmdChangelist(arena: Allocator, io: Io, args: []const []const u8, environ_map
process.exit(1);
}
const new_tree = try Ast.parse(arena, new_source, .zig);
const new_tree = try Ast.parse(arena, new_source, .zig, .{});
const new_zir = try AstGen.generate(arena, new_tree);
if (new_zir.loweringFailed()) {
test/standalone/build.zig
+1
View File
@@ -36,6 +36,7 @@ pub fn build(b: *std.Build) void {
"../../tools/fetch_them_macos_headers.zig",
"../../tools/gen_macos_headers_c.zig",
"../../tools/gen_outline_atomics.zig",
"../../tools/gen_parser_oracle.zig",
"../../tools/gen_spirv_spec.zig",
"../../tools/gen_stubs.zig",
"../../tools/generate_c_size_and_align_checks.zig",
tools/docgen.zig
+28 -4
View File
@@ -25,6 +25,7 @@ const usage =
\\
\\Options:
\\ --code-dir dir Path to directory containing code example outputs
\\ --grammar file Path to the PEG grammar definition
\\ -h, --help Print this help and exit
\\
;
@@ -37,6 +38,7 @@ pub fn main(init: std.process.Init) !void {
if (!args_it.skip()) @panic("expected self arg");
var opt_code_dir: ?[]const u8 = null;
var opt_grammar: ?[]const u8 = null;
var opt_input: ?[]const u8 = null;
var opt_output: ?[]const u8 = null;
@@ -51,6 +53,12 @@ pub fn main(init: std.process.Init) !void {
} else {
fatal("expected parameter after --code-dir", .{});
}
} else if (mem.eql(u8, arg, "--grammar")) {
if (args_it.next()) |param| {
opt_grammar = param;
} else {
fatal("expected parameter after --grammar", .{});
}
} else {
fatal("unrecognized option: '{s}'", .{arg});
}
@@ -65,6 +73,7 @@ pub fn main(init: std.process.Init) !void {
const input_path = opt_input orelse fatal("missing input file", .{});
const output_path = opt_output orelse fatal("missing output file", .{});
const code_dir_path = opt_code_dir orelse fatal("missing --code-dir argument", .{});
const grammar_path = opt_grammar orelse fatal("missing --grammar argument", .{});
var in_file = try Dir.cwd().openFile(io, input_path, .{});
defer in_file.close(io);
@@ -82,13 +91,16 @@ pub fn main(init: std.process.Init) !void {
};
defer code_dir.root_dir.handle.close(io);
const grammar = try Dir.cwd().readFileAlloc(io, grammar_path, init.gpa, .limited(max_doc_file_size));
defer init.gpa.free(grammar);
var in_file_reader = in_file.reader(io, &.{});
const input_file_bytes = try in_file_reader.interface.allocRemaining(arena, .limited(max_doc_file_size));
var tokenizer = Tokenizer.init(input_path, input_file_bytes);
var toc = try genToc(arena, &tokenizer);
try genHtml(arena, io, &tokenizer, &toc, code_dir, &out_file_writer.interface);
try genHtml(arena, io, &tokenizer, &toc, code_dir, grammar, &out_file_writer.interface);
try out_file_writer.end();
}
@@ -326,6 +338,7 @@ const Node = union(enum) {
HeaderOpen: HeaderOpen,
SeeAlso: []const SeeAlsoItem,
Code: Code,
Grammar,
Link: Link,
InlineSyntax: Token,
Shell: Token,
@@ -513,6 +526,9 @@ fn genToc(gpa: Allocator, tokenizer: *Tokenizer) !Toc {
.token = name_tok,
},
});
} else if (mem.eql(u8, tag_name, "grammar")) {
_ = try eatToken(tokenizer, .bracket_close);
try nodes.append(.Grammar);
} else if (mem.eql(u8, tag_name, "syntax")) {
_ = try eatToken(tokenizer, .bracket_close);
const content_tok = try eatToken(tokenizer, .content);
@@ -924,14 +940,14 @@ fn tokenizeAndPrint(
return tokenizeAndPrintRaw(allocator, docgen_tokenizer, out, source_token, raw_src);
}
fn printSourceBlock(allocator: Allocator, docgen_tokenizer: *Tokenizer, out: *Writer, syntax_block: SyntaxBlock) !void {
fn printSourceBlock(allocator: Allocator, docgen_tokenizer: *Tokenizer, out: *Writer, syntax_block: SyntaxBlock, content: ?[]const u8) !void {
const source_type = @tagName(syntax_block.source_type);
try out.print("<figure><figcaption class=\"{s}-cap\"><cite class=\"file\">{s}</cite></figcaption><pre>", .{ source_type, syntax_block.name });
switch (syntax_block.source_type) {
.zig => try tokenizeAndPrint(allocator, docgen_tokenizer, out, syntax_block.source_token),
else => {
const raw_source = docgen_tokenizer.buffer[syntax_block.source_token.start..syntax_block.source_token.end];
const raw_source = content orelse docgen_tokenizer.buffer[syntax_block.source_token.start..syntax_block.source_token.end];
const trimmed_raw_source = mem.trim(u8, raw_source, " \r\n");
try out.writeAll("<code>");
@@ -995,6 +1011,7 @@ fn genHtml(
tokenizer: *Tokenizer,
toc: *Toc,
code_dir: Path,
grammar: []const u8,
out: *Writer,
) !void {
for (toc.nodes) |node| {
@@ -1042,7 +1059,7 @@ fn genHtml(
try printShell(out, raw_shell_content, true);
},
.SyntaxBlock => |syntax_block| {
try printSourceBlock(allocator, tokenizer, out, syntax_block);
try printSourceBlock(allocator, tokenizer, out, syntax_block, null);
},
.Code => |code| {
const out_basename = try std.fmt.allocPrint(allocator, "{s}.out", .{
@@ -1062,6 +1079,13 @@ fn genHtml(
try out.writeAll(contents);
},
.Grammar => {
try printSourceBlock(allocator, tokenizer, out, .{
.source_type = .peg,
.name = "grammar.peg",
.source_token = undefined,
}, grammar);
},
}
}
}
tools/gen_parser_oracle.zig
+789
View File
@@ -0,0 +1,789 @@
//! Example usage:
//! zig build gen-parser-oracle
//! zig build check-parser-oracle
// This program implements a subset of the PEG grammar definition
// in the peg(1) man page.
//
// It generates a recursive descent parser that returns true if a given input is
// matched by the grammar. This generated parser is used as an oracle for fuzz testing.
const std = @import("std");
const assert = std.debug.assert;
const Io = std.Io;
const mem = std.mem;
const Allocator = mem.Allocator;
const log = std.log;
pub fn main(init: std.process.Init) !void {
const gpa = init.gpa;
const arena = init.arena.allocator();
const io = init.io;
const args = try init.minimal.args.toSlice(arena);
const grammar_path = args[1];
const out_path = args[2];
const check = args.len > 3 and mem.eql(u8, args[3], "--check");
const grammar = try Io.Dir.cwd().readFileAlloc(io, grammar_path, gpa, .unlimited);
defer gpa.free(grammar);
var parser: Parser = .init(gpa, grammar);
defer parser.deinit();
const root = try parser.parseGrammar() orelse {
log.err("Invalid grammar", .{});
return;
};
var buffer: Io.Writer.Allocating = .init(gpa);
defer buffer.deinit();
var g: Generator = .init(&buffer.writer, &parser);
try g.genRoot(root);
const generated = try buffer.toOwnedSliceSentinel(0);
defer gpa.free(generated);
// Parse the generated Zig code and render it in the canonical format
var tree = try std.zig.Ast.parse(gpa, generated, .zig, .{});
defer tree.deinit(gpa);
if (tree.errors.len != 0) {
// This should never be reached, but helps a lot when debugging this script.
try std.zig.printAstErrorsToStderr(gpa, io, tree, "generated", .auto);
return error.ParseError;
}
if (check) {
const current = try Io.Dir.cwd().readFileAlloc(io, out_path, gpa, .unlimited);
defer gpa.free(current);
var aw: Io.Writer.Allocating = .init(gpa);
defer aw.deinit();
try tree.render(gpa, &aw.writer, .{});
if (!mem.eql(u8, current, aw.written())) {
std.log.err("grammar.peg modified without regenerating oracle", .{});
std.log.info("Run zig build gen-parser-oracle to regenerate", .{});
std.process.exit(1);
}
} else {
var out_file = try Io.Dir.cwd().createFile(io, out_path, .{});
defer out_file.close(io);
var out_buffer: [4096]u8 = undefined;
var out_writer = out_file.writer(io, &out_buffer);
const out = &out_writer.interface;
try tree.render(gpa, out, .{});
try out.flush();
}
}
const Generator = struct {
w: *Io.Writer,
p: *const Parser,
/// Suffix for generated identifiers, incremented for each nested scope to avoid shadowing,
/// Decremented at end of each generated scope to give smaller git diffs when regenerating
/// lib/std/zig/parser_generated_oracle.zig.
suffix: usize,
fn init(w: *Io.Writer, p: *const Parser) Generator {
return .{ .w = w, .p = p, .suffix = 0 };
}
const Error = Io.Writer.Error;
const Node = Parser.Node;
fn genRoot(g: *Generator, node: Node.Index) Error!void {
try g.w.writeAll(
\\//! This file is generated, do not edit manually! To generate, run:
\\//! zig build gen-parser-oracle
\\
\\const std = @import("std");
\\
\\const Error = error{MaxDepth};
\\const max_depth = 5;
\\
\\/// Returns true if the input source is in the language defined by
\\/// the grammar.
\\/// Returns error.MaxDepth if more than `max_depth` levels of recursion/iteration are reached.
\\pub fn parse(source: []const u8) Error!bool {
\\ var p: Parser = .{ .source = source, .i = 0, .depths = @splat(1) };
\\ return p.parseRoot();
\\}
);
const defs = g.p.getExtra(node.get(g.p).root);
// This enum exists to minimize git diffs in the generated oracle when
// the grammar is modified. It allows mapping from def name (stable) to
// def index (unstable).
try g.w.writeAll("const Def = enum {");
for (defs) |n| {
const def = n.get(g.p).def;
const id = def.id.get(g.p).id;
try g.w.print("def{s},", .{id});
}
try g.w.writeAll("};");
try g.w.print(
\\const Parser = struct {{
\\ source: []const u8,
\\ i: usize,
\\ depths: [{d}]u8,
, .{defs.len});
for (defs) |def| {
try g.genDef(def);
}
try g.w.writeAll("};");
}
fn genDef(g: *Generator, node: Node.Index) Error!void {
const def = node.get(g.p).def;
const id = def.id.get(g.p).id;
assert(g.suffix == 0);
try g.w.print("pub fn parse{s}(p: *Parser) Error!bool {{", .{id});
try g.w.print(
\\const def_index = @intFromEnum(Def.def{s});
\\if (p.depths[def_index] > max_depth) return error.MaxDepth;
\\p.depths[def_index] += 1;
\\defer p.depths[def_index] -= 1;
, .{id});
try g.w.writeAll("return ");
try g.genExpr(def.expr);
try g.w.writeAll(";}");
}
fn genExpr(g: *Generator, node: Node.Index) Error!void {
const suffix = g.suffix;
g.suffix += 1;
defer g.suffix -= 1;
try g.w.print(
\\blk_{d}: {{
\\const pos_{d} = p.i;
, .{ suffix, suffix });
for (g.p.getExtra(node.get(g.p).expr)) |seq| {
try g.w.writeAll("if (");
try g.genSeq(seq);
try g.w.print(") break :blk_{d} true;", .{suffix});
try g.w.print("p.i = pos_{d};", .{suffix});
}
try g.w.print("break :blk_{d} false; }}", .{suffix});
}
fn genSeq(g: *Generator, node: Node.Index) Error!void {
const items = g.p.getExtra(node.get(g.p).seq);
for (items, 0..) |item, i| {
if (i > 0) try g.w.writeAll(" and ");
try g.genNode(item);
}
}
fn genNode(g: *Generator, node: Node.Index) Error!void {
const suffix = g.suffix;
g.suffix += 1;
defer g.suffix -= 1;
switch (node.get(g.p)) {
.id => |id| try g.w.print("try p.parse{s}()", .{id}),
.expr => try g.genExpr(node),
.@"&" => |child| {
// XXX forbid unbounded lookahead
try g.w.print(
\\blk_{d}: {{
\\const pos_{d} = p.i;
\\const match_{d} =
, .{ suffix, suffix, suffix });
try g.genNode(child);
try g.w.print(
\\;
\\p.i = pos_{d};
\\ break :blk_{d} match_{d};
\\}}
, .{ suffix, suffix, suffix });
},
.@"!" => |child| {
// XXX forbid unbounded lookahead
try g.w.print(
\\blk_{d}: {{
\\const pos_{d} = p.i;
\\const match_{d} =
, .{ suffix, suffix, suffix });
try g.genNode(child);
try g.w.print(
\\;
\\p.i = pos_{d};
\\ break :blk_{d} !match_{d};
\\}}
, .{ suffix, suffix, suffix });
},
.@"?" => |child| {
try g.w.writeAll("(");
try g.genNode(child);
try g.w.writeAll(" or true )");
},
.@"*" => |child| {
try g.w.print(
\\blk_{d}: {{
\\var i_{d}: usize = 0;
\\while (
, .{ suffix, suffix });
try g.genNode(child);
try g.w.print(
\\) {{
\\ if (i_{d} > max_depth) return error.MaxDepth;
\\ i_{d} += 1;
\\}}
\\break :blk_{d} true; }}
, .{ suffix, suffix, suffix });
},
.@"+" => |child| {
try g.w.print(
\\blk_{d}: {{
\\var match_{d} = false;
\\var i_{d}: usize = 0;
\\while (
, .{ suffix, suffix, suffix });
try g.genNode(child);
try g.w.print(
\\) {{
\\ match_{d} = true;
\\ if (i_{d} > max_depth) return error.MaxDepth;
\\ i_{d} += 1;
\\}}
\\break :blk_{d} match_{d}; }}
, .{ suffix, suffix, suffix, suffix, suffix });
},
.@"." => {
try g.w.print(
\\blk_{d}: {{
\\ if (p.i < p.source.len) {{
\\ p.i += 1;
\\ break :blk_{d} true;
\\ }}
\\ break :blk_{d} false;
\\}}
, .{ suffix, suffix, suffix });
},
.literal => |literal| {
const bytes = g.p.strings.items[literal.off..][0..literal.len];
try g.w.print(
\\blk_{d}: {{
\\if (std.mem.startsWith(u8, p.source[p.i..], "
, .{suffix});
try std.zig.stringEscape(bytes, g.w);
try g.w.print(
\\")) {{
\\p.i += {d};
\\ break :blk_{d} true;
\\}}
\\break :blk_{d} false;
\\}}
, .{ bytes.len, suffix, suffix });
},
.class => |ranges| {
try g.w.writeAll("(p.i < p.source.len and switch (p.source[p.i]) {");
for (g.p.getExtra(ranges)) |n| {
const range = n.get(g.p).range;
try g.w.writeAll("'");
try std.zig.charEscape(range.start, g.w);
try g.w.writeAll("'...'");
try std.zig.charEscape(range.end, g.w);
try g.w.writeAll("',");
}
try g.w.print(
\\=> blk_{d}: {{ p.i += 1; break :blk_{d} true; }},
\\else => false,
\\}})
, .{ suffix, suffix });
},
.sof => try g.w.writeAll("(p.i == 0)"),
else => unreachable,
}
}
};
/// Parser implements a subset of the PEG grammar definition.
/// We don't bother implementing the Action, BEGIN, and END rules
/// and also omit unneeded character escape sequences.
///
/// The full PEG grammar found in the peg(1) man page:
///
/// Grammar <- Spacing Definition+ EndOfFile
///
/// Definition <- Identifier LEFTARROW Expression
/// Expression <- Sequence ( SLASH Sequence )*
/// Sequence <- Prefix*
/// Prefix <- AND Action
/// / ( AND / NOT )? Suffix
/// Suffix <- Primary ( QUERY / STAR / PLUS )?
/// Primary <- Identifier !LEFTARROW
/// / OPEN Expression CLOSE
/// / Literal
/// / Class
/// / DOT
/// / Action
/// / BEGIN
/// / END
///
/// Identifier <- < IdentStart IdentCont* > Spacing
/// IdentStart <- [a-zA-Z_]
/// IdentCont <- IdentStart / [0-9]
/// Literal <- ['] < ( !['] Char )* > ['] Spacing
/// / ["] < ( !["] Char )* > ["] Spacing
/// Class <- '[' < ( !']' Range )* > ']' Spacing
/// Range <- Char '-' Char / Char
/// Char <- '\\' [abefnrtv'"\[\]\\]
/// / '\\' [0-3][0-7][0-7]
/// / '\\' [0-7][0-7]?
/// / '\\' '-'
/// / !'\\' .
/// LEFTARROW <- '<-' Spacing
/// SLASH <- '/' Spacing
/// AND <- '&' Spacing
/// NOT <- '!' Spacing
/// QUERY <- '?' Spacing
/// STAR <- '*' Spacing
/// PLUS <- '+' Spacing
/// OPEN <- '(' Spacing
/// CLOSE <- ')' Spacing
/// DOT <- '.' Spacing
/// Spacing <- ( Space / Comment )*
/// Comment <- '#' ( !EndOfLine . )* EndOfLine
/// Space <- ' ' / '\t' / EndOfLine
/// EndOfLine <- '\r\n' / '\n' / '\r'
/// EndOfFile <- !.
/// Action <- '{' < [^}]* > '}' Spacing
/// BEGIN <- '<' Spacing
/// END <- '>' Spacing
const Parser = struct {
gpa: Allocator,
/// PEG grammar source
source: []const u8,
/// Current index into source
i: u32,
nodes: std.ArrayList(Node),
extra: std.ArrayList(Node.Index),
strings: std.ArrayList(u8),
const Node = union(enum) {
/// Slice into extra
root: Slice,
def: struct {
id: Index,
expr: Index,
},
/// Slice into Parser.source
id: []const u8,
/// Slice into extra
expr: Slice,
/// Slice into extra
seq: Slice,
@"&": Index,
@"!": Index,
@"?": Index,
@"*": Index,
@"+": Index,
@".",
/// Slice into strings
literal: Slice,
/// Slice into extra
class: Slice,
range: struct {
start: u8,
end: u8,
},
/// Start of file
sof,
const Index = enum(u32) {
_,
fn get(index: Index, p: *const Parser) Node {
return p.nodes.items[@intFromEnum(index)];
}
};
const Slice = struct {
off: u32,
len: u32,
};
};
fn init(gpa: Allocator, source: []const u8) Parser {
return .{
.gpa = gpa,
.source = source,
.i = 0,
.nodes = .empty,
.extra = .empty,
.strings = .empty,
};
}
fn deinit(p: *Parser) void {
p.nodes.deinit(p.gpa);
p.extra.deinit(p.gpa);
p.strings.deinit(p.gpa);
}
// Grammar <- Spacing Definition+ EndOfFile
// EndOfFile <- !.
fn parseGrammar(p: *Parser) !?Node.Index {
var scratch: std.ArrayList(Node.Index) = .empty;
defer scratch.deinit(p.gpa);
_ = p.eatSpacing();
while (try p.parseDefinition()) |def| {
try scratch.append(p.gpa, def);
}
if (scratch.items.len == 0) return null;
if (p.peek() != null) return null;
const defs = try p.addExtra(scratch.items);
return try p.addNode(.{ .root = defs });
}
// Definition <- Identifier LEFTARROW Expression
fn parseDefinition(p: *Parser) !?Node.Index {
const id = try p.parseIdentifier() orelse return null;
if (!p.eatLeftArrow()) return null;
const expr = try p.parseExpression() orelse return null;
return try p.addNode(.{ .def = .{
.id = id,
.expr = expr,
} });
}
// Expression <- Sequence ( SLASH Sequence )*
fn parseExpression(p: *Parser) error{OutOfMemory}!?Node.Index {
var scratch: std.ArrayList(Node.Index) = .empty;
defer scratch.deinit(p.gpa);
while (try p.parseSequence()) |seq| {
try scratch.append(p.gpa, seq);
if (!p.eatSlash()) break;
}
if (scratch.items.len == 0) return null;
const seqs = try p.addExtra(scratch.items);
return try p.addNode(.{ .expr = seqs });
}
// Sequence <- Prefix*
fn parseSequence(p: *Parser) !?Node.Index {
var scratch: std.ArrayList(Node.Index) = .empty;
defer scratch.deinit(p.gpa);
while (try p.parsePrefix()) |primary| {
try scratch.append(p.gpa, primary);
}
const primaries = try p.addExtra(scratch.items);
return try p.addNode(.{ .seq = primaries });
}
// Prefix <- AND Action
// / ( AND / NOT )? Suffix
fn parsePrefix(p: *Parser) !?Node.Index {
if (p.eatAnd()) {
// We only support a single hardcoded "start of file" Action
if (p.eat('{')) {
// Action <- '{' < [^}]* > '}' Spacing
if (std.mem.startsWith(u8, p.source[p.i..], " (yy->__pos == 0) }")) {
while (!p.eat('}')) p.i += 1;
_ = p.eatSpacing();
return try p.addNode(.sof);
}
return null;
}
const suffix = try p.parseSuffix() orelse return null;
return try p.addNode(.{ .@"&" = suffix });
}
if (p.eatNot()) {
const suffix = try p.parseSuffix() orelse return null;
return try p.addNode(.{ .@"!" = suffix });
}
return try p.parseSuffix();
}
// Suffix <- Primary ( QUERY / STAR / PLUS )?
fn parseSuffix(p: *Parser) !?Node.Index {
const primary = try p.parsePrimary() orelse return null;
if (p.eatQuery()) {
return try p.addNode(.{ .@"?" = primary });
}
if (p.eatStar()) {
return try p.addNode(.{ .@"*" = primary });
}
if (p.eatPlus()) {
return try p.addNode(.{ .@"+" = primary });
}
return primary;
}
// Primary <- Identifier !LEFTARROW
// / OPEN Expression CLOSE
// / Literal
// / Class
// / DOT
// / Action
// / BEGIN
// / END
fn parsePrimary(p: *Parser) !?Node.Index {
const init_pos = p.savePos();
if (try p.parseIdentifier()) |id| {
const pos = p.savePos();
if (!p.eatLeftArrow()) {
p.restorePos(pos);
return id;
}
}
p.restorePos(init_pos);
if (p.eatOpen()) if (try p.parseExpression()) |expr| if (p.eatClose()) return expr;
p.restorePos(init_pos);
if (try p.parseLiteral()) |literal| return literal;
p.restorePos(init_pos);
if (try p.parseClass()) |class| return class;
p.restorePos(init_pos);
if (p.eatDot()) return try p.addNode(.@".");
// We don't implement Action, BEGIN, and END.
return null;
}
// Identifier <- < IdentStart IdentCont* > Spacing
// IdentStart <- [a-zA-Z_]
// IdentCont <- IdentStart / [0-9]
fn parseIdentifier(p: *Parser) !?Node.Index {
const start = p.i;
switch (p.next() orelse return null) {
'a'...'z', 'A'...'Z', '_' => {},
else => return null,
}
while (p.peek()) |cont| {
switch (cont) {
'a'...'z', 'A'...'Z', '_', '0'...'9' => p.i += 1,
else => break,
}
}
const id = p.source[start..p.i];
_ = p.eatSpacing();
return try p.addNode(.{ .id = id });
}
// Literal <- ['] < ( !['] Char )* > ['] Spacing
// / ["] < ( !["] Char )* > ["] Spacing
fn parseLiteral(p: *Parser) !?Node.Index {
const quote: u8 = if (p.eat('\'')) '\'' else if (p.eat('"')) '"' else return null;
const off = p.strings.items.len;
while (!p.eat(quote)) {
const byte = p.parseChar() orelse return null;
try p.strings.append(p.gpa, byte);
}
_ = p.eatSpacing();
return try p.addNode(.{ .literal = .{
.off = @intCast(off),
.len = @intCast(p.strings.items.len - off),
} });
}
// Class <- '[' < ( !']' Range )* > ']' Spacing
fn parseClass(p: *Parser) !?Node.Index {
var scratch: std.ArrayList(Node.Index) = .empty;
defer scratch.deinit(p.gpa);
if (!p.eat('[')) return null;
while (!p.eat(']')) {
const range = try p.parseRange() orelse return null;
try scratch.append(p.gpa, range);
}
_ = p.eatSpacing();
const ranges = try p.addExtra(scratch.items);
return try p.addNode(.{ .class = ranges });
}
// Range <- Char '-' Char / Char
fn parseRange(p: *Parser) !?Node.Index {
const start = p.parseChar() orelse return null;
const end = blk: {
if (p.eat('-')) {
break :blk p.parseChar() orelse return null;
}
break :blk start;
};
return try p.addNode(.{ .range = .{
.start = start,
.end = end,
} });
}
// Char <- '\\' [abefnrtv'"\[\]\\]
// / '\\' [0-3][0-7][0-7]
// / '\\' [0-7][0-7]?
// / '\\' '-'
// / !'\\' .
fn parseChar(p: *Parser) ?u8 {
if (p.eat('\\')) {
const c = p.next() orelse return null;
return switch (c) {
// Only the escape sequences actually used in the Zig grammar are implemented
'n' => '\n',
'r' => '\r',
't' => '\t',
'\'' => '\'',
'"' => '"',
'[' => '[',
']' => ']',
'\\' => '\\',
'-' => '-',
'0'...'7' => {
// octal
if (c <= '3') {
const c2 = p.next() orelse return null;
if (c2 < '0' or c2 > '7') return null;
const c3 = p.next() orelse return null;
if (c3 < '0' or c3 > '7') return null;
return (c - '0') * 8 * 8 + (c2 - '0') * 8 + (c3 - '0');
} else {
if (p.peek()) |c2| {
if (c2 >= '0' and c2 <= '7') {
p.i += 1;
return (c - '0') * 8 + (c2 - '0');
}
}
return (c - '0');
}
},
else => null,
};
} else {
return p.next();
}
}
// LEFTARROW <- '<-' Spacing
fn eatLeftArrow(p: *Parser) bool {
return p.eat('<') and p.eat('-') and p.eatSpacing();
}
// SLASH <- '/' Spacing
fn eatSlash(p: *Parser) bool {
return p.eat('/') and p.eatSpacing();
}
// AND <- '&' Spacing
fn eatAnd(p: *Parser) bool {
return p.eat('&') and p.eatSpacing();
}
// NOT <- '!' Spacing
fn eatNot(p: *Parser) bool {
return p.eat('!') and p.eatSpacing();
}
// QUERY <- '?' Spacing
fn eatQuery(p: *Parser) bool {
return p.eat('?') and p.eatSpacing();
}
// STAR <- '*' Spacing
fn eatStar(p: *Parser) bool {
return p.eat('*') and p.eatSpacing();
}
// PLUS <- '+' Spacing
fn eatPlus(p: *Parser) bool {
return p.eat('+') and p.eatSpacing();
}
// OPEN <- '(' Spacing
fn eatOpen(p: *Parser) bool {
return p.eat('(') and p.eatSpacing();
}
// CLOSE <- ')' Spacing
fn eatClose(p: *Parser) bool {
return p.eat(')') and p.eatSpacing();
}
// DOT <- '.' Spacing
fn eatDot(p: *Parser) bool {
return p.eat('.') and p.eatSpacing();
}
// Spacing <- ( Space / Comment )*
fn eatSpacing(p: *Parser) bool {
while (p.eatSpace() or p.eatComment()) {}
return true;
}
// Comment <- '#' ( !EndOfLine . )* EndOfLine
fn eatComment(p: *Parser) bool {
if (!p.eat('#')) return false;
while (!p.eatEndOfLine()) p.i += 1;
return true;
}
// Space <- ' ' / '\t' / EndOfLine
fn eatSpace(p: *Parser) bool {
return p.eat(' ') or p.eat('\t') or p.eatEndOfLine();
}
// EndOfLine <- '\r\n' / '\n' / '\r'
fn eatEndOfLine(p: *Parser) bool {
return p.eat('\n');
}
fn peek(p: *Parser) ?u8 {
if (p.i < p.source.len) {
return p.source[p.i];
}
return null;
}
fn next(p: *Parser) ?u8 {
if (p.i < p.source.len) {
defer p.i += 1;
return p.source[p.i];
}
return null;
}
fn eat(p: *Parser, byte: u8) bool {
if (p.i < p.source.len and p.source[p.i] == byte) {
p.i += 1;
return true;
}
return false;
}
fn addNode(p: *Parser, node: Node) !Node.Index {
try p.nodes.append(p.gpa, node);
return @enumFromInt(p.nodes.items.len - 1);
}
fn addExtra(p: *Parser, nodes: []const Node.Index) !Node.Slice {
const off = p.extra.items.len;
try p.extra.appendSlice(p.gpa, nodes);
return .{ .off = @intCast(off), .len = @intCast(p.extra.items.len - off) };
}
const Pos = struct {
i: u32,
nodes_len: u32,
extra_len: u32,
strings_len: u32,
};
fn savePos(p: *const Parser) Pos {
return .{
.i = p.i,
.nodes_len = @intCast(p.nodes.items.len),
.extra_len = @intCast(p.extra.items.len),
.strings_len = @intCast(p.strings.items.len),
};
}
fn restorePos(p: *Parser, pos: Pos) void {
assert(p.i >= pos.i);
p.i = pos.i;
p.nodes.shrinkRetainingCapacity(pos.nodes_len);
p.extra.shrinkRetainingCapacity(pos.extra_len);
p.strings.shrinkRetainingCapacity(pos.strings_len);
}
fn getExtra(p: *const Parser, s: Node.Slice) []const Node.Index {
return p.extra.items[s.off..][0..s.len];
}
};
tools/gen_spirv_spec.zig
+1 -1
View File
@@ -116,7 +116,7 @@ pub fn main(init: std.process.Init) !void {
try allocating.writer.writeByte(0);
const output = allocating.written()[0 .. allocating.written().len - 1 :0];
var tree = try std.zig.Ast.parse(arena, output, .zig);
var tree = try std.zig.Ast.parse(arena, output, .zig, .{});
if (tree.errors.len != 0) {
try std.zig.printAstErrorsToStderr(arena, io, tree, "", .auto);