diff --git a/parser/src/error.rs b/parser/src/error.rs index 9a8865710..fc53ebb0d 100644 --- a/parser/src/error.rs +++ b/parser/src/error.rs @@ -161,7 +161,7 @@ impl fmt::Display for ParseErrorType { ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok), ParseErrorType::InvalidToken => write!(f, "Got invalid token"), ParseErrorType::UnrecognizedToken(ref tok, _) => { - write!(f, "Got unexpected token: {:?}", tok) + write!(f, "Got unexpected token {}", tok) } ParseErrorType::Lexical(ref error) => write!(f, "{}", error), } diff --git a/parser/src/token.rs b/parser/src/token.rs index 982d8de92..09122bb83 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -1,6 +1,7 @@ //! Different token definitions. //! Loosely based on token.h from CPython source: use num_bigint::BigInt; +use std::fmt::{self, Write}; /// Python source code can be tokenized in a sequence of these tokens. #[derive(Clone, Debug, PartialEq)] @@ -103,3 +104,124 @@ pub enum Tok { With, Yield, } + +impl fmt::Display for Tok { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Tok::*; + match self { + Name { name } => write!(f, "'{}'", name), + Int { value } => write!(f, "'{}'", value), + Float { value } => write!(f, "'{}'", value), + Complex { real, imag } => write!(f, "{}j{}", real, imag), + String { value, is_fstring } => { + if *is_fstring { + write!(f, "f")? + } + write!(f, "{:?}", value) + } + Bytes { value } => { + write!(f, "b\"")?; + for i in value { + match i { + 0..=8 => write!(f, "\\x0{}", i)?, + 9 => f.write_str("\\t")?, + 10 => f.write_str("\\n")?, + 11 => write!(f, "\\x0{:x}", i)?, + 13 => f.write_str("\\r")?, + 32..=126 => f.write_char(*i as char)?, + _ => write!(f, "\\x{:x}", i)?, + } + } + f.write_str("\"") + } + Newline => f.write_str("Newline"), + Indent => f.write_str("Indent"), + Dedent => f.write_str("Dedent"), + StartProgram => f.write_str("StartProgram"), + StartStatement => f.write_str("StartStatement"), + StartExpression => f.write_str("StartExpression"), + EndOfFile => f.write_str("EOF"), + Lpar => f.write_str("'('"), + Rpar => f.write_str("')'"), + Lsqb => f.write_str("'['"), + Rsqb => f.write_str("']'"), + Colon => f.write_str("':'"), + Comma => f.write_str("','"), + Semi => f.write_str("';'"), + Plus => f.write_str("'+'"), + Minus => f.write_str("'-'"), + Star => f.write_str("'*'"), + Slash => f.write_str("'/'"), + Vbar => f.write_str("'|'"), + Amper => f.write_str("'&'"), + Less => f.write_str("'<'"), + Greater => f.write_str("'>'"), + Equal => f.write_str("'='"), + Dot => f.write_str("'.'"), + Percent => f.write_str("'%'"), + Lbrace => f.write_str("'{'"), + Rbrace => f.write_str("'}'"), + EqEqual => f.write_str("'=='"), + NotEqual => f.write_str("'!='"), + LessEqual => f.write_str("'<='"), + GreaterEqual => f.write_str("'>='"), + Tilde => f.write_str("'~'"), + CircumFlex => f.write_str("'^'"), + LeftShift => f.write_str("'<<'"), + RightShift => f.write_str("'>>'"), + DoubleStar => f.write_str("'**'"), + DoubleStarEqual => f.write_str("'**='"), + PlusEqual => f.write_str("'+='"), + MinusEqual => f.write_str("'-='"), + StarEqual => f.write_str("'*='"), + SlashEqual => f.write_str("'/='"), + PercentEqual => f.write_str("'%='"), + AmperEqual => f.write_str("'&='"), + VbarEqual => f.write_str("'|='"), + CircumflexEqual => f.write_str("'^='"), + LeftShiftEqual => f.write_str("'<<='"), + RightShiftEqual => f.write_str("'>>='"), + DoubleSlash => f.write_str("'//'"), + DoubleSlashEqual => f.write_str("'//='"), + At => f.write_str("'@'"), + AtEqual => f.write_str("'@='"), + Rarrow => f.write_str("'->'"), + Ellipsis => f.write_str("'...'"), + False => f.write_str("'False'"), + None => f.write_str("'None'"), + True => f.write_str("'True'"), + And => f.write_str("'and'"), + As => f.write_str("'as'"), + Assert => f.write_str("'assert'"), + Async => f.write_str("'async'"), + Await => f.write_str("'await'"), + Break => f.write_str("'break'"), + Class => f.write_str("'class'"), + Continue => f.write_str("'continue'"), + Def => f.write_str("'def'"), + Del => f.write_str("'del'"), + Elif => f.write_str("'elif'"), + Else => f.write_str("'else'"), + Except => f.write_str("'except'"), + Finally => f.write_str("'finally'"), + For => f.write_str("'for'"), + From => f.write_str("'from'"), + Global => f.write_str("'global'"), + If => f.write_str("'if'"), + Import => f.write_str("'import'"), + In => f.write_str("'in'"), + Is => f.write_str("'is'"), + Lambda => f.write_str("'lambda'"), + Nonlocal => f.write_str("'nonlocal'"), + Not => f.write_str("'not'"), + Or => f.write_str("'or'"), + Pass => f.write_str("'pass'"), + Raise => f.write_str("'raise'"), + Return => f.write_str("'return'"), + Try => f.write_str("'try'"), + While => f.write_str("'while'"), + With => f.write_str("'with'"), + Yield => f.write_str("'yield'"), + } + } +}