impl fmt::Display for Tok

parser/src/error.rs

@@ -161,7 +161,7 @@ impl fmt::Display for ParseErrorType {
             ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok),
             ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
             ParseErrorType::UnrecognizedToken(ref tok, _) => {
-                write!(f, "Got unexpected token: {:?}", tok)
+                write!(f, "Got unexpected token {}", tok)
             }
             ParseErrorType::Lexical(ref error) => write!(f, "{}", error),
         }

parser/src/token.rs

@@ -1,6 +1,7 @@
 //! Different token definitions.
 //! Loosely based on token.h from CPython source:
 use num_bigint::BigInt;
+use std::fmt::{self, Write};
 
 /// Python source code can be tokenized in a sequence of these tokens.
 #[derive(Clone, Debug, PartialEq)]
@@ -103,3 +104,124 @@ pub enum Tok {
     With,
     Yield,
 }
+
+impl fmt::Display for Tok {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use Tok::*;
+        match self {
+            Name { name } => write!(f, "'{}'", name),
+            Int { value } => write!(f, "'{}'", value),
+            Float { value } => write!(f, "'{}'", value),
+            Complex { real, imag } => write!(f, "{}j{}", real, imag),
+            String { value, is_fstring } => {
+                if *is_fstring {
+                    write!(f, "f")?
+                }
+                write!(f, "{:?}", value)
+            }
+            Bytes { value } => {
+                write!(f, "b\"")?;
+                for i in value {
+                    match i {
+                        0..=8 => write!(f, "\\x0{}", i)?,
+                        9 => f.write_str("\\t")?,
+                        10 => f.write_str("\\n")?,
+                        11 => write!(f, "\\x0{:x}", i)?,
+                        13 => f.write_str("\\r")?,
+                        32..=126 => f.write_char(*i as char)?,
+                        _ => write!(f, "\\x{:x}", i)?,
+                    }
+                }
+                f.write_str("\"")
+            }
+            Newline => f.write_str("Newline"),
+            Indent => f.write_str("Indent"),
+            Dedent => f.write_str("Dedent"),
+            StartProgram => f.write_str("StartProgram"),
+            StartStatement => f.write_str("StartStatement"),
+            StartExpression => f.write_str("StartExpression"),
+            EndOfFile => f.write_str("EOF"),
+            Lpar => f.write_str("'('"),
+            Rpar => f.write_str("')'"),
+            Lsqb => f.write_str("'['"),
+            Rsqb => f.write_str("']'"),
+            Colon => f.write_str("':'"),
+            Comma => f.write_str("','"),
+            Semi => f.write_str("';'"),
+            Plus => f.write_str("'+'"),
+            Minus => f.write_str("'-'"),
+            Star => f.write_str("'*'"),
+            Slash => f.write_str("'/'"),
+            Vbar => f.write_str("'|'"),
+            Amper => f.write_str("'&'"),
+            Less => f.write_str("'<'"),
+            Greater => f.write_str("'>'"),
+            Equal => f.write_str("'='"),
+            Dot => f.write_str("'.'"),
+            Percent => f.write_str("'%'"),
+            Lbrace => f.write_str("'{'"),
+            Rbrace => f.write_str("'}'"),
+            EqEqual => f.write_str("'=='"),
+            NotEqual => f.write_str("'!='"),
+            LessEqual => f.write_str("'<='"),
+            GreaterEqual => f.write_str("'>='"),
+            Tilde => f.write_str("'~'"),
+            CircumFlex => f.write_str("'^'"),
+            LeftShift => f.write_str("'<<'"),
+            RightShift => f.write_str("'>>'"),
+            DoubleStar => f.write_str("'**'"),
+            DoubleStarEqual => f.write_str("'**='"),
+            PlusEqual => f.write_str("'+='"),
+            MinusEqual => f.write_str("'-='"),
+            StarEqual => f.write_str("'*='"),
+            SlashEqual => f.write_str("'/='"),
+            PercentEqual => f.write_str("'%='"),
+            AmperEqual => f.write_str("'&='"),
+            VbarEqual => f.write_str("'|='"),
+            CircumflexEqual => f.write_str("'^='"),
+            LeftShiftEqual => f.write_str("'<<='"),
+            RightShiftEqual => f.write_str("'>>='"),
+            DoubleSlash => f.write_str("'//'"),
+            DoubleSlashEqual => f.write_str("'//='"),
+            At => f.write_str("'@'"),
+            AtEqual => f.write_str("'@='"),
+            Rarrow => f.write_str("'->'"),
+            Ellipsis => f.write_str("'...'"),
+            False => f.write_str("'False'"),
+            None => f.write_str("'None'"),
+            True => f.write_str("'True'"),
+            And => f.write_str("'and'"),
+            As => f.write_str("'as'"),
+            Assert => f.write_str("'assert'"),
+            Async => f.write_str("'async'"),
+            Await => f.write_str("'await'"),
+            Break => f.write_str("'break'"),
+            Class => f.write_str("'class'"),
+            Continue => f.write_str("'continue'"),
+            Def => f.write_str("'def'"),
+            Del => f.write_str("'del'"),
+            Elif => f.write_str("'elif'"),
+            Else => f.write_str("'else'"),
+            Except => f.write_str("'except'"),
+            Finally => f.write_str("'finally'"),
+            For => f.write_str("'for'"),
+            From => f.write_str("'from'"),
+            Global => f.write_str("'global'"),
+            If => f.write_str("'if'"),
+            Import => f.write_str("'import'"),
+            In => f.write_str("'in'"),
+            Is => f.write_str("'is'"),
+            Lambda => f.write_str("'lambda'"),
+            Nonlocal => f.write_str("'nonlocal'"),
+            Not => f.write_str("'not'"),
+            Or => f.write_str("'or'"),
+            Pass => f.write_str("'pass'"),
+            Raise => f.write_str("'raise'"),
+            Return => f.write_str("'return'"),
+            Try => f.write_str("'try'"),
+            While => f.write_str("'while'"),
+            With => f.write_str("'with'"),
+            Yield => f.write_str("'yield'"),
+        }
+    }
+}