Implemented hexadecimal, octal and binary literals. Also use bigint in the lexer to store big integer literals.

Cargo.lock

@@ -615,6 +615,8 @@ dependencies = [
  "lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 

parser/Cargo.toml

@@ -11,4 +11,6 @@ lalrpop="0.15.1"
 lalrpop-util="0.15.1"
 log="0.4.1"
 regex="0.2.2"
+num-bigint = "0.2"
+num-traits = "0.2"
 

parser/src/ast.rs

@@ -3,6 +3,7 @@
 //! Roughly equivalent to this: https://docs.python.org/3/library/ast.html
 
 pub use super::lexer::Location;
+use num_bigint::BigInt;
 /*
 #[derive(Debug)]
 
@@ -305,7 +306,7 @@ pub enum Comparison {
 
 #[derive(Debug, PartialEq)]
 pub enum Number {
-    Integer { value: i32 },
+    Integer { value: BigInt },
     Float { value: f64 },
     Complex { real: f64, imag: f64 },
 }

parser/src/lexer.rs

@@ -2,6 +2,8 @@
 //! code is translated into seperate tokens.
 
 pub use super::token::Tok;
+use num_bigint::BigInt;
+use num_traits::Num;
 use std::collections::HashMap;
 use std::str::FromStr;
 
@@ -280,10 +282,56 @@ where
     }
 
     fn lex_number(&mut self) -> Spanned<Tok> {
+        let start_pos = self.get_pos();
+        if self.chr0 == Some('0') {
+            if self.chr1 == Some('x') || self.chr1 == Some('X') {
+                // Hex!
+                self.next_char();
+                self.next_char();
+                self.lex_number_radix(start_pos, 16)
+            } else if self.chr1 == Some('o') || self.chr1 == Some('O') {
+                // Octal style!
+                self.next_char();
+                self.next_char();
+                self.lex_number_radix(start_pos, 8)
+            } else if self.chr1 == Some('b') || self.chr1 == Some('B') {
+                // Binary!
+                self.next_char();
+                self.next_char();
+                self.lex_number_radix(start_pos, 2)
+            } else {
+                self.lex_normal_number()
+            }
+        } else {
+            self.lex_normal_number()
+        }
+    }
+
+    fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> Spanned<Tok> {
         let mut value_text = String::new();
 
+        loop {
+            if self.is_number(radix) {
+                value_text.push(self.next_char().unwrap());
+            } else if self.chr0 == Some('_') {
+                self.next_char();
+            } else {
+                break;
+            }
+        }
+
+        let end_pos = self.get_pos();
+        let value = BigInt::from_str_radix(&value_text, radix).unwrap();
+        Ok((start_pos, Tok::Int { value: value }, end_pos))
+    }
+
+    fn lex_normal_number(&mut self) -> Spanned<Tok> {
         let start_pos = self.get_pos();
-        while self.is_number() {
+
+        let mut value_text = String::new();
+
+        // Normal number:
+        while self.is_number(10) {
             value_text.push(self.next_char().unwrap());
         }
 
@@ -292,7 +340,7 @@ where
             // Take '.':
             if self.chr0 == Some('.') {
                 value_text.push(self.next_char().unwrap());
-                while self.is_number() {
+                while self.is_number(10) {
                     value_text.push(self.next_char().unwrap());
                 }
             }
@@ -306,18 +354,47 @@ where
                     value_text.push(self.next_char().unwrap());
                 }
 
-                while self.is_number() {
+                while self.is_number(10) {
                     value_text.push(self.next_char().unwrap());
                 }
             }
 
-            let end_pos = self.get_pos();
             let value = f64::from_str(&value_text).unwrap();
-            Ok((start_pos, Tok::Float { value: value }, end_pos))
+            // Parse trailing 'j':
+            if self.chr0 == Some('j') {
+                self.next_char();
+                let end_pos = self.get_pos();
+                Ok((
+                    start_pos,
+                    Tok::Complex {
+                        real: 0.0,
+                        imag: value,
+                    },
+                    end_pos,
+                ))
+            } else {
+                let end_pos = self.get_pos();
+                Ok((start_pos, Tok::Float { value: value }, end_pos))
+            }
         } else {
-            let end_pos = self.get_pos();
-            let value = i32::from_str(&value_text).unwrap();
-            Ok((start_pos, Tok::Int { value: value }, end_pos))
+            // Parse trailing 'j':
+            if self.chr0 == Some('j') {
+                self.next_char();
+                let end_pos = self.get_pos();
+                let imag = f64::from_str(&value_text).unwrap();
+                Ok((
+                    start_pos,
+                    Tok::Complex {
+                        real: 0.0,
+                        imag: imag,
+                    },
+                    end_pos,
+                ))
+            } else {
+                let end_pos = self.get_pos();
+                let value = value_text.parse::<BigInt>().unwrap();
+                Ok((start_pos, Tok::Int { value: value }, end_pos))
+            }
         }
     }
 
@@ -445,10 +522,25 @@ where
         }
     }
 
-    fn is_number(&self) -> bool {
-        match self.chr0 {
-            Some('0'...'9') => return true,
-            _ => return false,
+    fn is_number(&self, radix: u32) -> bool {
+        match radix {
+            2 => match self.chr0 {
+                Some('0'...'1') => return true,
+                _ => return false,
+            },
+            8 => match self.chr0 {
+                Some('0'...'7') => return true,
+                _ => return false,
+            },
+            10 => match self.chr0 {
+                Some('0'...'9') => return true,
+                _ => return false,
+            },
+            16 => match self.chr0 {
+                Some('0'...'9') | Some('a'...'f') | Some('A'...'F') => return true,
+                _ => return false,
+            },
+            x => unimplemented!("Radix not implemented: {}", x),
         }
     }
 
@@ -920,6 +1012,7 @@ where
 #[cfg(test)]
 mod tests {
     use super::{make_tokenizer, NewlineHandler, Tok};
+    use num_bigint::BigInt;
     use std::iter::FromIterator;
     use std::iter::Iterator;
 
@@ -959,6 +1052,38 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_numbers() {
+        let source = String::from("0x2f 0b1101 0 123 0.2 2j 2.2j");
+        let tokens = lex_source(&source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Int {
+                    value: BigInt::from(47),
+                },
+                Tok::Int {
+                    value: BigInt::from(13),
+                },
+                Tok::Int {
+                    value: BigInt::from(0),
+                },
+                Tok::Int {
+                    value: BigInt::from(123),
+                },
+                Tok::Float { value: 0.2 },
+                Tok::Complex {
+                    real: 0.0,
+                    imag: 2.0,
+                },
+                Tok::Complex {
+                    real: 0.0,
+                    imag: 2.2,
+                },
+            ]
+        );
+    }
+
     macro_rules! test_line_comment {
         ($($name:ident: $eol:expr,)*) => {
             $(
@@ -966,7 +1091,7 @@ mod tests {
             fn $name() {
                 let source = String::from(format!(r"99232  # {}", $eol));
                 let tokens = lex_source(&source);
-                assert_eq!(tokens, vec![Tok::Int { value: 99232 }]);
+                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }]);
             }
             )*
         }
@@ -989,9 +1114,9 @@ mod tests {
                 assert_eq!(
                     tokens,
                     vec![
-                        Tok::Int { value: 123 },
+                        Tok::Int { value: BigInt::from(123) },
                         Tok::Newline,
-                        Tok::Int { value: 456 },
+                        Tok::Int { value: BigInt::from(456) },
                     ]
                 )
             }
@@ -1016,11 +1141,17 @@ mod tests {
                     name: String::from("avariable"),
                 },
                 Tok::Equal,
-                Tok::Int { value: 99 },
+                Tok::Int {
+                    value: BigInt::from(99)
+                },
                 Tok::Plus,
-                Tok::Int { value: 2 },
+                Tok::Int {
+                    value: BigInt::from(2)
+                },
                 Tok::Minus,
-                Tok::Int { value: 0 },
+                Tok::Int {
+                    value: BigInt::from(0)
+                },
             ]
         );
     }
@@ -1045,7 +1176,7 @@ mod tests {
                         Tok::Newline,
                         Tok::Indent,
                         Tok::Return,
-                        Tok::Int { value: 99 },
+                        Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
                         Tok::Dedent,
                     ]
@@ -1088,7 +1219,7 @@ mod tests {
                         Tok::Newline,
                         Tok::Indent,
                         Tok::Return,
-                        Tok::Int { value: 99 },
+                        Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
                         Tok::Dedent,
                         Tok::Dedent,
@@ -1120,9 +1251,9 @@ mod tests {
                         },
                         Tok::Equal,
                         Tok::Lsqb,
-                        Tok::Int { value: 1 },
+                        Tok::Int { value: BigInt::from(1) },
                         Tok::Comma,
-                        Tok::Int { value: 2 },
+                        Tok::Int { value: BigInt::from(2) },
                         Tok::Rsqb,
                         Tok::Newline,
                     ]

parser/src/lib.rs

@@ -1,6 +1,9 @@
 #[macro_use]
 extern crate log;
 
+extern crate num_bigint;
+extern crate num_traits;
+
 pub mod ast;
 pub mod lexer;
 pub mod parser;

parser/src/parser.rs

@@ -73,16 +73,19 @@ pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
 ///
 /// # Example
 /// ```
+/// extern crate num_bigint;
+/// extern crate rustpython_parser;
+/// use num_bigint::BigInt;
 /// use rustpython_parser::{parser, ast};
 /// let expr = parser::parse_expression("1+2").unwrap();
 ///
 /// assert_eq!(ast::Expression::Binop {
 ///         a: Box::new(ast::Expression::Number {
-///             value: ast::Number::Integer { value: 1 }
+///             value: ast::Number::Integer { value: BigInt::from(1) }
 ///         }),
 ///         op: ast::Operator::Add,
 ///         b: Box::new(ast::Expression::Number {
-///             value: ast::Number::Integer { value: 2 }
+///             value: ast::Number::Integer { value: BigInt::from(2) }
 ///         })
 ///     },
 ///     expr);
@@ -98,6 +101,7 @@ mod tests {
     use super::parse_expression;
     use super::parse_program;
     use super::parse_statement;
+    use num_bigint::BigInt;
 
     #[test]
     fn test_parse_empty() {
@@ -150,7 +154,9 @@ mod tests {
                                     value: String::from("Hello world"),
                                 },
                                 ast::Expression::Number {
-                                    value: ast::Number::Integer { value: 2 },
+                                    value: ast::Number::Integer {
+                                        value: BigInt::from(2)
+                                    },
                                 }
                             ],
                             keywords: vec![],
@@ -181,7 +187,9 @@ mod tests {
                             keywords: vec![ast::Keyword {
                                 name: Some("keyword".to_string()),
                                 value: ast::Expression::Number {
-                                    value: ast::Number::Integer { value: 2 },
+                                    value: ast::Number::Integer {
+                                        value: BigInt::from(2)
+                                    },
                                 }
                             }],
                         },
@@ -201,13 +209,17 @@ mod tests {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::If {
                     test: ast::Expression::Number {
-                        value: ast::Number::Integer { value: 1 },
+                        value: ast::Number::Integer {
+                            value: BigInt::from(1)
+                        },
                     },
                     body: vec![ast::LocatedStatement {
                         location: ast::Location::new(1, 7),
                         node: ast::Statement::Expression {
                             expression: ast::Expression::Number {
-                                value: ast::Number::Integer { value: 10 },
+                                value: ast::Number::Integer {
+                                    value: BigInt::from(10)
+                                },
                             }
                         },
                     },],
@@ -215,13 +227,17 @@ mod tests {
                         location: ast::Location::new(2, 1),
                         node: ast::Statement::If {
                             test: ast::Expression::Number {
-                                value: ast::Number::Integer { value: 2 },
+                                value: ast::Number::Integer {
+                                    value: BigInt::from(2)
+                                },
                             },
                             body: vec![ast::LocatedStatement {
                                 location: ast::Location::new(2, 9),
                                 node: ast::Statement::Expression {
                                     expression: ast::Expression::Number {
-                                        value: ast::Number::Integer { value: 20 },
+                                        value: ast::Number::Integer {
+                                            value: BigInt::from(20)
+                                        },
                                     },
                                 },
                             },],
@@ -229,7 +245,9 @@ mod tests {
                                 location: ast::Location::new(3, 7),
                                 node: ast::Statement::Expression {
                                     expression: ast::Expression::Number {
-                                        value: ast::Number::Integer { value: 30 },
+                                        value: ast::Number::Integer {
+                                            value: BigInt::from(30)
+                                        },
                                     },
                                 },
                             },]),
@@ -295,10 +313,14 @@ mod tests {
                     value: ast::Expression::Tuple {
                         elements: vec![
                             ast::Expression::Number {
-                                value: ast::Number::Integer { value: 4 }
+                                value: ast::Number::Integer {
+                                    value: BigInt::from(4)
+                                }
                             },
                             ast::Expression::Number {
-                                value: ast::Number::Integer { value: 5 }
+                                value: ast::Number::Integer {
+                                    value: BigInt::from(5)
+                                }
                             }
                         ]
                     }
@@ -441,7 +463,9 @@ mod tests {
                                 }),
                                 op: ast::Comparison::Less,
                                 b: Box::new(ast::Expression::Number {
-                                    value: ast::Number::Integer { value: 5 }
+                                    value: ast::Number::Integer {
+                                        value: BigInt::from(5)
+                                    }
                                 }),
                             },
                             ast::Expression::Compare {
@@ -450,7 +474,9 @@ mod tests {
                                 }),
                                 op: ast::Comparison::Greater,
                                 b: Box::new(ast::Expression::Number {
-                                    value: ast::Number::Integer { value: 10 }
+                                    value: ast::Number::Integer {
+                                        value: BigInt::from(10)
+                                    }
                                 }),
                             },
                         ],

parser/src/python.lalrpop

@@ -7,6 +7,7 @@
 use super::ast;
 use super::lexer;
 use std::iter::FromIterator;
+use num_bigint::BigInt;
 
 grammar;
 
@@ -981,12 +982,9 @@ OneOrMore<T>: Vec<T> = {
 };
 
 Number: ast::Number = {
-    <s:int> => {
-        ast::Number::Integer { value: s }
-    },
-    <s:float> => {
-        ast::Number::Float { value: s }
-    },
+    <s:int> => { ast::Number::Integer { value: s } },
+    <s:float> => { ast::Number::Float { value: s } },
+    <s:complex> => { ast::Number::Complex { real: s.0, imag: s.1 } },
 };
 
 StringConstant: ast::Expression = {
@@ -1089,8 +1087,9 @@ extern {
         "True" => lexer::Tok::True,
         "False" => lexer::Tok::False,
         "None" => lexer::Tok::None,
-        int => lexer::Tok::Int { value: <i32> },
+        int => lexer::Tok::Int { value: <BigInt> },
         float => lexer::Tok::Float { value: <f64> },
+        complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
         string => lexer::Tok::String { value: <String> },
         bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
         name => lexer::Tok::Name { name: <String> },

parser/src/token.rs

@@ -1,8 +1,12 @@
-// Loosely based on token.h from CPython source:
+//! Different token definitions.
+//! Loosely based on token.h from CPython source:
+use num_bigint::BigInt;
+
+/// Python source code can be tokenized in a sequence of these tokens.
 #[derive(Debug, PartialEq)]
 pub enum Tok {
     Name { name: String },
-    Int { value: i32 },
+    Int { value: BigInt },
     Float { value: f64 },
     Complex { real: f64, imag: f64 },
     String { value: String },

vm/src/bytecode.rs

@@ -11,6 +11,7 @@ let call_function = 0x64;
  * Primitive instruction type, which can be encoded and decoded.
  */
 
+use num_bigint::BigInt;
 use num_complex::Complex64;
 use rustpython_parser::ast;
 use std::collections::HashMap;
@@ -197,7 +198,7 @@ pub enum CallType {
 
 #[derive(Debug, Clone, PartialEq)]
 pub enum Constant {
-    Integer { value: i32 }, // TODO: replace by arbitrary big int math.
+    Integer { value: BigInt },
     Float { value: f64 },
     Complex { value: Complex64 },
     Boolean { value: bool },

vm/src/compile.rs

@@ -880,7 +880,9 @@ impl Compiler {
             }
             ast::Expression::Number { value } => {
                 let const_value = match value {
-                    ast::Number::Integer { value } => bytecode::Constant::Integer { value: *value },
+                    ast::Number::Integer { value } => bytecode::Constant::Integer {
+                        value: value.clone(),
+                    },
                     ast::Number::Float { value } => bytecode::Constant::Float { value: *value },
                     ast::Number::Complex { real, imag } => bytecode::Constant::Complex {
                         value: Complex64::new(*real, *imag),