From 17fc5a2345770dece96112463c58854faddb5520 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 4 Nov 2018 09:00:44 +0100 Subject: [PATCH] Implemented hexadecimal, octal and binary literals. Also use bigint in the lexer to store big integer literals. --- Cargo.lock | 2 + parser/Cargo.toml | 2 + parser/src/ast.rs | 3 +- parser/src/lexer.rs | 175 +++++++++++++++++++++++++++++++++----- parser/src/lib.rs | 3 + parser/src/parser.rs | 52 ++++++++--- parser/src/python.lalrpop | 13 ++- parser/src/token.rs | 8 +- vm/src/bytecode.rs | 3 +- vm/src/compile.rs | 4 +- 10 files changed, 218 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69378c986..9a29f9928 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -615,6 +615,8 @@ dependencies = [ "lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 745b14b98..d914a98a6 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -11,4 +11,6 @@ lalrpop="0.15.1" lalrpop-util="0.15.1" log="0.4.1" regex="0.2.2" +num-bigint = "0.2" +num-traits = "0.2" diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 31886e52b..582f39785 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -3,6 +3,7 @@ //! Roughly equivalent to this: https://docs.python.org/3/library/ast.html pub use super::lexer::Location; +use num_bigint::BigInt; /* #[derive(Debug)] @@ -305,7 +306,7 @@ pub enum Comparison { #[derive(Debug, PartialEq)] pub enum Number { - Integer { value: i32 }, + Integer { value: BigInt }, Float { value: f64 }, Complex { real: f64, imag: f64 }, } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 4087d34fd..dbde9b56b 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -2,6 +2,8 @@ //! code is translated into seperate tokens. pub use super::token::Tok; +use num_bigint::BigInt; +use num_traits::Num; use std::collections::HashMap; use std::str::FromStr; @@ -280,10 +282,56 @@ where } fn lex_number(&mut self) -> Spanned { + let start_pos = self.get_pos(); + if self.chr0 == Some('0') { + if self.chr1 == Some('x') || self.chr1 == Some('X') { + // Hex! + self.next_char(); + self.next_char(); + self.lex_number_radix(start_pos, 16) + } else if self.chr1 == Some('o') || self.chr1 == Some('O') { + // Octal style! + self.next_char(); + self.next_char(); + self.lex_number_radix(start_pos, 8) + } else if self.chr1 == Some('b') || self.chr1 == Some('B') { + // Binary! + self.next_char(); + self.next_char(); + self.lex_number_radix(start_pos, 2) + } else { + self.lex_normal_number() + } + } else { + self.lex_normal_number() + } + } + + fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> Spanned { let mut value_text = String::new(); + loop { + if self.is_number(radix) { + value_text.push(self.next_char().unwrap()); + } else if self.chr0 == Some('_') { + self.next_char(); + } else { + break; + } + } + + let end_pos = self.get_pos(); + let value = BigInt::from_str_radix(&value_text, radix).unwrap(); + Ok((start_pos, Tok::Int { value: value }, end_pos)) + } + + fn lex_normal_number(&mut self) -> Spanned { let start_pos = self.get_pos(); - while self.is_number() { + + let mut value_text = String::new(); + + // Normal number: + while self.is_number(10) { value_text.push(self.next_char().unwrap()); } @@ -292,7 +340,7 @@ where // Take '.': if self.chr0 == Some('.') { value_text.push(self.next_char().unwrap()); - while self.is_number() { + while self.is_number(10) { value_text.push(self.next_char().unwrap()); } } @@ -306,18 +354,47 @@ where value_text.push(self.next_char().unwrap()); } - while self.is_number() { + while self.is_number(10) { value_text.push(self.next_char().unwrap()); } } - let end_pos = self.get_pos(); let value = f64::from_str(&value_text).unwrap(); - Ok((start_pos, Tok::Float { value: value }, end_pos)) + // Parse trailing 'j': + if self.chr0 == Some('j') { + self.next_char(); + let end_pos = self.get_pos(); + Ok(( + start_pos, + Tok::Complex { + real: 0.0, + imag: value, + }, + end_pos, + )) + } else { + let end_pos = self.get_pos(); + Ok((start_pos, Tok::Float { value: value }, end_pos)) + } } else { - let end_pos = self.get_pos(); - let value = i32::from_str(&value_text).unwrap(); - Ok((start_pos, Tok::Int { value: value }, end_pos)) + // Parse trailing 'j': + if self.chr0 == Some('j') { + self.next_char(); + let end_pos = self.get_pos(); + let imag = f64::from_str(&value_text).unwrap(); + Ok(( + start_pos, + Tok::Complex { + real: 0.0, + imag: imag, + }, + end_pos, + )) + } else { + let end_pos = self.get_pos(); + let value = value_text.parse::().unwrap(); + Ok((start_pos, Tok::Int { value: value }, end_pos)) + } } } @@ -445,10 +522,25 @@ where } } - fn is_number(&self) -> bool { - match self.chr0 { - Some('0'...'9') => return true, - _ => return false, + fn is_number(&self, radix: u32) -> bool { + match radix { + 2 => match self.chr0 { + Some('0'...'1') => return true, + _ => return false, + }, + 8 => match self.chr0 { + Some('0'...'7') => return true, + _ => return false, + }, + 10 => match self.chr0 { + Some('0'...'9') => return true, + _ => return false, + }, + 16 => match self.chr0 { + Some('0'...'9') | Some('a'...'f') | Some('A'...'F') => return true, + _ => return false, + }, + x => unimplemented!("Radix not implemented: {}", x), } } @@ -920,6 +1012,7 @@ where #[cfg(test)] mod tests { use super::{make_tokenizer, NewlineHandler, Tok}; + use num_bigint::BigInt; use std::iter::FromIterator; use std::iter::Iterator; @@ -959,6 +1052,38 @@ mod tests { ); } + #[test] + fn test_numbers() { + let source = String::from("0x2f 0b1101 0 123 0.2 2j 2.2j"); + let tokens = lex_source(&source); + assert_eq!( + tokens, + vec![ + Tok::Int { + value: BigInt::from(47), + }, + Tok::Int { + value: BigInt::from(13), + }, + Tok::Int { + value: BigInt::from(0), + }, + Tok::Int { + value: BigInt::from(123), + }, + Tok::Float { value: 0.2 }, + Tok::Complex { + real: 0.0, + imag: 2.0, + }, + Tok::Complex { + real: 0.0, + imag: 2.2, + }, + ] + ); + } + macro_rules! test_line_comment { ($($name:ident: $eol:expr,)*) => { $( @@ -966,7 +1091,7 @@ mod tests { fn $name() { let source = String::from(format!(r"99232 # {}", $eol)); let tokens = lex_source(&source); - assert_eq!(tokens, vec![Tok::Int { value: 99232 }]); + assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }]); } )* } @@ -989,9 +1114,9 @@ mod tests { assert_eq!( tokens, vec![ - Tok::Int { value: 123 }, + Tok::Int { value: BigInt::from(123) }, Tok::Newline, - Tok::Int { value: 456 }, + Tok::Int { value: BigInt::from(456) }, ] ) } @@ -1016,11 +1141,17 @@ mod tests { name: String::from("avariable"), }, Tok::Equal, - Tok::Int { value: 99 }, + Tok::Int { + value: BigInt::from(99) + }, Tok::Plus, - Tok::Int { value: 2 }, + Tok::Int { + value: BigInt::from(2) + }, Tok::Minus, - Tok::Int { value: 0 }, + Tok::Int { + value: BigInt::from(0) + }, ] ); } @@ -1045,7 +1176,7 @@ mod tests { Tok::Newline, Tok::Indent, Tok::Return, - Tok::Int { value: 99 }, + Tok::Int { value: BigInt::from(99) }, Tok::Newline, Tok::Dedent, ] @@ -1088,7 +1219,7 @@ mod tests { Tok::Newline, Tok::Indent, Tok::Return, - Tok::Int { value: 99 }, + Tok::Int { value: BigInt::from(99) }, Tok::Newline, Tok::Dedent, Tok::Dedent, @@ -1120,9 +1251,9 @@ mod tests { }, Tok::Equal, Tok::Lsqb, - Tok::Int { value: 1 }, + Tok::Int { value: BigInt::from(1) }, Tok::Comma, - Tok::Int { value: 2 }, + Tok::Int { value: BigInt::from(2) }, Tok::Rsqb, Tok::Newline, ] diff --git a/parser/src/lib.rs b/parser/src/lib.rs index af4333cf2..cefb3938b 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,6 +1,9 @@ #[macro_use] extern crate log; +extern crate num_bigint; +extern crate num_traits; + pub mod ast; pub mod lexer; pub mod parser; diff --git a/parser/src/parser.rs b/parser/src/parser.rs index c67c81306..78657535c 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -73,16 +73,19 @@ pub fn parse_statement(source: &str) -> Result { /// /// # Example /// ``` +/// extern crate num_bigint; +/// extern crate rustpython_parser; +/// use num_bigint::BigInt; /// use rustpython_parser::{parser, ast}; /// let expr = parser::parse_expression("1+2").unwrap(); /// /// assert_eq!(ast::Expression::Binop { /// a: Box::new(ast::Expression::Number { -/// value: ast::Number::Integer { value: 1 } +/// value: ast::Number::Integer { value: BigInt::from(1) } /// }), /// op: ast::Operator::Add, /// b: Box::new(ast::Expression::Number { -/// value: ast::Number::Integer { value: 2 } +/// value: ast::Number::Integer { value: BigInt::from(2) } /// }) /// }, /// expr); @@ -98,6 +101,7 @@ mod tests { use super::parse_expression; use super::parse_program; use super::parse_statement; + use num_bigint::BigInt; #[test] fn test_parse_empty() { @@ -150,7 +154,9 @@ mod tests { value: String::from("Hello world"), }, ast::Expression::Number { - value: ast::Number::Integer { value: 2 }, + value: ast::Number::Integer { + value: BigInt::from(2) + }, } ], keywords: vec![], @@ -181,7 +187,9 @@ mod tests { keywords: vec![ast::Keyword { name: Some("keyword".to_string()), value: ast::Expression::Number { - value: ast::Number::Integer { value: 2 }, + value: ast::Number::Integer { + value: BigInt::from(2) + }, } }], }, @@ -201,13 +209,17 @@ mod tests { location: ast::Location::new(1, 1), node: ast::Statement::If { test: ast::Expression::Number { - value: ast::Number::Integer { value: 1 }, + value: ast::Number::Integer { + value: BigInt::from(1) + }, }, body: vec![ast::LocatedStatement { location: ast::Location::new(1, 7), node: ast::Statement::Expression { expression: ast::Expression::Number { - value: ast::Number::Integer { value: 10 }, + value: ast::Number::Integer { + value: BigInt::from(10) + }, } }, },], @@ -215,13 +227,17 @@ mod tests { location: ast::Location::new(2, 1), node: ast::Statement::If { test: ast::Expression::Number { - value: ast::Number::Integer { value: 2 }, + value: ast::Number::Integer { + value: BigInt::from(2) + }, }, body: vec![ast::LocatedStatement { location: ast::Location::new(2, 9), node: ast::Statement::Expression { expression: ast::Expression::Number { - value: ast::Number::Integer { value: 20 }, + value: ast::Number::Integer { + value: BigInt::from(20) + }, }, }, },], @@ -229,7 +245,9 @@ mod tests { location: ast::Location::new(3, 7), node: ast::Statement::Expression { expression: ast::Expression::Number { - value: ast::Number::Integer { value: 30 }, + value: ast::Number::Integer { + value: BigInt::from(30) + }, }, }, },]), @@ -295,10 +313,14 @@ mod tests { value: ast::Expression::Tuple { elements: vec![ ast::Expression::Number { - value: ast::Number::Integer { value: 4 } + value: ast::Number::Integer { + value: BigInt::from(4) + } }, ast::Expression::Number { - value: ast::Number::Integer { value: 5 } + value: ast::Number::Integer { + value: BigInt::from(5) + } } ] } @@ -441,7 +463,9 @@ mod tests { }), op: ast::Comparison::Less, b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { value: 5 } + value: ast::Number::Integer { + value: BigInt::from(5) + } }), }, ast::Expression::Compare { @@ -450,7 +474,9 @@ mod tests { }), op: ast::Comparison::Greater, b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { value: 10 } + value: ast::Number::Integer { + value: BigInt::from(10) + } }), }, ], diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 3df43431b..c5e530652 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -7,6 +7,7 @@ use super::ast; use super::lexer; use std::iter::FromIterator; +use num_bigint::BigInt; grammar; @@ -981,12 +982,9 @@ OneOrMore: Vec = { }; Number: ast::Number = { - => { - ast::Number::Integer { value: s } - }, - => { - ast::Number::Float { value: s } - }, + => { ast::Number::Integer { value: s } }, + => { ast::Number::Float { value: s } }, + => { ast::Number::Complex { real: s.0, imag: s.1 } }, }; StringConstant: ast::Expression = { @@ -1089,8 +1087,9 @@ extern { "True" => lexer::Tok::True, "False" => lexer::Tok::False, "None" => lexer::Tok::None, - int => lexer::Tok::Int { value: }, + int => lexer::Tok::Int { value: }, float => lexer::Tok::Float { value: }, + complex => lexer::Tok::Complex { real: , imag: }, string => lexer::Tok::String { value: }, bytes => lexer::Tok::Bytes { value: > }, name => lexer::Tok::Name { name: }, diff --git a/parser/src/token.rs b/parser/src/token.rs index 132429860..e4ee15ce8 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -1,8 +1,12 @@ -// Loosely based on token.h from CPython source: +//! Different token definitions. +//! Loosely based on token.h from CPython source: +use num_bigint::BigInt; + +/// Python source code can be tokenized in a sequence of these tokens. #[derive(Debug, PartialEq)] pub enum Tok { Name { name: String }, - Int { value: i32 }, + Int { value: BigInt }, Float { value: f64 }, Complex { real: f64, imag: f64 }, String { value: String }, diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index c29120069..6e442f3e2 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -11,6 +11,7 @@ let call_function = 0x64; * Primitive instruction type, which can be encoded and decoded. */ +use num_bigint::BigInt; use num_complex::Complex64; use rustpython_parser::ast; use std::collections::HashMap; @@ -197,7 +198,7 @@ pub enum CallType { #[derive(Debug, Clone, PartialEq)] pub enum Constant { - Integer { value: i32 }, // TODO: replace by arbitrary big int math. + Integer { value: BigInt }, Float { value: f64 }, Complex { value: Complex64 }, Boolean { value: bool }, diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 799ad20b1..8345b7737 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -880,7 +880,9 @@ impl Compiler { } ast::Expression::Number { value } => { let const_value = match value { - ast::Number::Integer { value } => bytecode::Constant::Integer { value: *value }, + ast::Number::Integer { value } => bytecode::Constant::Integer { + value: value.clone(), + }, ast::Number::Float { value } => bytecode::Constant::Float { value: *value }, ast::Number::Complex { real, imag } => bytecode::Constant::Complex { value: Complex64::new(*real, *imag),