diff --git a/Cargo.lock b/Cargo.lock index dd70824fc..b4507ae3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1620,6 +1620,7 @@ dependencies = [ "once_cell", "parking_lot", "rand", + "siphasher", ] [[package]] diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ca107100e..9cb95280d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -491,9 +491,7 @@ class SysModuleTest(unittest.TestCase): else: self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash24"}) else: - # PY_HASH_EXTERNAL - # TODO: RUSTPYTHON; use siphash24 - # self.assertEqual(algo, 0) + self.assertEqual(algo, 0) pass self.assertGreaterEqual(sys.hash_info.cutoff, 0) self.assertLess(sys.hash_info.cutoff, 8) diff --git a/common/Cargo.toml b/common/Cargo.toml index 7403b67b3..e0573e365 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -16,6 +16,5 @@ lexical-core = "0.7" hexf-parse = "0.1.0" cfg-if = "0.1" once_cell = "1.4.1" - -[dev-dependencies] +siphasher = "0.3" rand = "0.7.3" diff --git a/common/src/hash.rs b/common/src/hash.rs index 8f7f75f46..5f65a2c10 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -1,8 +1,9 @@ use num_bigint::BigInt; use num_complex::Complex64; use num_traits::ToPrimitive; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; +use siphasher::sip::SipHasher24; +use std::convert::TryInto; +use std::hash::{BuildHasher, Hash, Hasher}; use std::num::Wrapping; pub type PyHash = i64; @@ -16,24 +17,75 @@ pub const MODULUS: PyUHash = (1 << BITS) - 1; pub const INF: PyHash = 314_159; pub const NAN: PyHash = 0; pub const IMAG: PyHash = MULTIPLIER; -pub const ALGO: &str = "siphasher13"; +pub const ALGO: &str = "siphash24"; pub const HASH_BITS: usize = std::mem::size_of::() * 8; -// internally DefaultHasher uses 2 u64s as the seed, but -// that's not guaranteed to be consistent across Rust releases -// TODO: use something like the siphasher crate as our hash algorithm -pub const SEED_BITS: usize = std::mem::size_of::() * 2 * 8; +// SipHasher24 takes 2 u64s as a seed +pub const SEED_BITS: usize = std::mem::size_of::() * 2 * 8; // pub const CUTOFF: usize = 7; -#[inline] -pub fn mod_int(value: i64) -> PyHash { - value % MODULUS as i64 +pub struct HashSecret { + k0: u64, + k1: u64, } -pub fn hash_value(data: &T) -> PyHash { - let mut hasher = DefaultHasher::new(); - data.hash(&mut hasher); - mod_int(hasher.finish() as PyHash) +impl BuildHasher for HashSecret { + type Hasher = SipHasher24; + fn build_hasher(&self) -> Self::Hasher { + SipHasher24::new_with_keys(self.k0, self.k1) + } +} + +impl rand::distributions::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> HashSecret { + HashSecret { + k0: rng.gen(), + k1: rng.gen(), + } + } +} + +impl HashSecret { + pub fn new(seed: u32) -> Self { + let mut buf = [0u8; 16]; + lcg_urandom(seed, &mut buf); + let k0 = u64::from_le_bytes(buf[..8].try_into().unwrap()); + let k1 = u64::from_le_bytes(buf[8..].try_into().unwrap()); + Self { k0, k1 } + } +} + +impl HashSecret { + pub fn hash_value(&self, data: &T) -> PyHash { + let mut hasher = self.build_hasher(); + data.hash(&mut hasher); + mod_int(hasher.finish() as PyHash) + } + + pub fn hash_iter<'a, T: 'a, I, F, E>(&self, iter: I, hashf: F) -> Result + where + I: IntoIterator, + F: Fn(&'a T) -> Result, + { + let mut hasher = self.build_hasher(); + for element in iter { + let item_hash = hashf(element)?; + item_hash.hash(&mut hasher); + } + Ok(mod_int(hasher.finish() as PyHash)) + } + + pub fn hash_bytes(&self, value: &[u8]) -> PyHash { + if value.is_empty() { + 0 + } else { + self.hash_value(value) + } + } + + pub fn hash_str(&self, value: &str) -> PyHash { + self.hash_bytes(value.as_bytes()) + } } pub fn hash_float(value: f64) -> PyHash { @@ -84,21 +136,8 @@ pub fn hash_float(value: f64) -> PyHash { pub fn hash_complex(value: &Complex64) -> PyHash { let re_hash = hash_float(value.re); let im_hash = hash_float(value.im); - let ret = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(IMAG); - ret.0 -} - -pub fn hash_iter<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result -where - I: IntoIterator, - F: Fn(&'a T) -> Result, -{ - let mut hasher = DefaultHasher::new(); - for element in iter { - let item_hash = hashf(element)?; - item_hash.hash(&mut hasher); - } - Ok(mod_int(hasher.finish() as PyHash)) + let Wrapping(ret) = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(IMAG); + ret } pub fn hash_iter_unordered<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result @@ -126,6 +165,15 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { ) } -pub fn hash_str(value: &str) -> PyHash { - hash_value(value.as_bytes()) +#[inline] +pub fn mod_int(value: i64) -> PyHash { + value % MODULUS as i64 +} + +pub fn lcg_urandom(mut x: u32, buf: &mut [u8]) { + for b in buf { + x *= 214013; + x += 2531011; + *b = ((x >> 16) & 0xff) as u8; + } } diff --git a/src/main.rs b/src/main.rs index 80639aae6..9c4e075be 100644 --- a/src/main.rs +++ b/src/main.rs @@ -198,7 +198,7 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { fn create_settings(matches: &ArgMatches) -> PySettings { let ignore_environment = matches.is_present("ignore-environment") || matches.is_present("isolate"); - let mut settings: PySettings = Default::default(); + let mut settings = PySettings::default(); settings.ignore_environment = ignore_environment; // add the current directory to sys.path @@ -280,6 +280,16 @@ fn create_settings(matches: &ArgMatches) -> PySettings { vec!["".to_owned()] }; + let hash_seed = match env::var("PYTHONHASHSEED") { + Ok(s) if s == "random" => Some(None), + Ok(s) => s.parse::().ok().map(Some), + Err(_) => Some(None), + }; + settings.hash_seed = hash_seed.unwrap_or_else(|| { + error!("Fatal Python init error: PYTHONHASHSEED must be \"random\" or an integer in range [0; 4294967295]"); + process::exit(1) + }); + settings.argv = argv; settings diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 52570da65..46438d8b5 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -302,8 +302,8 @@ impl PyBytesInner { self.cmp(other, |a, b| a < b, vm) } - pub fn hash(&self) -> hash::PyHash { - hash::hash_value(&self.elements) + pub fn hash(&self, vm: &VirtualMachine) -> hash::PyHash { + vm.state.hash_secret.hash_bytes(&self.elements) } pub fn add(&self, other: PyBytesInner) -> Vec { diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index f8fffaff5..5c662b778 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -456,8 +456,8 @@ impl DictKey for PyObjectRef { } impl DictKey for PyStringRef { - fn key_hash(&self, _vm: &VirtualMachine) -> PyResult { - Ok(self.hash()) + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.hash(vm)) } fn key_is(&self, other: &PyObjectRef) -> bool { @@ -480,9 +480,9 @@ impl DictKey for PyStringRef { /// Implement trait for the str type, so that we can use strings /// to index dictionaries. impl DictKey for &str { - fn key_hash(&self, _vm: &VirtualMachine) -> PyResult { + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { // follow a similar route as the hashing of PyStringRef - Ok(hash::hash_str(*self)) + Ok(vm.state.hash_secret.hash_str(*self)) } fn key_is(&self, _other: &PyObjectRef) -> bool { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 5624d988a..6cb056946 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -131,8 +131,8 @@ impl PyBytes { } #[pymethod(name = "__hash__")] - fn hash(&self) -> PyHash { - self.inner.hash() + fn hash(&self, vm: &VirtualMachine) -> PyHash { + self.inner.hash(vm) } #[pymethod(name = "__iter__")] diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index ec2dea1c4..44607cb2c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -290,9 +290,9 @@ impl PyString { } #[pymethod(name = "__hash__")] - pub(crate) fn hash(&self) -> hash::PyHash { + pub(crate) fn hash(&self, vm: &VirtualMachine) -> hash::PyHash { self.hash.load().unwrap_or_else(|| { - let hash = hash::hash_str(&self.value); + let hash = vm.state.hash_secret.hash_str(&self.value); self.hash.store(Some(hash)); hash }) diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index c8d2f0c79..70b618a44 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -805,7 +805,7 @@ mod tests { #[test] fn test_linearise() { let context = PyContext::new(); - let object: PyClassRef = context.object(); + let object = &context.types.object_type; let type_type = &context.types.type_type; let a = new( diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d304e686d..07b41c653 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1403,7 +1403,7 @@ pub fn hash_iter<'a, I: IntoIterator>( iter: I, vm: &VirtualMachine, ) -> PyResult { - rustpython_common::hash::hash_iter(iter, |obj| vm._hash(obj)) + vm.state.hash_secret.hash_iter(iter, |obj| vm._hash(obj)) } pub fn hash_iter_unordered<'a, I: IntoIterator>( diff --git a/vm/src/vm.rs b/vm/src/vm.rs index d4f20e87b..de6020304 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -19,6 +19,7 @@ use rustpython_compiler::{ use crate::builtins::{self, to_ascii}; use crate::bytecode; +use crate::common::{hash::HashSecret, rc::PyRc}; use crate::exceptions::{self, PyBaseException, PyBaseExceptionRef}; use crate::frame::{ExecutionResult, Frame, FrameRef}; use crate::frozen; @@ -42,7 +43,6 @@ use crate::pyobject::{ use crate::scope::Scope; use crate::stdlib; use crate::sysmodule; -use rustpython_common::rc::PyRc; // use objects::objects; @@ -74,6 +74,7 @@ pub struct PyGlobalState { pub frozen: HashMap, pub stacksize: AtomicCell, pub thread_count: AtomicCell, + pub hash_secret: HashSecret, } pub const NSIG: usize = 64; @@ -123,6 +124,9 @@ pub struct PySettings { /// Initialization parameter to decide to initialize or not, /// and to decide the importer required external filesystem access or not pub initialization_parameter: InitParameter, + + /// PYTHONHASHSEED=x + pub hash_seed: Option, } /// Trace events for sys.settrace and sys.setprofile. @@ -157,6 +161,7 @@ impl Default for PySettings { path_list: vec![], argv: vec![], initialization_parameter: InitParameter::InitializeExternal, + hash_seed: None, } } } @@ -187,6 +192,11 @@ impl VirtualMachine { let stdlib_inits = stdlib::get_module_inits(); let frozen = frozen::get_module_inits(); + let hash_secret = match settings.hash_seed { + Some(seed) => HashSecret::new(seed), + None => rand::random(), + }; + let mut vm = VirtualMachine { builtins, sys_module: sysmod, @@ -207,6 +217,7 @@ impl VirtualMachine { frozen, stacksize: AtomicCell::new(0), thread_count: AtomicCell::new(0), + hash_secret, }), initialized: false, };