Merge pull request #2160 from RustPython/coolreader18/siphasher24

Use siphash24 + PYTHONHASHSEED

Cargo.lock

@@ -1620,6 +1620,7 @@ dependencies = [
  "once_cell",
  "parking_lot",
  "rand",
+ "siphasher",
 ]
 
 [[package]]

Lib/test/test_sys.py

@@ -491,9 +491,7 @@ class SysModuleTest(unittest.TestCase):
             else:
                 self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash24"})
         else:
-            # PY_HASH_EXTERNAL
-            # TODO: RUSTPYTHON; use siphash24
-            # self.assertEqual(algo, 0)
+            self.assertEqual(algo, 0)
             pass
         self.assertGreaterEqual(sys.hash_info.cutoff, 0)
         self.assertLess(sys.hash_info.cutoff, 8)

common/Cargo.toml

@@ -16,6 +16,5 @@ lexical-core = "0.7"
 hexf-parse = "0.1.0"
 cfg-if = "0.1"
 once_cell = "1.4.1"
-
-[dev-dependencies]
+siphasher = "0.3"
 rand = "0.7.3"

common/src/hash.rs

@@ -1,8 +1,9 @@
 use num_bigint::BigInt;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
-use std::collections::hash_map::DefaultHasher;
-use std::hash::{Hash, Hasher};
+use siphasher::sip::SipHasher24;
+use std::convert::TryInto;
+use std::hash::{BuildHasher, Hash, Hasher};
 use std::num::Wrapping;
 
 pub type PyHash = i64;
@@ -16,24 +17,75 @@ pub const MODULUS: PyUHash = (1 << BITS) - 1;
 pub const INF: PyHash = 314_159;
 pub const NAN: PyHash = 0;
 pub const IMAG: PyHash = MULTIPLIER;
-pub const ALGO: &str = "siphasher13";
+pub const ALGO: &str = "siphash24";
 pub const HASH_BITS: usize = std::mem::size_of::<PyHash>() * 8;
-// internally DefaultHasher uses 2 u64s as the seed, but
-// that's not guaranteed to be consistent across Rust releases
-// TODO: use something like the siphasher crate as our hash algorithm
-pub const SEED_BITS: usize = std::mem::size_of::<PyHash>() * 2 * 8;
+// SipHasher24 takes 2 u64s as a seed
+pub const SEED_BITS: usize = std::mem::size_of::<u64>() * 2 * 8;
 
 // pub const CUTOFF: usize = 7;
 
-#[inline]
-pub fn mod_int(value: i64) -> PyHash {
-    value % MODULUS as i64
+pub struct HashSecret {
+    k0: u64,
+    k1: u64,
 }
 
-pub fn hash_value<T: Hash + ?Sized>(data: &T) -> PyHash {
-    let mut hasher = DefaultHasher::new();
-    data.hash(&mut hasher);
-    mod_int(hasher.finish() as PyHash)
+impl BuildHasher for HashSecret {
+    type Hasher = SipHasher24;
+    fn build_hasher(&self) -> Self::Hasher {
+        SipHasher24::new_with_keys(self.k0, self.k1)
+    }
+}
+
+impl rand::distributions::Distribution<HashSecret> for rand::distributions::Standard {
+    fn sample<R: rand::Rng + ?Sized>(&self, rng: &mut R) -> HashSecret {
+        HashSecret {
+            k0: rng.gen(),
+            k1: rng.gen(),
+        }
+    }
+}
+
+impl HashSecret {
+    pub fn new(seed: u32) -> Self {
+        let mut buf = [0u8; 16];
+        lcg_urandom(seed, &mut buf);
+        let k0 = u64::from_le_bytes(buf[..8].try_into().unwrap());
+        let k1 = u64::from_le_bytes(buf[8..].try_into().unwrap());
+        Self { k0, k1 }
+    }
+}
+
+impl HashSecret {
+    pub fn hash_value<T: Hash + ?Sized>(&self, data: &T) -> PyHash {
+        let mut hasher = self.build_hasher();
+        data.hash(&mut hasher);
+        mod_int(hasher.finish() as PyHash)
+    }
+
+    pub fn hash_iter<'a, T: 'a, I, F, E>(&self, iter: I, hashf: F) -> Result<PyHash, E>
+    where
+        I: IntoIterator<Item = &'a T>,
+        F: Fn(&'a T) -> Result<PyHash, E>,
+    {
+        let mut hasher = self.build_hasher();
+        for element in iter {
+            let item_hash = hashf(element)?;
+            item_hash.hash(&mut hasher);
+        }
+        Ok(mod_int(hasher.finish() as PyHash))
+    }
+
+    pub fn hash_bytes(&self, value: &[u8]) -> PyHash {
+        if value.is_empty() {
+            0
+        } else {
+            self.hash_value(value)
+        }
+    }
+
+    pub fn hash_str(&self, value: &str) -> PyHash {
+        self.hash_bytes(value.as_bytes())
+    }
 }
 
 pub fn hash_float(value: f64) -> PyHash {
@@ -84,21 +136,8 @@ pub fn hash_float(value: f64) -> PyHash {
 pub fn hash_complex(value: &Complex64) -> PyHash {
     let re_hash = hash_float(value.re);
     let im_hash = hash_float(value.im);
-    let ret = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(IMAG);
-    ret.0
-}
-
-pub fn hash_iter<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result<PyHash, E>
-where
-    I: IntoIterator<Item = &'a T>,
-    F: Fn(&'a T) -> Result<PyHash, E>,
-{
-    let mut hasher = DefaultHasher::new();
-    for element in iter {
-        let item_hash = hashf(element)?;
-        item_hash.hash(&mut hasher);
-    }
-    Ok(mod_int(hasher.finish() as PyHash))
+    let Wrapping(ret) = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(IMAG);
+    ret
 }
 
 pub fn hash_iter_unordered<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result<PyHash, E>
@@ -126,6 +165,15 @@ pub fn hash_bigint(value: &BigInt) -> PyHash {
     )
 }
 
-pub fn hash_str(value: &str) -> PyHash {
-    hash_value(value.as_bytes())
+#[inline]
+pub fn mod_int(value: i64) -> PyHash {
+    value % MODULUS as i64
+}
+
+pub fn lcg_urandom(mut x: u32, buf: &mut [u8]) {
+    for b in buf {
+        x *= 214013;
+        x += 2531011;
+        *b = ((x >> 16) & 0xff) as u8;
+    }
 }

src/main.rs

@@ -198,7 +198,7 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> {
 fn create_settings(matches: &ArgMatches) -> PySettings {
     let ignore_environment =
         matches.is_present("ignore-environment") || matches.is_present("isolate");
-    let mut settings: PySettings = Default::default();
+    let mut settings = PySettings::default();
     settings.ignore_environment = ignore_environment;
 
     // add the current directory to sys.path
@@ -280,6 +280,16 @@ fn create_settings(matches: &ArgMatches) -> PySettings {
         vec!["".to_owned()]
     };
 
+    let hash_seed = match env::var("PYTHONHASHSEED") {
+        Ok(s) if s == "random" => Some(None),
+        Ok(s) => s.parse::<u32>().ok().map(Some),
+        Err(_) => Some(None),
+    };
+    settings.hash_seed = hash_seed.unwrap_or_else(|| {
+        error!("Fatal Python init error: PYTHONHASHSEED must be \"random\" or an integer in range [0; 4294967295]");
+        process::exit(1)
+    });
+
     settings.argv = argv;
 
     settings

vm/src/bytesinner.rs

@@ -302,8 +302,8 @@ impl PyBytesInner {
         self.cmp(other, |a, b| a < b, vm)
     }
 
-    pub fn hash(&self) -> hash::PyHash {
-        hash::hash_value(&self.elements)
+    pub fn hash(&self, vm: &VirtualMachine) -> hash::PyHash {
+        vm.state.hash_secret.hash_bytes(&self.elements)
     }
 
     pub fn add(&self, other: PyBytesInner) -> Vec<u8> {

vm/src/dictdatatype.rs

@@ -456,8 +456,8 @@ impl DictKey for PyObjectRef {
 }
 
 impl DictKey for PyStringRef {
-    fn key_hash(&self, _vm: &VirtualMachine) -> PyResult<HashValue> {
-        Ok(self.hash())
+    fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> {
+        Ok(self.hash(vm))
     }
 
     fn key_is(&self, other: &PyObjectRef) -> bool {
@@ -480,9 +480,9 @@ impl DictKey for PyStringRef {
 /// Implement trait for the str type, so that we can use strings
 /// to index dictionaries.
 impl DictKey for &str {
-    fn key_hash(&self, _vm: &VirtualMachine) -> PyResult<HashValue> {
+    fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> {
         // follow a similar route as the hashing of PyStringRef
-        Ok(hash::hash_str(*self))
+        Ok(vm.state.hash_secret.hash_str(*self))
     }
 
     fn key_is(&self, _other: &PyObjectRef) -> bool {

vm/src/obj/objbytes.rs

@@ -131,8 +131,8 @@ impl PyBytes {
     }
 
     #[pymethod(name = "__hash__")]
-    fn hash(&self) -> PyHash {
-        self.inner.hash()
+    fn hash(&self, vm: &VirtualMachine) -> PyHash {
+        self.inner.hash(vm)
     }
 
     #[pymethod(name = "__iter__")]

vm/src/obj/objstr.rs

@@ -290,9 +290,9 @@ impl PyString {
     }
 
     #[pymethod(name = "__hash__")]
-    pub(crate) fn hash(&self) -> hash::PyHash {
+    pub(crate) fn hash(&self, vm: &VirtualMachine) -> hash::PyHash {
         self.hash.load().unwrap_or_else(|| {
-            let hash = hash::hash_str(&self.value);
+            let hash = vm.state.hash_secret.hash_str(&self.value);
             self.hash.store(Some(hash));
             hash
         })

vm/src/obj/objtype.rs

@@ -805,7 +805,7 @@ mod tests {
     #[test]
     fn test_linearise() {
         let context = PyContext::new();
-        let object: PyClassRef = context.object();
+        let object = &context.types.object_type;
         let type_type = &context.types.type_type;
 
         let a = new(

vm/src/pyobject.rs

@@ -1403,7 +1403,7 @@ pub fn hash_iter<'a, I: IntoIterator<Item = &'a PyObjectRef>>(
     iter: I,
     vm: &VirtualMachine,
 ) -> PyResult<rustpython_common::hash::PyHash> {
-    rustpython_common::hash::hash_iter(iter, |obj| vm._hash(obj))
+    vm.state.hash_secret.hash_iter(iter, |obj| vm._hash(obj))
 }
 
 pub fn hash_iter_unordered<'a, I: IntoIterator<Item = &'a PyObjectRef>>(

vm/src/vm.rs

@@ -19,6 +19,7 @@ use rustpython_compiler::{
 
 use crate::builtins::{self, to_ascii};
 use crate::bytecode;
+use crate::common::{hash::HashSecret, rc::PyRc};
 use crate::exceptions::{self, PyBaseException, PyBaseExceptionRef};
 use crate::frame::{ExecutionResult, Frame, FrameRef};
 use crate::frozen;
@@ -42,7 +43,6 @@ use crate::pyobject::{
 use crate::scope::Scope;
 use crate::stdlib;
 use crate::sysmodule;
-use rustpython_common::rc::PyRc;
 
 // use objects::objects;
 
@@ -74,6 +74,7 @@ pub struct PyGlobalState {
     pub frozen: HashMap<String, bytecode::FrozenModule>,
     pub stacksize: AtomicCell<usize>,
     pub thread_count: AtomicCell<usize>,
+    pub hash_secret: HashSecret,
 }
 
 pub const NSIG: usize = 64;
@@ -123,6 +124,9 @@ pub struct PySettings {
     /// Initialization parameter to decide to initialize or not,
     /// and to decide the importer required external filesystem access or not
     pub initialization_parameter: InitParameter,
+
+    /// PYTHONHASHSEED=x
+    pub hash_seed: Option<u32>,
 }
 
 /// Trace events for sys.settrace and sys.setprofile.
@@ -157,6 +161,7 @@ impl Default for PySettings {
             path_list: vec![],
             argv: vec![],
             initialization_parameter: InitParameter::InitializeExternal,
+            hash_seed: None,
         }
     }
 }
@@ -187,6 +192,11 @@ impl VirtualMachine {
         let stdlib_inits = stdlib::get_module_inits();
         let frozen = frozen::get_module_inits();
 
+        let hash_secret = match settings.hash_seed {
+            Some(seed) => HashSecret::new(seed),
+            None => rand::random(),
+        };
+
         let mut vm = VirtualMachine {
             builtins,
             sys_module: sysmod,
@@ -207,6 +217,7 @@ impl VirtualMachine {
                 frozen,
                 stacksize: AtomicCell::new(0),
                 thread_count: AtomicCell::new(0),
+                hash_secret,
             }),
             initialized: false,
         };