diff --git a/src/uu/shuf/src/random_seed.rs b/src/uu/shuf/src/random_seed.rs deleted file mode 100644 index 009de965138..00000000000 --- a/src/uu/shuf/src/random_seed.rs +++ /dev/null @@ -1,115 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -use std::ops::RangeInclusive; - -use rand::Rng as _; -use rand_chacha::{ChaCha12Rng, rand_core::SeedableRng as _}; -use sha3::{Digest as _, Sha3_256}; - -/// Reproducible seeded random number generation. -/// -/// The behavior should stay the same between releases, so don't change it without -/// a very good reason. -/// -/// # How it works -/// -/// - Take a Unicode string as the seed. -/// -/// - Encode this seed as UTF-8. -/// -/// - Take the SHA3-256 hash of the encoded seed. -/// -/// - Use that hash as the input for a [`rand_chacha`] ChaCha12 RNG. -/// (We don't touch the nonce, so that's probably zero.) -/// -/// - Take 64-bit samples from the RNG. -/// -/// - Use Lemire's method to generate uniformly distributed integers and: -/// -/// - With --repeat, use these to pick elements from ranges. -/// -/// - Without --repeat, use these to do left-to-right modern Fisher-Yates. -/// -/// # Why it works like this -/// -/// - Unicode string: Greatest common denominator between platforms. Windows doesn't -/// let you pass raw bytes as a CLI argument and that would be bad practice anyway. -/// A decimal or hex number would work but this is much more flexible without being -/// unmanageable. -/// -/// (Footgun: if the user passes a filename we won't read from the file but the -/// command will run anyway.) -/// -/// - UTF-8: That's what Rust likes and it's the least unreasonable Unicode encoding. -/// -/// - SHA3-256: We want to make good use of the entire user input and SHA-3 is -/// state of the art. ChaCha12 takes a 256-bit seed. -/// -/// - ChaCha12: [`rand`]'s default rng as of writing. Seems state of the art. -/// -/// - 64-bit samples: We could often get away with 32-bit samples but let's keep things -/// simple and only use one width. (There doesn't seem to be much of a performance hit.) -/// -/// - Lemire, Fisher-Yates: These are very easy to implement and maintain ourselves. -/// `rand` provides fancier implementations but only promises reproducibility within -/// patch releases: -/// -/// Strictly speaking even `ChaCha12` is subject to breakage. But since it's a very -/// specific algorithm I assume it's safe in practice. -pub struct SeededRng(Box); - -impl SeededRng { - pub fn new(seed: &str) -> Self { - let mut hasher = Sha3_256::new(); - hasher.update(seed.as_bytes()); - let seed = hasher.finalize(); - let seed = seed.as_slice().try_into().unwrap(); - Self(Box::new(ChaCha12Rng::from_seed(seed))) - } - - #[allow(clippy::many_single_char_names)] // use original lemire names for easy comparison - fn generate_at_most(&mut self, at_most: u64) -> u64 { - if at_most == u64::MAX { - return self.0.next_u64(); - } - - // https://lemire.me/blog/2019/06/06/nearly-divisionless-random-integer-generation-on-various-systems/ - let s: u64 = at_most + 1; - let mut x: u64 = self.0.next_u64(); - let mut m: u128 = u128::from(x) * u128::from(s); - let mut l: u64 = m as u64; - if l < s { - let t: u64 = s.wrapping_neg() % s; - while l < t { - x = self.0.next_u64(); - m = u128::from(x) * u128::from(s); - l = m as u64; - } - } - (m >> 64) as u64 - } - - pub fn choose_from_range(&mut self, range: RangeInclusive) -> u64 { - let offset = self.generate_at_most(*range.end() - *range.start()); - *range.start() + offset - } - - pub fn choose_from_slice(&mut self, vals: &[T]) -> T { - assert!(!vals.is_empty()); - let idx = self.generate_at_most(vals.len() as u64 - 1) as usize; - vals[idx] - } - - pub fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> &'a mut [T] { - // Fisher-Yates shuffle. - let amount = amount.min(vals.len()); - for idx in 0..amount { - let other_idx = self.generate_at_most((vals.len() - idx - 1) as u64) as usize + idx; - vals.swap(idx, other_idx); - } - &mut vals[..amount] - } -} diff --git a/src/uu/shuf/src/shuf.rs b/src/uu/shuf/src/shuf.rs index b523001ce51..eeb24fed436 100644 --- a/src/uu/shuf/src/shuf.rs +++ b/src/uu/shuf/src/shuf.rs @@ -26,11 +26,9 @@ use uucore::translate; mod compat_random_source; mod nonrepeating_iterator; -mod random_seed; use compat_random_source::RandomSourceAdapter; use nonrepeating_iterator::NonrepeatingIterator; -use random_seed::SeededRng; enum Mode { Default(PathBuf), @@ -50,7 +48,7 @@ struct Options { enum RandomSource { None, - Seed(String), + //Seed(String), File(PathBuf), } @@ -60,7 +58,6 @@ mod options { pub static HEAD_COUNT: &str = "head-count"; pub static OUTPUT: &str = "output"; pub static RANDOM_SOURCE: &str = "random-source"; - pub static RANDOM_SEED: &str = "random-seed"; pub static REPEAT: &str = "repeat"; pub static ZERO_TERMINATED: &str = "zero-terminated"; pub static FILE_OR_ARGS: &str = "file-or-args"; @@ -96,8 +93,6 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let random_source = if let Some(filename) = matches.get_one(options::RANDOM_SOURCE).cloned() { RandomSource::File(filename) - } else if let Some(seed) = matches.get_one(options::RANDOM_SEED).cloned() { - RandomSource::Seed(seed) } else { RandomSource::None }; @@ -140,7 +135,6 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // file untouched, matching GNU and avoiding silent data loss. let mut rng = match options.random_source { RandomSource::None => WrappedRng::Default(rand::rng()), - RandomSource::Seed(ref seed) => WrappedRng::Seed(SeededRng::new(seed)), RandomSource::File(ref r) => { let file = File::open(r).map_err_context( || translate!("shuf-error-failed-to-open-random-source", "file" => r.quote()), @@ -218,15 +212,6 @@ pub fn uu_app() -> Command { .value_parser(ValueParser::path_buf()) .value_hint(clap::ValueHint::FilePath), ) - .arg( - Arg::new(options::RANDOM_SEED) - .long(options::RANDOM_SEED) - .value_name("STRING") - .help(translate!("shuf-help-random-seed")) - .value_parser(ValueParser::string()) - .value_hint(clap::ValueHint::Other) - .conflicts_with(options::RANDOM_SOURCE), - ) .arg( Arg::new(options::RANDOM_SOURCE) .long(options::RANDOM_SOURCE) @@ -461,7 +446,6 @@ fn parse_range(input_range: &str) -> Result, String> { enum WrappedRng { Default(ThreadRng), - Seed(SeededRng), File(RandomSourceAdapter>), } @@ -469,7 +453,6 @@ impl WrappedRng { fn choose(&mut self, vals: &[T]) -> UResult { match self { Self::Default(rng) => Ok(*vals.choose(rng).unwrap()), - Self::Seed(rng) => Ok(rng.choose_from_slice(vals)), Self::File(rng) => rng.choose_from_slice(vals), } } @@ -477,7 +460,6 @@ impl WrappedRng { fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> UResult<&'a mut [T]> { match self { Self::Default(rng) => Ok(vals.partial_shuffle(rng, amount).0), - Self::Seed(rng) => Ok(rng.shuffle(vals, amount)), Self::File(rng) => rng.shuffle(vals, amount), } } @@ -485,7 +467,6 @@ impl WrappedRng { fn choose_from_range(&mut self, range: RangeInclusive) -> UResult { match self { Self::Default(rng) => Ok(rng.random_range(range)), - Self::Seed(rng) => Ok(rng.choose_from_range(range)), Self::File(rng) => rng.choose_from_range(range), } } diff --git a/tests/by-util/test_shuf.rs b/tests/by-util/test_shuf.rs index 1246297b596..e1c1d20ecf2 100644 --- a/tests/by-util/test_shuf.rs +++ b/tests/by-util/test_shuf.rs @@ -4,7 +4,6 @@ // file that was distributed with this source code. // spell-checker:ignore (ToDO) unwritable GHSA -use std::fmt::Write; use uutests::at_and_ucmd; use uutests::new_ucmd; @@ -1070,95 +1069,6 @@ fn test_gnu_compat_range_no_repeat() { .stdout_is("10\n2\n8\n7\n3\n9\n6\n5\n1\n4\n"); } -// Test reproducibility of --random-seed. -// These results are arbitrary but they should not change unless we choose to break compatibility. - -#[test] -fn test_seed_args_repeat() { - new_ucmd!() - .arg("--random-seed=🌱") - .arg("-e") - .arg("-r") - .arg("-n10") - .args(&["foo", "bar", "baz", "qux"]) - .succeeds() - .no_stderr() - .stdout_is("qux\nbar\nbaz\nfoo\nbaz\nqux\nqux\nfoo\nqux\nqux\n"); -} - -#[test] -fn test_seed_args_no_repeat() { - new_ucmd!() - .arg("--random-seed=🌱") - .arg("-e") - .args(&["foo", "bar", "baz", "qux"]) - .succeeds() - .no_stderr() - .stdout_is("qux\nbaz\nfoo\nbar\n"); -} - -#[test] -fn test_seed_range_repeat() { - new_ucmd!() - .arg("--random-seed=🦀") - .arg("-r") - .arg("-i1-99") - .arg("-n10") - .succeeds() - .no_stderr() - .stdout_is("60\n44\n38\n41\n63\n43\n31\n71\n46\n90\n"); -} - -#[test] -fn test_seed_range_no_repeat() { - let expected = "8\n9\n1\n5\n2\n6\n4\n3\n10\n7\n"; - - new_ucmd!() - .arg("--random-seed=12345") - .arg("-i1-10") - .succeeds() - .no_stderr() - .stdout_is(expected); - - // Piping from e.g. seq gives identical results. - new_ucmd!() - .arg("--random-seed=12345") - .pipe_in("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n") - .succeeds() - .no_stderr() - .stdout_is(expected); -} - -// Test a longer input to exercise some more code paths in the sparse representation. -#[test] -fn test_seed_long_range_no_repeat() { - let expected = "\ - 1\n3\n35\n37\n36\n45\n72\n17\n18\n40\n67\n74\n81\n77\n14\n90\n\ - 7\n12\n80\n54\n23\n61\n29\n41\n15\n56\n6\n32\n82\n76\n11\n2\n100\n\ - 50\n60\n97\n73\n79\n91\n89\n85\n86\n66\n70\n22\n55\n8\n83\n39\n27\n"; - - new_ucmd!() - .arg("--random-seed=67890") - .arg("-i1-100") - .arg("-n50") - .succeeds() - .no_stderr() - .stdout_is(expected); - - let mut test_input = String::new(); - for n in 1..=100 { - writeln!(&mut test_input, "{n}").unwrap(); - } - - new_ucmd!() - .arg("--random-seed=67890") - .pipe_in(test_input.as_bytes()) - .arg("-n50") - .succeeds() - .no_stderr() - .stdout_is(expected); -} - #[test] fn test_empty_range_no_repeat() { new_ucmd!().arg("-i4-3").succeeds().no_output();