From 20880e03d1129b9deed9add4e566694a70761ed7 Mon Sep 17 00:00:00 2001 From: wtcpython <1762993226@qq.com> Date: Sun, 13 Sep 2026 16:04:49 +0800 Subject: [PATCH] uucore, expr: centralize regex in uucore --- Cargo.lock | 2 +- Cargo.toml | 6 +- fuzz/Cargo.lock | 2 +- src/uu/expr/Cargo.toml | 3 +- src/uu/expr/locales/en-US.ftl | 7 - src/uu/expr/locales/fr-FR.ftl | 7 - src/uu/expr/src/diagnostics.rs | 8 +- src/uu/expr/src/expr.rs | 16 +- src/uu/expr/src/syntax_tree.rs | 371 ++--------------- src/uucore/Cargo.toml | 6 + src/uucore/locales/errors/en-US.ftl | 10 + src/uucore/locales/errors/fr-FR.ftl | 10 + src/uucore/src/lib/features.rs | 2 + src/uucore/src/lib/features/regex.rs | 587 +++++++++++++++++++++++++++ src/uucore/src/lib/lib.rs | 2 + tests/by-util/test_expr.rs | 41 ++ 16 files changed, 697 insertions(+), 383 deletions(-) create mode 100644 src/uucore/src/lib/features/regex.rs diff --git a/Cargo.lock b/Cargo.lock index aaee6e9176c..b4b427ed89d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3560,7 +3560,6 @@ version = "0.11.0" dependencies = [ "clap", "codspeed-divan-compat", - "fancy-regex", "fluent", "num-bigint", "num-traits", @@ -4524,6 +4523,7 @@ dependencies = [ "digest 0.11.3", "dns-lookup", "dunce", + "fancy-regex", "fluent", "fluent-syntax", "glob", diff --git a/Cargo.toml b/Cargo.toml index f60ce433087..1fdac890976 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -456,11 +456,7 @@ notify = { version = "8.2.0", features = ["macos_kqueue"] } num-bigint = "0.4.4" num-prime = "0.5.0" num-traits = "0.2.19" -fancy-regex = { version = "0.19.2", default-features = false, features = [ - "std", - "unicode", - "leftmost_longest", -] } +fancy-regex = { version = "0.19.2", default-features = false } os_display = "0.1.3" parse_datetime = "0.16.0" phf = "0.14.0" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 56162893533..3ce31e314f4 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1953,7 +1953,6 @@ name = "uu_expr" version = "0.11.0" dependencies = [ "clap", - "fancy-regex", "fluent", "num-bigint", "num-traits", @@ -2068,6 +2067,7 @@ dependencies = [ "data-encoding-macro", "digest 0.11.3", "dunce", + "fancy-regex", "fluent", "fluent-syntax", "glob", diff --git a/src/uu/expr/Cargo.toml b/src/uu/expr/Cargo.toml index 83fed4854a3..dfbdb73d6d4 100644 --- a/src/uu/expr/Cargo.toml +++ b/src/uu/expr/Cargo.toml @@ -19,8 +19,7 @@ doctest = false clap = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } -fancy-regex = { workspace = true, default-features = false } -uucore = { workspace = true, features = ["i18n-collator"] } +uucore = { workspace = true, features = ["i18n-collator", "regex"] } thiserror = { workspace = true } fluent = { workspace = true } diff --git a/src/uu/expr/locales/en-US.ftl b/src/uu/expr/locales/en-US.ftl index 6c600aee0ba..b281e0b26e4 100644 --- a/src/uu/expr/locales/en-US.ftl +++ b/src/uu/expr/locales/en-US.ftl @@ -95,15 +95,8 @@ expr-error-missing-argument = syntax error: missing argument after { $arg } expr-error-non-integer-argument = non-integer argument expr-error-missing-operand = missing operand expr-error-division-by-zero = division by zero -expr-error-invalid-regex-expression = Invalid regex expression expr-error-expected-closing-brace-after = syntax error: expecting ')' after { $arg } expr-error-expected-closing-brace-instead-of = syntax error: expecting ')' instead of { $arg } -expr-error-unmatched-opening-parenthesis = Unmatched ( or \( -expr-error-unmatched-closing-parenthesis = Unmatched ) or \) -expr-error-unmatched-opening-brace = Unmatched {"\\{"} -expr-error-invalid-bracket-content = Invalid content of {"\\{\\}"} -expr-error-trailing-backslash = Trailing backslash -expr-error-too-big-range-quantifier-index = Regular expression too big expr-error-match-utf8 = match does not support invalid UTF-8 encoding in { $arg } # Diagnostic labels, used when errors are rendered with a source snippet diff --git a/src/uu/expr/locales/fr-FR.ftl b/src/uu/expr/locales/fr-FR.ftl index 8a376fa0186..793a2e20e27 100644 --- a/src/uu/expr/locales/fr-FR.ftl +++ b/src/uu/expr/locales/fr-FR.ftl @@ -95,15 +95,8 @@ expr-error-missing-argument = erreur de syntaxe : argument manquant après { $ar expr-error-non-integer-argument = argument non entier expr-error-missing-operand = opérande manquant expr-error-division-by-zero = division par zéro -expr-error-invalid-regex-expression = Expression regex invalide expr-error-expected-closing-brace-after = erreur de syntaxe : ')' attendu après { $arg } expr-error-expected-closing-brace-instead-of = erreur de syntaxe : ')' attendu au lieu de { $arg } -expr-error-unmatched-opening-parenthesis = Parenthèse ouvrante ( ou \( non appariée -expr-error-unmatched-closing-parenthesis = Parenthèse fermante ) ou \) non appariée -expr-error-unmatched-opening-brace = Accolade ouvrante {"\\{"} non appariée -expr-error-invalid-bracket-content = Contenu invalide de {"\\{\\}"} -expr-error-trailing-backslash = Barre oblique inverse en fin -expr-error-too-big-range-quantifier-index = Expression régulière trop grande expr-error-match-utf8 = match ne supporte pas l'encodage UTF-8 invalide dans { $arg } # Étiquettes de diagnostic, utilisées quand les erreurs sont rendues avec un extrait diff --git a/src/uu/expr/src/diagnostics.rs b/src/uu/expr/src/diagnostics.rs index 3a2893c9a8f..4284269920d 100644 --- a/src/uu/expr/src/diagnostics.rs +++ b/src/uu/expr/src/diagnostics.rs @@ -81,13 +81,7 @@ fn locate(snapshot: &Snapshot, err: &ExprError, at: &FailurePoint) -> Option return None, }; diff --git a/src/uu/expr/src/expr.rs b/src/uu/expr/src/expr.rs index d84eab9a525..4144d0fe8a3 100644 --- a/src/uu/expr/src/expr.rs +++ b/src/uu/expr/src/expr.rs @@ -42,26 +42,14 @@ pub enum ExprError { MissingOperand, #[error("{}", translate!("expr-error-division-by-zero"))] DivisionByZero, - #[error("{}", translate!("expr-error-invalid-regex-expression"))] - InvalidRegexExpression, #[error("{}", translate!("expr-error-expected-closing-brace-after", "arg" => _0.quote()))] ExpectedClosingBraceAfter(String), #[error("{}", translate!("expr-error-expected-closing-brace-instead-of", "arg" => _0.quote()))] ExpectedClosingBraceInsteadOf(String), - #[error("{}", translate!("expr-error-unmatched-opening-parenthesis"))] - UnmatchedOpeningParenthesis, - #[error("{}", translate!("expr-error-unmatched-closing-parenthesis"))] - UnmatchedClosingParenthesis, - #[error("{}", translate!("expr-error-unmatched-opening-brace"))] - UnmatchedOpeningBrace, - #[error("{}", translate!("expr-error-invalid-bracket-content"))] - InvalidBracketContent, - #[error("{}", translate!("expr-error-trailing-backslash"))] - TrailingBackslash, - #[error("{}", translate!("expr-error-too-big-range-quantifier-index"))] - TooBigRangeQuantifierIndex, #[error("{}", translate!("expr-error-match-utf8", "arg" => _0.quote()))] UnsupportedNonUtf8Match(String), + #[error(transparent)] + Regex(#[from] uucore::regex::RegexError), } impl UError for ExprError { diff --git a/src/uu/expr/src/syntax_tree.rs b/src/uu/expr/src/syntax_tree.rs index d3b42b4a3a8..c32d74b0eb3 100644 --- a/src/uu/expr/src/syntax_tree.rs +++ b/src/uu/expr/src/syntax_tree.rs @@ -7,9 +7,9 @@ use std::{cell::Cell, collections::BTreeMap}; -use fancy_regex::{Regex, RegexBuilder}; use num_bigint::BigInt; use num_traits::ToPrimitive; +use uucore::regex::{Regex, RegexBuilder, bre_to_ere}; use crate::{ ExprError, ExprResult, @@ -155,260 +155,6 @@ impl StringOp { } } -/// Check if regex pattern character iterator is at the end of a regex expression or subexpression -fn is_end_of_expression(pattern_chars: &I) -> bool -where - I: Iterator + Clone, -{ - let mut pattern_chars_clone = pattern_chars.clone(); - match pattern_chars_clone.next() { - Some('\\') => matches!(pattern_chars_clone.next(), Some(')' | '|')), - None => true, // No characters left - _ => false, - } -} - -/// Check if regex pattern character iterator is at the start of a valid range quantifier. -/// The iterator's start position is expected to be after the opening brace. -/// Range quantifier ends to closing brace. -/// -/// # Examples of valid range quantifiers -/// -/// - `r"\{3\}"` -/// - `r"\{3,\}"` -/// - `r"\{,6\}"` -/// - `r"\{3,6\}"` -/// - `r"\{,\}"` -fn verify_range_quantifier(pattern_chars: &I) -> Result<(), ExprError> -where - I: Iterator + Clone, -{ - let mut pattern_chars_clone = pattern_chars.clone().peekable(); - if pattern_chars_clone.peek().is_none() { - return Err(ExprError::UnmatchedOpeningBrace); - } - - // Parse the string between braces - let mut quantifier = String::new(); - let mut prev = '\0'; - let mut curr_is_escaped = false; - while let Some(curr) = pattern_chars_clone.next() { - curr_is_escaped = prev == '\\' && !curr_is_escaped; - if curr_is_escaped && curr == '}' { - break; - } - if pattern_chars_clone.peek().is_none() { - return Err(ExprError::UnmatchedOpeningBrace); - } - if prev != '\0' { - quantifier.push(prev); - } - prev = curr; - } - - // Check if parsed quantifier is valid - let re = Regex::new(r"^([0-9]*,[0-9]*|[0-9]+)$").expect("valid regular expression"); - if let Ok(Some(captures)) = re.captures(&quantifier) { - let matched = captures.get(0).map_or("", |m| m.as_str()); - match matched.split_once(',') { - Some(("", "")) => Ok(()), - Some((x, "") | ("", x)) if x.parse::().is_ok() => Ok(()), - Some((_, "") | ("", _)) => Err(ExprError::TooBigRangeQuantifierIndex), - Some((f, l)) => match (f.parse::(), l.parse::()) { - (Ok(f), Ok(l)) if f > l => Err(ExprError::InvalidBracketContent), - (Ok(_), Ok(_)) => Ok(()), - _ => Err(ExprError::TooBigRangeQuantifierIndex), - }, - None if matched.parse::().is_ok() => Ok(()), - None => Err(ExprError::TooBigRangeQuantifierIndex), - } - } else { - Err(ExprError::InvalidBracketContent) - } -} - -/// Check for errors in a supplied regular expression -/// -/// GNU coreutils shows messages for invalid regular expressions -/// differently from standard regex engines. -/// This method attempts to do these checks manually in one pass -/// through the regular expression. -/// -/// This method is not comprehensively checking all cases in which -/// a regular expression could be invalid; any cases not caught will -/// result in a [`ExprError::InvalidRegexExpression`]. This method is -/// intended to just identify a few situations for which GNU coreutils -/// has specific error messages. -fn check_posix_regex_errors(pattern: &str) -> ExprResult<()> { - let mut escaped_parens: u64 = 0; - let mut prev = '\0'; - let mut curr_is_escaped = false; - - for curr in pattern.chars() { - curr_is_escaped = prev == '\\' && !curr_is_escaped; - match (curr_is_escaped, curr) { - (true, '(') => escaped_parens += 1, - (true, ')') => { - escaped_parens = escaped_parens - .checked_sub(1) - .ok_or(ExprError::UnmatchedClosingParenthesis)?; - } - _ => {} - } - prev = curr; - } - - match escaped_parens { - 0 => Ok(()), - _ => Err(ExprError::UnmatchedOpeningParenthesis), - } -} - -/// Transpile the input pattern from BRE syntax to ERE for fancy-regex -fn transpile_bre_to_ere(pattern_str: &str) -> ExprResult { - let mut re_string = String::with_capacity(pattern_str.len() + 8); - let mut pattern_chars = pattern_str.chars().peekable(); - let mut prev = '\0'; - let mut prev_is_escaped = false; - let mut in_bracket = false; - let mut bracket_start_idx = 0; - let mut is_start_of_expression = true; - let mut after_anchor_caret = false; - - // All patterns are anchored so they begin with a caret (^) - if pattern_chars.peek() != Some(&'^') { - re_string.push('^'); - } - - while let Some(curr) = pattern_chars.next() { - let curr_is_escaped = prev == '\\' && !prev_is_escaped; - let mut next_is_start = false; - let mut next_after_anchor = false; - - if in_bracket { - let is_first = re_string.len() == bracket_start_idx + 1 - || (re_string.ends_with("[^") && re_string.len() == bracket_start_idx + 2); - if curr == ']' && !curr_is_escaped && !is_first { - in_bracket = false; - } - re_string.push(curr); - } else { - match curr { - '[' if !curr_is_escaped => { - in_bracket = true; - bracket_start_idx = re_string.len(); - re_string.push('['); - } - // In BRE, '(', ')', '|', '+', '?', '{', '}' are literal by default, - // and become operators only when escaped. ERE has the exact opposite convention. - '(' | ')' | '|' | '+' | '?' | '{' | '}' => { - if curr_is_escaped { - if re_string.ends_with('\\') { - re_string.pop(); - } - match curr { - '(' | '|' => { - re_string.push(curr); - next_is_start = true; - } - '+' | '?' => { - if is_start_of_expression || after_anchor_caret { - re_string.push('\\'); - } - re_string.push(curr); - } - '{' => { - // Handle '{' literally at the start of an expression - if is_start_of_expression || after_anchor_caret { - re_string.push_str(r"\{"); - } else { - // Check if the following section is a valid range quantifier - verify_range_quantifier(&pattern_chars)?; - re_string.push('{'); - // Set the lower bound of range quantifier to 0 if it is missing - if pattern_chars.peek() == Some(&',') { - re_string.push('0'); - } - } - } - _ => re_string.push(curr), // ')' and '}' - } - } else { - // Unescaped metacharacter in BRE -> literal in ERE - re_string.push('\\'); - re_string.push(curr); - } - } - '*' => { - if curr_is_escaped { - re_string.push('*'); - } else if is_start_of_expression || after_anchor_caret { - re_string.push_str(r"\*"); - } else { - re_string.push('*'); - } - } - // Character class negation "[^a]" - // Explicitly escaped caret "\^" - '^' => { - if curr_is_escaped { - re_string.push('^'); - } else if is_start_of_expression { - re_string.push('^'); - next_after_anchor = true; - } else if prev == '[' && !prev_is_escaped { - re_string.push('^'); - } else { - re_string.push_str(r"\^"); - } - } - '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { - re_string.push_str(r"\$"); - } - '<' if curr_is_escaped => { - if re_string.ends_with('\\') { - re_string.pop(); - } - re_string.push_str(r"\b(?=\w)"); - } - '>' if curr_is_escaped => { - if re_string.ends_with('\\') { - re_string.pop(); - } - re_string.push_str(r"\b(?<=\w)"); - } - '\\' if !curr_is_escaped => { - if pattern_chars.peek().is_none() { - return Err(ExprError::TrailingBackslash); - } - // Carry the expression-start / after-anchor state over the - // backslash so the escaped character is still treated as the - // first token of a (sub)expression. - next_is_start = is_start_of_expression; - next_after_anchor = after_anchor_caret; - re_string.push('\\'); - } - _ => { - if curr_is_escaped - && !"123456789.*^$[]\\wWsSbB".contains(curr) - && re_string.ends_with('\\') - { - re_string.pop(); - } - re_string.push(curr); - } - } - } - - is_start_of_expression = next_is_start; - after_anchor_caret = next_after_anchor; - prev_is_escaped = curr_is_escaped; - prev = curr; - } - - Ok(re_string) -} - /// Build a regex from a pattern string with locale-aware encoding fn build_regex(pattern_bytes: Vec) -> ExprResult { use uucore::i18n::UEncoding; @@ -421,15 +167,17 @@ fn build_regex(pattern_bytes: Vec) -> ExprResult { .unwrap_or_else(|_| String::from_utf8_lossy(&pattern_bytes).into()), UEncoding::Ascii => pattern_bytes.iter().map(|&b| b as char).collect(), }; - check_posix_regex_errors(&pattern_str)?; - let re_string = transpile_bre_to_ere(&pattern_str)?; + let re_string = bre_to_ere(&pattern_str, true)?; - RegexBuilder::new(&format!("(?s){re_string}")) + let regex = RegexBuilder::new(&format!("(?s){re_string}")) .oniguruma_mode(true) .leftmost_longest(true) + .seek(true) .build() - .map_err(|_| ExprError::InvalidRegexExpression) + .map_err(uucore::regex::RegexError::from)?; + + Ok(regex) } /// Find matches in the input using the compiled regex @@ -1115,12 +863,11 @@ pub fn is_truthy(s: &NumOrStr) -> bool { #[cfg(test)] mod test { - use crate::syntax_tree::verify_range_quantifier; use crate::{ExprError, ExprResult}; + use uucore::regex::RegexError; use super::{ - AstNode, AstNodeInner, BinOp, MaybeNonUtf8Str, NumericOp, RelationOp, StringOp, - check_posix_regex_errors, get_next_id, + AstNode, AstNodeInner, BinOp, MaybeNonUtf8Str, NumericOp, RelationOp, StringOp, get_next_id, }; /// Parse an expression, discarding how far the parser got. @@ -1337,78 +1084,6 @@ mod test { assert_eq!(result.eval_as_string(), b"0"); } - #[test] - fn check_regex_valid() { - assert!(check_posix_regex_errors(r"(a+b) \(a* b\)").is_ok()); - } - - #[test] - fn check_regex_simple_repeating_pattern() { - assert!(check_posix_regex_errors(r"\(a+b\)\{4\}").is_ok()); - } - - #[test] - fn check_regex_missing_closing() { - assert_eq!( - check_posix_regex_errors(r"\(abc"), - Err(ExprError::UnmatchedOpeningParenthesis) - ); - } - - #[test] - fn check_regex_missing_opening() { - assert_eq!( - check_posix_regex_errors(r"abc\)"), - Err(ExprError::UnmatchedClosingParenthesis) - ); - } - - #[test] - fn test_is_valid_range_quantifier() { - assert!(verify_range_quantifier(&"3\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"3,\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&",6\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"3,6\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&",\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"32767\\}anything".chars()).is_ok()); - assert_eq!( - verify_range_quantifier(&"\\{3,6\\}".chars()), - Err(ExprError::InvalidBracketContent) - ); - assert_eq!( - verify_range_quantifier(&"\\}".chars()), - Err(ExprError::InvalidBracketContent) - ); - assert_eq!( - verify_range_quantifier(&"".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3,".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&",6".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3,6".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&",".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"32768\\}".chars()), - Err(ExprError::TooBigRangeQuantifierIndex) - ); - } - #[test] fn test_evaluate_match_expression_basic() { use super::evaluate_match_expression; @@ -1573,23 +1248,29 @@ mod test { let result = evaluate_match_expression(b"hello".to_vec(), b"\\(hello".to_vec()); assert!(matches!( result, - Err(ExprError::UnmatchedOpeningParenthesis) + Err(ExprError::Regex(RegexError::UnmatchedOpeningParenthesis)) )); // Unmatched closing parenthesis let result = evaluate_match_expression(b"hello".to_vec(), b"hello\\)".to_vec()); assert!(matches!( result, - Err(ExprError::UnmatchedClosingParenthesis) + Err(ExprError::Regex(RegexError::UnmatchedClosingParenthesis)) )); // Trailing backslash let result = evaluate_match_expression(b"hello".to_vec(), b"hello\\".to_vec()); - assert!(matches!(result, Err(ExprError::TrailingBackslash))); + assert!(matches!( + result, + Err(ExprError::Regex(RegexError::TrailingBackslash)) + )); // Invalid bracket content let result = evaluate_match_expression(b"hello".to_vec(), b"a\\{invalid\\}".to_vec()); - assert!(matches!(result, Err(ExprError::InvalidBracketContent))); + assert!(matches!( + result, + Err(ExprError::Regex(RegexError::InvalidBracketContent)) + )); } #[test] @@ -1631,10 +1312,14 @@ mod test { use super::evaluate_match_expression; // This test verifies leftmost-longest (POSIX) match semantics. - // Pattern `(a|ab)` against `ab` should capture the longest alternative (`ab`) + // Pattern `\(a\|ab\)` against `ab` should capture the longest alternative (`ab`), // not the first (`a`). let result = evaluate_match_expression(b"ab".to_vec(), br"\(a\|ab\)".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"ab"); + + // `aaaaa\|a*` against `aaaaaa` should match all 6 characters + let result = evaluate_match_expression(b"aaaaaa".to_vec(), br"aaaaa\|a*".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"6"); } #[test] @@ -1657,6 +1342,14 @@ mod test { let result = evaluate_match_expression(b"b".to_vec(), br"b\>".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"1"); + // Beginning of buffer \` + let result = evaluate_match_expression(b"start".to_vec(), br"\`start".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"5"); + + // End of buffer \' + let result = evaluate_match_expression(b"end".to_vec(), br"end\'".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"3"); + // Escaped caret with quantifier \^* let result = evaluate_match_expression(b"^".to_vec(), br"\^*".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"1"); diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index 88a6ef9a2ba..d1e02aafc54 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -25,6 +25,11 @@ clap = { workspace = true } data-encoding = { workspace = true, optional = true } data-encoding-macro = { workspace = true, optional = true } dunce = { workspace = true, optional = true } +fancy-regex = { workspace = true, optional = true, features = [ + "std", + "unicode", + "leftmost_longest", +] } glob = { workspace = true, optional = true } itertools = { workspace = true, optional = true } jiff = { workspace = true, optional = true, features = [ @@ -182,6 +187,7 @@ process = ["libc", "nix/user", "rustix/process", "windows-sys"] proc-info = ["tty", "walkdir"] quoting-style = ["i18n-common"] ranges = [] +regex = ["dep:fancy-regex"] ringbuffer = [] safe-copy = [] safe-traversal = ["libc", "nix/fs", "nix/dir", "nix/user"] diff --git a/src/uucore/locales/errors/en-US.ftl b/src/uucore/locales/errors/en-US.ftl index 76d73436aeb..f5b6c3636ad 100644 --- a/src/uucore/locales/errors/en-US.ftl +++ b/src/uucore/locales/errors/en-US.ftl @@ -54,3 +54,13 @@ size-diag-help-syntax = a size is a number and an optional unit: K, M, G and so # in its own words. range-diag-label-too-large = this number is too large range-diag-label-inverted = this range ends before it starts + +# Regex parsing messages (expr, grep, sed, ...) +regex-error-unmatched-opening-parenthesis = Unmatched ( or \( +regex-error-unmatched-closing-parenthesis = Unmatched ) or \) +regex-error-trailing-backslash = Trailing backslash +regex-error-unmatched-opening-brace = Unmatched {"\\{"} +regex-error-invalid-bracket-content = Invalid content of {"\\{\\}"} +regex-error-too-big-range-quantifier-index = Regular expression too big +regex-error-invalid-character-class-name = Invalid character class name +regex-error-compilation-failed = Invalid regular expression: { $error } diff --git a/src/uucore/locales/errors/fr-FR.ftl b/src/uucore/locales/errors/fr-FR.ftl index 03b9eff60fe..8b3193e25b8 100644 --- a/src/uucore/locales/errors/fr-FR.ftl +++ b/src/uucore/locales/errors/fr-FR.ftl @@ -54,3 +54,13 @@ size-diag-help-syntax = une taille est un nombre suivi d'une unité facultative # compte, donc chaque utilitaire le dit avec ses propres mots. range-diag-label-too-large = ce nombre est trop grand range-diag-label-inverted = cet intervalle se termine avant de commencer + +# Messages d'analyse regex (expr, grep, sed, ...) +regex-error-unmatched-opening-parenthesis = Parenthèse ouvrante ( ou \( non appariée +regex-error-unmatched-closing-parenthesis = Parenthèse fermante ) ou \) non appariée +regex-error-trailing-backslash = Barre oblique inverse en fin +regex-error-unmatched-opening-brace = Accolade ouvrante {"\\{"} non appariée +regex-error-invalid-bracket-content = Contenu invalide de {"\\{\\}"} +regex-error-too-big-range-quantifier-index = Expression régulière trop grande +regex-error-invalid-character-class-name = Nom de classe de caractères invalide +regex-error-compilation-failed = Expression régulière invalide : { $error } diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index bfde5f93bf0..d6f4e559969 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -54,6 +54,8 @@ pub mod parser; pub mod quoting_style; #[cfg(feature = "ranges")] pub mod ranges; +#[cfg(feature = "regex")] +pub mod regex; #[cfg(feature = "ringbuffer")] pub mod ringbuffer; #[cfg(feature = "sum")] diff --git a/src/uucore/src/lib/features/regex.rs b/src/uucore/src/lib/features/regex.rs new file mode 100644 index 00000000000..9ddfbfa7381 --- /dev/null +++ b/src/uucore/src/lib/features/regex.rs @@ -0,0 +1,587 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Unified regular expression infrastructure for uutils. +//! +//! Provides a clean abstraction layer around [`fancy_regex`] with support for +//! POSIX Basic Regular Expressions (BRE) transpilation, GNU extensions, +//! and POSIX leftmost-longest matching semantics. + +use crate::translate; +pub use fancy_regex::{Regex, RegexBuilder}; + +/// Errors encountered when compiling or transpiling regular expressions. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum RegexError { + #[error("{}", translate!("regex-error-unmatched-opening-parenthesis"))] + UnmatchedOpeningParenthesis, + + #[error("{}", translate!("regex-error-unmatched-closing-parenthesis"))] + UnmatchedClosingParenthesis, + + #[error("{}", translate!("regex-error-trailing-backslash"))] + TrailingBackslash, + + #[error("{}", translate!("regex-error-unmatched-opening-brace"))] + UnmatchedOpeningBrace, + + #[error("{}", translate!("regex-error-invalid-bracket-content"))] + InvalidBracketContent, + + #[error("{}", translate!("regex-error-too-big-range-quantifier-index"))] + TooBigRangeQuantifierIndex, + + #[error("{}", translate!("regex-error-invalid-character-class-name"))] + InvalidCharacterClassName, + + #[error("{}", translate!("regex-error-compilation-failed", "error" => _0))] + CompilationFailed(String), +} + +impl crate::error::UError for RegexError { + fn code(&self) -> i32 { + 2 + } +} + +impl From for RegexError { + fn from(err: fancy_regex::Error) -> Self { + Self::CompilationFailed(err.to_string()) + } +} + +/// Map POSIX character class name (e.g. `"alpha"`, `"digit"`) to its Unicode property equivalent. +fn map_posix_class(name: &str) -> Option<&'static str> { + match name { + "alpha" => Some(r"\p{Alphabetic}"), + "lower" => Some(r"\p{Lowercase}"), + "upper" => Some(r"\p{Uppercase}"), + "alnum" => Some(r"\p{Alphabetic}0-9"), + "space" => Some(r"\p{White_Space}"), + "blank" => Some(r"\t\p{Zs}"), + "cntrl" => Some(r"\p{Control}"), + "digit" => Some("0-9"), + "xdigit" => Some("0-9A-Fa-f"), + "punct" => Some(r"\p{Punctuation}"), + "graph" => Some(r"\P{C}&&\P{Z}"), + "print" => Some(r"\P{C}"), + _ => None, + } +} + +/// Check if a regex character iterator is at the start of a valid range quantifier (`\{m,n\}`). +/// +/// The iterator's start position is expected to be immediately after the opening brace. +fn verify_range_quantifier(pattern_chars: &I) -> Result<(), RegexError> +where + I: Iterator + Clone, +{ + let mut pattern_chars_clone = pattern_chars.clone().peekable(); + if pattern_chars_clone.peek().is_none() { + return Err(RegexError::UnmatchedOpeningBrace); + } + + // Parse the string between braces + let mut quantifier = String::new(); + let mut prev = '\0'; + let mut curr_is_escaped = false; + while let Some(curr) = pattern_chars_clone.next() { + curr_is_escaped = prev == '\\' && !curr_is_escaped; + if curr_is_escaped && curr == '}' { + break; + } + if pattern_chars_clone.peek().is_none() { + return Err(RegexError::UnmatchedOpeningBrace); + } + if prev != '\0' { + quantifier.push(prev); + } + prev = curr; + } + + // Check if parsed quantifier is valid + let re = Regex::new(r"^([0-9]*,[0-9]*|[0-9]+)$").expect("valid regular expression"); + if let Ok(Some(captures)) = re.captures(&quantifier) { + let matched = captures.get(0).map_or("", |m| m.as_str()); + match matched.split_once(',') { + Some(("", "")) => Ok(()), + Some((x, "") | ("", x)) if x.parse::().is_ok() => Ok(()), + Some((_, "") | ("", _)) => Err(RegexError::TooBigRangeQuantifierIndex), + Some((f, l)) => match (f.parse::(), l.parse::()) { + (Ok(f), Ok(l)) if f > l => Err(RegexError::InvalidBracketContent), + (Ok(_), Ok(_)) => Ok(()), + _ => Err(RegexError::TooBigRangeQuantifierIndex), + }, + None if matched.parse::().is_ok() => Ok(()), + None => Err(RegexError::TooBigRangeQuantifierIndex), + } + } else { + Err(RegexError::InvalidBracketContent) + } +} + +/// Check for errors in a supplied regular expression +/// +/// GNU coreutils shows messages for invalid regular expressions +/// differently from standard regex engines. +/// This method attempts to do these checks manually in one pass +/// through the regular expression. +/// +/// This method is not comprehensively checking all cases in which +/// a regular expression could be invalid; any cases not caught will +/// fall through to `fancy-regex` compilation. This method is intended to +/// just identify a few situations for which GNU coreutils has specific +/// error messages. +fn check_posix_regex_errors(pattern: &str) -> Result<(), RegexError> { + let mut escaped_parens: u64 = 0; + let mut prev = '\0'; + let mut curr_is_escaped = false; + + for curr in pattern.chars() { + curr_is_escaped = prev == '\\' && !curr_is_escaped; + match (curr_is_escaped, curr) { + (true, '(') => escaped_parens += 1, + (true, ')') => { + escaped_parens = escaped_parens + .checked_sub(1) + .ok_or(RegexError::UnmatchedClosingParenthesis)?; + } + _ => {} + } + prev = curr; + } + + match escaped_parens { + 0 => Ok(()), + _ => Err(RegexError::UnmatchedOpeningParenthesis), + } +} + +/// Check if regex pattern character iterator is at the end of a regex expression or subexpression +fn is_end_of_expression(pattern_chars: &I) -> bool +where + I: Iterator + Clone, +{ + let mut pattern_chars_clone = pattern_chars.clone(); + match pattern_chars_clone.next() { + Some('\\') => matches!(pattern_chars_clone.next(), Some(')' | '|')), + None => true, // No characters left + _ => false, + } +} + +/// Transpile a POSIX Basic Regular Expression (BRE) into Extended Regular Expression (ERE) +/// compatible with `fancy-regex`. +/// +/// If `anchored` is `true`, the pattern is implicitly anchored at the start (`^`), as required by POSIX `expr`. +/// Otherwise, standard unanchored BRE transpilation is performed (e.g. for `grep` and `sed`). +pub fn bre_to_ere(pattern_str: &str, anchored: bool) -> Result { + check_posix_regex_errors(pattern_str)?; + + let mut re_string = String::with_capacity(pattern_str.len() + 8); + let mut pattern_chars = pattern_str.chars().peekable(); + let mut prev = '\0'; + let mut prev_is_escaped = false; + let mut in_bracket = false; + let mut is_start_of_expression = true; + let mut after_anchor_caret = false; + + if anchored && pattern_chars.peek() != Some(&'^') { + re_string.push('^'); + } + + while let Some(curr) = pattern_chars.next() { + let curr_is_escaped = prev == '\\' && !prev_is_escaped; + let mut next_is_start = false; + let mut next_after_anchor = false; + + if in_bracket { + if curr == '[' && pattern_chars.peek() == Some(&':') { + pattern_chars.next(); + let mut name = String::new(); + let mut closed = false; + while let Some(c) = pattern_chars.next() { + if c == ':' && pattern_chars.peek() == Some(&']') { + pattern_chars.next(); + closed = true; + break; + } + name.push(c); + } + if closed { + if let Some(unicode_class) = map_posix_class(&name) { + re_string.push_str(unicode_class); + prev = ']'; + prev_is_escaped = false; + continue; + } + return Err(RegexError::InvalidCharacterClassName); + } + re_string.push_str("[:"); + re_string.push_str(&name); + prev = name.chars().last().unwrap_or(':'); + prev_is_escaped = false; + continue; + } + if curr == ']' && re_string.ends_with(|c| c != '\\' && c != '[' && c != '^') { + in_bracket = false; + } + re_string.push(curr); + prev = curr; + prev_is_escaped = false; + continue; + } + + match curr { + '[' if !curr_is_escaped => { + in_bracket = true; + re_string.push('['); + } + // In BRE, '(', ')', '|', '+', '?', '{', '}' are literal by default, + // and become operators only when escaped. ERE has the exact opposite convention. + '(' | ')' | '|' | '+' | '?' | '{' | '}' => { + if curr_is_escaped { + if re_string.ends_with('\\') { + re_string.pop(); + } + match curr { + '(' | '|' => { + re_string.push(curr); + next_is_start = true; + } + '+' | '?' => { + if is_start_of_expression || after_anchor_caret { + re_string.push('\\'); + } + re_string.push(curr); + } + '{' => { + // Handle '{' literally at the start of an expression + if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\{"); + } else { + // Check if the following section is a valid range quantifier + verify_range_quantifier(&pattern_chars)?; + re_string.push('{'); + // Set the lower bound of range quantifier to 0 if it is missing + if pattern_chars.peek() == Some(&',') { + re_string.push('0'); + } + } + } + _ => re_string.push(curr), // ')' and '}' + } + } else { + // Unescaped metacharacter in BRE -> literal in ERE + re_string.push('\\'); + re_string.push(curr); + } + } + '*' => { + if curr_is_escaped { + re_string.push('*'); + } else if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\*"); + } else { + re_string.push('*'); + } + } + // Character class negation "[^a]" + // Explicitly escaped caret "\^" + '^' => { + if curr_is_escaped { + re_string.push('^'); + } else if is_start_of_expression { + re_string.push('^'); + next_after_anchor = true; + } else if prev == '[' && !prev_is_escaped { + re_string.push('^'); + } else { + re_string.push_str(r"\^"); + } + } + '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { + re_string.push_str(r"\$"); + } + '`' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\A"); + } + '\'' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\z"); + } + '<' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\b(?=\w)"); + } + '>' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\b(?<=\w)"); + } + '\\' if !curr_is_escaped => { + if pattern_chars.peek().is_none() { + return Err(RegexError::TrailingBackslash); + } + // Carry the expression-start / after-anchor state over the + // backslash so the escaped character is still treated as the + // first token of a (sub)expression. + next_is_start = is_start_of_expression; + next_after_anchor = after_anchor_caret; + re_string.push('\\'); + } + _ => { + if curr_is_escaped + && !"123456789.*^$[]\\wWsSbB".contains(curr) + && re_string.ends_with('\\') + { + re_string.pop(); + } + re_string.push(curr); + } + } + + is_start_of_expression = next_is_start; + after_anchor_caret = next_after_anchor; + prev_is_escaped = curr_is_escaped; + prev = curr; + } + Ok(re_string) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bre_groups() { + assert_eq!(bre_to_ere(r"foo\(bar\)baz", false).unwrap(), "foo(bar)baz"); + } + + #[test] + fn test_bre_intervals() { + assert_eq!(bre_to_ere(r"ab\{1,3\}c", false).unwrap(), "ab{1,3}c"); + assert_eq!(bre_to_ere(r"ab\{,3\}c", false).unwrap(), "ab{0,3}c"); + } + + #[test] + fn test_bre_alternation() { + assert_eq!(bre_to_ere(r"foo\|bar", false).unwrap(), "foo|bar"); + } + + #[test] + fn test_bre_literal_specials() { + assert_eq!( + bre_to_ere("a+b?c|d(e)f{g}", false).unwrap(), + r"a\+b\?c\|d\(e\)f\{g\}" + ); + } + + #[test] + fn test_bre_anchors() { + assert_eq!(bre_to_ere("^foo$", false).unwrap(), "^foo$"); + assert_eq!(bre_to_ere("a^b$c", false).unwrap(), r"a\^b\$c"); + } + + #[test] + fn test_bre_word_boundaries() { + assert_eq!( + bre_to_ere(r"\", false).unwrap(), + r"\b(?=\w)word\b(?<=\w)" + ); + } + + #[test] + fn test_bre_buffer_anchors() { + assert_eq!(bre_to_ere(r"\`start", false).unwrap(), r"\Astart"); + assert_eq!(bre_to_ere(r"end\'", false).unwrap(), r"end\z"); + } + + #[test] + fn test_bre_posix_classes() { + assert_eq!( + bre_to_ere(r"[[:alpha:]]", false).unwrap(), + r"[\p{Alphabetic}]" + ); + assert_eq!(bre_to_ere(r"[[:digit:]]", false).unwrap(), r"[0-9]"); + assert_eq!( + bre_to_ere(r"[^[:lower:]]", false).unwrap(), + r"[^\p{Lowercase}]" + ); + assert_eq!( + bre_to_ere(r"[[:alpha:][:digit:]]", false).unwrap(), + r"[\p{Alphabetic}0-9]" + ); + assert_eq!(bre_to_ere(r"[]a]", false).unwrap(), "[]a]"); + assert_eq!(bre_to_ere(r"[^]a]", false).unwrap(), "[^]a]"); + + // Unicode match test: [[:alpha:]] matches 'é' + let transpiled = bre_to_ere(r"[[:alpha:]]", true).unwrap(); + let re = Regex::new(&format!("(?s){transpiled}")).unwrap(); + assert!(re.is_match("é").unwrap()); + } + + #[test] + fn test_bre_invalid_posix_class() { + assert_eq!( + bre_to_ere(r"[[:bogus:]]", false).unwrap_err(), + RegexError::InvalidCharacterClassName + ); + assert_eq!( + bre_to_ere(r"[[:123:]]", false).unwrap_err(), + RegexError::InvalidCharacterClassName + ); + } + + #[test] + fn test_bre_backrefs() { + assert_eq!(bre_to_ere(r"\(foo\)\1", false).unwrap(), r"(foo)\1"); + } + + #[test] + fn test_bre_leading_quantifier() { + assert_eq!(bre_to_ere("*foo", false).unwrap(), r"\*foo"); + assert_eq!(bre_to_ere(r"\(*foo\)", false).unwrap(), r"(\*foo)"); + assert_eq!(bre_to_ere(r"a\|*b", false).unwrap(), r"a|\*b"); + } + + #[test] + fn test_bre_anchored() { + assert_eq!(bre_to_ere("abc", true).unwrap(), "^abc"); + assert_eq!(bre_to_ere("^abc", true).unwrap(), "^abc"); + assert_eq!(bre_to_ere(r"\(foo\)", true).unwrap(), "^(foo)"); + } + + #[test] + fn test_posix_errors() { + assert_eq!( + bre_to_ere(r"\(foo", false).unwrap_err(), + RegexError::UnmatchedOpeningParenthesis + ); + assert_eq!( + bre_to_ere(r"foo\)", false).unwrap_err(), + RegexError::UnmatchedClosingParenthesis + ); + assert_eq!( + bre_to_ere(r"foo\", false).unwrap_err(), + RegexError::TrailingBackslash + ); + assert_eq!( + bre_to_ere(r"foo\{1", false).unwrap_err(), + RegexError::UnmatchedOpeningBrace + ); + assert_eq!( + bre_to_ere(r"foo\{5,2\}", false).unwrap_err(), + RegexError::InvalidBracketContent + ); + } + + #[test] + fn test_leftmost_longest_semantics() { + // POSIX leftmost-longest requires (a|ab) against "ab" to match "ab" (longest), not "a". + let transpiled = bre_to_ere(r"\(a\|ab\)", false).unwrap(); + let re = RegexBuilder::new(&transpiled) + .oniguruma_mode(true) + .leftmost_longest(true) + .seek(true) + .build() + .unwrap(); + + let caps = re.captures("ab").unwrap().expect("should match"); + assert_eq!(caps.get(1).unwrap().as_str(), "ab"); + + // "aaaaa|a*" against "aaaaaa" should match all 6 "a"s + let transpiled2 = bre_to_ere(r"aaaaa\|a*", false).unwrap(); + let re2 = RegexBuilder::new(&transpiled2) + .oniguruma_mode(true) + .leftmost_longest(true) + .seek(true) + .build() + .unwrap(); + + let m = re2.find("aaaaaa").unwrap().expect("should match"); + assert_eq!(m.as_str(), "aaaaaa"); + } + + #[test] + fn check_regex_valid() { + assert!(check_posix_regex_errors(r"(a+b) \(a* b\)").is_ok()); + } + + #[test] + fn check_regex_simple_repeating_pattern() { + assert!(check_posix_regex_errors(r"\(a+b\)\{4\}").is_ok()); + } + + #[test] + fn check_regex_missing_closing() { + assert_eq!( + check_posix_regex_errors(r"\(abc"), + Err(RegexError::UnmatchedOpeningParenthesis) + ); + } + + #[test] + fn check_regex_missing_opening() { + assert_eq!( + check_posix_regex_errors(r"abc\)"), + Err(RegexError::UnmatchedClosingParenthesis) + ); + } + + #[test] + fn test_is_valid_range_quantifier() { + assert!(verify_range_quantifier(&"3\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"3,\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&",6\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"3,6\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&",\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"32767\\}anything".chars()).is_ok()); + assert_eq!( + verify_range_quantifier(&"\\{3,6\\}".chars()), + Err(RegexError::InvalidBracketContent) + ); + assert_eq!( + verify_range_quantifier(&"\\}".chars()), + Err(RegexError::InvalidBracketContent) + ); + assert_eq!( + verify_range_quantifier(&"".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3,".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&",6".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3,6".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&",".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"32768\\}".chars()), + Err(RegexError::TooBigRangeQuantifierIndex) + ); + } +} diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index d4c703d081a..068b5db39a4 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -73,6 +73,8 @@ pub use crate::features::parser; pub use crate::features::quoting_style; #[cfg(feature = "ranges")] pub use crate::features::ranges; +#[cfg(feature = "regex")] +pub use crate::features::regex; #[cfg(feature = "ringbuffer")] pub use crate::features::ringbuffer; #[cfg(feature = "sum")] diff --git a/tests/by-util/test_expr.rs b/tests/by-util/test_expr.rs index 21de966cda6..282986725e3 100644 --- a/tests/by-util/test_expr.rs +++ b/tests/by-util/test_expr.rs @@ -504,6 +504,47 @@ fn test_regex_leftmost_longest_match_semantics() { .args(&["ab", ":", "a\\|ab"]) .succeeds() .stdout_only("2\n"); + + new_ucmd!() + .args(&["aaaaaa", ":", "aaaaa\\|a*"]) + .succeeds() + .stdout_only("6\n"); + + new_ucmd!() + .args(&["ab", ":", "\\(a\\|ab\\)"]) + .succeeds() + .stdout_only("ab\n"); + + new_ucmd!() + .args(&["aaaaaa", ":", "\\(aaaaa\\|a*\\)"]) + .succeeds() + .stdout_only("aaaaaa\n"); +} + +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data, every locale is C")] +fn test_regex_posix_character_classes() { + new_ucmd!() + .args(&["é", ":", "[[:alpha:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["é", ":", "[[:lower:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["É", ":", "[[:upper:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .args(&["a", ":", "[[:bogus:]]"]) + .fails_with_code(2) + .stderr_contains("Invalid character class name"); } #[test]