Add not contains, matches & not matches to perspective-server and Virtual Servers

docs/md/explanation/view/config/selection_and_ordering.md

@@ -127,7 +127,14 @@ let view = table.view(Some(ViewConfigUpdate {
 The available filter operators depend on the column type:
 
 **String columns**: `==`, `!=`, `>`, `>=`, `<`, `<=`, `begins with`,
-`contains`, `ends with`, `in`, `not in`, `is not null`, `is null`.
+`not begins with`, `contains`, `not contains`, `ends with`, `not ends with`,
+`matches`, `not matches`, `in`, `not in`, `is not null`, `is null`.
+
+The string matching operators (`begins with`, `contains`, `ends with` and
+their negations) are case-insensitive, and `matches` / `not matches` are
+case-sensitive partial-match [RE2](https://github.com/google/re2) regular
+expressions. Null cells match none of these operators, including the negated
+forms - filter on `is null` to select them.
 
 **Numeric columns** (`integer`, `float`): `==`, `!=`, `>`, `>=`, `<`, `<=`,
 `is not null`, `is null`.

rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs

@@ -81,6 +81,27 @@ pub struct GenericSQLVirtualServerModelArgs {
     /// column-path separator is `"|"`, so any other value produces views the
     /// client will not interpret as column paths.
     column_separator: Option<String>,
+
+    /// Escape character emitted as an `ESCAPE` clause after generated
+    /// `ILIKE` patterns (Perspective's `begins with` / `contains` /
+    /// `ends with` filter ops and their negations). Dialects with no
+    /// default `LIKE` escape character (DuckDB) must pass `"\\"`; dialects
+    /// where backslash escaping is implicit and the `ESCAPE` clause is
+    /// unsupported (ClickHouse) must omit it.
+    like_escape_clause: Option<String>,
+
+    /// Whether the dialect's string literal parser consumes C-style
+    /// backslash escapes (ClickHouse), requiring backslashes in emitted
+    /// literals to be doubled. Dialects with standard-conforming literals
+    /// (DuckDB) omit it.
+    backslash_escaped_literals: Option<bool>,
+
+    /// Name of the dialect's partial-match regex function, emitted as
+    /// `{regex_fn}("col", 'pattern')` for the `matches` / `not matches`
+    /// filter ops — `"regexp_matches"` for DuckDB, `"match"` for
+    /// ClickHouse (both RE2, matching the engine's semantics). When
+    /// omitted, regex filter clauses are dropped.
+    regex_fn: Option<String>,
 }
 
 /// Recovers the source column of a pivoted view column name — the longest
@@ -369,11 +390,14 @@ impl GenericSQLVirtualServerModel {
         ))
     }
 
-    fn filter_term_to_sql(term: &FilterTerm) -> Option<String> {
+    fn filter_term_to_sql(term: &FilterTerm, backslash_escaped: bool) -> Option<String> {
         match term {
-            FilterTerm::Scalar(scalar) => Self::scalar_to_sql(scalar),
+            FilterTerm::Scalar(scalar) => Self::scalar_to_sql(scalar, backslash_escaped),
             FilterTerm::Array(scalars) => {
-                let values: Vec<String> = scalars.iter().filter_map(Self::scalar_to_sql).collect();
+                let values: Vec<String> = scalars
+                    .iter()
+                    .filter_map(|x| Self::scalar_to_sql(x, backslash_escaped))
+                    .collect();
                 if values.is_empty() {
                     None
                 } else {
@@ -383,12 +407,12 @@ impl GenericSQLVirtualServerModel {
         }
     }
 
-    fn scalar_to_sql(scalar: &Scalar) -> Option<String> {
+    fn scalar_to_sql(scalar: &Scalar, backslash_escaped: bool) -> Option<String> {
         match scalar {
             Scalar::Null => None,
             Scalar::Bool(b) => Some(if *b { "TRUE" } else { "FALSE" }.to_string()),
             Scalar::Float(f) => Some(f.to_string()),
-            Scalar::String(s) => Some(format!("'{}'", s.replace('\'', "''"))),
+            Scalar::String(s) => Some(table_make_view::string_literal(s, backslash_escaped)),
         }
     }
 }

rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs

@@ -12,8 +12,8 @@
 
 use super::GenericSQLError;
 use crate::config::{
-    Aggregate, GroupRollupMode, Sort, SortDir, ViewConfig, WindowAggregate, WindowFrame,
-    WindowSortDir, WindowSpec,
+    Aggregate, Filter, FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, ViewConfig,
+    WindowAggregate, WindowFrame, WindowSortDir, WindowSpec,
 };
 
 fn aggregate_to_string(agg: &Aggregate) -> String {
@@ -209,6 +209,34 @@ fn quote_literal(value: &str) -> String {
     value.replace('\'', "''")
 }
 
+/// Encodes a string as a quoted SQL literal. `backslash_escaped` doubles
+/// backslashes for dialects whose literal parser consumes C-style escapes
+/// (ClickHouse); standard-conforming dialects (DuckDB) leave them intact.
+pub(super) fn string_literal(value: &str, backslash_escaped: bool) -> String {
+    let value = if backslash_escaped {
+        value.replace('\\', "\\\\")
+    } else {
+        value.to_string()
+    };
+
+    format!("'{}'", quote_literal(&value))
+}
+
+/// Backslash-escapes the `LIKE` pattern metacharacters (`\`, `%`, `_`) so a
+/// filter term matches literally inside a generated `ILIKE` pattern.
+fn like_escape(term: &str) -> String {
+    let mut out = String::with_capacity(term.len());
+    for c in term.chars() {
+        if matches!(c, '\\' | '%' | '_') {
+            out.push('\\');
+        }
+
+        out.push(c);
+    }
+
+    out
+}
+
 /// Precomputed context for building a SQL view query from a [`ViewConfig`].
 ///
 /// Holds the resolved column names, grouping function, and row-path aliases
@@ -220,6 +248,9 @@ pub(crate) struct ViewQueryContext<'a> {
     group_col_names: Vec<String>,
     grouping_fn: &'a str,
     column_separator: &'a str,
+    like_escape_clause: Option<&'a str>,
+    backslash_escaped_literals: bool,
+    regex_fn: Option<&'a str>,
     row_path_aliases: Vec<String>,
 }
 
@@ -284,6 +315,9 @@ impl<'a> ViewQueryContext<'a> {
             group_col_names,
             grouping_fn,
             column_separator,
+            like_escape_clause: model.0.like_escape_clause.as_deref(),
+            backslash_escaped_literals: model.0.backslash_escaped_literals.unwrap_or(false),
+            regex_fn: model.0.regex_fn.as_deref(),
             row_path_aliases,
         })
     }
@@ -653,11 +687,7 @@ impl<'a> ViewQueryContext<'a> {
             .config
             .filter
             .iter()
-            .filter_map(|flt| {
-                super::GenericSQLVirtualServerModel::filter_term_to_sql(flt.term()).map(
-                    |term_lit| format!("{} {} {}", self.col_name(flt.column()), flt.op(), term_lit),
-                )
-            })
+            .filter_map(|flt| self.filter_clause_sql(flt))
             .collect();
 
         if clauses.is_empty() {
@@ -667,6 +697,76 @@ impl<'a> ViewQueryContext<'a> {
         }
     }
 
+    /// Translates one filter into a SQL `WHERE` clause, or `None` when it
+    /// cannot be expressed (missing term, regex with no `regex_fn`).
+    fn filter_clause_sql(&self, flt: &Filter) -> Option<String> {
+        let col = self.col_name(flt.column());
+        let op = flt.op();
+        match op {
+            "is null" => Some(format!("{col} IS NULL")),
+            "is not null" => Some(format!("{col} IS NOT NULL")),
+            "begins with" | "not begins with" | "ends with" | "not ends with" | "contains"
+            | "not contains" => {
+                let FilterTerm::Scalar(Scalar::String(term)) = flt.term() else {
+                    return None;
+                };
+
+                let term = like_escape(term);
+                let pattern = match op {
+                    "begins with" | "not begins with" => format!("{term}%"),
+                    "ends with" | "not ends with" => format!("%{term}"),
+                    _ => format!("%{term}%"),
+                };
+
+                let not = if op.starts_with("not ") { "NOT " } else { "" };
+                let escape = self
+                    .like_escape_clause
+                    .map(|c| {
+                        format!(
+                            " ESCAPE {}",
+                            string_literal(c, self.backslash_escaped_literals)
+                        )
+                    })
+                    .unwrap_or_default();
+
+                Some(format!(
+                    "{col} {not}ILIKE {}{escape}",
+                    string_literal(&pattern, self.backslash_escaped_literals)
+                ))
+            },
+            "matches" | "not matches" => {
+                let regex_fn = self.regex_fn?;
+                let FilterTerm::Scalar(Scalar::String(term)) = flt.term() else {
+                    return None;
+                };
+
+                let not = if op == "not matches" { "NOT " } else { "" };
+                Some(format!(
+                    "{not}{regex_fn}({col}, {})",
+                    string_literal(term, self.backslash_escaped_literals)
+                ))
+            },
+            "in" | "not in" => {
+                let term = super::GenericSQLVirtualServerModel::filter_term_to_sql(
+                    flt.term(),
+                    self.backslash_escaped_literals,
+                )?;
+
+                let sql_op = if op == "in" { "IN" } else { "NOT IN" };
+                Some(format!("{col} {sql_op} {term}"))
+            },
+            op => {
+                let term = super::GenericSQLVirtualServerModel::filter_term_to_sql(
+                    flt.term(),
+                    self.backslash_escaped_literals,
+                )?;
+
+                let sql_op = if op == "==" { "=" } else { op };
+                Some(format!("{col} {sql_op} {term}"))
+            },
+        }
+    }
+
     /// Builds the `ORDER BY` expression for the `ROW_NUMBER()` window
     /// function used inside `PIVOT` queries. Uses sort config if available,
     /// otherwise falls back to `rowid`.

rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs

@@ -654,9 +654,8 @@ fn test_table_make_view_pivoted_column_paths() {
 #[test]
 fn test_table_make_view_pivoted_custom_separator() {
     let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs {
-        create_entity: None,
-        grouping_fn: None,
         column_separator: Some("::".to_string()),
+        ..Default::default()
     });
 
     let mut config = ViewConfig::default();
@@ -1084,3 +1083,33 @@ fn test_table_make_view_window_ema_unsupported() {
         Err(GenericSQLError::UnsupportedOperation(_))
     ));
 }
+
+fn filters(json: serde_json::Value) -> Vec<crate::config::Filter> {
+    serde_json::from_value(json).unwrap()
+}
+
+fn duckdb_args() -> GenericSQLVirtualServerModelArgs {
+    GenericSQLVirtualServerModelArgs {
+        like_escape_clause: Some("\\".to_string()),
+        regex_fn: Some("regexp_matches".to_string()),
+        ..Default::default()
+    }
+}
+
+fn clickhouse_args() -> GenericSQLVirtualServerModelArgs {
+    GenericSQLVirtualServerModelArgs {
+        backslash_escaped_literals: Some(true),
+        regex_fn: Some("match".to_string()),
+        ..Default::default()
+    }
+}
+
+fn filter_sql(args: GenericSQLVirtualServerModelArgs, filter: serde_json::Value) -> String {
+    let builder = GenericSQLVirtualServerModel::new(args);
+    let mut config = ViewConfig::default();
+    config.columns = vec![Some("a".to_string())];
+    config.filter = filters(filter);
+    builder
+        .table_make_view("source_table", "dest_view", &config)
+        .unwrap()
+}

rust/perspective-js/src/ts/virtual_servers/clickhouse.ts

@@ -92,13 +92,34 @@ const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [
 const FILTER_OPS = [
     "==",
     "!=",
-    "LIKE",
     "IS DISTINCT FROM",
     "IS NOT DISTINCT FROM",
     ">=",
     "<=",
     ">",
     "<",
+    "is null",
+    "is not null",
+];
+
+// Perspective's canonical string ops (translated to `ILIKE` / `match` by the
+// SQL builder), plus ClickHouse's raw infix pattern ops spliced verbatim.
+const STRING_FILTER_OPS = [
+    ...FILTER_OPS,
+    "begins with",
+    "not begins with",
+    "contains",
+    "not contains",
+    "ends with",
+    "not ends with",
+    "matches",
+    "not matches",
+    "in",
+    "not in",
+    "LIKE",
+    "NOT LIKE",
+    "ILIKE",
+    "NOT ILIKE",
 ];
 
 function duckdbTypeToPsp(name: string): ColumnType {
@@ -241,6 +262,8 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler {
             create_entity: "VIEW",
             grouping_fn: "GROUPING",
             column_separator: "|",
+            backslash_escaped_literals: true,
+            regex_fn: "match",
         });
     }
 
@@ -266,7 +289,7 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler {
             filter_ops: {
                 integer: FILTER_OPS,
                 float: FILTER_OPS,
-                string: FILTER_OPS,
+                string: STRING_FILTER_OPS,
                 boolean: FILTER_OPS,
                 date: FILTER_OPS,
                 datetime: FILTER_OPS,

rust/perspective-js/src/ts/virtual_servers/duckdb.ts

@@ -93,13 +93,34 @@ const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [
 const FILTER_OPS = [
     "==",
     "!=",
-    "LIKE",
     "IS DISTINCT FROM",
     "IS NOT DISTINCT FROM",
     ">=",
     "<=",
     ">",
     "<",
+    "is null",
+    "is not null",
+];
+
+// Perspective's canonical string ops (translated to `ILIKE` / `regexp_matches`
+// by the SQL builder), plus DuckDB's raw infix pattern ops spliced verbatim.
+const STRING_FILTER_OPS = [
+    ...FILTER_OPS,
+    "begins with",
+    "not begins with",
+    "contains",
+    "not contains",
+    "ends with",
+    "not ends with",
+    "matches",
+    "not matches",
+    "in",
+    "not in",
+    "LIKE",
+    "NOT LIKE",
+    "ILIKE",
+    "NOT ILIKE",
 ];
 
 function duckdbTypeToPsp(name: string): ColumnType {
@@ -213,6 +234,8 @@ export class DuckDBHandler implements perspective.VirtualServerHandler {
         this.db = db;
         this.sqlBuilder = new mod!.GenericSQLVirtualServerModel({
             column_separator: "|",
+            like_escape_clause: "\\",
+            regex_fn: "regexp_matches",
         });
     }
 
@@ -235,7 +258,7 @@ export class DuckDBHandler implements perspective.VirtualServerHandler {
             filter_ops: {
                 integer: FILTER_OPS,
                 float: FILTER_OPS,
-                string: FILTER_OPS,
+                string: STRING_FILTER_OPS,
                 boolean: FILTER_OPS,
                 date: FILTER_OPS,
                 datetime: FILTER_OPS,