diff --git a/docs/md/explanation/view/config/selection_and_ordering.md b/docs/md/explanation/view/config/selection_and_ordering.md index 6fb20c66b9..32bc7f45d2 100644 --- a/docs/md/explanation/view/config/selection_and_ordering.md +++ b/docs/md/explanation/view/config/selection_and_ordering.md @@ -127,7 +127,14 @@ let view = table.view(Some(ViewConfigUpdate { The available filter operators depend on the column type: **String columns**: `==`, `!=`, `>`, `>=`, `<`, `<=`, `begins with`, -`contains`, `ends with`, `in`, `not in`, `is not null`, `is null`. +`not begins with`, `contains`, `not contains`, `ends with`, `not ends with`, +`matches`, `not matches`, `in`, `not in`, `is not null`, `is null`. + +The string matching operators (`begins with`, `contains`, `ends with` and +their negations) are case-insensitive, and `matches` / `not matches` are +case-sensitive partial-match [RE2](https://github.com/google/re2) regular +expressions. Null cells match none of these operators, including the negated +forms - filter on `is null` to select them. **Numeric columns** (`integer`, `float`): `==`, `!=`, `>`, `>=`, `<`, `<=`, `is not null`, `is null`. diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs index b2c0019ab7..e3949feade 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs @@ -81,6 +81,27 @@ pub struct GenericSQLVirtualServerModelArgs { /// column-path separator is `"|"`, so any other value produces views the /// client will not interpret as column paths. column_separator: Option, + + /// Escape character emitted as an `ESCAPE` clause after generated + /// `ILIKE` patterns (Perspective's `begins with` / `contains` / + /// `ends with` filter ops and their negations). Dialects with no + /// default `LIKE` escape character (DuckDB) must pass `"\\"`; dialects + /// where backslash escaping is implicit and the `ESCAPE` clause is + /// unsupported (ClickHouse) must omit it. + like_escape_clause: Option, + + /// Whether the dialect's string literal parser consumes C-style + /// backslash escapes (ClickHouse), requiring backslashes in emitted + /// literals to be doubled. Dialects with standard-conforming literals + /// (DuckDB) omit it. + backslash_escaped_literals: Option, + + /// Name of the dialect's partial-match regex function, emitted as + /// `{regex_fn}("col", 'pattern')` for the `matches` / `not matches` + /// filter ops — `"regexp_matches"` for DuckDB, `"match"` for + /// ClickHouse (both RE2, matching the engine's semantics). When + /// omitted, regex filter clauses are dropped. + regex_fn: Option, } /// Recovers the source column of a pivoted view column name — the longest @@ -369,11 +390,14 @@ impl GenericSQLVirtualServerModel { )) } - fn filter_term_to_sql(term: &FilterTerm) -> Option { + fn filter_term_to_sql(term: &FilterTerm, backslash_escaped: bool) -> Option { match term { - FilterTerm::Scalar(scalar) => Self::scalar_to_sql(scalar), + FilterTerm::Scalar(scalar) => Self::scalar_to_sql(scalar, backslash_escaped), FilterTerm::Array(scalars) => { - let values: Vec = scalars.iter().filter_map(Self::scalar_to_sql).collect(); + let values: Vec = scalars + .iter() + .filter_map(|x| Self::scalar_to_sql(x, backslash_escaped)) + .collect(); if values.is_empty() { None } else { @@ -383,12 +407,12 @@ impl GenericSQLVirtualServerModel { } } - fn scalar_to_sql(scalar: &Scalar) -> Option { + fn scalar_to_sql(scalar: &Scalar, backslash_escaped: bool) -> Option { match scalar { Scalar::Null => None, Scalar::Bool(b) => Some(if *b { "TRUE" } else { "FALSE" }.to_string()), Scalar::Float(f) => Some(f.to_string()), - Scalar::String(s) => Some(format!("'{}'", s.replace('\'', "''"))), + Scalar::String(s) => Some(table_make_view::string_literal(s, backslash_escaped)), } } } diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs index a04ddbd644..e1281c9f4e 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs @@ -12,8 +12,8 @@ use super::GenericSQLError; use crate::config::{ - Aggregate, GroupRollupMode, Sort, SortDir, ViewConfig, WindowAggregate, WindowFrame, - WindowSortDir, WindowSpec, + Aggregate, Filter, FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, ViewConfig, + WindowAggregate, WindowFrame, WindowSortDir, WindowSpec, }; fn aggregate_to_string(agg: &Aggregate) -> String { @@ -209,6 +209,34 @@ fn quote_literal(value: &str) -> String { value.replace('\'', "''") } +/// Encodes a string as a quoted SQL literal. `backslash_escaped` doubles +/// backslashes for dialects whose literal parser consumes C-style escapes +/// (ClickHouse); standard-conforming dialects (DuckDB) leave them intact. +pub(super) fn string_literal(value: &str, backslash_escaped: bool) -> String { + let value = if backslash_escaped { + value.replace('\\', "\\\\") + } else { + value.to_string() + }; + + format!("'{}'", quote_literal(&value)) +} + +/// Backslash-escapes the `LIKE` pattern metacharacters (`\`, `%`, `_`) so a +/// filter term matches literally inside a generated `ILIKE` pattern. +fn like_escape(term: &str) -> String { + let mut out = String::with_capacity(term.len()); + for c in term.chars() { + if matches!(c, '\\' | '%' | '_') { + out.push('\\'); + } + + out.push(c); + } + + out +} + /// Precomputed context for building a SQL view query from a [`ViewConfig`]. /// /// Holds the resolved column names, grouping function, and row-path aliases @@ -220,6 +248,9 @@ pub(crate) struct ViewQueryContext<'a> { group_col_names: Vec, grouping_fn: &'a str, column_separator: &'a str, + like_escape_clause: Option<&'a str>, + backslash_escaped_literals: bool, + regex_fn: Option<&'a str>, row_path_aliases: Vec, } @@ -284,6 +315,9 @@ impl<'a> ViewQueryContext<'a> { group_col_names, grouping_fn, column_separator, + like_escape_clause: model.0.like_escape_clause.as_deref(), + backslash_escaped_literals: model.0.backslash_escaped_literals.unwrap_or(false), + regex_fn: model.0.regex_fn.as_deref(), row_path_aliases, }) } @@ -653,11 +687,7 @@ impl<'a> ViewQueryContext<'a> { .config .filter .iter() - .filter_map(|flt| { - super::GenericSQLVirtualServerModel::filter_term_to_sql(flt.term()).map( - |term_lit| format!("{} {} {}", self.col_name(flt.column()), flt.op(), term_lit), - ) - }) + .filter_map(|flt| self.filter_clause_sql(flt)) .collect(); if clauses.is_empty() { @@ -667,6 +697,76 @@ impl<'a> ViewQueryContext<'a> { } } + /// Translates one filter into a SQL `WHERE` clause, or `None` when it + /// cannot be expressed (missing term, regex with no `regex_fn`). + fn filter_clause_sql(&self, flt: &Filter) -> Option { + let col = self.col_name(flt.column()); + let op = flt.op(); + match op { + "is null" => Some(format!("{col} IS NULL")), + "is not null" => Some(format!("{col} IS NOT NULL")), + "begins with" | "not begins with" | "ends with" | "not ends with" | "contains" + | "not contains" => { + let FilterTerm::Scalar(Scalar::String(term)) = flt.term() else { + return None; + }; + + let term = like_escape(term); + let pattern = match op { + "begins with" | "not begins with" => format!("{term}%"), + "ends with" | "not ends with" => format!("%{term}"), + _ => format!("%{term}%"), + }; + + let not = if op.starts_with("not ") { "NOT " } else { "" }; + let escape = self + .like_escape_clause + .map(|c| { + format!( + " ESCAPE {}", + string_literal(c, self.backslash_escaped_literals) + ) + }) + .unwrap_or_default(); + + Some(format!( + "{col} {not}ILIKE {}{escape}", + string_literal(&pattern, self.backslash_escaped_literals) + )) + }, + "matches" | "not matches" => { + let regex_fn = self.regex_fn?; + let FilterTerm::Scalar(Scalar::String(term)) = flt.term() else { + return None; + }; + + let not = if op == "not matches" { "NOT " } else { "" }; + Some(format!( + "{not}{regex_fn}({col}, {})", + string_literal(term, self.backslash_escaped_literals) + )) + }, + "in" | "not in" => { + let term = super::GenericSQLVirtualServerModel::filter_term_to_sql( + flt.term(), + self.backslash_escaped_literals, + )?; + + let sql_op = if op == "in" { "IN" } else { "NOT IN" }; + Some(format!("{col} {sql_op} {term}")) + }, + op => { + let term = super::GenericSQLVirtualServerModel::filter_term_to_sql( + flt.term(), + self.backslash_escaped_literals, + )?; + + let sql_op = if op == "==" { "=" } else { op }; + Some(format!("{col} {sql_op} {term}")) + }, + } + } + /// Builds the `ORDER BY` expression for the `ROW_NUMBER()` window /// function used inside `PIVOT` queries. Uses sort config if available, /// otherwise falls back to `rowid`. diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs index c78a097669..974d3556eb 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs @@ -654,9 +654,8 @@ fn test_table_make_view_pivoted_column_paths() { #[test] fn test_table_make_view_pivoted_custom_separator() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs { - create_entity: None, - grouping_fn: None, column_separator: Some("::".to_string()), + ..Default::default() }); let mut config = ViewConfig::default(); @@ -1084,3 +1083,33 @@ fn test_table_make_view_window_ema_unsupported() { Err(GenericSQLError::UnsupportedOperation(_)) )); } + +fn filters(json: serde_json::Value) -> Vec { + serde_json::from_value(json).unwrap() +} + +fn duckdb_args() -> GenericSQLVirtualServerModelArgs { + GenericSQLVirtualServerModelArgs { + like_escape_clause: Some("\\".to_string()), + regex_fn: Some("regexp_matches".to_string()), + ..Default::default() + } +} + +fn clickhouse_args() -> GenericSQLVirtualServerModelArgs { + GenericSQLVirtualServerModelArgs { + backslash_escaped_literals: Some(true), + regex_fn: Some("match".to_string()), + ..Default::default() + } +} + +fn filter_sql(args: GenericSQLVirtualServerModelArgs, filter: serde_json::Value) -> String { + let builder = GenericSQLVirtualServerModel::new(args); + let mut config = ViewConfig::default(); + config.columns = vec![Some("a".to_string())]; + config.filter = filters(filter); + builder + .table_make_view("source_table", "dest_view", &config) + .unwrap() +} diff --git a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts index eb1da682be..f10be40025 100644 --- a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts +++ b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts @@ -92,13 +92,34 @@ const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [ const FILTER_OPS = [ "==", "!=", - "LIKE", "IS DISTINCT FROM", "IS NOT DISTINCT FROM", ">=", "<=", ">", "<", + "is null", + "is not null", +]; + +// Perspective's canonical string ops (translated to `ILIKE` / `match` by the +// SQL builder), plus ClickHouse's raw infix pattern ops spliced verbatim. +const STRING_FILTER_OPS = [ + ...FILTER_OPS, + "begins with", + "not begins with", + "contains", + "not contains", + "ends with", + "not ends with", + "matches", + "not matches", + "in", + "not in", + "LIKE", + "NOT LIKE", + "ILIKE", + "NOT ILIKE", ]; function duckdbTypeToPsp(name: string): ColumnType { @@ -241,6 +262,8 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler { create_entity: "VIEW", grouping_fn: "GROUPING", column_separator: "|", + backslash_escaped_literals: true, + regex_fn: "match", }); } @@ -266,7 +289,7 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler { filter_ops: { integer: FILTER_OPS, float: FILTER_OPS, - string: FILTER_OPS, + string: STRING_FILTER_OPS, boolean: FILTER_OPS, date: FILTER_OPS, datetime: FILTER_OPS, diff --git a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts index cabc35b543..82d3bfdbfe 100644 --- a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts +++ b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts @@ -93,13 +93,34 @@ const WINDOW_AGGREGATES_ANY: WindowAggregate[] = [ const FILTER_OPS = [ "==", "!=", - "LIKE", "IS DISTINCT FROM", "IS NOT DISTINCT FROM", ">=", "<=", ">", "<", + "is null", + "is not null", +]; + +// Perspective's canonical string ops (translated to `ILIKE` / `regexp_matches` +// by the SQL builder), plus DuckDB's raw infix pattern ops spliced verbatim. +const STRING_FILTER_OPS = [ + ...FILTER_OPS, + "begins with", + "not begins with", + "contains", + "not contains", + "ends with", + "not ends with", + "matches", + "not matches", + "in", + "not in", + "LIKE", + "NOT LIKE", + "ILIKE", + "NOT ILIKE", ]; function duckdbTypeToPsp(name: string): ColumnType { @@ -213,6 +234,8 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { this.db = db; this.sqlBuilder = new mod!.GenericSQLVirtualServerModel({ column_separator: "|", + like_escape_clause: "\\", + regex_fn: "regexp_matches", }); } @@ -235,7 +258,7 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { filter_ops: { integer: FILTER_OPS, float: FILTER_OPS, - string: FILTER_OPS, + string: STRING_FILTER_OPS, boolean: FILTER_OPS, date: FILTER_OPS, datetime: FILTER_OPS, diff --git a/rust/perspective-js/test/js/duckdb/filter.spec.js b/rust/perspective-js/test/js/duckdb/filter.spec.js index 902c436e72..d1bf4ed360 100644 --- a/rust/perspective-js/test/js/duckdb/filter.spec.js +++ b/rust/perspective-js/test/js/duckdb/filter.spec.js @@ -133,6 +133,125 @@ describeDuckDB("filter", (getClient) => { await view.delete(); }); + test("filter with begins with is case-insensitive", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["State"], + filter: [["State", "begins with", "cal"]], + }); + const json = await view.to_columns(); + expect(json["State"].length).toBeGreaterThan(0); + expect(new Set(json["State"])).toEqual(new Set(["California"])); + await view.delete(); + }); + + test("filter with negated string ops complements the positive ops", async function () { + const table = await getClient().open_table("memory.superstore"); + const total = await table.size(); + for (const [op, term] of [ + ["begins with", "new"], + ["ends with", "as"], + ["contains", "as"], + ]) { + const pos = await table.view({ + columns: ["State"], + filter: [["State", op, term]], + }); + const neg = await table.view({ + columns: ["State"], + filter: [["State", `not ${op}`, term]], + }); + const pos_rows = await pos.num_rows(); + const neg_rows = await neg.num_rows(); + expect(pos_rows).toBeGreaterThan(0); + expect(pos_rows + neg_rows).toEqual(total); + await pos.delete(); + await neg.delete(); + } + }); + + test("filter with not ends with", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["State"], + filter: [["State", "not ends with", "as"]], + }); + const json = await view.to_columns(); + const suffixes = new Set( + json["State"].map((x) => x.slice(-2).toLowerCase()), + ); + expect(json["State"].length).toBeGreaterThan(0); + expect(suffixes.has("as")).toBe(false); + await view.delete(); + }); + + test("filter with matches", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["State"], + filter: [["State", "matches", "^Cal"]], + }); + const json = await view.to_columns(); + expect(json["State"].length).toBeGreaterThan(0); + expect(new Set(json["State"])).toEqual(new Set(["California"])); + await view.delete(); + }); + + test("filter with not matches complements matches", async function () { + const table = await getClient().open_table("memory.superstore"); + const total = await table.size(); + const pos = await table.view({ + columns: ["State"], + filter: [["State", "matches", "as$"]], + }); + const neg = await table.view({ + columns: ["State"], + filter: [["State", "not matches", "as$"]], + }); + const pos_rows = await pos.num_rows(); + const neg_rows = await neg.num_rows(); + expect(pos_rows).toBeGreaterThan(0); + expect(pos_rows + neg_rows).toEqual(total); + await pos.delete(); + await neg.delete(); + }); + + test("filter with is null and is not null", async function () { + const table = await getClient().open_table("memory.superstore"); + const total = await table.size(); + const nulls = await table.view({ + columns: ["State"], + filter: [["State", "is null", null]], + }); + const not_nulls = await table.view({ + columns: ["State"], + filter: [["State", "is not null", null]], + }); + expect(await nulls.num_rows()).toEqual(0); + expect(await not_nulls.num_rows()).toEqual(total); + await nulls.delete(); + await not_nulls.delete(); + }); + + test("filter with in and not in", async function () { + const table = await getClient().open_table("memory.superstore"); + const total = await table.size(); + const pos = await table.view({ + columns: ["Region"], + filter: [["Region", "in", ["West", "East"]]], + }); + const neg = await table.view({ + columns: ["Region"], + filter: [["Region", "not in", ["West", "East"]]], + }); + const pos_rows = await pos.num_rows(); + const neg_rows = await neg.num_rows(); + expect(pos_rows).toBeGreaterThan(0); + expect(pos_rows + neg_rows).toEqual(total); + await pos.delete(); + await neg.delete(); + }); + test("multiple filters", async function () { const table = await getClient().open_table("memory.superstore"); const view = await table.view({ diff --git a/rust/perspective-js/test/js/filters.spec.js b/rust/perspective-js/test/js/filters.spec.js index 613cd9af85..dedda9308b 100644 --- a/rust/perspective-js/test/js/filters.spec.js +++ b/rust/perspective-js/test/js/filters.spec.js @@ -498,6 +498,102 @@ const datetime_data_local = [ }); }); + test.describe("negated string ops", function () { + const string_data = [ + { x: "Cat" }, + { x: "cathedral" }, + { x: "dog" }, + { x: null }, + ]; + + test("x not contains 'at' excludes matches and nulls", async function () { + const table = await perspective.table(string_data); + const view = await table.view({ + filter: [["x", "not contains", "at"]], + }); + expect(await view.to_columns()).toEqual({ x: ["dog"] }); + view.delete(); + table.delete(); + }); + + test("x not begins with 'cat' is case-insensitive", async function () { + const table = await perspective.table(string_data); + const view = await table.view({ + filter: [["x", "not begins with", "cat"]], + }); + expect(await view.to_columns()).toEqual({ x: ["dog"] }); + view.delete(); + table.delete(); + }); + + test("x not ends with 'at'", async function () { + const table = await perspective.table(string_data); + const view = await table.view({ + filter: [["x", "not ends with", "at"]], + }); + expect(await view.to_columns()).toEqual({ + x: ["cathedral", "dog"], + }); + view.delete(); + table.delete(); + }); + }); + + test.describe("matches", function () { + const string_data = [ + { x: "Cat" }, + { x: "cathedral" }, + { x: "dog" }, + { x: null }, + ]; + + test("x matches '^ca' is a case-sensitive partial match", async function () { + const table = await perspective.table(string_data); + const view = await table.view({ + filter: [["x", "matches", "^ca"]], + }); + expect(await view.to_columns()).toEqual({ x: ["cathedral"] }); + view.delete(); + table.delete(); + }); + + test("x matches character class", async function () { + const table = await perspective.table(string_data); + const view = await table.view({ + filter: [["x", "matches", "d[aeiou]g"]], + }); + expect(await view.to_columns()).toEqual({ x: ["dog"] }); + view.delete(); + table.delete(); + }); + + test("x not matches 'at' excludes matches and nulls", async function () { + const table = await perspective.table(string_data); + const view = await table.view({ + filter: [["x", "not matches", "at"]], + }); + expect(await view.to_columns()).toEqual({ x: ["dog"] }); + view.delete(); + table.delete(); + }); + + test("an invalid pattern matches nothing for both ops", async function () { + const table = await perspective.table(string_data); + const view = await table.view({ + filter: [["x", "matches", "["]], + }); + expect(await view.to_json()).toEqual([]); + view.delete(); + + const view2 = await table.view({ + filter: [["x", "not matches", "["]], + }); + expect(await view2.to_json()).toEqual([]); + view2.delete(); + table.delete(); + }); + }); + test.describe("Arrow types", function () { // https://github.com/perspective-dev/perspective/issues/2881 test("Arrow float32 filters", async function () { diff --git a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp index 1cc8314f17..0f7b145596 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp @@ -337,6 +337,21 @@ filter_op_to_str(t_filter_op op) { case FILTER_OP_CONTAINS: { return "contains"; } break; + case FILTER_OP_NOT_BEGINS_WITH: { + return "not startswith"; + } break; + case FILTER_OP_NOT_ENDS_WITH: { + return "not endswith"; + } break; + case FILTER_OP_NOT_CONTAINS: { + return "not contains"; + } break; + case FILTER_OP_MATCHES: { + return "matches"; + } break; + case FILTER_OP_NOT_MATCHES: { + return "not matches"; + } break; case FILTER_OP_OR: { return "or"; } break; @@ -386,12 +401,27 @@ str_to_filter_op(const std::string& str) { if (str == "ends with" || str == "endswith") { return t_filter_op::FILTER_OP_ENDS_WITH; } + if (str == "not begins with" || str == "not startswith") { + return t_filter_op::FILTER_OP_NOT_BEGINS_WITH; + } + if (str == "not ends with" || str == "not endswith") { + return t_filter_op::FILTER_OP_NOT_ENDS_WITH; + } if (str == "in") { return t_filter_op::FILTER_OP_IN; } if (str == "contains") { return t_filter_op::FILTER_OP_CONTAINS; } + if (str == "not contains") { + return t_filter_op::FILTER_OP_NOT_CONTAINS; + } + if (str == "matches") { + return t_filter_op::FILTER_OP_MATCHES; + } + if (str == "not matches") { + return t_filter_op::FILTER_OP_NOT_MATCHES; + } if (str == "not in") { return t_filter_op::FILTER_OP_NOT_IN; } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/filter.cpp b/rust/perspective-server/cpp/perspective/src/cpp/filter.cpp index 824f6bbdf0..ad03c33f05 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/filter.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/filter.cpp @@ -35,6 +35,7 @@ t_fterm::t_fterm( m_is_primary(is_primary) { m_use_interned = (op == FILTER_OP_EQ || op == FILTER_OP_NE) && threshold.m_type == DTYPE_STR; + compile_pattern(); } t_fterm::t_fterm( @@ -51,6 +52,16 @@ t_fterm::t_fterm( m_is_primary(false) { m_use_interned = (op == FILTER_OP_EQ || op == FILTER_OP_NE) && threshold.m_type == DTYPE_STR; + compile_pattern(); +} + +void +t_fterm::compile_pattern() { + if ((m_op == FILTER_OP_MATCHES || m_op == FILTER_OP_NOT_MATCHES) + && m_threshold.m_type == DTYPE_STR) { + m_pattern = + std::make_shared(m_threshold.to_string(), RE2::Quiet); + } } void @@ -74,7 +85,10 @@ t_fterm::get_expr() const { case FILTER_OP_GTEQ: case FILTER_OP_EQ: case FILTER_OP_NE: - case FILTER_OP_CONTAINS: { + case FILTER_OP_CONTAINS: + case FILTER_OP_NOT_CONTAINS: + case FILTER_OP_MATCHES: + case FILTER_OP_NOT_MATCHES: { ss << filter_op_to_str(m_op) << " "; ss << m_threshold.to_string(true); } break; @@ -87,7 +101,9 @@ t_fterm::get_expr() const { ss << " )"; } break; case FILTER_OP_BEGINS_WITH: - case FILTER_OP_ENDS_WITH: { + case FILTER_OP_ENDS_WITH: + case FILTER_OP_NOT_BEGINS_WITH: + case FILTER_OP_NOT_ENDS_WITH: { ss << "." << filter_op_to_str(m_op) << "( " << m_threshold.to_string(true) << " )"; } break; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp b/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp index ba0eea736f..b096eab424 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp @@ -1617,6 +1617,18 @@ t_tscalar::cmp(t_filter_op op, const t_tscalar& other) const { case FILTER_OP_CONTAINS: { return value.contains(other); } break; + case FILTER_OP_NOT_BEGINS_WITH: { + return m_status == STATUS_VALID && other.m_status == STATUS_VALID + && !value.begins_with(other); + } break; + case FILTER_OP_NOT_ENDS_WITH: { + return m_status == STATUS_VALID && other.m_status == STATUS_VALID + && !value.ends_with(other); + } break; + case FILTER_OP_NOT_CONTAINS: { + return m_status == STATUS_VALID && other.m_status == STATUS_VALID + && !value.contains(other); + } break; case FILTER_OP_IS_NULL: { return m_status != STATUS_VALID; } break; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index ecec9a9705..a103932587 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -1461,8 +1461,13 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { opts.add_options("<"); opts.add_options("<="); opts.add_options("begins with"); + opts.add_options("not begins with"); opts.add_options("contains"); + opts.add_options("not contains"); opts.add_options("ends with"); + opts.add_options("not ends with"); + opts.add_options("matches"); + opts.add_options("not matches"); opts.add_options("in"); opts.add_options("not in"); opts.add_options("is not null"); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/base.h b/rust/perspective-server/cpp/perspective/src/include/perspective/base.h index 0df6c435f3..127bc5c3c2 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/base.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/base.h @@ -227,6 +227,11 @@ enum t_filter_op { FILTER_OP_BEGINS_WITH, FILTER_OP_ENDS_WITH, FILTER_OP_CONTAINS, + FILTER_OP_NOT_BEGINS_WITH, + FILTER_OP_NOT_ENDS_WITH, + FILTER_OP_NOT_CONTAINS, + FILTER_OP_MATCHES, + FILTER_OP_NOT_MATCHES, FILTER_OP_OR, FILTER_OP_IN, FILTER_OP_NOT_IN, diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/filter.h b/rust/perspective-server/cpp/perspective/src/include/perspective/filter.h index b54d642a84..7f71bb597b 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/filter.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/filter.h @@ -18,7 +18,9 @@ #include #include #include +#include #include +#include #include namespace perspective { @@ -158,6 +160,16 @@ struct PERSPECTIVE_EXPORT t_fterm { case FILTER_OP_IN: { rv = std::find(m_bag.begin(), m_bag.end(), s) != m_bag.end(); } break; + case FILTER_OP_MATCHES: + case FILTER_OP_NOT_MATCHES: { + if (s.m_status != STATUS_VALID || s.m_type != DTYPE_STR + || !m_pattern || !m_pattern->ok()) { + rv = false; + } else { + bool match = RE2::PartialMatch(s.to_string(), *m_pattern); + rv = m_op == FILTER_OP_MATCHES ? match : !match; + } + } break; default: { rv = s.cmp(m_op, m_threshold); } break; @@ -170,10 +182,13 @@ struct PERSPECTIVE_EXPORT t_fterm { void coerce_numeric(t_dtype dtype); + void compile_pattern(); + std::string m_colname; t_filter_op m_op; t_tscalar m_threshold; std::vector m_bag; + std::shared_ptr m_pattern; bool m_negated; bool m_is_primary; bool m_use_interned;