Allow setnafill() to accept logical vectors in cols (#4113)

NEWS.md

@@ -36,6 +36,8 @@
 
 7. Rows can now be deleted by reference using `DT[i, .ROW := NULL]`, avoiding a full copy of the table for large row-removal operations, [#635](https://github.com/Rdatatable/data.table/issues/635). This has been one of data.table's most requested features. Target rows must be selected with the `i` expression, `by`/`keyby` are not supported, and keys/indices are cleared after deletion. The new experimental helper `setallocrow()` prepares columns for by-reference row operations. Thanks @arunsrinivasan for the feature request, @ben-schwen for the implementation, and @aitap for review and assistance.
 
+8. `setnafill()` now accepts a logical vector for the `cols` argument, which must be the same length as the number of columns in `x`, [#4113](https://github.com/Rdatatable/data.table/issues/4113). Thanks to @MichaelChirico for the suggestion and @venom1204 for the PR.
+
 ### BUG FIXES
 
 1. `fread()` with `skip=0` and `(header=TRUE|FALSE)` no longer skips the first row when it has fewer fields than subsequent rows, [#7463](https://github.com/Rdatatable/data.table/issues/7463). Thanks @emayerhofer for the report and @ben-schwen for the fix.

R/shift.R

@@ -33,5 +33,7 @@
 
 setnafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA, cols=seq_along(x)) {
   type = match.arg(type)
+  if (!is.list(x)) stop("in-place update is supported only for list")
+  cols = .Call(CcolnamesInt, x, cols, FALSE, FALSE)
   invisible(.Call(CnafillR, x, type, fill, nan_is_na(nan), TRUE, cols))
 }

inst/tests/tests.Rraw

@@ -21850,3 +21850,10 @@ a = c(a=1L)
 test(2379.12, as.IDate(d) + 1L, as.IDate(as.Date(d) + 1L))
 test(2379.13, as.IDate("2020-01-01") + a, as.IDate(as.Date("2020-01-01") + a))
 test(2379.14, as.IDate(d) - 1L, as.IDate(as.Date(d) - 1L))
+
+# #4113 setnafill could accept cols=logical(ncol(x))
+test(2380.01, {DT = data.table(a=c(1,NA,3), b=c(4,NA,6), c=c(7,NA,9)); DT1 = copy(DT); DT2 = copy(DT); setnafill(DT1, type="locf", cols=c(TRUE,FALSE,TRUE)); setnafill(DT2, type="locf", cols=c(1L,3L)); identical(DT1, DT2)}, TRUE)
+test(2380.02, {DT = data.table(a=c(1,NA), b=c(2,NA), c=c(3,NA)); DT1 = copy(DT); DT2 = copy(DT); setnafill(DT1, type="locf", cols=c(TRUE,FALSE,TRUE)); setnafill(DT2, type="locf", cols=c("a","c")); identical(DT1, DT2)}, TRUE)
+test(2380.03, {DT = data.table(a=c(1,NA), b=c(2,NA)); before = copy(DT); setnafill(DT, type="locf", cols=c(FALSE,FALSE)); identical(DT, before)}, TRUE)
+test(2380.04, {DT = data.table(a=c(1,NA), b=c("x",NA), c=c(3,NA)); cols = sapply(DT, is.numeric); setnafill(DT, type="locf", cols=cols); DT}, data.table(a=c(1,1), b=c("x",NA), c=c(3,3)))
+test(2380.05, setnafill(data.table(a=1,b=2,c=3), type="locf", cols=c(TRUE,NA,FALSE)), error="argument specifying columns is a logical vector containing NA at position 2")

man/assign.Rd

@@ -96,6 +96,7 @@ Since \code{[.data.table} incurs overhead to check the existence and type of arg
     \code{DT[a > 4, b := c]} is different from \code{DT[a > 4][, b := c]}. The first expression updates (or adds) column \code{b} with the value \code{c} on those rows where \code{a > 4} evaluates to \code{TRUE}. \code{X} is updated \emph{by reference}, therefore no assignment needed.  Note that this does not apply when \code{i} is missing, i.e. \code{DT[]}.
 
     The second expression on the other hand updates a \emph{new} \code{data.table} that's returned by the subset operation. Since the subsetted data.table is ephemeral (it is not assigned to a symbol), the result would be lost; unless the result is assigned, for example, as follows: \code{ans <- DT[a > 4][, b := c]}.
+    Note that \samp{:=} modifies its input in-place. When reusing a \code{data.table} in loops or multi-level tests, use \code{\link{copy}} to ensure a fresh state.
 }
 \value{
 \code{DT} is modified by reference and returned invisibly. If you require a copy, take a \code{\link{copy}} first (using \code{DT2 = copy(DT)}).
@@ -191,4 +192,3 @@ system.time(for (i in 1:1000) set(DT, i, 1L, i))
 
 }
 \keyword{ data }
-

man/nafill.Rd

@@ -18,7 +18,7 @@ setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x)
   \item{type}{ Character, one of \emph{"const"}, \emph{"locf"} or \emph{"nocb"}. Defaults to \code{"const"}. }
   \item{fill}{ Value to be used to replace missing observations. See examples. }
   \item{nan}{ Either \code{NaN} or \code{NA}; if the former, \code{NaN} is treated as distinct from \code{NA}, otherwise, they are treated the same during replacement. See Examples. }
-  \item{cols}{ Numeric or character vector specifying columns to be updated. }
+  \item{cols}{ Numeric, character or logical vector specifying columns to be updated. A logical vector must be the same length as the number of columns in \code{x}. }
 }
 \details{
   Supported types are \emph{logical}, \emph{integer}, \emph{double}, \emph{character}, and \emph{factor}, as well as classes built on top of these such as \code{Date}, \code{IDate}, and \code{POSIXct}.

man/test.Rd

@@ -25,7 +25,7 @@ test(num, x, y = TRUE,
 \item{env}{ A named list of environment variables to set for the duration of the test, much like \code{options}. A list entry set to \code{NULL} will unset (i.e., \code{\link{Sys.unsetenv}}) the corresponding variable. }
 \item{context}{ String, default \code{NULL}. Used to provide context where this is useful, e.g. in a test run in a loop where we can't just search for the test number. }
 \item{requires_utf8}{ \code{FALSE} (default), \code{TRUE}, or a character string. When set, the test is skipped if UTF-8 characters cannot be represented in the native encoding. Use \code{TRUE} for default UTF-8 test characters or provide a custom string of test characters. }
-\item{optimize}{ A vector of different optimization levels to test. The code in \code{x} will be run once for each optimization level, with \code{options(datatable.optimize=optimize)} set accordingly. All optimization levels must pass the test for the overall test to pass. If no \code{y} is supplied, the results from the different levels are compared to each other for equality. If a \code{y} is supplied, the results from each level are compared to \code{y}. }
+\item{optimize}{ A vector of different optimization levels to test. The code in \code{x} will be run once for each optimization level, with \code{options(datatable.optimize=optimize)} set accordingly. All optimization levels must pass the test for the overall test to pass. If no \code{y} is supplied, the results from the different levels are compared to each other for equality. If a \code{y} is supplied, the results from each level are compared to \code{y}. Note that since \code{x} is evaluated multiple times, any in-place modifications to a \code{data.table} (e.g. \code{:=}) will persist across runs; use \code{\link{copy}} if a fresh state is required for each level. }
 }
 \note{
    \code{NA_real_} and \code{NaN} are treated as equal, use \code{identical} if distinction is needed. See examples below.

src/utils.c

@@ -156,6 +156,20 @@ SEXP colnamesInt(SEXP x, SEXP cols, SEXP check_dups, SEXP skip_absent) {
       else if(bskip_absent && icols[i]>nx)
         icols[i] = 0L;
     }
+  } else if (isLogical(cols) && nc == nx) { 
+    int count = 0;
+    const int *bcols = LOGICAL_RO(cols);
+    for (int i=0; i<nc; i++) {
+      if (bcols[i] == 1) count++;
+      else if (bcols[i] == NA_LOGICAL)
+        error(_("argument specifying columns is a logical vector containing NA at position %d"), i+1);
+    }
+    ricols = PROTECT(allocVector(INTSXP, count)); protecti++;
+    int *icols = INTEGER(ricols);
+    int target = 0;
+    for (int i=0; i<nc; i++) {
+      if (bcols[i] == 1) icols[target++] = i + 1;
+    }
   } else if (isString(cols)) {
     SEXP xnames = PROTECT(getAttrib(x, R_NamesSymbol)); protecti++;
     if (isNull(xnames))

vignettes/datatable-reference-semantics.Rmd

@@ -410,6 +410,28 @@ To select a single column as a vector, remember:
 - `DT[, mycol]` is safer as it always returns a new, independent copy.
 - `DT$mycol` is fast but may return a reference. Use `copy(DT$mycol)` to guarantee independence.
 
+### d) Side effects and testing
+
+Because `:=` modifies its input in-place, the changes persist. If the same *data.table* is reused, for example in a loop or when using `test()` with multiple `optimization` levels, subsequent runs will start with the table as modified by the previous run. Use `copy()` to ensure each run starts with the same data for every evaluation.
+
+```{r}
+DT = data.table(a = 1L)
+
+# Simulation of a test running under two optimization levels:
+# Optimization level 1
+DT[, a := a + 1L]
+# Optimization level 2 starts with the modified table (a = 2)
+DT[, a := a + 1L]
+DT  # 'a' is now 3, not 2!
+
+# Correct approach: use copy() to isolate evaluations for each level
+DT = data.table(a = 1L)
+# Optimization level 1
+copy(DT)[, a := a + 1L]
+# Optimization level 2
+copy(DT)[, a := a + 1L]
+```
+
 ## Summary
 
 #### The `:=` operator