feat: execute wildcard exclusion projection

.github/workflows/qodana_code_quality.yml

@@ -1,15 +1,10 @@
 name: Qodana
 on:
   workflow_dispatch:
-  pull_request:
-  push:
-    branches: # Specify your branches here
-      - main # The 'main' branch
-      - 'releases/*' # The release branches
 
 jobs:
   qodana:
-    if: false
+    if: ${{ vars.QODANA_ENABLED == 'true' }}
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -26,4 +21,4 @@ jobs:
           pr-mode: false
         env:
           QODANA_TOKEN: ${{ secrets.QODANA_TOKEN_703174159 }}
-          QODANA_ENDPOINT: 'https://qodana.cloud'
+          QODANA_ENDPOINT: 'https://qodana.cloud'

end2end/go.mod

@@ -5,7 +5,7 @@ go 1.26.0
 toolchain go1.27.1
 
 require (
-	github.com/dal-go/dalgo v0.80.11
+	github.com/dal-go/dalgo v0.81.1-0.20260920182151-52a700c3b99a
 	github.com/dal-go/dalgo2sql v0.9.6 // No version as we alway replace it with local version
 	github.com/mattn/go-sqlite3 v1.14.52
 )

end2end/go.sum

@@ -14,6 +14,8 @@ github.com/dal-go/dalgo v0.80.4 h1:qlbxd+k0vH1LlcKWIddkRVDXo2OwnBRiowUBytYw64c=
 github.com/dal-go/dalgo v0.80.4/go.mod h1:u7sD8jlXshi2qHrlPUm9GzR2uyPRaBI4Ro9ef5+WFzY=
 github.com/dal-go/dalgo v0.80.11 h1:jvbqrsnzOcYAdmgIW/bSogTg4/ERtQTk6eFiotPg1+w=
 github.com/dal-go/dalgo v0.80.11/go.mod h1:+6651I975Yh2hoh+eyiqrI19VHC2lWFQNhN0iQmjZQ0=
+github.com/dal-go/dalgo v0.81.1-0.20260920182151-52a700c3b99a h1:t6huw80nzExaDGr4mBowB0Km6jVDD0te31fqHSgCTpE=
+github.com/dal-go/dalgo v0.81.1-0.20260920182151-52a700c3b99a/go.mod h1:+6651I975Yh2hoh+eyiqrI19VHC2lWFQNhN0iQmjZQ0=
 github.com/dal-go/record v0.1.3 h1:K85k/kSX08lTefMiMmPpC12nmCY2TVJk7+ZyC5PD9XU=
 github.com/dal-go/record v0.1.3/go.mod h1:quwsVJTT0f6y3Mhx+yHpTobY7luX1M6kyO6fdJ/AFYE=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

go.mod

@@ -8,7 +8,7 @@ toolchain go1.27.1
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.2
-	github.com/dal-go/dalgo v0.80.11
+	github.com/dal-go/dalgo v0.81.1-0.20260920182151-52a700c3b99a
 	github.com/dal-go/record v0.1.3
 	github.com/georgysavva/scany/v2 v2.1.4
 	modernc.org/sqlite v1.59.0

go.sum

@@ -14,6 +14,8 @@ github.com/dal-go/dalgo v0.80.4 h1:qlbxd+k0vH1LlcKWIddkRVDXo2OwnBRiowUBytYw64c=
 github.com/dal-go/dalgo v0.80.4/go.mod h1:u7sD8jlXshi2qHrlPUm9GzR2uyPRaBI4Ro9ef5+WFzY=
 github.com/dal-go/dalgo v0.80.11 h1:jvbqrsnzOcYAdmgIW/bSogTg4/ERtQTk6eFiotPg1+w=
 github.com/dal-go/dalgo v0.80.11/go.mod h1:+6651I975Yh2hoh+eyiqrI19VHC2lWFQNhN0iQmjZQ0=
+github.com/dal-go/dalgo v0.81.1-0.20260920182151-52a700c3b99a h1:t6huw80nzExaDGr4mBowB0Km6jVDD0te31fqHSgCTpE=
+github.com/dal-go/dalgo v0.81.1-0.20260920182151-52a700c3b99a/go.mod h1:+6651I975Yh2hoh+eyiqrI19VHC2lWFQNhN0iQmjZQ0=
 github.com/dal-go/record v0.1.3 h1:K85k/kSX08lTefMiMmPpC12nmCY2TVJk7+ZyC5PD9XU=
 github.com/dal-go/record v0.1.3/go.mod h1:quwsVJTT0f6y3Mhx+yHpTobY7luX1M6kyO6fdJ/AFYE=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

projection.go

@@ -0,0 +1,95 @@
+package dalgo2sql
+
+import (
+	"fmt"
+
+	"github.com/dal-go/dalgo/dal"
+)
+
+type wildcardProjectionPlan struct {
+	qualifier     string
+	excluded      map[string]struct{}
+	explicitCount int
+}
+
+func planWildcardProjection(q dal.StructuredQuery) (*wildcardProjectionPlan, error) {
+	columns := q.Columns()
+	var projection *dal.WildcardProjection
+	for i, column := range columns {
+		if column.Wildcard == nil {
+			continue
+		}
+		if projection != nil {
+			return nil, fmt.Errorf("multiple wildcard projections require join support")
+		}
+		if i != 0 {
+			return nil, fmt.Errorf("wildcard projection must precede explicit columns")
+		}
+		if column.Expression != nil {
+			return nil, fmt.Errorf("wildcard projection cannot also contain an expression")
+		}
+		if column.Alias != "" {
+			return nil, fmt.Errorf("wildcard projection cannot have an alias")
+		}
+		projection = column.Wildcard
+	}
+	if projection == nil {
+		return nil, nil
+	}
+	if len(projection.Exclude) == 0 {
+		return nil, fmt.Errorf("wildcard projection requires at least one exclusion")
+	}
+	if q.From() == nil || q.From().Base() == nil {
+		return nil, fmt.Errorf("wildcard projection requires a source")
+	}
+	if len(q.From().Joins()) != 0 {
+		return nil, fmt.Errorf("wildcard projection with joins is not supported")
+	}
+	base := q.From().Base()
+	qualifier := ""
+	if projection.Source != "" {
+		if projection.Source != base.Alias() && projection.Source != base.Name() {
+			return nil, fmt.Errorf("wildcard projection source %q does not match source name or alias", projection.Source)
+		}
+		qualifier = base.Alias()
+		if qualifier == "" {
+			qualifier = base.Name()
+		}
+	}
+	excluded := make(map[string]struct{}, len(projection.Exclude))
+	for i, name := range projection.Exclude {
+		if name == "" {
+			return nil, fmt.Errorf("wildcard projection exclusion %d is empty", i)
+		}
+		excluded[name] = struct{}{}
+	}
+	return &wildcardProjectionPlan{
+		qualifier:     qualifier,
+		excluded:      excluded,
+		explicitCount: len(columns) - 1,
+	}, nil
+}
+
+func (p wildcardProjectionPlan) sqlExpression(quote func(string) string) string {
+	if p.qualifier == "" {
+		return "*"
+	}
+	return quote(p.qualifier) + ".*"
+}
+
+func (p wildcardProjectionPlan) visibleIndexes(columnNames []string) ([]int, error) {
+	wildcardEnd := len(columnNames) - p.explicitCount
+	if wildcardEnd < 0 {
+		return nil, fmt.Errorf("SQL result has %d columns, fewer than %d explicit projections", len(columnNames), p.explicitCount)
+	}
+	indexes := make([]int, 0, len(columnNames))
+	for i, name := range columnNames {
+		if i < wildcardEnd {
+			if _, excluded := p.excluded[name]; excluded {
+				continue
+			}
+		}
+		indexes = append(indexes, i)
+	}
+	return indexes, nil
+}

projection_test.go

@@ -0,0 +1,225 @@
+package dalgo2sql
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/dal-go/dalgo/dal"
+)
+
+func TestCompileStructuredSQLWildcardExclusion(t *testing.T) {
+	tests := []struct {
+		name string
+		q    dal.StructuredQuery
+		want string
+	}{
+		{
+			name: "unqualified",
+			q: dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().
+				SelectColumns(dal.AllColumnsExcept("email", "missing")),
+			want: "SELECT * FROM `customers`",
+		},
+		{
+			name: "qualified",
+			q: dal.From(dal.NewRootCollectionRef("customers", "c")).NewQuery().
+				SelectColumns(dal.AllColumnsExceptFrom("c", "email")),
+			want: "SELECT `c`.* FROM `customers` AS `c`",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, args, err := compileStructuredSQL(tt.q)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tt.want || len(args) != 0 {
+				t.Fatalf("compileStructuredSQL() = %q, %#v; want %q, no args", got, args, tt.want)
+			}
+		})
+	}
+}
+
+func TestEmitSQLQualifiedWildcardExclusionUsesLegacySingleSourceWildcard(t *testing.T) {
+	q := dal.From(dal.NewRootCollectionRef("customers", "c")).NewQuery().
+		SelectColumns(dal.AllColumnsExceptFrom("c", "email"))
+	if got, want := emitSQL(q), "SELECT * FROM customers"; got != want {
+		t.Fatalf("emitSQL() = %q, want %q", got, want)
+	}
+}
+
+func TestWildcardExclusionRecordsReader(t *testing.T) {
+	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeDatabase(t, db)
+
+	q := dal.From(dal.NewRootCollectionRef("customers", "c")).NewQuery().
+		SelectColumns(dal.AllColumnsExceptFrom("c", "email", "password_hash", "email"))
+	mock.ExpectQuery("SELECT `c`.* FROM `customers` AS `c`").
+		WillReturnRows(sqlmock.NewRows([]string{"id", "name", "email", "created_at"}).
+			AddRow("c1", "Ada", "ada@example.test", "2026-09-20"))
+
+	r, err := getRecordsReaderWithOptions(context.Background(), q, db.QueryContext, DbOptions{
+		StructuredQueryDialect: "sqlite",
+		Recordsets: map[string]*Recordset{
+			"customers": NewRecordset("customers", Table, []dal.FieldRef{dal.Field("id")}),
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = r.Close() }()
+	record, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := map[string]any{"id": "c1", "name": "Ada", "created_at": "2026-09-20"}
+	if got := record.Data().(map[string]any); !reflect.DeepEqual(got, want) {
+		t.Fatalf("record data = %#v, want %#v", got, want)
+	}
+	if record.Key().ID != "c1" {
+		t.Fatalf("record key ID = %#v, want c1", record.Key().ID)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestWildcardExclusionRecordsetReaderPreservesOrder(t *testing.T) {
+	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeDatabase(t, db)
+
+	q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().
+		SelectColumns(dal.AllColumnsExcept("email", "missing"))
+	mock.ExpectQuery("SELECT * FROM `customers`").
+		WillReturnRows(sqlmock.NewRows([]string{"id", "name", "email", "created_at"}).
+			AddRow([]byte("c1"), []byte("Ada"), []byte("ada@example.test"), []byte("2026-09-20")))
+
+	r, err := getRecordsetReaderWithDialect(context.Background(), q, db.QueryContext, "sqlite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = r.Close() }()
+	if got := r.Recordset().Columns(); len(got) != 3 || got[0].Name() != "id" || got[1].Name() != "name" || got[2].Name() != "created_at" {
+		t.Fatalf("recordset columns = %#v", got)
+	}
+	row, rs, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, want := range []string{"c1", "Ada", "2026-09-20"} {
+		value, valueErr := row.GetValueByIndex(i, rs)
+		if valueErr != nil {
+			t.Fatal(valueErr)
+		}
+		if stringValue := valueString(value); stringValue != want {
+			t.Fatalf("value %d = %q, want %q", i, stringValue, want)
+		}
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func valueString(value any) string {
+	if bytes, ok := value.([]byte); ok {
+		return string(bytes)
+	}
+	return value.(string)
+}
+
+func TestWildcardProjectionValidation(t *testing.T) {
+	base := dal.From(dal.NewRootCollectionRef("customers", "c")).NewQuery()
+	tests := []struct {
+		name    string
+		columns []dal.Column
+	}{
+		{name: "empty exclusions", columns: []dal.Column{dal.AllColumnsExcept()}},
+		{name: "empty name", columns: []dal.Column{dal.AllColumnsExcept("")}},
+		{name: "unknown source", columns: []dal.Column{dal.AllColumnsExceptFrom("x", "email")}},
+		{name: "alias", columns: []dal.Column{{Alias: "rest", Wildcard: &dal.WildcardProjection{Exclude: []string{"email"}}}}},
+		{name: "expression", columns: []dal.Column{{Expression: dal.Field("id"), Wildcard: &dal.WildcardProjection{Exclude: []string{"email"}}}}},
+		{name: "multiple", columns: []dal.Column{dal.AllColumnsExcept("email"), dal.AllColumnsExcept("secret")}},
+		{name: "not first", columns: []dal.Column{{Expression: dal.Field("id")}, dal.AllColumnsExcept("email")}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if _, _, err := compileStructuredSQL(base.SelectColumns(tt.columns...)); err == nil {
+				t.Fatal("expected validation error")
+			}
+		})
+	}
+}
+
+func TestWildcardProjectionRejectsJoinedSources(t *testing.T) {
+	from := dal.From(dal.NewRootCollectionRef("customers", "c")).Join(
+		dal.NewJoinedSource(dal.NewRootCollectionRef("orders", "o"), dal.JoinInner),
+	)
+	q := from.NewQuery().SelectColumns(dal.AllColumnsExceptFrom("c", "email"))
+	if _, err := planWildcardProjection(q); err == nil {
+		t.Fatal("expected joined wildcard projection to be rejected")
+	}
+}
+
+func TestWildcardProjectionKeepsExplicitTail(t *testing.T) {
+	q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().SelectColumns(
+		dal.AllColumnsExcept("email"),
+		dal.Column{Expression: dal.Field("id"), Alias: "helper"},
+	)
+	plan, err := planWildcardProjection(q)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := plan.visibleIndexes([]string{"id", "email", "helper"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, []int{0, 2}) {
+		t.Fatalf("visible indexes = %#v, want [0 2]", got)
+	}
+}
+
+func TestWildcardExclusionIdentityHelperDoesNotHideSameNamedSourceColumn(t *testing.T) {
+	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeDatabase(t, db)
+
+	q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().
+		SelectColumns(dal.AllColumnsExcept("id"))
+	mock.ExpectQuery("SELECT *, `id` AS `__dalgo_record_id` FROM `customers`").
+		WillReturnRows(sqlmock.NewRows([]string{"id", "__dalgo_record_id", "__dalgo_record_id"}).
+			AddRow("c1", "source-value", "c1"))
+
+	r, err := getRecordsReaderWithOptions(context.Background(), q, db.QueryContext, DbOptions{
+		StructuredQueryDialect: "sqlite",
+		Recordsets: map[string]*Recordset{
+			"customers": NewRecordset("customers", Table, []dal.FieldRef{dal.Field("id")}),
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = r.Close() }()
+	record, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if record.Key().ID != "c1" {
+		t.Fatalf("record key ID = %#v, want c1", record.Key().ID)
+	}
+	want := map[string]any{"__dalgo_record_id": "source-value"}
+	if got := record.Data().(map[string]any); !reflect.DeepEqual(got, want) {
+		t.Fatalf("record data = %#v, want %#v", got, want)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatal(err)
+	}
+}