diff --git a/.claude/.gitignore b/.claude/.gitignore new file mode 100644 index 000000000..a13633799 --- /dev/null +++ b/.claude/.gitignore @@ -0,0 +1 @@ +*.pdf diff --git a/.claude/dynamic_doc_queries_and_iac.md b/.claude/dynamic_doc_queries_and_iac.md new file mode 100644 index 000000000..87468b0ee --- /dev/null +++ b/.claude/dynamic_doc_queries_and_iac.md @@ -0,0 +1,252 @@ + +# Dynamic Queries and IAC in omnisdk + +### Calling it from Go + +The CLI is a thin consumer of the facade; a client such as stackql imports `pkg/omnisdk` and reaches +the same functions. Both paths return the same `Plan`/`Rows` a single-method query returns, so a +consumer iterates one cursor shape whether it is reading or provisioning. + +**Auth and document location behave exactly as elsewhere.** The provider document declares the scheme +— `aws_signing_v4`, `service_account`, `oauth2` — and the credential comes from `Args.Auth`, falling +back to the canonical `AWS_*`, `AZURE_*` and `GOOGLE_*` variables. Only the credential a document +actually declares is required, so a run touching AWS alone does not fail because a Google key +elsewhere is stale; a credential that IS present but unusable says so rather than reporting as +absent. The registry root is a parameter rather than configuration, and is required — which document +set a run resolves against is scope, and scope is never inferred. + +**1. `stackql_dynamic` A dynamic query across provider documents** — the `doc-graph` equivalent: + +```go +const vpcs, subnets = "stackql_unstable_aws.ec2.vpcs", "stackql_unstable_aws.ec2.subnets" + +g, err := omnisdk.NewGraph( + []string{vpcs, subnets}, + []omnisdk.Wiring{omnisdk.NewWiring( + subnets, // the consumer + []omnisdk.Inbound{omnisdk.NewInbound(vpcs, "VpcId", "vpc_id")}, // β: src → inbox label + gotemplate.TypeJSON1, // T_in, or "" for identity + `{"Filter.1.Name":"vpc-id","Filter.1.Value.1":"{{ .vpc_id }}"}`, + "Filter.1.Name", "Filter.1.Value.1", // what T_in provides + )}, + // Corrections to what a document says about its response, where it is wrong for this engine: + // omnisdk.NewOverride(addr, "$.items", "", "", ""), +) +if err != nil { + return err +} + +pl, err := omnisdk.NewGraphQuery(registryRoot, g, omnisdk.Args{ + Auth: auth, // nil falls back to the env + Params: map[string]string{"region": "us-east-1"}, // scope +}) +rows, err := pl.Open(ctx) +defer rows.Close() +for rows.Next() { + row := rows.Row() // {"VpcId":…, "SubnetId":…, "CidrBlock":…} +} +return rows.Err() +``` + +**2. `stackql_iac` An idempotent IaC run** — the `iac-apply` equivalent: + +```go +res := []omnisdk.ManagedResource{ + omnisdk.NewResource( + "aws/ec2/vpc", // key within the collection + "aws", "ec2.vpcs", // registry provider, document address + []byte(`{"CidrBlock":"10.42.0.0/16"}`), + map[string]string{ + "TagSpecification.1.ResourceType": "vpc", + "TagSpecification.1.Tag.1.Key": "omnisdk:key", + }, + nil, "", "", // inbound, T_in type, T_in program + "line_items.VpcId", // where the minted id sits in the projected response + "VpcId", // the parameter addressing an existing object + "TagSpecification.1.Tag.1.Value", // the parameter taking the correlation stamp + ), + omnisdk.NewResource( + "aws/ec2/subnet", "aws", "ec2.subnets", + []byte(`{"CidrBlock":"10.42.1.0/24"}`), nil, + []omnisdk.Arrival{{From: "aws/ec2/vpc", As: "VpcId"}}, "", "", + "line_items.SubnetId", "SubnetId", "", + ), +} + +pl, err := omnisdk.Converge(registryRoot, "scratch", stateDir, "" /* runID: timestamp */, res, + omnisdk.Args{Auth: auth, Params: map[string]string{"region": "us-east-1"}}) +if err != nil { + return err +} +rows, err := pl.Open(ctx) // the run happens here +``` + +Rows report `{"key":…, "identity":…, "status":…}`, with a final row carrying `error`, +`compensated` and `outstanding` when a run failed. A non-empty `outstanding` means the run is +*partially* compensated — something it created is still there and could not be removed. + +Precanned deployments are reachable the same way, and are only a way of building that slice: + +```go +bp, ok := omnisdk.BlueprintFor("aws-vpc-subnet") +res, err := bp.Resources(map[string]string{ + "region": "us-east-1", "vpc_cidr": "10.42.0.0/16", "subnet_cidr": "10.42.1.0/24", +}) +``` + +`omnisdk.Blueprints()` lists them with the inputs each declares, which is what `iac-handles` prints. + + +## stackql invocations + +Let us look at some `omnicli` invocations and their `stackql` equivalents. + + +### Bindings across exchanges + + +Here is the example `omnicli` functionality: + +```bash +omnicli doc-graph /path/to/registry '{ + "addresses": ["stackql_unstable_aws.ec2.vpcs", "stackql_unstable_aws.ec2.subnets"], + "wirings": [{ + "to": "stackql_unstable_aws.ec2.subnets", + "inbound": [{"from": "stackql_unstable_aws.ec2.vpcs", "src": "VpcId", "as": "vpc_id"}], + "via_type": "golang_template_json_v0.1.0", + "via": "{\"Filter.1.Name\":\"vpc-id\",\"Filter.1.Value.1\":\"{{ .vpc_id }}\"}", + "provides": ["Filter.1.Name", "Filter.1.Value.1"] + }] +}' --aws-region us-east-1 +``` + + +Here is the equivalent `stackql` functionality: + +```bash +stackql exec --preview '{"unstable":true}' "$(cat <<'SQL' +SELECT * FROM stackql_dynamic.graph.query +WHERE region = 'us-east-1' + AND spec = '{ + "addresses": ["stackql_unstable_aws.ec2.vpcs", "stackql_unstable_aws.ec2.subnets"], + "wirings": [{ + "to": "stackql_unstable_aws.ec2.subnets", + "inbound": [{"from": "stackql_unstable_aws.ec2.vpcs", "src": "VpcId", "as": "vpc_id"}], + "viaType": "golang_template_json_v0.1.0", + "viaProgram": "{\\"Filter.1.Name\\":\\"vpc-id\\",\\"Filter.1.Value.1\\":\\"{{ .vpc_id }}\\"}", + "provides": ["Filter.1.Name", "Filter.1.Value.1"] + }] + }'; +SQL +)" +``` + +Three things differ from the `omnicli` form: + +- The region is a predicate rather than a flag. Every predicate left after the + control ones are read becomes `Args.Params`, which is where `--aws-region` + lands. +- The wiring keys are `viaType` and `viaProgram`, where `omnicli` writes + `via_type` and `via`. Unknown keys are dropped in silence, so a document + pasted across unchanged loses its transform rather than failing. +- The backslashes in `viaProgram` are **doubled**. The SQL string literal + consumes one level of escaping before the JSON parser sees the value, so + `\"` in the `omnicli` payload must be written `\\"` here. + +Credentials are unchanged: the provider document declares the scheme, and the +credential comes from stackql's `--auth` for that provider, falling back to the +canonical `AWS_*`, `AZURE_*` and `GOOGLE_*` variables. + +Rows arrive exactly as a single-method select's do. A document declares no +egress schema, so the columns are the ones the first row carries, sorted by +name; `SELECT VpcId, SubnetId` projects over them in the order asked for. + + +### IAC with exchanges + + +Here is the example `omnicli` functionality: + +```bash +omnicli iac-apply /path/to/registry '{ + "name": "scratch", "state": "cicd/work/iac-state", + "resources": [ + {"key": "aws/ec2/vpc", "provider": "aws", "address": "ec2.vpcs", + "desired": {"CidrBlock": "10.42.0.0/16"}, + "params": {"TagSpecification.1.ResourceType": "vpc", + "TagSpecification.1.Tag.1.Key": "omnisdk:key"}, + "identity": "line_items.VpcId", "addressed_by": "VpcId", + "correlation_param": "TagSpecification.1.Tag.1.Value"}, + {"key": "aws/ec2/subnet", "provider": "aws", "address": "ec2.subnets", + "desired": {"CidrBlock": "10.42.1.0/24"}, + "inbound": [{"from": "aws/ec2/vpc", "as": "VpcId"}], + "identity": "line_items.SubnetId", "addressed_by": "SubnetId"} + ], + "args": {"params": {"region": "us-east-1"}} +}' --aws-region us-east-1 +``` + +Here is the equivalent `stackql` functionality: + +```bash +stackql exec --preview '{"unstable":true}' "$(cat <<'SQL' +SELECT * FROM stackql_iac.converge.run +WHERE collection = 'scratch' + AND state = 'cicd/work/iac-state' + AND region = 'us-east-1' + AND resources = '[ + {"key": "aws/ec2/vpc", "provider": "aws", "address": "ec2.vpcs", + "desired": {"CidrBlock": "10.42.0.0/16"}, + "params": {"TagSpecification.1.ResourceType": "vpc", + "TagSpecification.1.Tag.1.Key": "omnisdk:key"}, + "identity": "line_items.VpcId", "addressedBy": "VpcId", + "correlationParam": "TagSpecification.1.Tag.1.Value"}, + {"key": "aws/ec2/subnet", "provider": "aws", "address": "ec2.subnets", + "desired": {"CidrBlock": "10.42.1.0/24"}, + "inbound": [{"from": "aws/ec2/vpc", "as": "VpcId"}], + "identity": "line_items.SubnetId", "addressedBy": "SubnetId"} + ]'; +SQL +)" +``` + +The mapping onto `iac-apply` is one predicate per top-level field: `name` +becomes `collection`, `state` stays `state`, and `args.params` becomes the +predicates left over once the control ones are read. `run_id` is accepted too, +and defaults to a UTC timestamp. `state` may be omitted, in which case the +ledger and run journals sit under `/iac`. + +Two key names differ from the `omnicli` payload, and are dropped in silence if +pasted across unchanged: `addressedBy` for `addressed_by`, and +`correlationParam` for `correlation_param`. + +A blueprint is the same relation with the resources named rather than stated: + +```bash +stackql exec --preview '{"unstable":true}' \ + "SELECT * FROM stackql_iac.converge.run + WHERE collection = 'scratch' AND blueprint = 'aws-vpc-subnet' + AND region = 'us-east-1' AND vpc_cidr = '10.42.0.0/16' + AND subnet_cidr = '10.42.1.0/24';" +``` + +`SHOW RESOURCES IN stackql_iac.blueprints;` lists the handles and +`DESCRIBE stackql_iac.blueprints.aws_vpc_subnet;` the inputs each declares, +which is what `iac-handles` prints. A handle is written with hyphens, which no +unquoted SQL identifier can carry, so it is addressed as `aws_vpc_subnet` in +the relation name and accepted either way in the predicate. + +Opening the cursor performs the run, so the rows are the report: `key`, +`identity` and `status`, with a failed run carrying `error`, `compensated` and +`outstanding` as well. A non-empty `outstanding` means the run is *partially* +compensated - something it created is still there and could not be removed. +Columns are sorted by name, as everywhere the cursor infers them from the first +row. A run whose endpoint refused the connection reports: + +```text +|-------------|--------------------------------|---------------------|-----------------------|-------------------------------| +| compensated | error | key | outstanding | status | +|-------------|--------------------------------|---------------------|-----------------------|-------------------------------| +| [] | docrun: ... connection refused | scratch/aws/ec2/vpc | [scratch/aws/ec2/vpc] | failed; partially compensated | +|-------------|--------------------------------|---------------------|-----------------------|-------------------------------| +``` diff --git a/cicd/work/.gitignore b/cicd/work/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/cicd/work/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/go.mod b/go.mod index b3332f33f..afc177b44 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.1-alpha06 + github.com/stackql-labs/omnisdk v0.1.2-alpha02 github.com/stackql/any-sdk v0.5.5-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.2-beta01 diff --git a/go.sum b/go.sum index 19a02ba4c..18c9f8002 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.1-alpha06 h1:jVetiuZa6J1uWJ4vPc/YQtol4l7Lgoq281UWvYXw6GM= -github.com/stackql-labs/omnisdk v0.1.1-alpha06/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= +github.com/stackql-labs/omnisdk v0.1.2-alpha02 h1:BZVB5+NiqGE4pMmds1IyUvrYjcF1qSEVHTqyhJkDP6w= +github.com/stackql-labs/omnisdk v0.1.2-alpha02/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= github.com/stackql/any-sdk v0.5.5-alpha01 h1:Omj/EuF0hx8oai54aRZMrWkjsV11YK532lq4Spfvjiw= github.com/stackql/any-sdk v0.5.5-alpha01/go.mod h1:DV6KrDVMIpbuWHm4YAdMtCsszbpdzLL0JUXpEX3wDMU= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= diff --git a/internal/stackql/handler/handler.go b/internal/stackql/handler/handler.go index 24f97c717..26e0c32f4 100644 --- a/internal/stackql/handler/handler.go +++ b/internal/stackql/handler/handler.go @@ -316,6 +316,16 @@ func (hc *standardHandlerContext) GetSupportedProviders(extended bool) (map[stri "name": intrinsic.ProviderName, "version": intrinsic.ProviderVersion, } + // The alias providers read documents straight from disk, so they appear + // alongside the document-driven bundles rather than unconditionally. + if intrinsic.IsUnstableEnabled() { + for _, alias := range []string{intrinsic.DynamicProviderName, intrinsic.IaCProviderName} { + retVal[alias] = map[string]interface{}{ + "name": alias, + "version": intrinsic.ProviderVersion, + } + } + } // Supporting SQL data sources // These will be overwritten by any documented providers with the same name for k := range hc.sqlDataSources { diff --git a/internal/stackql/intrinsic/alias.go b/internal/stackql/intrinsic/alias.go new file mode 100644 index 000000000..89f78854e --- /dev/null +++ b/internal/stackql/intrinsic/alias.go @@ -0,0 +1,328 @@ +package intrinsic + +// The alias providers present omnisdk's document-driven capabilities as +// relations: stackql_dynamic for a query spanning several exchanges, stackql_iac +// for an idempotent converge run. Both are deliberately thin. A predicate +// carries the specification, the SDK does the work, and the rows it yields +// stream through the same cursor a single-method select already uses. +// +// They are gated behind the same opt-in as the document-driven providers, +// because that is what they read: documents straight from disk, with none of +// the registry's curation behind them. + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/any-sdk/public/formulation" + "github.com/stackql/psql-wire/pkg/sqldata" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/util" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" +) + +const ( + // DynamicProviderName addresses a query wired across several exchanges. + DynamicProviderName = "stackql_dynamic" + // IaCProviderName addresses an idempotent converge run and the blueprints + // that render one. + IaCProviderName = "stackql_iac" +) + +const ( + graphService = "graph" + graphRelation = "query" + convergeService = "converge" + convergeRelation = "run" + blueprintService = "blueprints" +) + +const ( + specPredicate = "spec" + collectionPredicate = "collection" + blueprintPredicate = "blueprint" + resourcesPredicate = "resources" + statePredicate = "state" + runIDPredicate = "run_id" +) + +// aliasProvider canonicalises an alias provider name, or reports false. The +// aliases read documents from disk, so they appear only once that was opted +// into, exactly as the unstable providers do. +func aliasProvider(name string) (string, bool) { + if !IsUnstableEnabled() { + return "", false + } + switch trimmed := strings.TrimSpace(name); { + case strings.EqualFold(trimmed, DynamicProviderName): + return DynamicProviderName, true + case strings.EqualFold(trimmed, IaCProviderName): + return IaCProviderName, true + default: + return "", false + } +} + +// aliasServices names the services an alias provider presents. +func aliasServices(provider string) []string { + if provider == IaCProviderName { + return []string{blueprintService, convergeService} + } + return []string{graphService} +} + +// registryRoot is the provider-document root omnisdk resolves addresses +// against. It is the directory holding "//provider.yaml", +// which is where stackql's own document root keeps them. +func registryRoot(ctx queryContext) string { + return filepath.Join(localDocRoot(ctx.GetRuntimeContext()), "src") +} + +// popPredicate takes a predicate out of the parameter map. What remains after +// every control predicate is popped is the run's scope, which rides through to +// omnisdk untouched. +func popPredicate(params map[string]string, key string) string { + value := params[key] + delete(params, key) + return value +} + +// streamPlan opens a plan and hands its cursor to the caller. Neither alias +// declares an egress schema, so the columns are the ones the first row carries +// and the projection is applied over them. +func streamPlan( + ctx queryContext, + plan omnisdk.Plan, + relation string, + exprs sqlparser.SelectExprs, +) internaldto.ExecutorOutput { + rows, openErr := plan.Open(context.Background()) + if openErr != nil { + return internaldto.NewErroneousExecutorOutput(openErr) + } + input := previewCfg + stream := &rowStream{ + rows: rows, + batchSize: input.getBatchSize(), + flushInterval: input.getFlushInterval(), + table: sqldata.NewSQLTable(0, relation), + typCfg: ctx.GetTypingConfig(), + projection: exprs, + } + primed, readErr := newPrimedStream(stream) + if readErr != nil { + return internaldto.NewErroneousExecutorOutput(readErr) + } + return internaldto.NewExecutorOutput(primed, nil, nil, nil, nil) +} + +// aliasArgs assembles the SDK arguments common to both aliases: the scope left +// over after the control predicates, the credential for the cloud in play, and +// the backend tuning. +func aliasArgs(ctx queryContext, cloud string, params map[string]string) omnisdk.Args { + input := previewCfg + return omnisdk.Args{ + Params: params, + Auth: omnisdkAuth(providerAuthContext(ctx, cloud)), + Endpoint: input.getEndpoint(), + InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(), + } +} + +// aliasSelectFunc routes a SELECT over an alias relation. +func aliasSelectFunc( + ctx queryContext, + node *sqlparser.Select, + provider, service, resource string, +) (func() internaldto.ExecutorOutput, bool) { + if provider == DynamicProviderName { + return dynamicSelectFunc(ctx, node, service, resource) + } + return iacSelectFunc(ctx, node, service, resource) +} + +// aliasPredicates reads a select's WHERE clause as the flat equality map both +// aliases take, refusing anything the streaming path cannot honour. +func aliasPredicates( + node *sqlparser.Select, + relation string, +) (map[string]string, func() internaldto.ExecutorOutput) { + if unsupported := unsupportedClauses(node); len(unsupported) > 0 { + return nil, refuse(fmt.Errorf( + "relation '%s' streams its rows, so %s cannot be applied; remove %s from the query", + relation, strings.Join(unsupported, ", "), pluralClause(len(unsupported)))) + } + params, bad := equalityPredicates(node.Where) + if len(bad) > 0 { + return nil, refuse(fmt.Errorf( + "relation '%s' streams its rows, so only equality predicates are applied; "+ + "%s cannot be honoured", + relation, strings.Join(bad, ", "))) + } + return params, nil +} + +// showAliasFunc answers SHOW for the alias providers. +func showAliasFunc( + ctx queryContext, + node *sqlparser.Show, + currentProvider string, + extended bool, +) (func() internaldto.ExecutorOutput, bool) { + switch strings.ToUpper(strings.TrimSpace(node.Type)) { + case "SERVICES": + provider, isAlias := aliasProvider(resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider)) + if !isAlias { + return nil, false + } + return func() internaldto.ExecutorOutput { + return showAliasServices(ctx, provider, extended) + }, true + case "RESOURCES": + provider, isAlias := aliasProvider( + resolveProvider(node.OnTable.Qualifier.GetRawVal(), currentProvider)) + if !isAlias { + return nil, false + } + service := node.OnTable.Name.GetRawVal() + return func() internaldto.ExecutorOutput { + return showAliasResources(ctx, provider, service, extended) + }, true + case "METHODS": + // Every alias relation is select-only, so the method list does not vary + // by service or resource. + if _, isAlias := aliasProvider( + resolveProvider(node.OnTable.QualifierSecond.GetRawVal(), currentProvider)); !isAlias { + return nil, false + } + return func() internaldto.ExecutorOutput { return showAliasMethods(ctx, extended) }, true + } + return nil, false +} + +func showAliasServices(ctx queryContext, provider string, extended bool) internaldto.ExecutorOutput { + services := aliasServices(provider) + rows := make(map[string]map[string]interface{}, len(services)) + for i, service := range services { + row := map[string]interface{}{ + "id": fmt.Sprintf("%s.%s", provider, service), + "name": service, + "title": service, + } + if extended { + row["description"] = aliasServiceDescription(provider, service) + row["version"] = ProviderVersion + row["preferred"] = nil + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, formulation.GetServicesHeader(extended), rows, util.DefaultRowSort) +} + +func aliasServiceDescription(provider, service string) string { + switch { + case provider == DynamicProviderName: + return "a query wired across several document-declared exchanges" + case service == blueprintService: + return "precanned deployments, and the inputs each declares" + default: + return "an idempotent converge run over a collection of resources" + } +} + +func showAliasResources( + ctx queryContext, provider, service string, extended bool) internaldto.ExecutorOutput { + tables, err := aliasTables(provider, service) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows := make(map[string]map[string]interface{}, len(tables)) + for i, tbl := range tables { + row := map[string]interface{}{ + "id": fmt.Sprintf("%s.%s.%s", provider, tbl.service, tbl.name), + "name": tbl.name, + } + if extended { + row["description"] = tbl.description + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, formulation.GetResourcesHeader(extended), rows, util.DefaultRowSort) +} + +func showAliasMethods(ctx queryContext, extended bool) internaldto.ExecutorOutput { + columnOrder := []string{"MethodName", "RequiredParams", "SQLVerb"} + if extended { + columnOrder = append(columnOrder, "description") + } + row := map[string]interface{}{ + "MethodName": selectMethodName, + "RequiredParams": "", + "SQLVerb": strings.ToUpper(selectMethodName), + } + if extended { + row["description"] = "select-only intrinsic method" + } + return prepare(ctx, columnOrder, + map[string]map[string]interface{}{"000001": row}, util.DefaultRowSort) +} + +// aliasTables presents an alias service's relations. +func aliasTables(provider, service string) ([]table, error) { + switch { + case provider == DynamicProviderName && strings.EqualFold(service, graphService): + return []table{{ + service: graphService, + name: graphRelation, + isData: true, + description: "a graph query; the '" + specPredicate + "' predicate carries its wiring", + }}, nil + case provider == IaCProviderName && strings.EqualFold(service, convergeService): + return []table{{ + service: convergeService, + name: convergeRelation, + isData: true, + description: "an idempotent converge run over a named collection", + }}, nil + case provider == IaCProviderName && strings.EqualFold(service, blueprintService): + return blueprintTables(), nil + } + return nil, fmt.Errorf("provider '%s' has no service '%s'", provider, service) +} + +// describeAliasTableFunc answers DESCRIBE for the alias providers. Only a +// blueprint has columns worth describing: its relations take a specification +// rather than a column list, and yield whatever the run reports. +func describeAliasTableFunc( + ctx queryContext, + node *sqlparser.DescribeTable, + currentProvider string, +) (func() internaldto.ExecutorOutput, bool) { + provider, isAlias := aliasProvider( + resolveProvider(node.Table.QualifierSecond.GetRawVal(), currentProvider)) + if !isAlias { + return nil, false + } + service := node.Table.Qualifier.GetRawVal() + resource := node.Table.Name.GetRawVal() + extended := isExtended(node.Extended) + if provider == IaCProviderName && strings.EqualFold(service, blueprintService) { + blueprint, ok := blueprintFor(resource) + if !ok { + return refuse(fmt.Errorf( + "'%s.%s' has no blueprint '%s'; run SHOW RESOURCES IN %s.%s to list them", + IaCProviderName, blueprintService, resource, IaCProviderName, blueprintService)), true + } + return func() internaldto.ExecutorOutput { + return describeTable(ctx, table{columns: blueprintColumns(blueprint)}, extended) + }, true + } + return refuse(fmt.Errorf( + "relation '%s.%s.%s' takes a specification rather than columns; "+ + "run SHOW RESOURCES IN %s.%s for what it accepts", + provider, service, resource, provider, service)), true +} diff --git a/internal/stackql/intrinsic/alias_test.go b/internal/stackql/intrinsic/alias_test.go new file mode 100644 index 000000000..9e116c3b0 --- /dev/null +++ b/internal/stackql/intrinsic/alias_test.go @@ -0,0 +1,259 @@ +package intrinsic //nolint:testpackage // tests unexported alias plumbing + +import ( + "strings" + "testing" +) + +// withUnstable opts the document-driven surface in for one test and restores +// whatever the package had afterwards. +func withUnstable(t *testing.T, enabled bool) { + t.Helper() + previous := previewCfg + previewCfg = newBackendInput(previewCfgDTO{Unstable: enabled}) + t.Cleanup(func() { previewCfg = previous }) +} + +func TestAliasProviderRequiresOptIn(t *testing.T) { + withUnstable(t, false) + for _, name := range []string{DynamicProviderName, IaCProviderName} { + if _, ok := aliasProvider(name); ok { + t.Errorf("aliasProvider(%q) = true without the unstable opt-in", name) + } + if IsProvider(name) { + t.Errorf("IsProvider(%q) = true without the unstable opt-in", name) + } + } +} + +func TestAliasProviderCanonicalises(t *testing.T) { + withUnstable(t, true) + cases := map[string]string{ + "stackql_dynamic": DynamicProviderName, + "STACKQL_DYNAMIC": DynamicProviderName, + " stackql_iac ": IaCProviderName, + "StAcKqL_iAc": IaCProviderName, + "stackql_preview": "", + "stackql_dynamics": "", + "stackql_iac_extra": "", + } + for input, want := range cases { + got, ok := aliasProvider(input) + if want == "" { + if ok { + t.Errorf("aliasProvider(%q) = %q, want no match", input, got) + } + continue + } + if !ok || got != want { + t.Errorf("aliasProvider(%q) = %q, %v; want %q, true", input, got, ok, want) + } + } +} + +func TestAliasServices(t *testing.T) { + withUnstable(t, true) + dynamic := aliasServices(DynamicProviderName) + if len(dynamic) != 1 || dynamic[0] != graphService { + t.Errorf("aliasServices(dynamic) = %v, want [%s]", dynamic, graphService) + } + iac := aliasServices(IaCProviderName) + if len(iac) != 2 || iac[0] != blueprintService || iac[1] != convergeService { + t.Errorf("aliasServices(iac) = %v, want [%s %s]", iac, blueprintService, convergeService) + } +} + +func TestPopPredicateRemovesControlKeys(t *testing.T) { + params := map[string]string{ + collectionPredicate: "scratch", + "region": "us-east-1", + } + if got := popPredicate(params, collectionPredicate); got != "scratch" { + t.Errorf("popPredicate = %q, want %q", got, "scratch") + } + if _, still := params[collectionPredicate]; still { + t.Error("popPredicate left the control key in the scope") + } + if got := popPredicate(params, statePredicate); got != "" { + t.Errorf("popPredicate of an absent key = %q, want empty", got) + } + if len(params) != 1 || params["region"] != "us-east-1" { + t.Errorf("scope after popping = %v, want only the region", params) + } +} + +func TestBuildGraphMapsSpecOntoSDK(t *testing.T) { + spec := `{ + "addresses": ["stackql_unstable_aws.ec2.vpcs", "stackql_unstable_aws.ec2.subnets"], + "wirings": [{ + "to": "stackql_unstable_aws.ec2.subnets", + "inbound": [{"from": "stackql_unstable_aws.ec2.vpcs", "src": "VpcId", "as": "vpc_id"}], + "viaType": "golang_template_json_v0.1.0", + "viaProgram": "{\"Filter.1.Name\":\"vpc-id\"}", + "provides": ["Filter.1.Name", "Filter.1.Value.1"] + }], + "overrides": [{"address": "stackql_unstable_aws.ec2.vpcs", "objectKey": "$.items"}] + }` + graph, err := buildGraph(spec) + if err != nil { + t.Fatalf("buildGraph: %v", err) + } + if got := len(graph.Addresses()); got != 2 { + t.Fatalf("addresses = %d, want 2", got) + } + wirings := graph.Wirings() + if len(wirings) != 1 { + t.Fatalf("wirings = %d, want 1", len(wirings)) + } + wiring := wirings[0] + if wiring.To() != "stackql_unstable_aws.ec2.subnets" { + t.Errorf("wiring To = %q", wiring.To()) + } + inbound := wiring.Inbound() + if len(inbound) != 1 || inbound[0].Src() != "VpcId" || inbound[0].As() != "vpc_id" { + t.Errorf("inbound = %+v, want one VpcId -> vpc_id", inbound) + } + viaType, viaProgram := wiring.Via() + if viaType != "golang_template_json_v0.1.0" || viaProgram == "" { + t.Errorf("via = %q, %q", viaType, viaProgram) + } + if got := len(wiring.Provides()); got != 2 { + t.Errorf("provides = %d, want 2", got) + } + overrides := graph.Overrides() + if len(overrides) != 1 || overrides[0].ObjectKey() != "$.items" { + t.Errorf("overrides = %+v, want one $.items correction", overrides) + } +} + +// An identity wiring is the common case: a value already shaped for the +// consumer passes straight through, and NewInbound names it after its source. +func TestBuildGraphIdentityInboundTakesSourceName(t *testing.T) { + graph, err := buildGraph(`{ + "addresses": ["a.b.c", "d.e.f"], + "wirings": [{"to": "d.e.f", "inbound": [{"from": "a.b.c", "src": "VpcId"}]}] + }`) + if err != nil { + t.Fatalf("buildGraph: %v", err) + } + inbound := graph.Wirings()[0].Inbound()[0] + if inbound.As() != "VpcId" { + t.Errorf("As = %q, want the source name %q", inbound.As(), "VpcId") + } +} + +func TestBuildGraphRejectsMalformedSpec(t *testing.T) { + if _, err := buildGraph("{ not json"); err == nil { + t.Fatal("buildGraph accepted a malformed specification") + } +} + +func TestSpecResourcesMapsOntoSDK(t *testing.T) { + spec := `[{ + "key": "aws/ec2/vpc", + "provider": "aws", + "address": "ec2.vpcs", + "desired": {"CidrBlock": "10.42.0.0/16"}, + "params": {"TagSpecification.1.ResourceType": "vpc"}, + "identity": "line_items.VpcId", + "addressedBy": "VpcId", + "correlationParam": "TagSpecification.1.Tag.1.Value" + }, { + "key": "aws/ec2/subnet", + "provider": "aws", + "address": "ec2.subnets", + "desired": {"CidrBlock": "10.42.1.0/24"}, + "inbound": [{"From": "aws/ec2/vpc", "As": "VpcId"}], + "identity": "line_items.SubnetId", + "addressedBy": "SubnetId" + }]` + resources, err := specResources(spec) + if err != nil { + t.Fatalf("specResources: %v", err) + } + if len(resources) != 2 { + t.Fatalf("resources = %d, want 2", len(resources)) + } + vpc := resources[0] + if vpc.Key() != "aws/ec2/vpc" || vpc.Provider() != "aws" || vpc.Address() != "ec2.vpcs" { + t.Errorf("vpc addressing = %q %q %q", vpc.Key(), vpc.Provider(), vpc.Address()) + } + if vpc.Identity() != "line_items.VpcId" || vpc.AddressedBy() != "VpcId" { + t.Errorf("vpc identity = %q, addressedBy = %q", vpc.Identity(), vpc.AddressedBy()) + } + if vpc.CorrelationParam() != "TagSpecification.1.Tag.1.Value" { + t.Errorf("vpc correlation = %q", vpc.CorrelationParam()) + } + // Desired is opaque: nothing between the predicate and the wire parses it, + // so it must arrive as the bytes the caller wrote. + if string(vpc.Desired()) != `{"CidrBlock": "10.42.0.0/16"}` { + t.Errorf("vpc desired = %q, want the caller's bytes unaltered", vpc.Desired()) + } + subnet := resources[1] + arrivals := subnet.Inbound() + if len(arrivals) != 1 || arrivals[0].From != "aws/ec2/vpc" || arrivals[0].As != "VpcId" { + t.Errorf("subnet inbound = %+v, want one aws/ec2/vpc -> VpcId", arrivals) + } +} + +func TestSpecResourcesRejectsMalformedSpec(t *testing.T) { + if _, err := specResources("nonsense"); err == nil { + t.Fatal("specResources accepted a malformed specification") + } +} + +func TestBlueprintHandleIsSQLAddressable(t *testing.T) { + if got := blueprintHandle("aws-vpc-subnet"); got != "aws_vpc_subnet" { + t.Errorf("blueprintHandle = %q, want %q", got, "aws_vpc_subnet") + } +} + +// A blueprint is reachable by the handle as written and by the relation name it +// takes, so a caller who read SHOW RESOURCES can paste what they saw. +func TestBlueprintForAcceptsEitherSpelling(t *testing.T) { + for _, name := range []string{"aws-vpc-subnet", "aws_vpc_subnet", "AWS_VPC_SUBNET"} { + blueprint, ok := blueprintFor(name) + if !ok { + t.Errorf("blueprintFor(%q) found nothing", name) + continue + } + if blueprint.Handle() != "aws-vpc-subnet" { + t.Errorf("blueprintFor(%q).Handle() = %q", name, blueprint.Handle()) + } + } + if _, ok := blueprintFor("no-such-thing"); ok { + t.Error("blueprintFor resolved a handle that does not exist") + } +} + +func TestBlueprintColumnsMarkRequiredInputs(t *testing.T) { + blueprint, ok := blueprintFor("aws-vpc-subnet") + if !ok { + t.Fatal("aws-vpc-subnet blueprint is absent") + } + columns := blueprintColumns(blueprint) + if len(columns) != len(blueprint.Params()) { + t.Fatalf("columns = %d, want %d", len(columns), len(blueprint.Params())) + } + byName := make(map[string]column, len(columns)) + for _, col := range columns { + byName[col.name] = col + } + region, present := byName["region"] + if !present { + t.Fatal("region column is absent") + } + if region.dataType != "string" { + t.Errorf("region type = %q, want %q", region.dataType, "string") + } + if !strings.HasPrefix(region.description, "required; ") { + t.Errorf("region description = %q, want it marked required", region.description) + } + tags, present := byName["vpc_tags"] + if !present { + t.Fatal("vpc_tags column is absent") + } + if strings.HasPrefix(tags.description, "required; ") { + t.Errorf("vpc_tags description = %q, want it unmarked", tags.description) + } +} diff --git a/internal/stackql/intrinsic/dynamic.go b/internal/stackql/intrinsic/dynamic.go new file mode 100644 index 000000000..9efa0435d --- /dev/null +++ b/internal/stackql/intrinsic/dynamic.go @@ -0,0 +1,128 @@ +package intrinsic + +// stackql_dynamic presents omnisdk's multi-exchange query as a single relation. +// A document describes one provider and cannot state a relationship spanning +// two, so the caller states it: which exchanges take part, and what flows +// between them. That statement is the 'spec' predicate, and it maps onto the +// SDK's own constructors one field at a time. + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" +) + +// graphSpecDTO is the wire shape of the 'spec' predicate. It is omnisdk's Graph +// with the interfaces flattened into data, and nothing else: every field here +// is handed to a constructor unaltered. +type graphSpecDTO struct { + Addresses []string `json:"addresses"` + Wirings []wiringDTO `json:"wirings"` + Overrides []overrideDTO `json:"overrides"` +} + +// wiringDTO is everything arriving at one consumer, and how its inbox becomes +// its inputs. ViaType and ViaProgram are T_in; empty is identity, which is the +// common case. +type wiringDTO struct { + To string `json:"to"` + Inbound []inboundDTO `json:"inbound"` + ViaType string `json:"viaType"` + ViaProgram string `json:"viaProgram"` + Provides []string `json:"provides"` +} + +// inboundDTO is one value arriving at a consumer: an attribute a producer +// emits, landing in the consumer's inbox under a name of the caller's choosing. +type inboundDTO struct { + From string `json:"from"` + Src string `json:"src"` + As string `json:"as"` +} + +// overrideDTO corrects what a document says about one exchange's response, +// where it is wrong for this engine. +type overrideDTO struct { + Address string `json:"address"` + ObjectKey string `json:"objectKey"` + MediaType string `json:"mediaType"` + ProgramType string `json:"programType"` + ProgramBody string `json:"programBody"` +} + +// dynamicSelectFunc routes a SELECT over stackql_dynamic.graph.query. +func dynamicSelectFunc( + ctx queryContext, + node *sqlparser.Select, + service, resource string, +) (func() internaldto.ExecutorOutput, bool) { + if !strings.EqualFold(service, graphService) || !strings.EqualFold(resource, graphRelation) { + return nil, false + } + relation := fmt.Sprintf("%s.%s.%s", DynamicProviderName, graphService, graphRelation) + params, refusal := aliasPredicates(node, relation) + if refusal != nil { + return refusal, true + } + spec := popPredicate(params, specPredicate) + if strings.TrimSpace(spec) == "" { + return refuse(fmt.Errorf( + "relation '%s' needs a '%s' predicate naming the exchanges and their wiring", + relation, specPredicate)), true + } + return func() internaldto.ExecutorOutput { + graph, err := buildGraph(spec) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + plan, planErr := omnisdk.NewGraphQuery( + registryRoot(ctx), graph, aliasArgs(ctx, graphCloud(graph.Addresses()), params)) + if planErr != nil { + return internaldto.NewErroneousExecutorOutput(planErr) + } + return streamPlan(ctx, plan, graphRelation, node.SelectExprs) + }, true +} + +// buildGraph turns the 'spec' predicate into the SDK's graph. Validation is the +// SDK's: an edge naming an exchange the query does not run is caught there, +// where it can name the address. +func buildGraph(spec string) (omnisdk.Graph, error) { + var dtoSpec graphSpecDTO + if err := json.Unmarshal([]byte(spec), &dtoSpec); err != nil { + return nil, fmt.Errorf("intrinsic: '%s' is not a valid graph specification: %w", specPredicate, err) + } + wirings := make([]omnisdk.Wiring, 0, len(dtoSpec.Wirings)) + for _, wiring := range dtoSpec.Wirings { + inbound := make([]omnisdk.Inbound, 0, len(wiring.Inbound)) + for _, arrival := range wiring.Inbound { + inbound = append(inbound, omnisdk.NewInbound(arrival.From, arrival.Src, arrival.As)) + } + wirings = append(wirings, omnisdk.NewWiring( + wiring.To, inbound, wiring.ViaType, wiring.ViaProgram, wiring.Provides...)) + } + overrides := make([]omnisdk.Override, 0, len(dtoSpec.Overrides)) + for _, override := range dtoSpec.Overrides { + overrides = append(overrides, omnisdk.NewOverride( + override.Address, override.ObjectKey, override.MediaType, + override.ProgramType, override.ProgramBody)) + } + return omnisdk.NewGraph(dtoSpec.Addresses, wirings, overrides...) +} + +// graphCloud is the cloud whose credential a graph runs under. omnisdk takes +// one credential per run, so a graph spanning two clouds carries the first +// address's and leaves the rest to the canonical environment variables, which +// is the SDK's own fallback. +func graphCloud(addresses []string) string { + if len(addresses) == 0 { + return "" + } + cloud, _, _ := strings.Cut(strings.TrimPrefix(addresses[0], UnstablePrefix), ".") + return cloud +} diff --git a/internal/stackql/intrinsic/iac.go b/internal/stackql/intrinsic/iac.go new file mode 100644 index 000000000..0c4325ce9 --- /dev/null +++ b/internal/stackql/intrinsic/iac.go @@ -0,0 +1,230 @@ +package intrinsic + +// stackql_iac presents omnisdk's converge run as a relation, and the blueprints +// that render one as a catalogue. A run is issued as an imperative - these +// resources, under this collection name - and idempotence, ordering, locking and +// compensation are the SDK's problem, not stackql's. + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" +) + +// stateDirName is where the ledger and run journals live when a query names no +// state directory. It sits under the application root, which is already local +// disk: O_EXCL and link are unreliable on a network share. +const stateDirName = "iac" + +// resourceDTO is the wire shape of one entry in the 'resources' predicate. It +// is omnisdk's ManagedResource with the interface flattened into data. +type resourceDTO struct { + Key string `json:"key"` + Provider string `json:"provider"` + Address string `json:"address"` + Desired json.RawMessage `json:"desired"` + Params map[string]string `json:"params"` + Inbound []arrivalDTO `json:"inbound"` + ViaType string `json:"viaType"` + ViaProgram string `json:"viaProgram"` + Identity string `json:"identity"` + AddressedBy string `json:"addressedBy"` + CorrelationParam string `json:"correlationParam"` +} + +// arrivalDTO is one value reaching a resource from a sibling in the same +// collection. +type arrivalDTO struct { + From string `json:"from"` + As string `json:"as"` +} + +// iacSelectFunc routes a SELECT over stackql_iac. Opening the plan performs the +// run, so a select over the converge relation is a mutation in every sense +// except its syntax. +func iacSelectFunc( + ctx queryContext, + node *sqlparser.Select, + service, resource string, +) (func() internaldto.ExecutorOutput, bool) { + if strings.EqualFold(service, blueprintService) { + return blueprintSelectFunc(ctx, node, resource) + } + if !strings.EqualFold(service, convergeService) || !strings.EqualFold(resource, convergeRelation) { + return nil, false + } + relation := fmt.Sprintf("%s.%s.%s", IaCProviderName, convergeService, convergeRelation) + params, refusal := aliasPredicates(node, relation) + if refusal != nil { + return refusal, true + } + collection := popPredicate(params, collectionPredicate) + if strings.TrimSpace(collection) == "" { + return refuse(fmt.Errorf( + "relation '%s' needs a '%s' predicate; it is the ledger key prefix, the lease scope "+ + "and the correlation stamp, and it is how a later run addresses the same resources", + relation, collectionPredicate)), true + } + handle := popPredicate(params, blueprintPredicate) + spec := popPredicate(params, resourcesPredicate) + state := popPredicate(params, statePredicate) + runID := popPredicate(params, runIDPredicate) + if (handle == "") == (spec == "") { + return refuse(fmt.Errorf( + "relation '%s' needs exactly one of '%s' or '%s'; "+ + "a blueprint renders the resources, a specification states them", + relation, blueprintPredicate, resourcesPredicate)), true + } + if state == "" { + state = filepath.Join(ctx.GetRuntimeContext().ApplicationFilesRootPath, stateDirName) + } + return func() internaldto.ExecutorOutput { + resources, err := convergeResources(handle, spec, params) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + plan, planErr := omnisdk.Converge( + registryRoot(ctx), collection, state, runID, resources, + aliasArgs(ctx, convergeCloud(resources), params)) + if planErr != nil { + return internaldto.NewErroneousExecutorOutput(planErr) + } + return streamPlan(ctx, plan, convergeRelation, node.SelectExprs) + }, true +} + +// convergeResources renders what a run converges, from whichever of the two +// predicates was supplied. +func convergeResources( + handle, spec string, params map[string]string) ([]omnisdk.ManagedResource, error) { + if handle != "" { + return blueprintResources(handle, params) + } + return specResources(spec) +} + +// blueprintResources renders a named blueprint. Only the inputs it declares are +// passed: it rejects unknown ones rather than ignoring them, and the remaining +// predicates are the run's scope rather than the blueprint's. +func blueprintResources(handle string, params map[string]string) ([]omnisdk.ManagedResource, error) { + blueprint, ok := blueprintFor(handle) + if !ok { + return nil, fmt.Errorf( + "intrinsic: no blueprint '%s'; run SHOW RESOURCES IN %s.%s to list them", + handle, IaCProviderName, blueprintService) + } + inputs := make(map[string]string, len(params)) + for _, param := range blueprint.Params() { + if value, supplied := params[param.Name]; supplied { + inputs[param.Name] = value + } + } + return blueprint.Resources(inputs) +} + +// specResources reads the 'resources' predicate. +func specResources(spec string) ([]omnisdk.ManagedResource, error) { + var specs []resourceDTO + if err := json.Unmarshal([]byte(spec), &specs); err != nil { + return nil, fmt.Errorf( + "intrinsic: '%s' is not a valid resource specification: %w", resourcesPredicate, err) + } + out := make([]omnisdk.ManagedResource, 0, len(specs)) + for _, resource := range specs { + arrivals := make([]omnisdk.Arrival, 0, len(resource.Inbound)) + for _, arrival := range resource.Inbound { + arrivals = append(arrivals, omnisdk.Arrival{From: arrival.From, As: arrival.As}) + } + out = append(out, omnisdk.NewResource( + resource.Key, resource.Provider, resource.Address, resource.Desired, resource.Params, + arrivals, resource.ViaType, resource.ViaProgram, + resource.Identity, resource.AddressedBy, resource.CorrelationParam)) + } + return out, nil +} + +// convergeCloud is the cloud whose credential a run uses. omnisdk takes one +// credential per run, so a collection spanning two clouds carries the first +// resource's and leaves the rest to the canonical environment variables. +func convergeCloud(resources []omnisdk.ManagedResource) string { + if len(resources) == 0 { + return "" + } + return resources[0].Provider() +} + +// blueprintHandle is the relation name a blueprint takes. A handle is written +// with hyphens, which no unquoted SQL identifier can carry. +func blueprintHandle(name string) string { + return strings.ReplaceAll(name, "-", "_") +} + +// blueprintFor resolves a blueprint by handle, accepting either the handle as +// written or the relation name it takes. +func blueprintFor(name string) (omnisdk.Blueprint, bool) { + if blueprint, ok := omnisdk.BlueprintFor(name); ok { + return blueprint, true + } + for _, blueprint := range omnisdk.Blueprints() { + if strings.EqualFold(blueprintHandle(blueprint.Handle()), blueprintHandle(name)) { + return blueprint, true + } + } + return nil, false +} + +// blueprintTables presents the blueprints as relations, which is what makes +// them discoverable in the same shape as the query catalogue. +func blueprintTables() []table { + blueprints := omnisdk.Blueprints() + out := make([]table, 0, len(blueprints)) + for _, blueprint := range blueprints { + out = append(out, table{ + service: blueprintService, + name: blueprintHandle(blueprint.Handle()), + description: blueprint.Summary(), + }) + } + return out +} + +// blueprintColumns presents a blueprint's declared inputs as columns, so +// DESCRIBE answers "what does this deployment need". +func blueprintColumns(blueprint omnisdk.Blueprint) []column { + params := blueprint.Params() + out := make([]column, 0, len(params)) + for _, param := range params { + description := param.Description + if param.Required { + description = "required; " + description + } + out = append(out, column{ + name: param.Name, + dataType: param.Type.Name, + description: description, + }) + } + return out +} + +// blueprintSelectFunc refuses a select over a blueprint relation. A blueprint is +// a way of building a resource set, not a thing to read: naming it in a converge +// run is how it is applied. +func blueprintSelectFunc( + _ queryContext, + _ *sqlparser.Select, + resource string, +) (func() internaldto.ExecutorOutput, bool) { + return refuse(fmt.Errorf( + "'%s.%s.%s' renders a deployment rather than rows; "+ + "apply it with SELECT ... FROM %s.%s.%s WHERE %s = '%s' AND %s = ''", + IaCProviderName, blueprintService, resource, + IaCProviderName, convergeService, convergeRelation, + blueprintPredicate, resource, collectionPredicate)), true +} diff --git a/internal/stackql/intrinsic/intrinsic.go b/internal/stackql/intrinsic/intrinsic.go index 72a2daf7d..151ef5fe5 100644 --- a/internal/stackql/intrinsic/intrinsic.go +++ b/internal/stackql/intrinsic/intrinsic.go @@ -81,6 +81,9 @@ func IsProvider(name string) bool { if strings.EqualFold(strings.TrimSpace(name), ProviderName) { return true } + if _, isAlias := aliasProvider(name); isAlias { + return true + } _, isDoc := docProvider(name) return isDoc } @@ -136,6 +139,9 @@ func showFunc( currentProvider string, ) (func() internaldto.ExecutorOutput, bool) { extended := isExtended(node.Extended) + if fn, isAlias := showAliasFunc(ctx, node, currentProvider, extended); isAlias { + return fn, true + } switch strings.ToUpper(strings.TrimSpace(node.Type)) { case "SERVICES": provider := resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider) @@ -185,6 +191,9 @@ func describeTableFunc( node *sqlparser.DescribeTable, currentProvider string, ) (func() internaldto.ExecutorOutput, bool) { + if fn, isAlias := describeAliasTableFunc(ctx, node, currentProvider); isAlias { + return fn, true + } tbl, ok := lookupTable( node.Table.QualifierSecond.GetRawVal(), node.Table.Qualifier.GetRawVal(), diff --git a/internal/stackql/intrinsic/omnisdk.go b/internal/stackql/intrinsic/omnisdk.go index 95696e09a..1f83ebb69 100644 --- a/internal/stackql/intrinsic/omnisdk.go +++ b/internal/stackql/intrinsic/omnisdk.go @@ -319,6 +319,11 @@ func selectFunc( if !ok { return nil, false } + if alias, isAlias := aliasProvider( + resolveProvider(tableName.QualifierSecond.GetRawVal(), currentProvider)); isAlias { + return aliasSelectFunc(ctx, node, alias, + tableName.Qualifier.GetRawVal(), tableName.Name.GetRawVal()) + } if bundle, isDoc := docProvider( resolveProvider(tableName.QualifierSecond.GetRawVal(), currentProvider)); isDoc { return docSelectFunc(ctx, node, bundle, diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 5c7182d28..19f7d90ec 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -11120,3 +11120,226 @@ OTel Output Zero Row Statement Emits Only Completion ... ${expected} ... stdout=${CURDIR}${/}tmp${/}OTel-Output-Zero-Rows.tmp ... stderr=${CURDIR}${/}tmp${/}OTel-Output-Zero-Rows-stderr.tmp + +# =========================================================================== +# The "stackql_dynamic" and "stackql_iac" aliases present omnisdk's graph query +# and converge run as relations. Discovery is answered in process, and a run is +# specified by predicate rather than by column, so these assert the catalogue +# and the refusals; the runs themselves reach live clouds and are covered by +# omnisdk's own suite. +# =========================================================================== + +Dynamic Alias Show Services Returns Graph Service + ${preview} = Set Variable {"unstable":true} + Should StackQL Exec Inline Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... show services in stackql_dynamic; + ... id,name,title\nstackql_dynamic.graph,graph,graph + ... \-o\=csv + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Dynamic-Alias-Show-Services.tmp + ... stderr=${CURDIR}${/}tmp${/}Dynamic-Alias-Show-Services-stderr.tmp + +IaC Alias Show Services Returns Blueprints And Converge + ${preview} = Set Variable {"unstable":true} + Should StackQL Exec Inline Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... show services in stackql_iac; + ... id,name,title\nstackql_iac.blueprints,blueprints,blueprints\nstackql_iac.converge,converge,converge + ... \-o\=csv + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}IaC-Alias-Show-Services.tmp + ... stderr=${CURDIR}${/}tmp${/}IaC-Alias-Show-Services-stderr.tmp + +IaC Alias Show Resources Lists Blueprint Handles + [Documentation] A blueprint handle is written with hyphens, which no + ... unquoted SQL identifier can carry, so it is presented + ... under the relation name it takes. + ${preview} = Set Variable {"unstable":true} + Should StackQL Exec Inline Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... show resources in stackql_iac.blueprints; + ... name,id\naws_vpc_subnet,stackql_iac.blueprints.aws_vpc_subnet + ... \-o\=csv + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}IaC-Alias-Show-Resources.tmp + ... stderr=${CURDIR}${/}tmp${/}IaC-Alias-Show-Resources-stderr.tmp + +IaC Alias Describe Blueprint Reports Declared Inputs + ${preview} = Set Variable {"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... name,type + ... region,string + ... vpc_cidr,string + ... subnet_cidr,string + ... vpc_tags,object + ... subnet_tags,object + Should StackQL Exec Inline Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... describe stackql_iac.blueprints.aws_vpc_subnet; + ... ${expected} + ... \-o\=csv + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}IaC-Alias-Describe-Blueprint.tmp + ... stderr=${CURDIR}${/}tmp${/}IaC-Alias-Describe-Blueprint-stderr.tmp + +Dynamic Alias Select Without Spec Is Refused + ${preview} = Set Variable {"unstable":true} + ${expected} = Catenate SEPARATOR=${SPACE} + ... relation 'stackql_dynamic.graph.query' needs a 'spec' predicate + ... naming the exchanges and their wiring + Should StackQL Exec Inline Equal Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... select * from stackql_dynamic.graph.query; + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Dynamic-Alias-No-Spec.tmp + ... stderr=${CURDIR}${/}tmp${/}Dynamic-Alias-No-Spec-stderr.tmp + +IaC Alias Converge Without Collection Is Refused + ${preview} = Set Variable {"unstable":true} + ${query} = Catenate SEPARATOR=${SPACE} + ... select * from stackql_iac.converge.run + ... where blueprint = 'aws-vpc-subnet'; + ${expected} = Catenate SEPARATOR=${SPACE} + ... relation 'stackql_iac.converge.run' needs a 'collection' predicate; + ... it is the ledger key prefix, the lease scope and the correlation + ... stamp, and it is how a later run addresses the same resources + Should StackQL Exec Inline Equal Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}IaC-Alias-No-Collection.tmp + ... stderr=${CURDIR}${/}tmp${/}IaC-Alias-No-Collection-stderr.tmp + +IaC Alias Converge Needs Exactly One Of Blueprint Or Resources + ${preview} = Set Variable {"unstable":true} + ${query} = Catenate SEPARATOR=${SPACE} + ... select * from stackql_iac.converge.run where collection = 'scratch'; + ${expected} = Catenate SEPARATOR=${SPACE} + ... relation 'stackql_iac.converge.run' needs exactly one of 'blueprint' + ... or 'resources'; a blueprint renders the resources, a specification + ... states them + Should StackQL Exec Inline Equal Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}IaC-Alias-Ambiguous-Source.tmp + ... stderr=${CURDIR}${/}tmp${/}IaC-Alias-Ambiguous-Source-stderr.tmp + +IaC Alias Unknown Blueprint Names The Catalogue + ${preview} = Set Variable {"unstable":true} + ${query} = Catenate SEPARATOR=${SPACE} + ... select * from stackql_iac.converge.run + ... where collection = 'scratch' and blueprint = 'no-such-thing'; + ${expected} = Catenate SEPARATOR=${SPACE} + ... intrinsic: no blueprint 'no-such-thing'; run SHOW RESOURCES IN + ... stackql_iac.blueprints to list them + Should StackQL Exec Inline Equal Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}IaC-Alias-Unknown-Blueprint.tmp + ... stderr=${CURDIR}${/}tmp${/}IaC-Alias-Unknown-Blueprint-stderr.tmp + +Dynamic Alias Wiring Outside Graph Is Refused By Omnisdk + [Documentation] The 'spec' predicate maps onto omnisdk's own constructors + ... one field at a time, so its validation is what a caller + ... sees: an edge naming an exchange the query does not run is + ... caught where it can name the address. + ${preview} = Set Variable {"unstable":true} + ${spec} = Catenate SEPARATOR= + ... {"addresses":["stackql_unstable_aws.ec2.vpcs"],"wirings": + ... [{"to":"stackql_unstable_aws.ec2.subnets","inbound": + ... [{"from":"stackql_unstable_aws.ec2.vpcs","src":"VpcId","as":"vpc_id"}]}]} + ${query} = Catenate SEPARATOR=${SPACE} + ... select * from stackql_dynamic.graph.query where spec = '${spec}'; + ${expected} = Catenate SEPARATOR=${SPACE} + ... omnisdk: wiring targets "stackql_unstable_aws.ec2.subnets", + ... which the graph does not include + Should StackQL Exec Inline Equal Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Dynamic-Alias-Wiring-Outside-Graph.tmp + ... stderr=${CURDIR}${/}tmp${/}Dynamic-Alias-Wiring-Outside-Graph-stderr.tmp + +IaC Alias Resource Without Provider Is Refused By Omnisdk + [Documentation] The 'resources' predicate maps onto omnisdk's NewResource + ... one field at a time; this asserts a malformed entry is + ... caught by the SDK before any effect is attempted. + ${preview} = Set Variable {"unstable":true} + ${query} = Catenate SEPARATOR=${SPACE} + ... select * from stackql_iac.converge.run where collection = 'scratch' + ... and resources = '[{"key":"a/b/c"}]'; + Should StackQL Exec Inline Equal Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... omnisdk: resource "a/b/c" names no provider or address + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}IaC-Alias-Resource-No-Provider.tmp + ... stderr=${CURDIR}${/}tmp${/}IaC-Alias-Resource-No-Provider-stderr.tmp